diff --git a/config.php.sample b/config.php.sample index a2c5801f45..cc2e4c3c14 100644 --- a/config.php.sample +++ b/config.php.sample @@ -112,6 +112,14 @@ $config['sphinx']['port'] = 3312; #background. See the README for details. #$config['queue']['enabled'] = true; +#Queue subsystem +#subsystems: internal (default) or stomp +#using stomp requires an external message queue server +#$config['queue']['subsystem'] = 'stomp'; +#$config['queue']['stomp_server'] = 'tcp://localhost:61613'; +#use different queue_basename for each laconica instance managed by the server +#$config['queue']['queue_basename'] = 'laconica'; + #The following customise the behaviour of the various daemons: #$config['daemon']['piddir'] = '/var/run'; #$config['daemon']['user'] = false; diff --git a/extlib/Stomp.php b/extlib/Stomp.php new file mode 100644 index 0000000000..9e1c97b3b3 --- /dev/null +++ b/extlib/Stomp.php @@ -0,0 +1,594 @@ + + * @author Dejan Bosanac + * @author Michael Caplan + * @version $Revision: 43 $ + */ +class Stomp +{ + /** + * Perform request synchronously + * + * @var boolean + */ + public $sync = false; + + /** + * Default prefetch size + * + * @var int + */ + public $prefetchSize = 1; + + /** + * Client id used for durable subscriptions + * + * @var string + */ + public $clientId = null; + + protected $_brokerUri = null; + protected $_socket = null; + protected $_hosts = array(); + protected $_params = array(); + protected $_subscriptions = array(); + protected $_defaultPort = 61613; + protected $_currentHost = - 1; + protected $_attempts = 10; + protected $_username = ''; + protected $_password = ''; + protected $_sessionId; + protected $_read_timeout_seconds = 60; + protected $_read_timeout_milliseconds = 0; + + /** + * Constructor + * + * @param string $brokerUri Broker URL + * @throws Stomp_Exception + */ + public function __construct ($brokerUri) + { + $this->_brokerUri = $brokerUri; + $this->_init(); + } + /** + * Initialize connection + * + * @throws Stomp_Exception + */ + protected function _init () + { + $pattern = "|^(([a-zA-Z]+)://)+\(*([a-zA-Z0-9\.:/i,-]+)\)*\??([a-zA-Z0-9=]*)$|i"; + if (preg_match($pattern, $this->_brokerUri, $regs)) { + $scheme = $regs[2]; + $hosts = $regs[3]; + $params = $regs[4]; + if ($scheme != "failover") { + $this->_processUrl($this->_brokerUri); + } else { + $urls = explode(",", $hosts); + foreach ($urls as $url) { + $this->_processUrl($url); + } + } + if ($params != null) { + parse_str($params, $this->_params); + } + } else { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception("Bad Broker URL {$this->_brokerUri}"); + } + } + /** + * Process broker URL + * + * @param string $url Broker URL + * @throws Stomp_Exception + * @return boolean + */ + protected function _processUrl ($url) + { + $parsed = parse_url($url); + if ($parsed) { + array_push($this->_hosts, array($parsed['host'] , $parsed['port'] , $parsed['scheme'])); + } else { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception("Bad Broker URL $url"); + } + } + /** + * Make socket connection to the server + * + * @throws Stomp_Exception + */ + protected function _makeConnection () + { + if (count($this->_hosts) == 0) { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception("No broker defined"); + } + + // force disconnect, if previous established connection exists + $this->disconnect(); + + $i = $this->_currentHost; + $att = 0; + $connected = false; + while (! $connected && $att ++ < $this->_attempts) { + if (isset($this->_params['randomize']) && $this->_params['randomize'] == 'true') { + $i = rand(0, count($this->_hosts) - 1); + } else { + $i = ($i + 1) % count($this->_hosts); + } + $broker = $this->_hosts[$i]; + $host = $broker[0]; + $port = $broker[1]; + $scheme = $broker[2]; + if ($port == null) { + $port = $this->_defaultPort; + } + if ($this->_socket != null) { + fclose($this->_socket); + $this->_socket = null; + } + $this->_socket = @fsockopen($scheme . '://' . $host, $port); + if (!is_resource($this->_socket) && $att >= $this->_attempts && !array_key_exists($i + 1, $this->_hosts)) { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception("Could not connect to $host:$port ($att/{$this->_attempts})"); + } else if (is_resource($this->_socket)) { + $connected = true; + $this->_currentHost = $i; + break; + } + } + if (! $connected) { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception("Could not connect to a broker"); + } + } + /** + * Connect to server + * + * @param string $username + * @param string $password + * @return boolean + * @throws Stomp_Exception + */ + public function connect ($username = '', $password = '') + { + $this->_makeConnection(); + if ($username != '') { + $this->_username = $username; + } + if ($password != '') { + $this->_password = $password; + } + $headers = array('login' => $this->_username , 'passcode' => $this->_password); + if ($this->clientId != null) { + $headers["client-id"] = $this->clientId; + } + $frame = new Stomp_Frame("CONNECT", $headers); + $this->_writeFrame($frame); + $frame = $this->readFrame(); + if ($frame instanceof Stomp_Frame && $frame->command == 'CONNECTED') { + $this->_sessionId = $frame->headers["session"]; + return true; + } else { + require_once 'Stomp/Exception.php'; + if ($frame instanceof Stomp_Frame) { + throw new Stomp_Exception("Unexpected command: {$frame->command}", 0, $frame->body); + } else { + throw new Stomp_Exception("Connection not acknowledged"); + } + } + } + + /** + * Check if client session has ben established + * + * @return boolean + */ + public function isConnected () + { + return !empty($this->_sessionId) && is_resource($this->_socket); + } + /** + * Current stomp session ID + * + * @return string + */ + public function getSessionId() + { + return $this->_sessionId; + } + /** + * Send a message to a destination in the messaging system + * + * @param string $destination Destination queue + * @param string|Stomp_Frame $msg Message + * @param array $properties + * @param boolean $sync Perform request synchronously + * @return boolean + */ + public function send ($destination, $msg, $properties = null, $sync = null) + { + if ($msg instanceof Stomp_Frame) { + $msg->headers['destination'] = $destination; + $msg->headers = array_merge($msg->headers, $properties); + $frame = $msg; + } else { + $headers = $properties; + $headers['destination'] = $destination; + $frame = new Stomp_Frame('SEND', $headers, $msg); + } + $this->_prepareReceipt($frame, $sync); + $this->_writeFrame($frame); + return $this->_waitForReceipt($frame, $sync); + } + /** + * Prepair frame receipt + * + * @param Stomp_Frame $frame + * @param boolean $sync + */ + protected function _prepareReceipt (Stomp_Frame $frame, $sync) + { + $receive = $this->sync; + if ($sync !== null) { + $receive = $sync; + } + if ($receive == true) { + $frame->headers['receipt'] = md5(microtime()); + } + } + /** + * Wait for receipt + * + * @param Stomp_Frame $frame + * @param boolean $sync + * @return boolean + * @throws Stomp_Exception + */ + protected function _waitForReceipt (Stomp_Frame $frame, $sync) + { + + $receive = $this->sync; + if ($sync !== null) { + $receive = $sync; + } + if ($receive == true) { + $id = (isset($frame->headers['receipt'])) ? $frame->headers['receipt'] : null; + if ($id == null) { + return true; + } + $frame = $this->readFrame(); + if ($frame instanceof Stomp_Frame && $frame->command == 'RECEIPT') { + if ($frame->headers['receipt-id'] == $id) { + return true; + } else { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception("Unexpected receipt id {$frame->headers['receipt-id']}", 0, $frame->body); + } + } else { + require_once 'Stomp/Exception.php'; + if ($frame instanceof Stomp_Frame) { + throw new Stomp_Exception("Unexpected command {$frame->command}", 0, $frame->body); + } else { + throw new Stomp_Exception("Receipt not received"); + } + } + } + return true; + } + /** + * Register to listen to a given destination + * + * @param string $destination Destination queue + * @param array $properties + * @param boolean $sync Perform request synchronously + * @return boolean + * @throws Stomp_Exception + */ + public function subscribe ($destination, $properties = null, $sync = null) + { + $headers = array('ack' => 'client'); + $headers['activemq.prefetchSize'] = $this->prefetchSize; + if ($this->clientId != null) { + $headers["activemq.subcriptionName"] = $this->clientId; + } + if (isset($properties)) { + foreach ($properties as $name => $value) { + $headers[$name] = $value; + } + } + $headers['destination'] = $destination; + $frame = new Stomp_Frame('SUBSCRIBE', $headers); + $this->_prepareReceipt($frame, $sync); + $this->_writeFrame($frame); + if ($this->_waitForReceipt($frame, $sync) == true) { + $this->_subscriptions[$destination] = $properties; + return true; + } else { + return false; + } + } + /** + * Remove an existing subscription + * + * @param string $destination + * @param array $properties + * @param boolean $sync Perform request synchronously + * @return boolean + * @throws Stomp_Exception + */ + public function unsubscribe ($destination, $properties = null, $sync = null) + { + $headers = array(); + if (isset($properties)) { + foreach ($properties as $name => $value) { + $headers[$name] = $value; + } + } + $headers['destination'] = $destination; + $frame = new Stomp_Frame('UNSUBSCRIBE', $headers); + $this->_prepareReceipt($frame, $sync); + $this->_writeFrame($frame); + if ($this->_waitForReceipt($frame, $sync) == true) { + unset($this->_subscriptions[$destination]); + return true; + } else { + return false; + } + } + /** + * Start a transaction + * + * @param string $transactionId + * @param boolean $sync Perform request synchronously + * @return boolean + * @throws Stomp_Exception + */ + public function begin ($transactionId = null, $sync = null) + { + $headers = array(); + if (isset($transactionId)) { + $headers['transaction'] = $transactionId; + } + $frame = new Stomp_Frame('BEGIN', $headers); + $this->_prepareReceipt($frame, $sync); + $this->_writeFrame($frame); + return $this->_waitForReceipt($frame, $sync); + } + /** + * Commit a transaction in progress + * + * @param string $transactionId + * @param boolean $sync Perform request synchronously + * @return boolean + * @throws Stomp_Exception + */ + public function commit ($transactionId = null, $sync = null) + { + $headers = array(); + if (isset($transactionId)) { + $headers['transaction'] = $transactionId; + } + $frame = new Stomp_Frame('COMMIT', $headers); + $this->_prepareReceipt($frame, $sync); + $this->_writeFrame($frame); + return $this->_waitForReceipt($frame, $sync); + } + /** + * Roll back a transaction in progress + * + * @param string $transactionId + * @param boolean $sync Perform request synchronously + */ + public function abort ($transactionId = null, $sync = null) + { + $headers = array(); + if (isset($transactionId)) { + $headers['transaction'] = $transactionId; + } + $frame = new Stomp_Frame('ABORT', $headers); + $this->_prepareReceipt($frame, $sync); + $this->_writeFrame($frame); + return $this->_waitForReceipt($frame, $sync); + } + /** + * Acknowledge consumption of a message from a subscription + * Note: This operation is always asynchronous + * + * @param string|Stomp_Frame $messageMessage ID + * @param string $transactionId + * @return boolean + * @throws Stomp_Exception + */ + public function ack ($message, $transactionId = null) + { + if ($message instanceof Stomp_Frame) { + $frame = new Stomp_Frame('ACK', $message->headers); + $this->_writeFrame($frame); + return true; + } else { + $headers = array(); + if (isset($transactionId)) { + $headers['transaction'] = $transactionId; + } + $headers['message-id'] = $message; + $frame = new Stomp_Frame('ACK', $headers); + $this->_writeFrame($frame); + return true; + } + } + /** + * Graceful disconnect from the server + * + */ + public function disconnect () + { + $header = array(); + + if ($this->clientId != null) { + $headers["client-id"] = $this->clientId; + } + + if (is_resource($this->_socket)) { + $this->_writeFrame(new Stomp_Frame('DISCONNECT', $headers)); + fclose($this->_socket); + } + $this->_socket = null; + $this->_sessionId = null; + $this->_currentHost = -1; + $this->_subscriptions = array(); + $this->_username = ''; + $this->_password = ''; + } + /** + * Write frame to server + * + * @param Stomp_Frame $stompFrame + */ + protected function _writeFrame (Stomp_Frame $stompFrame) + { + if (!is_resource($this->_socket)) { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception('Socket connection hasn\'t been established'); + } + + $data = $stompFrame->__toString(); + $r = fwrite($this->_socket, $data, strlen($data)); + if ($r === false || $r == 0) { + $this->_reconnect(); + $this->_writeFrame($stompFrame); + } + } + + /** + * Set timeout to wait for content to read + * + * @param int $seconds_to_wait Seconds to wait for a frame + * @param int $milliseconds Milliseconds to wait for a frame + */ + public function setReadTimeout($seconds, $milliseconds = 0) + { + $this->_read_timeout_seconds = $seconds; + $this->_read_timeout_milliseconds = $milliseconds; + } + + /** + * Read responce frame from server + * + * @return Stomp_Frame|Stomp_Message_Map|boolean False when no frame to read + */ + public function readFrame () + { + if (!$this->hasFrameToRead()) { + return false; + } + + $rb = 1024; + $data = ''; + do { + $read = fgets($this->_socket, $rb); + if ($read === false) { + $this->_reconnect(); + return $this->readFrame(); + } + $data .= $read; + $len = strlen($data); + } while (($len < 2 || ! ($data[$len - 2] == "\x00" && $data[$len - 1] == "\n"))); + + list ($header, $body) = explode("\n\n", $data, 2); + $header = explode("\n", $header); + $headers = array(); + $command = null; + foreach ($header as $v) { + if (isset($command)) { + list ($name, $value) = explode(':', $v, 2); + $headers[$name] = $value; + } else { + $command = $v; + } + } + $frame = new Stomp_Frame($command, $headers, trim($body)); + if (isset($frame->headers['amq-msg-type']) && $frame->headers['amq-msg-type'] == 'MapMessage') { + require_once 'Stomp/Message/Map.php'; + return new Stomp_Message_Map($frame); + } else { + return $frame; + } + } + + /** + * Check if there is a frame to read + * + * @return boolean + */ + public function hasFrameToRead() + { + $read = array($this->_socket); + $write = null; + $except = null; + + $has_frame_to_read = stream_select($read, $write, $except, $this->_read_timeout_seconds, $this->_read_timeout_milliseconds); + + if ($has_frame_to_read === false) { + throw new Stomp_Exception('Check failed to determin if the socket is readable'); + } else if ($has_frame_to_read > 0) { + return true; + } else { + return false; + } + } + + /** + * Reconnects and renews subscriptions (if there were any) + * Call this method when you detect connection problems + */ + protected function _reconnect () + { + $subscriptions = $this->_subscriptions; + + $this->connect($this->_username, $this->_password); + foreach ($subscriptions as $dest => $properties) { + $this->subscribe($dest, $properties); + } + } + /** + * Graceful object desruction + * + */ + public function __destruct() + { + $this->disconnect(); + } +} +?> diff --git a/extlib/Stomp/Exception.php b/extlib/Stomp/Exception.php new file mode 100644 index 0000000000..e6870bc15d --- /dev/null +++ b/extlib/Stomp/Exception.php @@ -0,0 +1,57 @@ + + * @version $Revision: 23 $ + */ +class Stomp_Exception extends Exception +{ + protected $_details; + + /** + * Constructor + * + * @param string $message Error message + * @param int $code Error code + * @param string $details Stomp server error details + */ + public function __construct($message = null, $code = 0, $details = '') + { + $this->_details = $details; + + parent::__construct($message, $code); + } + + /** + * Stomp server error details + * + * @return string + */ + public function getDetails() + { + return $this->_details; + } +} +?> \ No newline at end of file diff --git a/extlib/Stomp/Frame.php b/extlib/Stomp/Frame.php new file mode 100644 index 0000000000..dc59c1cb7f --- /dev/null +++ b/extlib/Stomp/Frame.php @@ -0,0 +1,80 @@ + + * @author Dejan Bosanac + * @author Michael Caplan + * @version $Revision: 36 $ + */ +class Stomp_Frame +{ + public $command; + public $headers = array(); + public $body; + + /** + * Constructor + * + * @param string $command + * @param array $headers + * @param string $body + */ + public function __construct ($command = null, $headers = null, $body = null) + { + $this->_init($command, $headers, $body); + } + + protected function _init ($command = null, $headers = null, $body = null) + { + $this->command = $command; + if ($headers != null) { + $this->headers = $headers; + } + $this->body = $body; + + if ($this->command == 'ERROR') { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception($this->headers['message'], 0, $this->body); + } + } + + /** + * Convert frame to transportable string + * + * @return string + */ + public function __toString() + { + $data = $this->command . "\n"; + + foreach ($this->headers as $name => $value) { + $data .= $name . ": " . $value . "\n"; + } + + $data .= "\n"; + $data .= $this->body; + return $data .= "\x00\n"; + } +} +?> \ No newline at end of file diff --git a/extlib/Stomp/Message.php b/extlib/Stomp/Message.php new file mode 100644 index 0000000000..6bcad3efd9 --- /dev/null +++ b/extlib/Stomp/Message.php @@ -0,0 +1,37 @@ + + * @version $Revision: 23 $ + */ +class Stomp_Message extends Stomp_Frame +{ + public function __construct ($body, $headers = null) + { + $this->_init("SEND", $headers, $body); + } +} +?> \ No newline at end of file diff --git a/extlib/Stomp/Message/Bytes.php b/extlib/Stomp/Message/Bytes.php new file mode 100644 index 0000000000..c75f23e43a --- /dev/null +++ b/extlib/Stomp/Message/Bytes.php @@ -0,0 +1,47 @@ + + * @version $Revision: 23 $ + */ +class Stomp_Message_Bytes extends Stomp_Message +{ + /** + * Constructor + * + * @param string $body + * @param array $headers + */ + function __construct ($body, $headers = null) + { + $this->_init("SEND", $headers, $body); + if ($this->headers == null) { + $this->headers = array(); + } + $this->headers['content-length'] = count($body); + } +} +?> \ No newline at end of file diff --git a/extlib/Stomp/Message/Map.php b/extlib/Stomp/Message/Map.php new file mode 100644 index 0000000000..288456a849 --- /dev/null +++ b/extlib/Stomp/Message/Map.php @@ -0,0 +1,55 @@ + + * @version $Revision: 23 $ + */ +class Stomp_Message_Map extends Stomp_Message +{ + public $map; + + /** + * Constructor + * + * @param Stomp_Frame|string $msg + * @param array $headers + */ + function __construct ($msg, $headers = null) + { + if ($msg instanceof Stomp_Frame) { + $this->_init($msg->command, $msg->headers, $msg->body); + $this->map = json_decode($msg->body); + } else { + $this->_init("SEND", $headers, $msg); + if ($this->headers == null) { + $this->headers = array(); + } + $this->headers['amq-msg-type'] = 'MapMessage'; + $this->body = json_encode($msg); + } + } +} +?> \ No newline at end of file diff --git a/lib/queuehandler.php b/lib/queuehandler.php index 9ce9e32b3b..752bbc6d26 100644 --- a/lib/queuehandler.php +++ b/lib/queuehandler.php @@ -75,15 +75,9 @@ class QueueHandler extends Daemon return true; } - function run() - { - if (!$this->start()) { - return false; - } - $this->log(LOG_INFO, 'checking for queued notices'); - $transport = $this->transport(); + function db_dispatch() { do { - $qi = Queue_item::top($transport); + $qi = Queue_item::top($this->transport()); if ($qi) { $this->log(LOG_INFO, 'Got item enqueued '.common_exact_date($qi->created)); $notice = Notice::staticGet($qi->notice_id); @@ -115,6 +109,68 @@ class QueueHandler extends Daemon $this->idle(5); } } while (true); + } + + function stomp_dispatch() { + require("Stomp.php"); + $con = new Stomp(common_config('queue','stomp_server')); + if (!$con->connect()) { + $this->log(LOG_ERR, 'Failed to connect to queue server'); + return false; + } + $queue_basename = common_config('queue','queue_basename'); + // subscribe to the relevant queue (format: basename-transport) + $con->subscribe('/queue/'.$queue_basename.'-'.$this->transport()); + + do { + $frame = $con->readFrame(); + if ($frame) { + $this->log(LOG_INFO, 'Got item enqueued '.common_exact_date($frame->headers['created'])); + + // XXX: Now the queue handler receives only the ID of the + // notice, and it has to get it from the DB + // A massive improvement would be avoid DB query by transmitting + // all the notice details via queue server... + $notice = Notice::staticGet($frame->body); + + if ($notice) { + $this->log(LOG_INFO, 'broadcasting notice ID = ' . $notice->id); + $result = $this->handle_notice($notice); + if ($result) { + // if the msg has been handled positively, ack it + // and the queue server will remove it from the queue + $con->ack($frame); + $this->log(LOG_INFO, 'finished broadcasting notice ID = ' . $notice->id); + } + else { + // no ack + $this->log(LOG_WARNING, 'Failed broadcast for notice ID = ' . $notice->id); + } + $notice->free(); + unset($notice); + $notice = null; + } else { + $this->log(LOG_WARNING, 'queue item for notice that does not exist'); + } + } + } while (true); + + $con->disconnect(); + } + + + function run() + { + if (!$this->start()) { + return false; + } + $this->log(LOG_INFO, 'checking for queued notices'); + if (common_config('queue','subsystem') == 'stomp') { + $this->stomp_dispatch(); + } + else { + $this->db_dispatch(); + } if (!$this->finish()) { return false; } @@ -143,4 +199,4 @@ class QueueHandler extends Daemon common_log($level, $this->class_name() . ' ('. $this->get_id() .'): '.$msg); } } - \ No newline at end of file + diff --git a/lib/util.php b/lib/util.php index c5a092f630..8ef19f5796 100644 --- a/lib/util.php +++ b/lib/util.php @@ -1138,18 +1138,44 @@ function common_twitter_broadcast($notice, $flink) function common_enqueue_notice($notice) { - foreach (array('jabber', 'omb', 'sms', 'public') as $transport) { - $qi = new Queue_item(); - $qi->notice_id = $notice->id; - $qi->transport = $transport; - $qi->created = $notice->created; - $result = $qi->insert(); - if (!$result) { - $last_error = &PEAR::getStaticProperty('DB_DataObject','lastError'); - common_log(LOG_ERR, 'DB error inserting queue item: ' . $last_error->message); - return false; + if (common_config('queue','subsystem') == 'stomp') { + // use an external message queue system via STOMP + require_once("Stomp.php"); + $con = new Stomp(common_config('queue','stomp_server')); + if (!$con->connect()) { + common_log(LOG_ERR, 'Failed to connect to queue server'); + return false; + } + $queue_basename = common_config('queue','queue_basename'); + foreach (array('jabber', 'omb', 'sms', 'public') as $transport) { + if (!$con->send( + '/queue/'.$queue_basename.'-'.$transport, // QUEUE + $notice->id, // BODY of the message + array ( // HEADERS of the msg + 'created' => $notice->created + ))) { + common_log(LOG_ERR, 'Error sending to '.$transport.' queue'); + return false; + } + common_log(LOG_DEBUG, 'complete remote queueing notice ID = ' . $notice->id . ' for ' . $transport); + } + $result = true; + } + else { + // in any other case, 'internal' + foreach (array('jabber', 'omb', 'sms', 'public') as $transport) { + $qi = new Queue_item(); + $qi->notice_id = $notice->id; + $qi->transport = $transport; + $qi->created = $notice->created; + $result = $qi->insert(); + if (!$result) { + $last_error = &PEAR::getStaticProperty('DB_DataObject','lastError'); + common_log(LOG_ERR, 'DB error inserting queue item: ' . $last_error->message); + return false; + } + common_log(LOG_DEBUG, 'complete queueing notice ID = ' . $notice->id . ' for ' . $transport); } - common_log(LOG_DEBUG, 'complete queueing notice ID = ' . $notice->id . ' for ' . $transport); } return $result; }