Stomp queue restructuring for mass scalability:

- Multiplexing queues into groups and for multiple sites.
- Sharing vs breakout configurable per site and per queue via $config['queue']['breakout']
- Detect how many times a message is redelivered, discard if it's killed too many daemons
 - count configurable with $config['queue']['max_retries']
 - can dump the items to files in $config['queue']['dead_letter_dir']

Queue daemon memory & resource leak fixes:
- avoid unnecessary reconnections to memcached server (switch persistent connections back in on second initialization, assuming it's child process)
- monkey-patch for leaky .ini loads in DB_DataObject::databaseStructure() - was leaking 200k per active switch
- applied leak fixes to Status_network as well, using intermediate base Safe_DataObject for both it and Memcache_DataObject

Misc queue fixes:
- correct handling of child processes exiting due to signal termination instead of regular exit
- shutdown instead of infinite respawn loop if we're already past the soft memory limit at startup
- Added --all option for xmppdaemon... still opens one xmpp connection per site that has xmpp active

Cache updates:
- add Cache::increment() method with native support for memcached atomic increment

classes/Memcached_DataObject.php

@@ -19,57 +19,8 @@
 
 if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); }
 
-class Memcached_DataObject extends DB_DataObject
+class Memcached_DataObject extends Safe_DataObject
 {
-    /**
-     * Destructor to free global memory resources associated with
-     * this data object when it's unset or goes out of scope.
-     * DB_DataObject doesn't do this yet by itself.
-     */
-
-    function __destruct()
-    {
-        $this->free();
-        if (method_exists('DB_DataObject', '__destruct')) {
-            parent::__destruct();
-        }
-    }
-
-    /**
-     * Magic function called at serialize() time.
-     *
-     * We use this to drop a couple process-specific references
-     * from DB_DataObject which can cause trouble in future
-     * processes.
-     *
-     * @return array of variable names to include in serialization.
-     */
-    function __sleep()
-    {
-        $vars = array_keys(get_object_vars($this));
-        $skip = array('_DB_resultid', '_link_loaded');
-        return array_diff($vars, $skip);
-    }
-
-    /**
-     * Magic function called at unserialize() time.
-     *
-     * Clean out some process-specific variables which might
-     * be floating around from a previous process's cached
-     * objects.
-     *
-     * Old cached objects may still have them.
-     */
-    function __wakeup()
-    {
-        // Refers to global state info from a previous process.
-        // Clear this out so we don't accidentally break global
-        // state in *this* process.
-        $this->_DB_resultid = null;
-        // We don't have any local DBO refs, so clear these out.
-        $this->_link_loaded = false;
-    }
-
     /**
      * Wrapper for DB_DataObject's static lookup using memcached
      * as backing instead of an in-process cache array.
@@ -579,3 +530,4 @@ class Memcached_DataObject extends DB_DataObject
         return $c->set($cacheKey, $value);
     }
 }
+

classes/Safe_DataObject.php

@@ -0,0 +1,250 @@
+<?php
+/*
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); }
+
+/**
+ * Extended DB_DataObject to improve a few things:
+ * - free global resources from destructor
+ * - remove bogus global references from serialized objects
+ * - don't leak memory when loading already-used .ini files
+ *   (eg when using the same schema on thousands of databases)
+ */
+class Safe_DataObject extends DB_DataObject
+{
+    /**
+     * Destructor to free global memory resources associated with
+     * this data object when it's unset or goes out of scope.
+     * DB_DataObject doesn't do this yet by itself.
+     */
+
+    function __destruct()
+    {
+        $this->free();
+        if (method_exists('DB_DataObject', '__destruct')) {
+            parent::__destruct();
+        }
+    }
+
+    /**
+     * Magic function called at serialize() time.
+     *
+     * We use this to drop a couple process-specific references
+     * from DB_DataObject which can cause trouble in future
+     * processes.
+     *
+     * @return array of variable names to include in serialization.
+     */
+    function __sleep()
+    {
+        $vars = array_keys(get_object_vars($this));
+        $skip = array('_DB_resultid', '_link_loaded');
+        return array_diff($vars, $skip);
+    }
+
+    /**
+     * Magic function called at unserialize() time.
+     *
+     * Clean out some process-specific variables which might
+     * be floating around from a previous process's cached
+     * objects.
+     *
+     * Old cached objects may still have them.
+     */
+    function __wakeup()
+    {
+        // Refers to global state info from a previous process.
+        // Clear this out so we don't accidentally break global
+        // state in *this* process.
+        $this->_DB_resultid = null;
+        // We don't have any local DBO refs, so clear these out.
+        $this->_link_loaded = false;
+    }
+
+
+    /**
+     * Work around memory-leak bugs...
+     * Had to copy-paste the whole function in order to patch a couple lines of it.
+     * Would be nice if this code was better factored.
+     *     
+     * @param optional string  name of database to assign / read
+     * @param optional array   structure of database, and keys
+     * @param optional array  table links
+     *
+     * @access public
+     * @return true or PEAR:error on wrong paramenters.. or false if no file exists..
+     *              or the array(tablename => array(column_name=>type)) if called with 1 argument.. (databasename)
+     */
+    function databaseStructure()
+    {
+
+        global $_DB_DATAOBJECT;
+        
+        // Assignment code 
+        
+        if ($args = func_get_args()) {
+        
+            if (count($args) == 1) {
+                
+                // this returns all the tables and their structure..
+                if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) {
+                    $this->debug("Loading Generator as databaseStructure called with args",1);
+                }
+                
+                $x = new DB_DataObject;
+                $x->_database = $args[0];
+                $this->_connect();
+                $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5];
+       
+                $tables = $DB->getListOf('tables');
+                class_exists('DB_DataObject_Generator') ? '' : 
+                    require_once 'DB/DataObject/Generator.php';
+                    
+                foreach($tables as $table) {
+                    $y = new DB_DataObject_Generator;
+                    $y->fillTableSchema($x->_database,$table);
+                }
+                return $_DB_DATAOBJECT['INI'][$x->_database];            
+            } else {
+        
+                $_DB_DATAOBJECT['INI'][$args[0]] = isset($_DB_DATAOBJECT['INI'][$args[0]]) ?
+                    $_DB_DATAOBJECT['INI'][$args[0]] + $args[1] : $args[1];
+                
+                if (isset($args[1])) {
+                    $_DB_DATAOBJECT['LINKS'][$args[0]] = isset($_DB_DATAOBJECT['LINKS'][$args[0]]) ?
+                        $_DB_DATAOBJECT['LINKS'][$args[0]] + $args[2] : $args[2];
+                }
+                return true;
+            }
+          
+        }
+        
+        
+        
+        if (!$this->_database) {
+            $this->_connect();
+        }
+        
+        // loaded already?
+        if (!empty($_DB_DATAOBJECT['INI'][$this->_database])) {
+            
+            // database loaded - but this is table is not available..
+            if (
+                    empty($_DB_DATAOBJECT['INI'][$this->_database][$this->__table]) 
+                    && !empty($_DB_DATAOBJECT['CONFIG']['proxy'])
+                ) {
+                if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) {
+                    $this->debug("Loading Generator to fetch Schema",1);
+                }
+                class_exists('DB_DataObject_Generator') ? '' : 
+                    require_once 'DB/DataObject/Generator.php';
+                    
+                
+                $x = new DB_DataObject_Generator;
+                $x->fillTableSchema($this->_database,$this->__table);
+            }
+            return true;
+        }
+        
+        
+        if (empty($_DB_DATAOBJECT['CONFIG'])) {
+            DB_DataObject::_loadConfig();
+        }
+        
+        // if you supply this with arguments, then it will take those
+        // as the database and links array...
+         
+        $schemas = isset($_DB_DATAOBJECT['CONFIG']['schema_location']) ?
+            array("{$_DB_DATAOBJECT['CONFIG']['schema_location']}/{$this->_database}.ini") :
+            array() ;
+                 
+        if (isset($_DB_DATAOBJECT['CONFIG']["ini_{$this->_database}"])) {
+            $schemas = is_array($_DB_DATAOBJECT['CONFIG']["ini_{$this->_database}"]) ?
+                $_DB_DATAOBJECT['CONFIG']["ini_{$this->_database}"] :
+                explode(PATH_SEPARATOR,$_DB_DATAOBJECT['CONFIG']["ini_{$this->_database}"]);
+        }
+                    
+         
+        /* BEGIN CHANGED FROM UPSTREAM */
+        $_DB_DATAOBJECT['INI'][$this->_database] = $this->parseIniFiles($schemas);
+        /* END CHANGED FROM UPSTREAM */
+
+        // now have we loaded the structure.. 
+        
+        if (!empty($_DB_DATAOBJECT['INI'][$this->_database][$this->__table])) {
+            return true;
+        }
+        // - if not try building it..
+        if (!empty($_DB_DATAOBJECT['CONFIG']['proxy'])) {
+            class_exists('DB_DataObject_Generator') ? '' : 
+                require_once 'DB/DataObject/Generator.php';
+                
+            $x = new DB_DataObject_Generator;
+            $x->fillTableSchema($this->_database,$this->__table);
+            // should this fail!!!???
+            return true;
+        }
+        $this->debug("Cant find database schema: {$this->_database}/{$this->__table} \n".
+                    "in links file data: " . print_r($_DB_DATAOBJECT['INI'],true),"databaseStructure",5);
+        // we have to die here!! - it causes chaos if we dont (including looping forever!)
+        $this->raiseError( "Unable to load schema for database and table (turn debugging up to 5 for full error message)", DB_DATAOBJECT_ERROR_INVALIDARGS, PEAR_ERROR_DIE);
+        return false;
+    }
+
+    /** For parseIniFiles */
+    protected static $iniCache = array();
+
+    /**
+     * When switching site configurations, DB_DataObject was loading its
+     * .ini files over and over, leaking gobs of memory.
+     * This refactored helper function uses a local cache of .ini files
+     * to minimize the leaks.
+     *
+     * @param array of .ini file names $schemas
+     * @return array
+     */
+    protected function parseIniFiles($schemas)
+    {
+        $key = implode("|", $schemas);
+        if (!isset(Safe_DataObject::$iniCache[$key])) {
+            $data = array();
+            foreach ($schemas as $ini) {
+                if (file_exists($ini) && is_file($ini)) {
+                    $data = array_merge($data, parse_ini_file($ini, true));
+
+                    if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) { 
+                        if (!is_readable ($ini)) {
+                            $this->debug("ini file is not readable: $ini","databaseStructure",1);
+                        } else {
+                            $this->debug("Loaded ini file: $ini","databaseStructure",1);
+                        }
+                    }
+                } else {
+                    if (!empty($_DB_DATAOBJECT['CONFIG']['debug'])) {
+                        $this->debug("Missing ini file: $ini","databaseStructure",1);
+                    }
+                }
+            }
+            Safe_DataObject::$iniCache[$key] = $data;
+        }
+
+        return Safe_DataObject::$iniCache[$key];
+    }
+}
+

classes/Status_network.php

@@ -21,7 +21,7 @@
 
 if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); }
 
-class Status_network extends DB_DataObject
+class Status_network extends Safe_DataObject
 {
     ###START_AUTOCODE
     /* the code below is auto generated do not remove the above tag */
@@ -57,6 +57,7 @@ class Status_network extends DB_DataObject
     ###END_AUTOCODE
 
     static $cache = null;
+    static $cacheInitialized = false;
     static $base = null;
     static $wildcard = null;
 
@@ -78,11 +79,15 @@ class Status_network extends DB_DataObject
         if (class_exists('Memcache')) {
             self::$cache = new Memcache();
 
-            // Can't close persistent connections, making forking painful.
+            // If we're a parent command-line process we need
+            // to be able to close out the connection after
+            // forking, so disable persistence.
             //
-            // @fixme only do this in *parent* CLI processes.
+            // We'll turn it back on again the second time
-            // single-process and child-processes *should* use persistent.
+            // through which will either be in a child process,
-            $persist = php_sapi_name() != 'cli';
+            // or a single-process script which is switching
+            // configurations.
+            $persist = php_sapi_name() != 'cli' || self::$cacheInitialized;
             if (is_array($servers)) {
                 foreach($servers as $server) {
                     self::$cache->addServer($server, 11211, $persist);
@@ -90,6 +95,7 @@ class Status_network extends DB_DataObject
             } else {
                 self::$cache->addServer($servers, 11211, $persist);
             }
+            self::$cacheInitialized = true;
         }
 
         self::$base = $dbname;

lib/cache.php

@@ -157,6 +157,32 @@ class Cache
         return $success;
     }
 
+    /**
+     * Atomically increment an existing numeric value.
+     * Existing expiration time should remain unchanged, if any.
+     *
+     * @param string  $key    The key to use for lookups
+     * @param int     $step   Amount to increment (default 1)
+     *
+     * @return mixed incremented value, or false if not set.
+     */
+    function increment($key, $step=1)
+    {
+        $value = false;
+        if (Event::handle('StartCacheIncrement', array(&$key, &$step, &$value))) {
+            // Fallback is not guaranteed to be atomic,
+            // and may original expiry value.
+            $value = $this->get($key);
+            if ($value !== false) {
+                $value += $step;
+                $ok = $this->set($key, $value);
+                $got = $this->get($key);
+            }
+            Event::handle('EndCacheIncrement', array($key, $step, $value));
+        }
+        return $value;
+    }
+
     /**
      * Delete the value associated with a key
      *

lib/dbqueuemanager.php

@@ -72,7 +72,7 @@ class DBQueueManager extends QueueManager
     public function poll()
     {
         $this->_log(LOG_DEBUG, 'Checking for notices...');
-        $qi = Queue_item::top($this->getQueues());
+        $qi = Queue_item::top($this->activeQueues());
         if (empty($qi)) {
             $this->_log(LOG_DEBUG, 'No notices waiting; idling.');
             return false;
@@ -142,9 +142,4 @@ class DBQueueManager extends QueueManager
 
         $this->stats('error', $queue);
     }
-
-    protected function _log($level, $msg)
-    {
-        common_log($level, 'DBQueueManager: '.$msg);
-    }
 }

lib/default.php

@@ -81,7 +81,7 @@ $default =
               'subsystem' => 'db', # default to database, or 'stomp'
               'stomp_server' => null,
               'queue_basename' => '/queue/statusnet/',
-              'control_channel' => '/topic/statusnet-control', // broadcasts to all queue daemons
+              'control_channel' => '/topic/statusnet/control', // broadcasts to all queue daemons
               'stomp_username' => null,
               'stomp_password' => null,
               'stomp_persistent' => true, // keep items across queue server restart, if persistence is enabled
@@ -91,6 +91,12 @@ $default =
               'spawndelay' => 1, // Wait at least N seconds between (re)spawns of child processes to avoid slamming the queue server with subscription startup
               'debug_memory' => false, // true to spit memory usage to log
               'inboxes' => true, // true to do inbox distribution & output queueing from in background via 'distrib' queue
+              'breakout' => array('*' => 'shared'), // set global or per-handler queue breakout
+                                      // 'shared': use a shared queue for all sites
+                                      // 'handler': share each/this handler over multiple sites
+                                      // 'site': break out for each/this handler on this site
+              'max_retries' => 10, // drop messages after N failed attempts to process (Stomp)
+              'dead_letter_dir' => false, // set to directory to save dropped messages into (Stomp)
               ),
         'license' =>
         array('type' => 'cc', # can be 'cc', 'allrightsreserved', 'private'

lib/iomanager.php

@@ -59,9 +59,10 @@ abstract class IoManager
      * your manager about each site you'll have to handle so you
      * can do any necessary per-site setup.
      *
-     * @param string $site target site server name
+     * The new site will be the currently live configuration during
+     * this call.
      */
-    public function addSite($site)
+    public function addSite()
     {
         /* no-op */
     }

lib/iomaster.php

@@ -56,9 +56,9 @@ abstract class IoMaster
             $this->multiSite = $multiSite;
         }
         if ($this->multiSite) {
-            $this->sites = $this->findAllSites();
+            $this->sites = StatusNet::findAllSites();
         } else {
-            $this->sites = array(common_config('site', 'server'));
+            $this->sites = array(StatusNet::currentSite());
         }
 
         if (empty($this->sites)) {
@@ -66,9 +66,7 @@ abstract class IoMaster
         }
 
         foreach ($this->sites as $site) {
-            if ($site != common_config('site', 'server')) {
+            StatusNet::switchSite($site);
-                StatusNet::init($site);
-            }
             $this->initManagers();
         }
     }
@@ -81,58 +79,32 @@ abstract class IoMaster
      */
     abstract function initManagers();
 
-    /**
-     * Pull all local sites from status_network table.
-     * @return array of hostnames
-     */
-    protected function findAllSites()
-    {
-        $hosts = array();
-        $sn = new Status_network();
-        $sn->find();
-        while ($sn->fetch()) {
-            $hosts[] = $sn->getServerName();
-        }
-        return $hosts;
-    }
-
     /**
      * Instantiate an i/o manager class for the current site.
      * If a multi-site capable handler is already present,
      * we don't need to build a new one.
      *
-     * @param string $class
+     * @param mixed $manager class name (to run $class::get()) or object
      */
-    protected function instantiate($class)
+    protected function instantiate($manager)
     {
-        if (isset($this->singletons[$class])) {
+        if (is_string($manager)) {
-            // Already instantiated a multi-site-capable handler.
+            $manager = call_user_func(array($class, 'get'));
-            // Just let it know it should listen to this site too!
-            $this->singletons[$class]->addSite(common_config('site', 'server'));
-            return;
         }
 
-        $manager = $this->getManager($class);
-
-        if ($this->multiSite) {
         $caps = $manager->multiSite();
         if ($caps == IoManager::SINGLE_ONLY) {
+            if ($this->multiSite) {
                 throw new Exception("$class can't run with --all; aborting.");
             }
-            if ($caps == IoManager::INSTANCE_PER_PROCESS) {
+        } else if ($caps == IoManager::INSTANCE_PER_PROCESS) {
-                // Save this guy for later!
+            $manager->addSite();
-                // We'll only need the one to cover multiple sites.
-                $this->singletons[$class] = $manager;
-                $manager->addSite(common_config('site', 'server'));
-            }
         }
 
+        if (!in_array($manager, $this->managers, true)) {
+            // Only need to save singletons once
             $this->managers[] = $manager;
         }
-    
-    protected function getManager($class)
-    {
-        return call_user_func(array($class, 'get'));
     }
 
     /**
@@ -146,6 +118,7 @@ abstract class IoMaster
     {
         $this->logState('init');
         $this->start();
+        $this->checkMemory(false);
 
         while (!$this->shutdown) {
             $timeouts = array_values($this->pollTimeouts);
@@ -209,17 +182,24 @@ abstract class IoMaster
     /**
      * Check runtime memory usage, possibly triggering a graceful shutdown
      * and thread respawn if we've crossed the soft limit.
+     *
+     * @param boolean $respawn if false we'll shut down instead of respawning
      */
-    protected function checkMemory()
+    protected function checkMemory($respawn=true)
     {
         $memoryLimit = $this->softMemoryLimit();
         if ($memoryLimit > 0) {
             $usage = memory_get_usage();
             if ($usage > $memoryLimit) {
                 common_log(LOG_INFO, "Queue thread hit soft memory limit ($usage > $memoryLimit); gracefully restarting.");
+                if ($respawn) {
                     $this->requestRestart();
+                } else {
+                    $this->requestShutdown();
+                }
             } else if (common_config('queue', 'debug_memory')) {
-                common_log(LOG_DEBUG, "Memory usage $usage");
+                $fmt = number_format($usage);
+                common_log(LOG_DEBUG, "Memory usage $fmt");
             }
         }
     }

lib/queued_xmpp.php

@@ -63,7 +63,7 @@ class Queued_XMPP extends XMPPHP_XMPP
      */
     public function send($msg, $timeout=NULL)
     {
-        $qm = QueueManager::get();
+        $qm = QueueManager::get('xmppout');
         $qm->enqueue(strval($msg), 'xmppout');
     }
 

lib/queuemanager.php

@@ -39,9 +39,10 @@ abstract class QueueManager extends IoManager
 {
     static $qm = null;
 
-    public $master = null;
+    protected $master = null;
-    public $handlers = array();
+    protected $handlers = array();
-    public $groups = array();
+    protected $groups = array();
+    protected $activeGroups = array();
 
     /**
      * Factory function to pull the appropriate QueueManager object
@@ -215,55 +216,64 @@ abstract class QueueManager extends IoManager
             if (class_exists($class)) {
                 return new $class();
             } else {
-                common_log(LOG_ERR, "Nonexistent handler class '$class' for queue '$queue'");
+                $this->_log(LOG_ERR, "Nonexistent handler class '$class' for queue '$queue'");
             }
         } else {
-            common_log(LOG_ERR, "Requested handler for unkown queue '$queue'");
+            $this->_log(LOG_ERR, "Requested handler for unkown queue '$queue'");
         }
         return null;
     }
 
     /**
      * Get a list of registered queue transport names to be used
-     * for this daemon.
+     * for listening in this daemon.
      *
      * @return array of strings
      */
-    function getQueues()
+    function activeQueues()
     {
-        $group = $this->activeGroup();
+        $queues = array();
-        return array_keys($this->groups[$group]);
+        foreach ($this->activeGroups as $group) {
+            if (isset($this->groups[$group])) {
+                $queues = array_merge($queues, $this->groups[$group]);
+            }
+        }
+
+        return array_keys($queues);
     }
 
     /**
-     * Initialize the list of queue handlers
+     * Initialize the list of queue handlers for the current site.
      *
      * @event StartInitializeQueueManager
      * @event EndInitializeQueueManager
      */
     function initialize()
     {
-        // @fixme we'll want to be able to listen to particular queues...
+        $this->handlers = array();
+        $this->groups = array();
+        $this->groupsByTransport = array();
+
         if (Event::handle('StartInitializeQueueManager', array($this))) {
-            $this->connect('plugin', 'PluginQueueHandler');
+            $this->connect('distrib', 'DistribQueueHandler');
             $this->connect('omb', 'OmbQueueHandler');
             $this->connect('ping', 'PingQueueHandler');
-            $this->connect('distrib', 'DistribQueueHandler');
             if (common_config('sms', 'enabled')) {
                 $this->connect('sms', 'SmsQueueHandler');
             }
 
             // XMPP output handlers...
+            if (common_config('xmpp', 'enabled')) {
+                // Delivery prep, read by queuedaemon.php:
                 $this->connect('jabber', 'JabberQueueHandler');
                 $this->connect('public', 'PublicQueueHandler');
-            // @fixme this should get an actual queue
+
-            //$this->connect('confirm', 'XmppConfirmHandler');
+                // Raw output, read by xmppdaemon.php:
+                $this->connect('xmppout', 'XmppOutQueueHandler', 'xmpp');
+            }
 
             // For compat with old plugins not registering their own handlers.
             $this->connect('plugin', 'PluginQueueHandler');
-
-            $this->connect('xmppout', 'XmppOutQueueHandler', 'xmppdaemon');
-
         }
         Event::handle('EndInitializeQueueManager', array($this));
     }
@@ -276,25 +286,41 @@ abstract class QueueManager extends IoManager
      * @param string $class
      * @param string $group
      */
-    public function connect($transport, $class, $group='queuedaemon')
+    public function connect($transport, $class, $group='main')
     {
         $this->handlers[$transport] = $class;
         $this->groups[$group][$transport] = $class;
+        $this->groupsByTransport[$transport] = $group;
     }
 
     /**
-     * @return string queue group to use for this request
+     * Set the active group which will be used for listening.
+     * @param string $group
      */
-    function activeGroup()
+    function setActiveGroup($group)
     {
-        $group = 'queuedaemon';
+        $this->activeGroups = array($group);
-        if ($this->master) {
-            // hack hack
-            if ($this->master instanceof XmppMaster) {
-                return 'xmppdaemon';
     }
+
+    /**
+     * Set the active group(s) which will be used for listening.
+     * @param array $groups
+     */
+    function setActiveGroups($groups)
+    {
+        $this->activeGroups = $groups;
+    }
+
+    /**
+     * @return string queue group for this queue
+     */
+    function queueGroup($queue)
+    {
+        if (isset($this->groupsByTransport[$queue])) {
+            return $this->groupsByTransport[$queue];
+        } else {
+            throw new Exception("Requested group for unregistered transport $queue");
         }
-        return $group;
     }
 
     /**
@@ -318,4 +344,15 @@ abstract class QueueManager extends IoManager
             $monitor->stats($key, $owners);
         }
     }
+
+    protected function _log($level, $msg)
+    {
+        $class = get_class($this);
+        if ($this->activeGroups) {
+            $groups = ' (' . implode(',', $this->activeGroups) . ')';
+        } else {
+            $groups = '';
+        }
+        common_log($level, "$class$groups: $msg");
+    }
 }

lib/spawningdaemon.php

@@ -90,18 +90,24 @@ abstract class SpawningDaemon extends Daemon
         while (count($children) > 0) {
             $status = null;
             $pid = pcntl_wait($status);
-            if ($pid > 0 && pcntl_wifexited($status)) {
+            if ($pid > 0) {
-                $exitCode = pcntl_wexitstatus($status);
-
                 $i = array_search($pid, $children);
                 if ($i === false) {
-                    $this->log(LOG_ERR, "Unrecognized child pid $pid exited with status $exitCode");
+                    $this->log(LOG_ERR, "Ignoring exit of unrecognized child pid $pid");
                     continue;
                 }
+                if (pcntl_wifexited($status)) {
+                    $exitCode = pcntl_wexitstatus($status);
+                    $info = "status $exitCode";
+                } else if (pcntl_wifsignaled($status)) {
+                    $exitCode = self::EXIT_ERR;
+                    $signal = pcntl_wtermsig($status);
+                    $info = "signal $signal";
+                }
                 unset($children[$i]);
 
                 if ($this->shouldRespawn($exitCode)) {
-                    $this->log(LOG_INFO, "Thread $i pid $pid exited with status $exitCode; respawing.");
+                    $this->log(LOG_INFO, "Thread $i pid $pid exited with $info; respawing.");
 
                     $pid = pcntl_fork();
                     if ($pid < 0) {

lib/statusnet.php

@@ -101,6 +101,60 @@ class StatusNet
         self::initPlugins();
     }
 
+    /**
+     * Get identifier of the currently active site configuration
+     * @return string
+     */
+    public static function currentSite()
+    {
+        return common_config('site', 'nickname');
+    }
+
+    /**
+     * Change site configuration to site specified by nickname,
+     * if set up via Status_network. If not, sites other than
+     * the current will fail horribly.
+     *
+     * May throw exception or trigger a fatal error if the given
+     * site is missing or configured incorrectly.
+     *
+     * @param string $nickname
+     */
+    public static function switchSite($nickname)
+    {
+        if ($nickname == StatusNet::currentSite()) {
+            return true;
+        }
+
+        $sn = Status_network::staticGet($nickname);
+        if (empty($sn)) {
+            return false;
+            throw new Exception("No such site nickname '$nickname'");
+        }
+
+        $server = $sn->getServerName();
+        StatusNet::init($server);
+    }
+
+    /**
+     * Pull all local sites from status_network table.
+     *
+     * Behavior undefined if site is not configured via Status_network.
+     *
+     * @return array of nicknames
+     */
+    public static function findAllSites()
+    {
+        $sites = array();
+        $sn = new Status_network();
+        $sn->find();
+        while ($sn->fetch()) {
+            $sites[] = $sn->nickname;
+        }
+        return $sites;
+    }
+
+
     /**
      * Fire initialization events for all instantiated plugins.
      */

lib/stompqueuemanager.php

@@ -63,6 +63,7 @@ class StompQueueManager extends QueueManager
         $this->password = common_config('queue', 'stomp_password');
         $this->base     = common_config('queue', 'queue_basename');
         $this->control  = common_config('queue', 'control_channel');
+        $this->subscriptions = array($this->control => $this->control);
     }
 
     /**
@@ -75,17 +76,25 @@ class StompQueueManager extends QueueManager
     }
 
     /**
-     * Record each site we'll be handling input for in this process,
+     * Record queue subscriptions we'll need to handle the current site.
-     * so we can listen to the necessary queues for it.
-     *
-     * @fixme possibly actually do subscription here to save another
-     *        loop over all sites later?
-     * @fixme possibly don't assume it's the current site
      */
-    public function addSite($server)
+    public function addSite()
     {
-        $this->sites[] = $server;
+        $this->sites[] = StatusNet::currentSite();
+
+        // Set up handlers active for this site...
         $this->initialize();
+
+        foreach ($this->activeGroups as $group) {
+            if (isset($this->groups[$group])) {
+                // Actual queues may be broken out or consolidated...
+                // Subscribe to all the target queues we'll need.
+                foreach ($this->groups[$group] as $transport => $class) {
+                    $target = $this->queueName($transport);
+                    $this->subscriptions[$target] = $target;
+                }
+            }
+        }
     }
 
     /**
@@ -121,59 +130,11 @@ class StompQueueManager extends QueueManager
     }
 
     /**
-     * Instantiate the appropriate QueueHandler class for the given queue.
+     * Saves an object into the queue item table.
      *
+     * @param mixed $object
      * @param string $queue
-     * @return mixed QueueHandler or null
-     */
-    function getHandler($queue)
-    {
-        $handlers = $this->handlers[$this->currentSite()];
-        if (isset($handlers[$queue])) {
-            $class = $handlers[$queue];
-            if (class_exists($class)) {
-                return new $class();
-            } else {
-                common_log(LOG_ERR, "Nonexistent handler class '$class' for queue '$queue'");
-            }
-        } else {
-            common_log(LOG_ERR, "Requested handler for unkown queue '$queue'");
-        }
-        return null;
-    }
-
-    /**
-     * Get a list of all registered queue transport names.
      *
-     * @return array of strings
-     */
-    function getQueues()
-    {
-        $group = $this->activeGroup();
-        $site = $this->currentSite();
-        if (empty($this->groups[$site][$group])) {
-            return array();
-        } else {
-            return array_keys($this->groups[$site][$group]);
-        }
-    }
-
-    /**
-     * Register a queue transport name and handler class for your plugin.
-     * Only registered transports will be reliably picked up!
-     *
-     * @param string $transport
-     * @param string $class
-     * @param string $group
-     */
-    public function connect($transport, $class, $group='queuedaemon')
-    {
-        $this->handlers[$this->currentSite()][$transport] = $class;
-        $this->groups[$this->currentSite()][$group][$transport] = $class;
-    }
-
-    /**
-     * Saves a notice object reference into the queue item table.
      * @return boolean true on success
      * @throws StompException on connection or send error
      */
@@ -192,8 +153,11 @@ class StompQueueManager extends QueueManager
      */
     protected function _doEnqueue($object, $queue, $idx)
     {
-        $msg = $this->encode($object);
         $rep = $this->logrep($object);
+        $envelope = array('site' => common_config('site', 'nickname'),
+                          'handler' => $queue,
+                          'payload' => $this->encode($object));
+        $msg = serialize($envelope);
 
         $props = array('created' => common_sql_now());
         if ($this->isPersistent($queue)) {
@@ -205,11 +169,11 @@ class StompQueueManager extends QueueManager
         $result = $con->send($this->queueName($queue), $msg, $props);
 
         if (!$result) {
-            common_log(LOG_ERR, "Error sending $rep to $queue queue on $host");
+            $this->_log(LOG_ERR, "Error sending $rep to $queue queue on $host");
             return false;
         }
 
-        common_log(LOG_DEBUG, "complete remote queueing $rep for $queue on $host");
+        $this->_log(LOG_DEBUG, "complete remote queueing $rep for $queue on $host");
         $this->stats('enqueued', $queue);
         return true;
     }
@@ -275,12 +239,14 @@ class StompQueueManager extends QueueManager
         $idx = $this->connectionFromSocket($socket);
         $con = $this->cons[$idx];
         $host = $con->getServer();
+        $this->defaultIdx = $idx;
 
         $ok = true;
         try {
             $frames = $con->readFrames();
         } catch (StompException $e) {
-            common_log(LOG_ERR, "Lost connection to $host: " . $e->getMessage());
+            $this->_log(LOG_ERR, "Lost connection to $host: " . $e->getMessage());
+            fclose($socket); // ???
             $this->cons[$idx] = null;
             $this->transaction[$idx] = null;
             $this->disconnect[$idx] = time();
@@ -289,14 +255,17 @@ class StompQueueManager extends QueueManager
         foreach ($frames as $frame) {
             $dest = $frame->headers['destination'];
             if ($dest == $this->control) {
-                if (!$this->handleControlSignal($idx, $frame)) {
+                if (!$this->handleControlSignal($frame)) {
                     // We got a control event that requests a shutdown;
                     // close out and stop handling anything else!
                     break;
                 }
             } else {
-                $ok = $ok && $this->handleItem($idx, $frame);
+                $ok = $this->handleItem($frame) && $ok;
             }
+            $this->ack($idx, $frame);
+            $this->commit($idx);
+            $this->begin($idx);
         }
         return $ok;
     }
@@ -333,22 +302,9 @@ class StompQueueManager extends QueueManager
         parent::start($master);
         $this->_connectAll();
 
-        common_log(LOG_INFO, "Subscribing to $this->control");
-        foreach ($this->cons as $con) {
-            if ($con) {
-                $con->subscribe($this->control);
-            }
-        }
-        if ($this->sites) {
-            foreach ($this->sites as $server) {
-                StatusNet::init($server);
-                $this->doSubscribe();
-            }
-        } else {
-            $this->doSubscribe();
-        }
         foreach ($this->cons as $i => $con) {
             if ($con) {
+                $this->doSubscribe($con);
                 $this->begin($i);
             }
         }
@@ -356,9 +312,7 @@ class StompQueueManager extends QueueManager
     }
 
     /**
-     * Subscribe to all the queues we're going to need to handle...
+     * Close out any active connections.
-     *
-     * Side effects: in multi-site mode, may reset site configuration.
      *
      * @return bool return false on failure
      */
@@ -376,15 +330,6 @@ class StompQueueManager extends QueueManager
         return true;
     }
 
-    /**
-     * Get identifier of the currently active site configuration
-     * @return string
-     */
-    protected function currentSite()
-    {
-        return common_config('site', 'server'); // @fixme switch to nickname
-    }
-
     /**
      * Lazy open a single connection to Stomp queue server.
      * If multiple servers are configured, we let the Stomp client library
@@ -441,6 +386,10 @@ class StompQueueManager extends QueueManager
         }
     }
 
+    /**
+     * Attempt to manually reconnect to the Stomp server for the given
+     * slot. If successful, set up our subscriptions on it.
+     */
     protected function _reconnect($idx)
     {
         try {
@@ -453,17 +402,7 @@ class StompQueueManager extends QueueManager
             $this->cons[$idx] = $con;
             $this->disconnect[$idx] = null;
 
-            // now we have to listen to everything...
+            $this->doSubscribe($con);
-            // @fixme refactor this nicer. :P
-            $host = $con->getServer();
-            $this->_log(LOG_INFO, "Resubscribing to $this->control on $host");
-            $con->subscribe($this->control);
-            foreach ($this->subscriptions as $site => $queues) {
-                foreach ($queues as $queue) {
-                    $this->_log(LOG_INFO, "Resubscribing to $queue on $host");
-                    $con->subscribe($queue);
-                }
-            }
             $this->begin($idx);
         } else {
             // Try again later...
@@ -487,41 +426,15 @@ class StompQueueManager extends QueueManager
     }
 
     /**
-     * Subscribe to all enabled notice queues for the current site.
+     * Set up all our raw queue subscriptions on the given connection
+     * @param LiberalStomp $con
      */
-    protected function doSubscribe()
+    protected function doSubscribe(LiberalStomp $con)
     {
-        $site = $this->currentSite();
+        $host = $con->getServer();
-        $this->_connect();
+        foreach ($this->subscriptions as $queue) {
-        foreach ($this->getQueues() as $queue) {
+            $this->_log(LOG_INFO, "Subscribing to $queue on $host");
-            $rawqueue = $this->queueName($queue);
+            $con->subscribe($queue);
-            $this->subscriptions[$site][$queue] = $rawqueue;
-            $this->_log(LOG_INFO, "Subscribing to $rawqueue");
-            foreach ($this->cons as $con) {
-                if ($con) {
-                    $con->subscribe($rawqueue);
-                }
-            }
-        }
-    }
-
-    /**
-     * Subscribe from all enabled notice queues for the current site.
-     */
-    protected function doUnsubscribe()
-    {
-        $site = $this->currentSite();
-        $this->_connect();
-        if (!empty($this->subscriptions[$site])) {
-            foreach ($this->subscriptions[$site] as $queue => $rawqueue) {
-                $this->_log(LOG_INFO, "Unsubscribing from $rawqueue");
-                foreach ($this->cons as $con) {
-                    if ($con) {
-                        $con->unsubscribe($rawqueue);
-                    }
-                }
-                unset($this->subscriptions[$site][$queue]);
-            }
         }
     }
 
@@ -534,25 +447,29 @@ class StompQueueManager extends QueueManager
      * Side effects: in multi-site mode, may reset site configuration to
      * match the site that queued the event.
      *
-     * @param int $idx connection index
      * @param StompFrame $frame
-     * @return bool
+     * @return bool success
      */
-    protected function handleItem($idx, $frame)
+    protected function handleItem($frame)
     {
-        $this->defaultIdx = $idx;
+        $host = $this->cons[$this->defaultIdx]->getServer();
+        $message = unserialize($frame->body);
+        $site = $message['site'];
+        $queue = $message['handler'];
 
-        list($site, $queue) = $this->parseDestination($frame->headers['destination']);
+        if ($this->isDeadletter($frame, $message)) {
-        if ($site != $this->currentSite()) {
+            $this->stats('deadletter', $queue);
-            $this->stats('switch');
+	        return false;
-            StatusNet::init($site);
         }
 
-        $host = $this->cons[$idx]->getServer();
+        // @fixme detect failing site switches
-        $item = $this->decode($frame->body);
+        $this->switchSite($site);
+
+        $item = $this->decode($message['payload']);
         if (empty($item)) {
             $this->_log(LOG_ERR, "Skipping empty or deleted item in queue $queue from $host");
-            return true;
+            $this->stats('baditem', $queue);
+            return false;
         }
         $info = $this->logrep($item) . " posted at " .
                 $frame->headers['created'] . " in queue $queue from $host";
@@ -561,16 +478,10 @@ class StompQueueManager extends QueueManager
         $handler = $this->getHandler($queue);
         if (!$handler) {
             $this->_log(LOG_ERR, "Missing handler class; skipping $info");
-            $this->ack($idx, $frame);
-            $this->commit($idx);
-            $this->begin($idx);
             $this->stats('badhandler', $queue);
             return false;
         }
 
-        // If there's an exception when handling,
-        // log the error and let it get requeued.
-
         try {
             $ok = $handler->handle($item);
         } catch (Exception $e) {
@@ -578,25 +489,80 @@ class StompQueueManager extends QueueManager
             $ok = false;
         }
 
-        if (!$ok) {
+        if ($ok) {
+            $this->_log(LOG_INFO, "Successfully handled $info");
+            $this->stats('handled', $queue);
+        } else {
             $this->_log(LOG_WARNING, "Failed handling $info");
-            // FIXME we probably shouldn't have to do
+            // Requeing moves the item to the end of the line for its next try.
-            // this kind of queue management ourselves;
+            // @fixme add a manual retry count
-            // if we don't ack, it should resend...
-            $this->ack($idx, $frame);
             $this->enqueue($item, $queue);
-            $this->commit($idx);
-            $this->begin($idx);
             $this->stats('requeued', $queue);
+        }
+
+        return $ok;
+    }
+
+    /**
+     * Check if a redelivered message has been run through enough
+     * that we're going to give up on it.
+     *
+     * @param StompFrame $frame
+     * @param array $message unserialized message body
+     * @return boolean true if we should discard
+     */
+    protected function isDeadLetter($frame, $message)
+    {
+        if (isset($frame->headers['redelivered']) && $frame->headers['redelivered'] == 'true') {
+	        // Message was redelivered, possibly indicating a previous failure.
+            $msgId = $frame->headers['message-id'];
+            $site = $message['site'];
+            $queue = $message['handler'];
+	        $msgInfo = "message $msgId for $site in queue $queue";
+
+	        $deliveries = $this->incDeliveryCount($msgId);
+	        if ($deliveries > common_config('queue', 'max_retries')) {
+		        $info = "DEAD-LETTER FILE: Gave up after retry $deliveries on $msgInfo";
+
+		        $outdir = common_config('queue', 'dead_letter_dir');
+		        if ($outdir) {
+    		        $filename = $outdir . "/$site-$queue-" . rawurlencode($msgId);
+    		        $info .= ": dumping to $filename";
+    		        file_put_contents($filename, $message['payload']);
+		        }
+
+		        common_log(LOG_ERR, $info);
+		        return true;
+	        } else {
+	            common_log(LOG_INFO, "retry $deliveries on $msgInfo");
+	        }
+        }
         return false;
     }
 
-        $this->_log(LOG_INFO, "Successfully handled $info");
+    /**
-        $this->ack($idx, $frame);
+     * Update count of times we've re-encountered this message recently,
-        $this->commit($idx);
+     * triggered when we get a message marked as 'redelivered'.
-        $this->begin($idx);
+     *
-        $this->stats('handled', $queue);
+     * Requires a CLI-friendly cache configuration.
-        return true;
+     *
+     * @param string $msgId message-id header from message
+     * @return int number of retries recorded
+     */
+    function incDeliveryCount($msgId)
+    {
+	    $count = 0;
+	    $cache = common_memcache();
+	    if ($cache) {
+		    $key = 'statusnet:stomp:message-retries:' . $msgId;
+		    $count = $cache->increment($key);
+		    if (!$count) {
+			    $count = 1;
+			    $cache->set($key, $count, null, 3600);
+			    $got = $cache->get($key);
+		    }
+	    }
+	    return $count;
     }
 
     /**
@@ -629,13 +595,22 @@ class StompQueueManager extends QueueManager
         } else {
             $this->_log(LOG_ERR, "Ignoring unrecognized control message: $message");
         }
-
-        $this->ack($idx, $frame);
-        $this->commit($idx);
-        $this->begin($idx);
         return $shutdown;
     }
 
+    /**
+     * Switch site, if necessary, and reset current handler assignments
+     * @param string $site
+     */
+    function switchSite($site)
+    {
+        if ($site != StatusNet::currentSite()) {
+            $this->stats('switch');
+            StatusNet::switchSite($site);
+            $this->initialize();
+        }
+    }
+
     /**
      * Set us up with queue subscriptions for a new site added at runtime,
      * triggered by a broadcast to the 'statusnet-control' topic.
@@ -648,22 +623,17 @@ class StompQueueManager extends QueueManager
         if (empty($this->sites)) {
             if ($nickname == common_config('site', 'nickname')) {
                 StatusNet::init(common_config('site', 'server'));
-                $this->doUnsubscribe();
-                $this->doSubscribe();
             } else {
                 $this->_log(LOG_INFO, "Ignoring update ping for other site $nickname");
             }
         } else {
             $sn = Status_network::staticGet($nickname);
             if ($sn) {
-                $server = $sn->getServerName(); // @fixme do config-by-nick
+                $this->switchSite($nickname);
-                StatusNet::init($server);
+                if (!in_array($nickname, $this->sites)) {
-                if (empty($this->sites[$server])) {
+                    $this->addSite();
-                    $this->addSite($server);
                 }
-                $this->_log(LOG_INFO, "(Re)subscribing to queues for site $nickname / $server");
+                // @fixme update subscriptions, if applicable
-                $this->doUnsubscribe();
-                $this->doSubscribe();
                 $this->stats('siteupdate');
             } else {
                 $this->_log(LOG_ERR, "Ignoring ping for unrecognized new site $nickname");
@@ -673,42 +643,47 @@ class StompQueueManager extends QueueManager
 
     /**
      * Combines the queue_basename from configuration with the
-     * site server name and queue name to give eg:
+     * group name for this queue to give eg:
      *
-     * /queue/statusnet/identi.ca/sms
+     * /queue/statusnet/main
      *
      * @param string $queue
      * @return string
      */
     protected function queueName($queue)
     {
-        return common_config('queue', 'queue_basename') .
+        $base = common_config('queue', 'queue_basename');
-            $this->currentSite() . '/' . $queue;
+        $group = $this->queueGroup($queue);
+        $breakout = $this->breakoutMode($queue);
+        if ($breakout == 'shared') {
+            return $base . "$group";
+        } else if ($breakout == 'handler') {
+            return $base . "$group/$queue";
+        } else if ($breakout == 'site') {
+            $site = StatusNet::currentSite();
+            return $base . "$group/$queue/$site";
+        }
+        throw Exception("Unrecognized queue breakout mode '$breakout' for '$queue'");
     }
 
     /**
-     * Returns the site and queue name from the server-side queue.
+     * Get the breakout mode for the given queue on the current site.
      *
-     * @param string queue destination (eg '/queue/statusnet/identi.ca/sms')
+     * @param string $queue
-     * @return array of site and queue: ('identi.ca','sms') or false if unrecognized
+     * @return string one of 'shared', 'handler', 'site'
      */
-    protected function parseDestination($dest)
+    protected function breakoutMode($queue)
     {
-        $prefix = common_config('queue', 'queue_basename');
+        $breakout = common_config('queue', 'breakout');
-        if (substr($dest, 0, strlen($prefix)) == $prefix) {
+        if (isset($breakout[$queue])) {
-            $rest = substr($dest, strlen($prefix));
+            return $breakout[$queue];
-            return explode("/", $rest, 2);
+        } else if (isset($breakout['*'])) {
+            return $breakout['*'];
         } else {
-            common_log(LOG_ERR, "Got a message from unrecognized stomp queue: $dest");
+            return 'shared';
-            return array(false, false);
         }
     }
 
-    function _log($level, $msg)
-    {
-        common_log($level, 'StompQueueManager: '.$msg);
-    }
-
     protected function begin($idx)
     {
         if ($this->useTransactions) {

lib/xmppmanager.php

@@ -48,7 +48,7 @@ class XmppManager extends IoManager
     public static function get()
     {
         if (common_config('xmpp', 'enabled')) {
-            $site = common_config('site', 'server');
+            $site = StatusNet::currentSite();
             if (empty(self::$singletons[$site])) {
                 self::$singletons[$site] = new XmppManager();
             }
@@ -69,7 +69,7 @@ class XmppManager extends IoManager
 
     function __construct()
     {
-        $this->site = common_config('site', 'server');
+        $this->site = StatusNet::currentSite();
         $this->resource = common_config('xmpp', 'resource') . 'daemon';
     }
 
@@ -476,10 +476,10 @@ class XmppManager extends IoManager
      */
     protected function switchSite()
     {
-        if ($this->site != common_config('site', 'server')) {
+        if ($this->site != StatusNet::currentSite()) {
             common_log(LOG_DEBUG, __METHOD__ . ": switching to site $this->site");
             $this->stats('switch');
-            StatusNet::init($this->site);
+            StatusNet::switchSite($this->site);
         }
     }
 }

plugins/MemcachePlugin.php

@@ -51,6 +51,8 @@ if (!defined('STATUSNET')) {
 
 class MemcachePlugin extends Plugin
 {
+    static $cacheInitialized = false;
+
     private $_conn  = null;
     public $servers = array('127.0.0.1;11211');
 
@@ -71,10 +73,21 @@ class MemcachePlugin extends Plugin
 
     function onInitializePlugin()
     {
-        if (is_null($this->persistent)) {
+        if (self::$cacheInitialized) {
+            $this->persistent = true;
+        } else {
+            // If we're a parent command-line process we need
+            // to be able to close out the connection after
+            // forking, so disable persistence.
+            //
+            // We'll turn it back on again the second time
+            // through which will either be in a child process,
+            // or a single-process script which is switching
+            // configurations.
             $this->persistent = (php_sapi_name() == 'cli') ? false : true;
         }
         $this->_ensureConn();
+        self::$cacheInitialized = true;
         return true;
     }
 
@@ -121,6 +134,24 @@ class MemcachePlugin extends Plugin
         return false;
     }
 
+    /**
+     * Atomically increment an existing numeric key value.
+     * Existing expiration time will not be changed.
+     *
+     * @param string &$key    in; Key to use for lookups
+     * @param int    &$step   in; Amount to increment (default 1)
+     * @param mixed  &$value  out; Incremented value, or false if key not set.
+     *
+     * @return boolean hook success
+     */
+    function onStartCacheIncrement(&$key, &$step, &$value)
+    {
+        $this->_ensureConn();
+        $value = $this->_conn->increment($key, $step);
+        Event::handle('EndCacheIncrement', array($key, $step, $value));
+        return false;
+    }
+
     /**
      * Delete a value associated with a key
      *

scripts/queuedaemon.php

@@ -74,8 +74,6 @@ require_once(INSTALLDIR.'/lib/daemon.php');
 require_once(INSTALLDIR.'/classes/Queue_item.php');
 require_once(INSTALLDIR.'/classes/Notice.php');
 
-define('CLAIM_TIMEOUT', 1200);
-
 /**
  * Queue handling daemon...
  *
@@ -92,7 +90,7 @@ class QueueDaemon extends SpawningDaemon
     function __construct($id=null, $daemonize=true, $threads=1, $allsites=false)
     {
         parent::__construct($id, $daemonize, $threads);
-        $this->all = $allsites;
+        $this->allsites = $allsites;
     }
 
     /**
@@ -108,7 +106,7 @@ class QueueDaemon extends SpawningDaemon
         $this->log(LOG_INFO, 'checking for queued notices');
 
         $master = new QueueMaster($this->get_id());
-        $master->init($this->all);
+        $master->init($this->allsites);
         try {
             $master->service();
         } catch (Exception $e) {
@@ -133,14 +131,16 @@ class QueueMaster extends IoMaster
      */
     function initManagers()
     {
-        $classes = array();
+        $managers = array();
-        if (Event::handle('StartQueueDaemonIoManagers', array(&$classes))) {
+        if (Event::handle('StartQueueDaemonIoManagers', array(&$managers))) {
-            $classes[] = 'QueueManager';
+            $qm = QueueManager::get();
+            $qm->setActiveGroup('main');
+            $managers[] = $qm;
         }
-        Event::handle('EndQueueDaemonIoManagers', array(&$classes));
+        Event::handle('EndQueueDaemonIoManagers', array(&$managers));
 
-        foreach ($classes as $class) {
+        foreach ($managers as $manager) {
-            $this->instantiate($class);
+            $this->instantiate($manager);
         }
     }
 }

scripts/xmppdaemon.php

@@ -20,13 +20,15 @@
 
 define('INSTALLDIR', realpath(dirname(__FILE__) . '/..'));
 
-$shortoptions = 'fi::';
+$shortoptions = 'fi::a';
-$longoptions = array('id::', 'foreground');
+$longoptions = array('id::', 'foreground', 'all');
 
 $helptext = <<<END_OF_XMPP_HELP
 Daemon script for receiving new notices from Jabber users.
 
     -i --id           Identity (default none)
+    -a --all          Handle XMPP for all local sites
+                      (requires Stomp queue handler, status_network setup)
     -f --foreground   Stay in the foreground (default background)
 
 END_OF_XMPP_HELP;
@@ -37,13 +39,16 @@ require_once INSTALLDIR . '/lib/jabber.php';
 
 class XMPPDaemon extends SpawningDaemon
 {
-    function __construct($id=null, $daemonize=true, $threads=1)
+    protected $allsites = false;
+
+    function __construct($id=null, $daemonize=true, $threads=1, $allsites=false)
     {
         if ($threads != 1) {
             // This should never happen. :)
             throw new Exception("XMPPDaemon can must run single-threaded");
         }
         parent::__construct($id, $daemonize, $threads);
+        $this->allsites = $allsites;
     }
 
     function runThread()
@@ -51,7 +56,7 @@ class XMPPDaemon extends SpawningDaemon
         common_log(LOG_INFO, 'Waiting to listen to XMPP and queues');
 
         $master = new XmppMaster($this->get_id());
-        $master->init();
+        $master->init($this->allsites);
         $master->service();
 
         common_log(LOG_INFO, 'terminating normally');
@@ -69,15 +74,19 @@ class XmppMaster extends IoMaster
      */
     function initManagers()
     {
-        // @fixme right now there's a hack in QueueManager to determine
+        if (common_config('xmpp', 'enabled')) {
-        // which queues to subscribe to based on the master class.
+            $qm = QueueManager::get();
-        $this->instantiate('QueueManager');
+            $qm->setActiveGroup('xmpp');
-        $this->instantiate('XmppManager');
+            $this->instantiate($qm);
+            $this->instantiate(XmppManager::get());
+        }
     }
 }
 
 // Abort immediately if xmpp is not enabled, otherwise the daemon chews up
 // lots of CPU trying to connect to unconfigured servers
+// @fixme do this check after we've run through the site list so we
+// don't have to find an XMPP site to start up when using --all mode.
 if (common_config('xmpp','enabled')==false) {
     print "Aborting daemon - xmpp is disabled\n";
     exit();
@@ -92,7 +101,8 @@ if (have_option('i', 'id')) {
 }
 
 $foreground = have_option('f', 'foreground');
+$all = have_option('a') || have_option('--all');
 
-$daemon = new XMPPDaemon($id, !$foreground);
+$daemon = new XMPPDaemon($id, !$foreground, 1, $all);
 
 $daemon->runOnce();