gnu-social/lib/dbqueuemanager.php
Brion Vibber c74aea589d Stomp queue restructuring for mass scalability:
- Multiplexing queues into groups and for multiple sites.
- Sharing vs breakout configurable per site and per queue via $config['queue']['breakout']
- Detect how many times a message is redelivered, discard if it's killed too many daemons
 - count configurable with $config['queue']['max_retries']
 - can dump the items to files in $config['queue']['dead_letter_dir']

Queue daemon memory & resource leak fixes:
- avoid unnecessary reconnections to memcached server (switch persistent connections back in on second initialization, assuming it's child process)
- monkey-patch for leaky .ini loads in DB_DataObject::databaseStructure() - was leaking 200k per active switch
- applied leak fixes to Status_network as well, using intermediate base Safe_DataObject for both it and Memcache_DataObject

Misc queue fixes:
- correct handling of child processes exiting due to signal termination instead of regular exit
- shutdown instead of infinite respawn loop if we're already past the soft memory limit at startup
- Added --all option for xmppdaemon... still opens one xmpp connection per site that has xmpp active

Cache updates:
- add Cache::increment() method with native support for memcached atomic increment
2010-02-16 09:16:51 -08:00

146 lines
4.4 KiB
PHP

<?php
/**
* StatusNet, the distributed open-source microblogging tool
*
* Simple-minded queue manager for storing items in the database
*
* PHP version 5
*
* LICENCE: This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @category QueueManager
* @package StatusNet
* @author Evan Prodromou <evan@status.net>
* @author Brion Vibber <brion@status.net>
* @copyright 2009-2010 StatusNet, Inc.
* @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
* @link http://status.net/
*/
class DBQueueManager extends QueueManager
{
/**
* Saves an object reference into the queue item table.
* @return boolean true on success
* @throws ServerException on failure
*/
public function enqueue($object, $queue)
{
$qi = new Queue_item();
$qi->frame = $this->encode($object);
$qi->transport = $queue;
$qi->created = common_sql_now();
$result = $qi->insert();
if (!$result) {
common_log_db_error($qi, 'INSERT', __FILE__);
throw new ServerException('DB error inserting queue item');
}
$this->stats('enqueued', $queue);
return true;
}
/**
* Poll every 10 seconds for new events during idle periods.
* We'll look in more often when there's data available.
*
* @return int seconds
*/
public function pollInterval()
{
return 10;
}
/**
* Run a polling cycle during idle processing in the input loop.
* @return boolean true if we should poll again for more data immediately
*/
public function poll()
{
$this->_log(LOG_DEBUG, 'Checking for notices...');
$qi = Queue_item::top($this->activeQueues());
if (empty($qi)) {
$this->_log(LOG_DEBUG, 'No notices waiting; idling.');
return false;
}
$queue = $qi->transport;
$item = $this->decode($qi->frame);
if ($item) {
$rep = $this->logrep($item);
$this->_log(LOG_INFO, "Got $rep for transport $queue");
$handler = $this->getHandler($queue);
if ($handler) {
if ($handler->handle($item)) {
$this->_log(LOG_INFO, "[$queue:$rep] Successfully handled item");
$this->_done($qi);
} else {
$this->_log(LOG_INFO, "[$queue:$rep] Failed to handle item");
$this->_fail($qi);
}
} else {
$this->_log(LOG_INFO, "[$queue:$rep] No handler for queue $queue; discarding.");
$this->_done($qi);
}
} else {
$this->_log(LOG_INFO, "[$queue] Got empty/deleted item, discarding");
$this->_fail($qi);
}
return true;
}
/**
* Delete our claimed item from the queue after successful processing.
*
* @param QueueItem $qi
*/
protected function _done($qi)
{
$queue = $qi->transport;
if (empty($qi->claimed)) {
$this->_log(LOG_WARNING, "Reluctantly releasing unclaimed queue item $qi->id from $qi->queue");
}
$qi->delete();
$this->stats('handled', $queue);
}
/**
* Free our claimed queue item for later reprocessing in case of
* temporary failure.
*
* @param QueueItem $qi
*/
protected function _fail($qi)
{
$queue = $qi->transport;
if (empty($qi->claimed)) {
$this->_log(LOG_WARNING, "[$queue:item $qi->id] Ignoring failure for unclaimed queue item");
} else {
$orig = clone($qi);
$qi->claimed = null;
$qi->update($orig);
}
$this->stats('error', $queue);
}
}