From 9400795a5f4d5aa8b46c82cd7b021330d1f10e3a Mon Sep 17 00:00:00 2001 From: Mikael Nordfeldth Date: Wed, 20 Nov 2013 21:20:42 +0100 Subject: [PATCH] Cron split into Cronish and OpportunisticQM /main/cron changed to /main/runqueue The key-required functionality is not throughly tested yet. --- lib/default.php | 3 +- lib/queuemanager.php | 4 +- plugins/Cronish/CronishPlugin.php | 42 ++++++++ plugins/Cronish/lib/cronish.php | 58 ++++++++++ .../OpportunisticQM/OpportunisticQMPlugin.php | 46 ++++++++ plugins/OpportunisticQM/actions/runqueue.php | 61 +++++++++++ .../lib/opportunisticqueuemanager.php | 102 ++++++++++++++++++ .../lib/runqueuebadkeyexception.php | 39 +++++++ .../lib/runqueueoutofworkexception.php | 36 +++++++ 9 files changed, 388 insertions(+), 3 deletions(-) create mode 100644 plugins/Cronish/CronishPlugin.php create mode 100644 plugins/Cronish/lib/cronish.php create mode 100644 plugins/OpportunisticQM/OpportunisticQMPlugin.php create mode 100644 plugins/OpportunisticQM/actions/runqueue.php create mode 100644 plugins/OpportunisticQM/lib/opportunisticqueuemanager.php create mode 100644 plugins/OpportunisticQM/lib/runqueuebadkeyexception.php create mode 100644 plugins/OpportunisticQM/lib/runqueueoutofworkexception.php diff --git a/lib/default.php b/lib/default.php index 5ce0f45061..27c26c15a2 100644 --- a/lib/default.php +++ b/lib/default.php @@ -294,8 +294,9 @@ $default = 'plugins' => array('core' => array( 'AuthCrypt' => array(), - 'Cron' => array(), + 'Cronish' => array(), 'LRDD' => array(), + 'OpportunisticQM' => array(), 'StrictTransportSecurity' => array(), ), 'default' => array( diff --git a/lib/queuemanager.php b/lib/queuemanager.php index c8409df2e4..1fb6c3eb45 100644 --- a/lib/queuemanager.php +++ b/lib/queuemanager.php @@ -67,8 +67,8 @@ abstract class QueueManager extends IoManager self::$qm = new UnQueueManager(); } else { switch ($type) { - case 'cron': - self::$qm = new CronQueueManager(); + case 'opportunistic': + self::$qm = new OpportunisticQueueManager(); break; case 'db': self::$qm = new DBQueueManager(); diff --git a/plugins/Cronish/CronishPlugin.php b/plugins/Cronish/CronishPlugin.php new file mode 100644 index 0000000000..b0c0095542 --- /dev/null +++ b/plugins/Cronish/CronishPlugin.php @@ -0,0 +1,42 @@ +callTimedEvents(); + + return true; + } + + public function onPluginVersion(&$versions) + { + $versions[] = array('name' => 'Cronish', + 'version' => GNUSOCIAL_VERSION, + 'author' => 'Mikael Nordfeldth', + 'homepage' => 'http://www.gnu.org/software/social/', + 'description' => + // TRANS: Plugin description. + _m('Cronish plugin that executes events on a near-hour/day/week basis.')); + return true; + } +} diff --git a/plugins/Cronish/lib/cronish.php b/plugins/Cronish/lib/cronish.php new file mode 100644 index 0000000000..c7d79eeb5f --- /dev/null +++ b/plugins/Cronish/lib/cronish.php @@ -0,0 +1,58 @@ + + * @copyright 2013 Free Software Foundation, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0 + * @link http://status.net/ + */ +class Cronish +{ + /** + * Will call events as close as it gets to one hour. Event handlers + * which use this MUST be as quick as possible, maybe only adding a + * queue item to be handled later or something. Otherwise execution + * will timeout for PHP - or at least cause unnecessary delays for + * the unlucky user who visits the site exactly at one of these events. + */ + public function callTimedEvents() + { + $timers = array('hourly' => 3600, + 'daily' => 86400, + 'weekly' => 604800); + + foreach($timers as $name=>$interval) { + $run = false; + + $lastrun = new Config(); + $lastrun->section = 'cron'; + $lastrun->setting = 'last_' . $name; + $found = $lastrun->find(true); + + if (!$found) { + $lastrun->value = time(); + if ($lastrun->insert() === false) { + common_log(LOG_WARNING, "Could not save 'cron' setting '{$name}'"); + continue; + } + $run = true; + } elseif ($lastrun->value < time() - $interval) { + $orig = clone($lastrun); + $lastrun->value = time(); + $lastrun->update($orig); + $run = true; + } + + if ($run === true) { + // such as CronHourly, CronDaily, CronWeekly + Event::handle('Cron' . ucfirst($name)); + } + } + } +} diff --git a/plugins/OpportunisticQM/OpportunisticQMPlugin.php b/plugins/OpportunisticQM/OpportunisticQMPlugin.php new file mode 100644 index 0000000000..89bd15a1a0 --- /dev/null +++ b/plugins/OpportunisticQM/OpportunisticQMPlugin.php @@ -0,0 +1,46 @@ +connect('main/runqueue', array('action' => 'runqueue')); + } + + /** + * When the page has finished rendering, let's do some cron jobs + * if we have the time. + */ + public function onEndActionExecute($status, Action $action) + { + if ($action instanceof RunqueueAction) { + return true; + } + + global $_startTime; + + $args = array( + 'qmkey' => common_config('opportunisticqm', 'qmkey'), + 'max_execution_time' => $this->secs_per_action, + 'started_at' => $this->rel_to_pageload ? $_startTime : null, + ); + $qm = new OpportunisticQueueManager($args); + $qm->runQueue(); + return true; + } + + public function onPluginVersion(&$versions) + { + $versions[] = array('name' => 'OpportunisticQM', + 'version' => GNUSOCIAL_VERSION, + 'author' => 'Mikael Nordfeldth', + 'homepage' => 'http://www.gnu.org/software/social/', + 'description' => + // TRANS: Plugin description. + _m('Opportunistic queue manager plugin for background processing.')); + return true; + } +} diff --git a/plugins/OpportunisticQM/actions/runqueue.php b/plugins/OpportunisticQM/actions/runqueue.php new file mode 100644 index 0000000000..e6ce5b84c0 --- /dev/null +++ b/plugins/OpportunisticQM/actions/runqueue.php @@ -0,0 +1,61 @@ +. + */ + +if (!defined('GNUSOCIAL')) { exit(1); } + +class RunqueueAction extends Action +{ + protected $qm = null; + + protected function prepare(array $args=array()) + { + parent::prepare($args); + + $args = array(); + + foreach (array('qmkey') as $key) { + if ($this->arg($key) !== null) { + $args[$key] = $this->arg($key); + } + } + + try { + $this->qm = new OpportunisticQueueManager($args); + } catch (RunQueueBadKeyException $e) { + return false; + } + + header('Content-type: text/plain; charset=utf-8'); + + return true; + } + + protected function handle() { + // We don't need any of the parent functionality from parent::handle() here. + + // runQueue is a loop that works until limits have passed or there is no more work + if ($this->qm->runQueue() === true) { + // We don't have any more work + $this->text('0'); + } else { + // There were still items left in queue when we aborted + $this->text('1'); + } + } +} diff --git a/plugins/OpportunisticQM/lib/opportunisticqueuemanager.php b/plugins/OpportunisticQM/lib/opportunisticqueuemanager.php new file mode 100644 index 0000000000..5b09be8f9c --- /dev/null +++ b/plugins/OpportunisticQM/lib/opportunisticqueuemanager.php @@ -0,0 +1,102 @@ + + * @copyright 2013 Free Software Foundation, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0 + * @link http://status.net/ + */ +class OpportunisticQueueManager extends DBQueueManager +{ + protected $qmkey = false; + protected $max_execution_time = null; + protected $max_queue_items = null; + + protected $started_at = null; + protected $handled_items = 0; + + const MAXEXECTIME = 30; // typically just used for the /main/cron action + + public function __construct(array $args=array()) { + foreach (get_class_vars(get_class($this)) as $key=>$val) { + if (array_key_exists($key, $args)) { + $this->$key = $args[$key]; + } + } + $this->verifyKey(); + + if ($this->started_at === null) { + $this->started_at = time(); + } + + if ($this->max_execution_time === null) { + $this->max_execution_time = ini_get('max_execution_time') ?: self::MAXEXECTIME; + } + + return parent::__construct(); + } + + protected function verifyKey() + { + if ($this->qmkey !== common_config('opportunisticqm', 'qmkey')) { + throw new RunQueueBadKeyException($this->qmkey); + } + } + + public function canContinue() + { + $time_passed = time() - $this->started_at; + + // Only continue if limit values are sane + if ($time_passed <= 0 && (!is_null($this->max_queue_items) && $this->max_queue_items <= 0)) { + return false; + } + // If too much time has passed, stop + if ($time_passed >= $this->max_execution_time) { + return false; + } + // If we have a max-item-limit, check if it has been passed + if (!is_null($this->max_queue_items) && $this->handled_items >= $this->max_queue_items) { + return false; + } + + return true; + } + + public function poll() + { + $this->handled_items++; + if (!parent::poll()) { + throw new RunQueueOutOfWorkException(); + } + return true; + } + + /** + * Takes care of running through the queue items, returning when + * the limits setup in __construct are met. + * + * @return true on workqueue finished, false if there are still items in the queue + */ + public function runQueue() + { + while ($this->canContinue()) { + try { + $this->poll(); + } catch (RunQueueOutOfWorkException $e) { + common_debug('Opportunistic queue manager finished.'); + return true; + } + } + common_debug('Opportunistic queue manager passed execution time/item handling limit without being out of work.'); + return false; + } +} diff --git a/plugins/OpportunisticQM/lib/runqueuebadkeyexception.php b/plugins/OpportunisticQM/lib/runqueuebadkeyexception.php new file mode 100644 index 0000000000..1e17b5f23e --- /dev/null +++ b/plugins/OpportunisticQM/lib/runqueuebadkeyexception.php @@ -0,0 +1,39 @@ +. + * + * @category Exception + * @package GNUsocial + * @author Mikael Nordfeldth + * @copyright 2013 Free Software Foundation, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPLv3 + * @link http://www.gnu.org/software/social/ + */ + +if (!defined('GNUSOCIAL')) { exit(1); } + +class RunQueueBadKeyException extends ClientException +{ + public $qmkey; + + public function __construct($qmkey) + { + $this->qmkey = $qmkey; + parent::__construct(_('Bad queue manager key was used.')); + } +} diff --git a/plugins/OpportunisticQM/lib/runqueueoutofworkexception.php b/plugins/OpportunisticQM/lib/runqueueoutofworkexception.php new file mode 100644 index 0000000000..f81aeda5c2 --- /dev/null +++ b/plugins/OpportunisticQM/lib/runqueueoutofworkexception.php @@ -0,0 +1,36 @@ +. + * + * @category Exception + * @package GNUsocial + * @author Mikael Nordfeldth + * @copyright 2013 Free Software Foundation, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPLv3 + * @link http://www.gnu.org/software/social/ + */ + +if (!defined('GNUSOCIAL')) { exit(1); } + +class RunQueueOutOfWorkException extends ServerException +{ + public function __construct() + { + parent::__construct(_('Opportunistic queue manager is out of work (no more items).')); + } +}