From 8594a2ba16a6884bbac6dbe0951c263ce940944f Mon Sep 17 00:00:00 2001 From: Mikael Nordfeldth Date: Wed, 14 Jan 2015 01:16:28 +0100 Subject: [PATCH] FeedPoller plugin, for hubless feeds --- plugins/FeedPoller/FeedPollerPlugin.php | 61 ++++++++++++++++ plugins/FeedPoller/lib/feedpoll.php | 69 ++++++++++++++++++ .../FeedPoller/lib/feedpollqueuehandler.php | 41 +++++++++++ plugins/FeedPoller/scripts/pollfeed.php | 71 +++++++++++++++++++ plugins/OStatus/classes/FeedSub.php | 2 +- 5 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 plugins/FeedPoller/FeedPollerPlugin.php create mode 100644 plugins/FeedPoller/lib/feedpoll.php create mode 100644 plugins/FeedPoller/lib/feedpollqueuehandler.php create mode 100644 plugins/FeedPoller/scripts/pollfeed.php diff --git a/plugins/FeedPoller/FeedPollerPlugin.php b/plugins/FeedPoller/FeedPollerPlugin.php new file mode 100644 index 0000000000..c844d99c1b --- /dev/null +++ b/plugins/FeedPoller/FeedPollerPlugin.php @@ -0,0 +1,61 @@ + + * @copyright 2013 Free Software Foundation, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0 + * @link http://www.gnu.org/software/social/ + */ + +if (!defined('GNUSOCIAL')) { exit(1); } + +class FeedPollerPlugin extends Plugin { + public $interval = 5; // interval in minutes for feed checks + + public function onEndInitializeQueueManager(QueueManager $qm) + { + $qm->connect(FeedPoll::QUEUE_CHECK, 'FeedPollQueueHandler'); + return true; + } + + public function onCronMinutely() + { + $args = array('interval'=>$this->interval); + FeedPoll::enqueueNewFeeds($args); + return true; + } + + public function onFeedSubscribe(FeedSub $feedsub) + { + if (!$feedsub->isPuSH()) { + FeedPoll::setupFeedSub($feedsub, $this->interval*60); + return false; // We're polling this feed, so stop processing FeedSubscribe + } + return true; + } + + public function onFeedUnsubscribe(FeedSub $feedsub) + { + if (!$feedsub->isPuSH()) { + // removes sub_state setting and such + $feedsub->confirmUnsubscribe(); + return false; + } + return true; + } + + public function onPluginVersion(&$versions) + { + $versions[] = array('name' => 'FeedPoller', + 'version' => GNUSOCIAL_VERSION, + 'author' => 'Mikael Nordfeldth', + 'homepage' => 'http://www.gnu.org/software/social/', + 'description' => + // TRANS: Plugin description. + _m('Feed polling plugin to avoid using external push hubs.')); + return true; + } +} diff --git a/plugins/FeedPoller/lib/feedpoll.php b/plugins/FeedPoller/lib/feedpoll.php new file mode 100644 index 0000000000..1ac5fad0cf --- /dev/null +++ b/plugins/FeedPoller/lib/feedpoll.php @@ -0,0 +1,69 @@ + + * @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3 + * @link http://www.gnu.org/software/social/ + */ + +if (!defined('GNUSOCIAL')) { exit(1); } + +class FeedPoll { + const DEFAULT_INTERVAL = 5; // in minutes + + const QUEUE_CHECK = 'feedpoll-check'; + + // TODO: Find some smart way to add feeds only once, so they don't get more than 1 feedpoll in the queue each + // probably through sub_start sub_end trickery. + public static function enqueueNewFeeds(array $args=array()) { + if (!isset($args['interval']) || !is_int($args['interval']) || $args['interval']<=0) { + $args['interval'] = self::DEFAULT_INTERVAL; + } + + $args['interval'] *= 60; // minutes to seconds + + $feedsub = new FeedSub(); + $feedsub->sub_state = 'nohub'; + // Find feeds that haven't been polled within the desired interval, + // though perhaps we're abusing the "last_update" field here? + $feedsub->whereAdd(sprintf('last_update < "%s"', common_sql_date(time()-$args['interval']))); + $feedsub->find(); + + $qm = QueueManager::get(); + while ($feedsub->fetch()) { + $orig = clone($feedsub); + $item = array('id' => $feedsub->id); + $qm->enqueue($item, self::QUEUE_CHECK); + common_debug('Enqueueing FeedPoll feeds, currently: '.$feedsub->uri); + $feedsub->last_update = common_sql_now(); + $feedsub->update($orig); + } + } + + public function setupFeedSub(FeedSub $feedsub, $interval=300) + { + $orig = clone($feedsub); + $feedsub->sub_state = 'nohub'; + $feedsub->sub_start = common_sql_date(time()); + $feedsub->sub_end = ''; + $feedsub->last_update = common_sql_date(time()-$interval); // force polling as soon as we can + $feedsub->update($orig); + } + + public function checkUpdates(FeedSub $feedsub) + { + $request = new HTTPClient(); + common_debug('Enqueueing FeedPoll feeds went well, now checking updates for: '.$feedsub->getUri()); + $feed = $request->get($feedsub->uri); + if (!$feed->isOk()) { + throw new ServerException('FeedSub could not fetch id='.$feedsub->id.' (Error '.$feed->getStatus().': '.$feed->getBody()); + } + $feedsub->receive($feed->getBody(), null); + } +} diff --git a/plugins/FeedPoller/lib/feedpollqueuehandler.php b/plugins/FeedPoller/lib/feedpollqueuehandler.php new file mode 100644 index 0000000000..c80253dcba --- /dev/null +++ b/plugins/FeedPoller/lib/feedpollqueuehandler.php @@ -0,0 +1,41 @@ + + */ +class FeedPollQueueHandler extends QueueHandler +{ + public function transport() + { + return FeedPoll::QUEUE_CHECK; + } + + public function handle($item) + { + common_debug('Enqueueing FeedPoll feeds but actually running the queue handler!'); + $feedsub = FeedSub::getKV('id', $item['id']); + if (!$feedsub instanceof FeedSub) { + // Removed from the feedsub table I guess + return true; + } + if (!$feedsub->sub_state == 'nohub') { + // We're not supposed to poll this (either it's PuSH or it's unsubscribed) + return true; + } + + common_debug('Enqueueing FeedPoll feeds but actually checking updates'); + try { + FeedPoll::checkUpdates($feedsub); + } catch (Exception $e) { + common_log(LOG_ERR, "Failed to check feedsub id= ".$feedsub->id.' ("'.$e->getMessage().'")'); + } + + common_debug('Enqueueing FeedPoll feeds but actually done with '.$feedsub->id); + + return true; + } +} diff --git a/plugins/FeedPoller/scripts/pollfeed.php b/plugins/FeedPoller/scripts/pollfeed.php new file mode 100644 index 0000000000..3aa8296416 --- /dev/null +++ b/plugins/FeedPoller/scripts/pollfeed.php @@ -0,0 +1,71 @@ +#!/usr/bin/env php +. + */ + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/../../..')); + +$helptext = <<sub_state != 'nohub') { + echo "Feed is a PuSH feed, so we will not poll it.\n"; + exit(1); +} + +showSub($feedsub); + +try { + FeedPoll::checkUpdates($feedsub); +} catch (Exception $e) { + echo "Could not check updates for feed: ".$e->getMessage(); + echo $e->getTraceAsString(); + exit(1); +} + +function showSub(FeedSub $sub) +{ + echo " Subscription state: $sub->sub_state\n"; + echo " Signature secret: $sub->secret\n"; + echo " Sub start date: $sub->sub_start\n"; + echo " Record created: $sub->created\n"; + echo " Record modified: $sub->modified\n"; +} diff --git a/plugins/OStatus/classes/FeedSub.php b/plugins/OStatus/classes/FeedSub.php index 338c2f5144..25a826a552 100644 --- a/plugins/OStatus/classes/FeedSub.php +++ b/plugins/OStatus/classes/FeedSub.php @@ -435,7 +435,7 @@ class FeedSub extends Managed_DataObject { common_log(LOG_INFO, __METHOD__ . ": packet for \"" . $this->getUri() . "\"! $hmac $post"); - if ($this->sub_state != 'active') { + if (!in_array($this->sub_state, array('active', 'nohub'))) { common_log(LOG_ERR, __METHOD__ . ": ignoring PuSH for inactive feed " . $this->getUri() . " (in state '$this->sub_state')"); return; }