From 22fead1b46118a4fcb45b20a3bce620dd3283b65 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 20 Oct 2011 10:40:39 -0400 Subject: [PATCH] Squashed commit of the following: commit fb1dfa9e98ded23fb5bdebae6465424a8cb8acd6 Author: Evan Prodromou Date: Thu Oct 20 10:40:07 2011 -0400 Use popular notice stream for favorited page commit e1d409ff738e39061ad35589d546ce9bed456975 Author: Evan Prodromou Date: Thu Oct 20 10:32:23 2011 -0400 Use a caching stream for popular notice section Instead of a big cached query, we now use a caching notice stream for the popular notice section. It uses a single-table query at the bottom, then scopes the notices and filters for silenced users. This should be much nicer to our database servers. Also clears the popular cache when someone favors or disfavors something. A nice optimization would be to save the last weights and re-calculate them at invalidation time, adding the new notice (or not) depending on its own score. That will have to wait for another day, though. commit e9b7ab4c26c95e755adaff53c3957dcfca31c16b Author: Evan Prodromou Date: Thu Oct 20 10:31:14 2011 -0400 Let CachingNoticeStream users skip the ';last' optimization --- actions/favorited.php | 7 +-- actions/public.php | 4 +- classes/Fave.php | 2 + lib/cachingnoticestream.php | 43 ++++++++------ lib/popularnoticesection.php | 19 +++--- lib/popularnoticestream.php | 109 +++++++++++++++++++++++++++++++++++ 6 files changed, 152 insertions(+), 32 deletions(-) create mode 100644 lib/popularnoticestream.php diff --git a/actions/favorited.php b/actions/favorited.php index 17c2a58c94..ff4a99cd60 100644 --- a/actions/favorited.php +++ b/actions/favorited.php @@ -172,11 +172,8 @@ class FavoritedAction extends Action */ function showContent() { - $pop = new Popularity(); - $pop->offset = ($this->page - 1) * NOTICES_PER_PAGE; - $pop->limit = NOTICES_PER_PAGE; - $pop->expiry = 600; - $notice = $pop->getNotices(); + $stream = new PopularNoticeStream(Profile::current()); + $notice = $stream->getNotices(($this->page-1)*NOTICES_PER_PAGE, NOTICES_PER_PAGE+1); $nl = new NoticeList($notice, $this); diff --git a/actions/public.php b/actions/public.php index cf732fe464..7bcdd3fae2 100644 --- a/actions/public.php +++ b/actions/public.php @@ -255,7 +255,9 @@ class PublicAction extends Action $ibs->show(); } - $pop = new PopularNoticeSection($this); + $p = Profile::current(); + + $pop = new PopularNoticeSection($this, $p); $pop->show(); if (!common_config('performance', 'high')) { $cloud = new PublicTagCloudSection($this); diff --git a/classes/Fave.php b/classes/Fave.php index 467465ba05..59a1e00318 100644 --- a/classes/Fave.php +++ b/classes/Fave.php @@ -74,6 +74,7 @@ class Fave extends Managed_DataObject return false; } self::blow('fave:list-ids:notice_id:%d', $fave->notice_id); + self::blow('popular'); Event::handle('EndFavorNotice', array($profile, $notice)); } @@ -92,6 +93,7 @@ class Fave extends Managed_DataObject $result = parent::delete(); self::blow('fave:list-ids:notice_id:%d', $this->notice_id); + self::blow('popular'); if ($result) { Event::handle('EndDisfavorNotice', array($profile, $notice)); diff --git a/lib/cachingnoticestream.php b/lib/cachingnoticestream.php index f8ab2a85af..e68bb3a1f2 100644 --- a/lib/cachingnoticestream.php +++ b/lib/cachingnoticestream.php @@ -51,11 +51,13 @@ class CachingNoticeStream extends NoticeStream public $stream = null; public $cachekey = null; + public $useLast = true; - function __construct($stream, $cachekey) + function __construct($stream, $cachekey, $useLast = true) { $this->stream = $stream; $this->cachekey = $cachekey; + $this->useLast = $useLast; } function getNoticeIds($offset, $limit, $sinceId, $maxId) @@ -85,29 +87,31 @@ class CachingNoticeStream extends NoticeStream return $ids; } - // Check the cache to see if we have a "last-known-good" version. - // The actual cache gets blown away when new notices are added, but - // the "last" value holds a lot of info. We might need to only generate - // a few at the "tip", which can bound our queries and save lots - // of time. + if ($this->useLast) { + // Check the cache to see if we have a "last-known-good" version. + // The actual cache gets blown away when new notices are added, but + // the "last" value holds a lot of info. We might need to only generate + // a few at the "tip", which can bound our queries and save lots + // of time. - $laststr = $cache->get($idkey.';last'); + $laststr = $cache->get($idkey.';last'); - if ($laststr !== false) { - $window = explode(',', $laststr); - $last_id = $window[0]; - $new_ids = $this->stream->getNoticeIds(0, self::CACHE_WINDOW, $last_id, 0); + if ($laststr !== false) { + $window = explode(',', $laststr); + $last_id = $window[0]; + $new_ids = $this->stream->getNoticeIds(0, self::CACHE_WINDOW, $last_id, 0); - $new_window = array_merge($new_ids, $window); + $new_window = array_merge($new_ids, $window); - $new_windowstr = implode(',', $new_window); + $new_windowstr = implode(',', $new_window); - $result = $cache->set($idkey, $new_windowstr); - $result = $cache->set($idkey . ';last', $new_windowstr); + $result = $cache->set($idkey, $new_windowstr); + $result = $cache->set($idkey . ';last', $new_windowstr); - $ids = array_slice($new_window, $offset, $limit); + $ids = array_slice($new_window, $offset, $limit); - return $ids; + return $ids; + } } // No cache hits :( Generate directly and stick the results @@ -118,7 +122,10 @@ class CachingNoticeStream extends NoticeStream $windowstr = implode(',', $window); $result = $cache->set($idkey, $windowstr); - $result = $cache->set($idkey . ';last', $windowstr); + + if ($this->useLast) { + $result = $cache->set($idkey . ';last', $windowstr); + } // Return just the slice that was requested diff --git a/lib/popularnoticesection.php b/lib/popularnoticesection.php index e66df8f423..2000d302d4 100644 --- a/lib/popularnoticesection.php +++ b/lib/popularnoticesection.php @@ -22,7 +22,7 @@ * @category Widget * @package StatusNet * @author Evan Prodromou - * @copyright 2009 StatusNet, Inc. + * @copyright 2009,2011 StatusNet, Inc. * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 * @link http://status.net/ */ @@ -45,15 +45,18 @@ if (!defined('STATUSNET') && !defined('LACONICA')) { */ class PopularNoticeSection extends NoticeSection { + protected $viewer; + + function __construct($out, $viewer) + { + parent::__construct($out); + $this->viewer = $viewer; + } + function getNotices() { - $pop = new Popularity(); - if (!empty($this->out->tag)) { - $pop->tag = $this->out->tag; - } - $pop->limit = NOTICES_PER_SECTION; - $pop->expiry = 1200; - return $pop->getNotices(); + $stream = new PopularNoticeStream($this->viewer); + return $stream->getNotices(0, NOTICES_PER_SECTION + 1); } function title() diff --git a/lib/popularnoticestream.php b/lib/popularnoticestream.php new file mode 100644 index 0000000000..794fd87557 --- /dev/null +++ b/lib/popularnoticestream.php @@ -0,0 +1,109 @@ +. + * + * @category Popular + * @package StatusNet + * @author Evan Prodromou + * @copyright 2011 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0 + * @link http://status.net/ + */ + +if (!defined('STATUSNET')) { + // This check helps protect against security problems; + // your code file can't be executed directly from the web. + exit(1); +} + +/** + * Stream of notices sorted by popularity + * + * @category Popular + * @package StatusNet + * @author Evan Prodromou + * @copyright 2011 StatusNet, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0 + * @link http://status.net/ + */ + +class PopularNoticeStream extends ScopingNoticeStream +{ + function __construct($profile=null) + { + parent::__construct(new HideSilencedStream(new CachingNoticeStream(new RawPopularNoticeStream(), + 'popular', + false)), + $profile); + } +} + +class HideSilencedStream extends FilteringNoticeStream +{ + /** + * Only return notices where the profile is in scope + * + * @param Notice $notice The notice to check + * + * @return boolean whether to include the notice + */ + + function filter($notice) + { + $author = $notice->getProfile(); + return !$author->isSilenced(); + } +} + +class RawPopularNoticeStream extends NoticeStream +{ + function getNoticeIds($offset, $limit, $since_id, $max_id) + { + $weightexpr = common_sql_weight('modified', common_config('popular', 'dropoff')); + $cutoff = sprintf("modified > '%s'", + common_sql_date(time() - common_config('popular', 'cutoff'))); + + $fave = new Fave(); + $fave->selectAdd(); + $fave->selectAdd('notice_id'); + $fave->selectAdd("$weightexpr as weight"); + $fave->whereAdd($cutoff); + $fave->orderBy('weight DESC'); + $fave->groupBy('notice_id'); + + if (!is_null($offset)) { + $fave->limit($offset, $limit); + } + + // FIXME: $since_id, $max_id are ignored + + $ids = array(); + + if ($fave->find()) { + while ($fave->fetch()) { + $ids[] = $fave->notice_id; + } + } + + return $ids; + } +} +