From 7a7e2162dd7eed59e60d9360d8692abc111d940c Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 3 Feb 2010 14:58:29 -0500 Subject: [PATCH 01/18] Script to update profile URLs --- scripts/updateprofileurl.php | 99 ++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 scripts/updateprofileurl.php diff --git a/scripts/updateprofileurl.php b/scripts/updateprofileurl.php new file mode 100644 index 0000000000..2fc6828e60 --- /dev/null +++ b/scripts/updateprofileurl.php @@ -0,0 +1,99 @@ +#!/usr/bin/env php +. + */ + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); + +$shortoptions = 'i:n:a'; +$longoptions = array('id=', 'nickname=', 'all'); + +$helptext = <<find()) { + while ($user->fetch()) { + updateProfileURL($user); + } + } + } else { + show_help(); + exit(1); + } +} catch (Exception $e) { + print $e->getMessage()."\n"; + exit(1); +} + +function updateProfileURL($user) +{ + $profile = $user->getProfile(); + + if (empty($profile)) { + throw new Exception("Can't find profile for user $user->nickname ($user->id)"); + } + + $orig = clone($profile); + + $profile->profileurl = common_profile_url($user->nickname); + + if (!have_option('q', 'quiet')) { + print "Updating profile url for $user->nickname ($user->id) ". + "from $orig->profileurl to $profile->profileurl..."; + } + + $result = $profile->update($orig); + + if (!$result) { + print "FAIL.\n"; + common_log_db_error($profile, 'UPDATE', __FILE__); + throw new Exception("Can't update profile for user $user->nickname ($user->id)"); + } + + common_broadcast_profile($profile); + + print "OK.\n"; +} From 9ca4fd69b3d5a0f86e1307746dda74e2b02bd668 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Thu, 4 Feb 2010 01:53:08 +0000 Subject: [PATCH 02/18] - Fix cache handling in TwitterStatusFetcher - Other stability fixes --- .../daemons/twitterstatusfetcher.php | 53 ++++++++++++++++--- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php index 36732ce46a..bff657eb68 100755 --- a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php +++ b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php @@ -2,7 +2,7 @@ is_local = Notice::GATEWAY; if (Event::handle('StartNoticeSave', array(&$notice))) { - $id = $notice->insert(); + $notice->insert(); Event::handle('EndNoticeSave', array($notice)); } @@ -270,17 +270,41 @@ class TwitterStatusFetcher extends ParallelizingDaemon Inbox::insertNotice($flink->user_id, $notice->id); - $notice->blowCaches(); + $notice->blowOnInsert(); return $notice; } + /** + * Look up a Profile by profileurl field. Profile::staticGet() was + * not working consistently. + * + * @param string $url the profile url + * + * @return mixed the first profile with that url, or null + */ + + function getProfileByUrl($nickname, $profileurl) + { + $profile = new Profile(); + $profile->nickname = $nickname; + $profile->profileurl = $profileurl; + $profile->limit(1); + + if ($profile->find()) { + $profile->fetch(); + return $profile; + } + + return null; + } + function ensureProfile($user) { // check to see if there's already a profile for this user $profileurl = 'http://twitter.com/' . $user->screen_name; - $profile = Profile::staticGet('profileurl', $profileurl); + $profile = $this->getProfileByUrl($user->screen_name, $profileurl); if (!empty($profile)) { common_debug($this->name() . @@ -292,6 +316,7 @@ class TwitterStatusFetcher extends ParallelizingDaemon return $profile->id; } else { + common_debug($this->name() . ' - Adding profile and remote profile ' . "for Twitter user: $profileurl."); @@ -306,7 +331,11 @@ class TwitterStatusFetcher extends ParallelizingDaemon $profile->profileurl = $profileurl; $profile->created = common_sql_now(); - $id = $profile->insert(); + try { + $id = $profile->insert(); + } catch(Exception $e) { + common_log(LOG_WARNING, $this->name . ' Couldn\'t insert profile - ' . $e->getMessage()); + } if (empty($id)) { common_log_db_error($profile, 'INSERT', __FILE__); @@ -326,7 +355,11 @@ class TwitterStatusFetcher extends ParallelizingDaemon $remote_pro->uri = $profileurl; $remote_pro->created = common_sql_now(); - $rid = $remote_pro->insert(); + try { + $rid = $remote_pro->insert(); + } catch (Exception $e) { + common_log(LOG_WARNING, $this->name() . ' Couldn\'t save remote profile - ' . $e->getMessage()); + } if (empty($rid)) { common_log_db_error($profile, 'INSERT', __FILE__); @@ -446,7 +479,7 @@ class TwitterStatusFetcher extends ParallelizingDaemon if ($this->fetchAvatar($url, $filename)) { $this->newAvatar($id, $size, $mediatype, $filename); } else { - common_log(LOG_WARNING, $this->id() . + common_log(LOG_WARNING, $id() . " - Problem fetching Avatar: $url"); } } @@ -507,7 +540,11 @@ class TwitterStatusFetcher extends ParallelizingDaemon $avatar->created = common_sql_now(); - $id = $avatar->insert(); + try { + $id = $avatar->insert(); + } catch (Exception $e) { + common_log(LOG_WARNING, $this->name() . ' Couldn\'t insert avatar - ' . $e->getMessage()); + } if (empty($id)) { common_log_db_error($avatar, 'INSERT', __FILE__); From d2dc3e41c571cb4e9edef18b3bb1918633a9ce39 Mon Sep 17 00:00:00 2001 From: Sarven Capadisli Date: Thu, 4 Feb 2010 16:27:34 +0000 Subject: [PATCH 03/18] Fixes minor remote subscription profile layout --- actions/userauthorization.php | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/actions/userauthorization.php b/actions/userauthorization.php index 4321f1302e..7f71c60dbe 100644 --- a/actions/userauthorization.php +++ b/actions/userauthorization.php @@ -127,10 +127,10 @@ class UserauthorizationAction extends Action $location = $params->getLocation(); $avatar = $params->getAvatarURL(); - $this->elementStart('div', array('class' => 'profile')); $this->elementStart('div', 'entity_profile vcard'); - $this->elementStart('a', array('href' => $profile, - 'class' => 'url')); + $this->elementStart('dl', 'entity_depiction'); + $this->element('dt', null, _('Photo')); + $this->elementStart('dd'); if ($avatar) { $this->element('img', array('src' => $avatar, 'class' => 'photo avatar', @@ -138,11 +138,19 @@ class UserauthorizationAction extends Action 'height' => AVATAR_PROFILE_SIZE, 'alt' => $nickname)); } + $this->elementEnd('dd'); + $this->elementEnd('dl'); + + $this->elementStart('dl', 'entity_nickname'); + $this->element('dt', null, _('Nickname')); + $this->elementStart('dd'); $hasFN = ($fullname !== '') ? 'nickname' : 'fn nickname'; - $this->elementStart('span', $hasFN); + $this->elementStart('a', array('href' => $profile, + 'class' => 'url '.$hasFN)); $this->raw($nickname); - $this->elementEnd('span'); $this->elementEnd('a'); + $this->elementEnd('dd'); + $this->elementEnd('dl'); if (!is_null($fullname)) { $this->elementStart('dl', 'entity_fn'); @@ -214,7 +222,6 @@ class UserauthorizationAction extends Action $this->elementEnd('li'); $this->elementEnd('ul'); $this->elementEnd('div'); - $this->elementEnd('div'); } function sendAuthorization() @@ -350,4 +357,4 @@ class UserauthorizationAction extends Action } } } -} \ No newline at end of file +} From c56250fb33e2e43a7c0f16bb78810e4ed70cf792 Mon Sep 17 00:00:00 2001 From: Sarven Capadisli Date: Thu, 4 Feb 2010 16:51:51 +0000 Subject: [PATCH 04/18] Added accept and reject icons to remote subscription authorization --- theme/base/images/icons/icons-01.gif | Bin 3758 -> 3870 bytes .../images/icons/twotone/green/against.gif | Bin 0 -> 85 bytes .../images/icons/twotone/green/checkmark.gif | Bin 0 -> 76 bytes theme/default/css/display.css | 9 ++++++++- theme/identica/css/display.css | 9 ++++++++- 5 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 theme/base/images/icons/twotone/green/against.gif create mode 100644 theme/base/images/icons/twotone/green/checkmark.gif diff --git a/theme/base/images/icons/icons-01.gif b/theme/base/images/icons/icons-01.gif index 01a729c10b92b7669ed745051227990269f70fe2..6f284f023ee7c7c1fef480c26e7ccfd2b27ba32c 100644 GIT binary patch delta 171 zcmV;c096039iAQxM@dFFIbj+A(g^4Nu?+qK6aWAJA^8LW3IP8AEC2ui02%<&vmFEY z3IYC;77mhj>6VmPnU3MbOL5+!)erc0<`I&NwL7_=retDXM8JP8^nvltv zM;Vfnd6~3Xo3hhj~J*c|oTsoWog|*~XWGNtnh-oXUxsjoF*f Z8J)%1M8FxGVFa8l2N|0$NnZg006RIlO_cxu delta 59 zcmbOyw@#MF-P6s&GEs`bkK?)jMxK96!tqlC6o0ZXaxwg8&|v@qpwwnr<{w;)( Date: Thu, 4 Feb 2010 11:06:01 -0800 Subject: [PATCH 05/18] Add time-based cutoffs for public tag cloud, favorited lists to speed up those queries. Defaulting to only looking at last 90 days of activity, can be adjusted up or down. $config['tag']['cutoff'] = 86400 * 90; $config['popular']['cutoff'] = 86400 * 90; Per-user and per-group tag clouds do not use the cutoff (and it doesn't help with indexing on them). --- actions/favorited.php | 3 +++ actions/publictagcloud.php | 3 +++ lib/default.php | 6 ++++-- lib/grouptagcloudsection.php | 1 + lib/personaltagcloudsection.php | 1 + lib/popularnoticesection.php | 7 +++++-- 6 files changed, 17 insertions(+), 4 deletions(-) diff --git a/actions/favorited.php b/actions/favorited.php index 9ffa5b8445..d8980440d1 100644 --- a/actions/favorited.php +++ b/actions/favorited.php @@ -186,10 +186,13 @@ class FavoritedAction extends Action function showContent() { $weightexpr = common_sql_weight('fave.modified', common_config('popular', 'dropoff')); + $cutoff = sprintf("fave.modified > '%s'", + common_sql_date(time() - common_config('popular', 'cutoff'))); $qry = 'SELECT notice.*, '. $weightexpr . ' as weight ' . 'FROM notice JOIN fave ON notice.id = fave.notice_id ' . + "WHERE $cutoff " . 'GROUP BY id,profile_id,uri,content,rendered,url,created,notice.modified,reply_to,is_local,source,notice.conversation ' . 'ORDER BY weight DESC'; diff --git a/actions/publictagcloud.php b/actions/publictagcloud.php index 9e4478dbb1..9993b2d3fd 100644 --- a/actions/publictagcloud.php +++ b/actions/publictagcloud.php @@ -106,7 +106,10 @@ class PublictagcloudAction extends Action #Add the aggregated columns... $tags->selectAdd('max(notice_id) as last_notice_id'); $calc = common_sql_weight('created', common_config('tag', 'dropoff')); + $cutoff = sprintf("notice_tag.created > '%s'", + common_sql_date(time() - common_config('tag', 'cutoff'))); $tags->selectAdd($calc . ' as weight'); + $tags->addWhere($cutoff); $tags->groupBy('tag'); $tags->orderBy('weight DESC'); diff --git a/lib/default.php b/lib/default.php index 2bedc4bf08..485a08ba44 100644 --- a/lib/default.php +++ b/lib/default.php @@ -144,9 +144,11 @@ $default = 'invite' => array('enabled' => true), 'tag' => - array('dropoff' => 864000.0), + array('dropoff' => 864000.0, # controls weighting based on age + 'cutoff' => 86400 * 90), # only look at notices posted in last 90 days 'popular' => - array('dropoff' => 864000.0), + array('dropoff' => 864000.0, # controls weighting based on age + 'cutoff' => 86400 * 90), # only look at notices favorited in last 90 days 'daemon' => array('piddir' => '/var/run', 'user' => false, diff --git a/lib/grouptagcloudsection.php b/lib/grouptagcloudsection.php index 14ceda0850..f1106cc7bf 100644 --- a/lib/grouptagcloudsection.php +++ b/lib/grouptagcloudsection.php @@ -59,6 +59,7 @@ class GroupTagCloudSection extends TagCloudSection function getTags() { $weightexpr = common_sql_weight('notice_tag.created', common_config('tag', 'dropoff')); + // @fixme should we use the cutoff too? Doesn't help with indexing per-group. $names = $this->group->getAliases(); diff --git a/lib/personaltagcloudsection.php b/lib/personaltagcloudsection.php index 091425f926..5ea3f188db 100644 --- a/lib/personaltagcloudsection.php +++ b/lib/personaltagcloudsection.php @@ -59,6 +59,7 @@ class PersonalTagCloudSection extends TagCloudSection function getTags() { $weightexpr = common_sql_weight('notice_tag.created', common_config('tag', 'dropoff')); + // @fixme should we use the cutoff too? Doesn't help with indexing per-user. $qry = 'SELECT notice_tag.tag, '. $weightexpr . ' as weight ' . diff --git a/lib/popularnoticesection.php b/lib/popularnoticesection.php index fbf9a60ab8..296ddbbb50 100644 --- a/lib/popularnoticesection.php +++ b/lib/popularnoticesection.php @@ -59,12 +59,15 @@ class PopularNoticeSection extends NoticeSection } } $weightexpr = common_sql_weight('fave.modified', common_config('popular', 'dropoff')); + $cutoff = sprintf("fave.modified > '%s'", + common_sql_date(time() - common_config('popular', 'cutoff'))); $qry = "SELECT notice.*, $weightexpr as weight "; if(isset($tag)) { $qry .= 'FROM notice_tag, notice JOIN fave ON notice.id = fave.notice_id ' . - "WHERE notice.id = notice_tag.notice_id and '$tag' = notice_tag.tag"; + "WHERE $cutoff and notice.id = notice_tag.notice_id and '$tag' = notice_tag.tag"; } else { - $qry .= 'FROM notice JOIN fave ON notice.id = fave.notice_id'; + $qry .= 'FROM notice JOIN fave ON notice.id = fave.notice_id ' . + "WHERE $cutoff"; } $qry .= ' GROUP BY notice.id,notice.profile_id,notice.content,notice.uri,' . 'notice.rendered,notice.url,notice.created,notice.modified,' . From 5bdc6fa5d456c3f520d8124950684220d8f440a3 Mon Sep 17 00:00:00 2001 From: Sarven Capadisli Date: Thu, 4 Feb 2010 19:39:46 +0000 Subject: [PATCH 06/18] Moved hardcoded identica theme out of MobileProfile. In this case, it will use whichever theme is loaded as its base and then add its own mobile styles. Of course, if a theme comes with its own mobile styles, it will use that instead as an addition to its own base. --- plugins/MobileProfile/MobileProfilePlugin.php | 10 ++++++++++ plugins/MobileProfile/mp-screen.css | 5 +---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/plugins/MobileProfile/MobileProfilePlugin.php b/plugins/MobileProfile/MobileProfilePlugin.php index 5c913836dc..cd2531fa72 100644 --- a/plugins/MobileProfile/MobileProfilePlugin.php +++ b/plugins/MobileProfile/MobileProfilePlugin.php @@ -240,6 +240,8 @@ class MobileProfilePlugin extends WAP20Plugin return true; } + $action->cssLink('css/display.css'); + if (file_exists(Theme::file('css/mp-screen.css'))) { $action->cssLink('css/mp-screen.css', null, 'screen'); } else { @@ -256,6 +258,14 @@ class MobileProfilePlugin extends WAP20Plugin } + function onStartShowUAStyles($action) { + if (!$this->serveMobile) { + return true; + } + + return false; + } + function onStartShowHeader($action) { if (!$this->serveMobile) { diff --git a/plugins/MobileProfile/mp-screen.css b/plugins/MobileProfile/mp-screen.css index 04fa5fb002..0fc801612b 100644 --- a/plugins/MobileProfile/mp-screen.css +++ b/plugins/MobileProfile/mp-screen.css @@ -1,15 +1,12 @@ /** theme: mobile profile screen * * @package StatusNet - * @author Sarven Capadisli + * @author Sarven Capadisli * @copyright 2009 StatusNet, Inc. * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 * @link http://status.net/ */ -@import url(../../theme/base/css/display.css); -@import url(../../theme/identica/css/display.css); - #wrap { min-width:0; max-width:100%; From 239b88025ef1368bb871871ee903d6b078493f76 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Thu, 4 Feb 2010 13:08:34 -0800 Subject: [PATCH 07/18] Should fix spurious 'nickname taken' and 'email taken' errors on registration. Form's checks for existing nicks & emails would incorrectly return true on the second lookup due to bad interaction with negative caching. (was checking $obj !== false but we return null now on negative cache hits, with false for cache misses) --- actions/register.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/actions/register.php b/actions/register.php index 6981373467..ccab76cf01 100644 --- a/actions/register.php +++ b/actions/register.php @@ -280,7 +280,7 @@ class RegisterAction extends Action function nicknameExists($nickname) { $user = User::staticGet('nickname', $nickname); - return ($user !== false); + return is_object($user); } /** @@ -300,7 +300,7 @@ class RegisterAction extends Action return false; } $user = User::staticGet('email', $email); - return ($user !== false); + return is_object($user); } // overrrided to add entry-title class From 37f3a3d558ba55a085c9ee5427948b572c197bc3 Mon Sep 17 00:00:00 2001 From: Eric Helgeson Date: Sat, 16 Jan 2010 11:56:07 -0500 Subject: [PATCH 08/18] Missed change when refactoring groups. Thanks macno --- actions/apigroupjoin.php | 2 +- actions/apigroupleave.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/actions/apigroupjoin.php b/actions/apigroupjoin.php index 3309d63e7b..374cf83df0 100644 --- a/actions/apigroupjoin.php +++ b/actions/apigroupjoin.php @@ -145,7 +145,7 @@ class ApiGroupJoinAction extends ApiAuthAction switch($this->format) { case 'xml': - $this->show_single_xml_group($this->group); + $this->showSingleXmlGroup($this->group); break; case 'json': $this->showSingleJsonGroup($this->group); diff --git a/actions/apigroupleave.php b/actions/apigroupleave.php index 6f8d40527b..9848ece053 100644 --- a/actions/apigroupleave.php +++ b/actions/apigroupleave.php @@ -131,7 +131,7 @@ class ApiGroupLeaveAction extends ApiAuthAction switch($this->format) { case 'xml': - $this->show_single_xml_group($this->group); + $this->showSingleXmlGroup($this->group); break; case 'json': $this->showSingleJsonGroup($this->group); From 2eadeca74515802dccc63d7ec84af1bc7d1338d9 Mon Sep 17 00:00:00 2001 From: Eric Helgeson Date: Sat, 16 Jan 2010 11:56:07 -0500 Subject: [PATCH 09/18] Missed change when refactoring groups. Thanks macno --- actions/apigroupjoin.php | 2 +- actions/apigroupleave.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/actions/apigroupjoin.php b/actions/apigroupjoin.php index 3309d63e7b..374cf83df0 100644 --- a/actions/apigroupjoin.php +++ b/actions/apigroupjoin.php @@ -145,7 +145,7 @@ class ApiGroupJoinAction extends ApiAuthAction switch($this->format) { case 'xml': - $this->show_single_xml_group($this->group); + $this->showSingleXmlGroup($this->group); break; case 'json': $this->showSingleJsonGroup($this->group); diff --git a/actions/apigroupleave.php b/actions/apigroupleave.php index 6f8d40527b..9848ece053 100644 --- a/actions/apigroupleave.php +++ b/actions/apigroupleave.php @@ -131,7 +131,7 @@ class ApiGroupLeaveAction extends ApiAuthAction switch($this->format) { case 'xml': - $this->show_single_xml_group($this->group); + $this->showSingleXmlGroup($this->group); break; case 'json': $this->showSingleJsonGroup($this->group); From 9554b4ccbf0783516ea1735a7c999919be33c280 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Thu, 4 Feb 2010 14:50:20 -0800 Subject: [PATCH 10/18] Inbox::streamNotices() with deletion compensation: inbox paging should more or less work with deleted items now. No change in efficiency for the common case where nothing's deleted: does the same bulk fetch of just the notices we think we'll need as before, then if we turned up short keeps checking one by one until we've filled up to our $limit. This can leave us with overlap between pages, but we already have that when new messages come in between clicks; seems to be the lesser of evils versus not getting a 'before' button. More permanent fix for that will be to switch timeline paging in the UI to use notice IDs. --- classes/Inbox.php | 66 ++++++++++++++++++++++++++++++++++++++++++++--- classes/User.php | 14 +++------- 2 files changed, 67 insertions(+), 13 deletions(-) diff --git a/classes/Inbox.php b/classes/Inbox.php index 26b27d2b58..0294107990 100644 --- a/classes/Inbox.php +++ b/classes/Inbox.php @@ -32,6 +32,7 @@ require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; class Inbox extends Memcached_DataObject { const BOXCAR = 128; + const MAX_NOTICES = 1024; ###START_AUTOCODE /* the code below is auto generated do not remove the above tag */ @@ -81,7 +82,7 @@ class Inbox extends Memcached_DataObject $ni->selectAdd(); $ni->selectAdd('notice_id'); $ni->orderBy('notice_id DESC'); - $ni->limit(0, 1024); + $ni->limit(0, self::MAX_NOTICES); if ($ni->find()) { while($ni->fetch()) { @@ -115,9 +116,11 @@ class Inbox extends Memcached_DataObject $result = $inbox->query(sprintf('UPDATE inbox '. 'set notice_ids = concat(cast(0x%08x as binary(4)), '. - 'substr(notice_ids, 1, 4092)) '. + 'substr(notice_ids, 1, %d)) '. 'WHERE user_id = %d', - $notice_id, $user_id)); + $notice_id, + 4 * (self::MAX_NOTICES - 1), + $user_id)); if ($result) { self::blow('inbox:user_id:%d', $user_id); @@ -173,4 +176,61 @@ class Inbox extends Memcached_DataObject return $ids; } + + /** + * Wrapper for Inbox::stream() and Notice::getStreamByIds() returning + * additional items up to the limit if we were short due to deleted + * notices still being listed in the inbox. + * + * The fast path (when no items are deleted) should be just as fast; the + * offset parameter is applied *before* lookups for maximum efficiency. + * + * This means offset-based paging may show duplicates, but similar behavior + * already exists when new notices are posted between page views, so we + * think people will be ok with this until id-based paging is introduced + * to the user interface. + * + * @param int $user_id + * @param int $offset skip past the most recent N notices (after since_id checks) + * @param int $limit + * @param mixed $since_id return only notices after but not including this id + * @param mixed $max_id return only notices up to and including this id + * @param mixed $since obsolete/ignored + * @param mixed $own ignored? + * @return array of Notice objects + * + * @todo consider repacking the inbox when this happens? + */ + function streamNotices($user_id, $offset, $limit, $since_id, $max_id, $since, $own=false) + { + $ids = self::stream($user_id, $offset, self::MAX_NOTICES, $since_id, $max_id, $since, $own); + + // Do a bulk lookup for the first $limit items + // Fast path when nothing's deleted. + $firstChunk = array_slice($ids, 0, $limit); + $notices = Notice::getStreamByIds($firstChunk); + + $wanted = count($firstChunk); // raw entry count in the inbox up to our $limit + if ($notices->N >= $wanted) { + common_log(LOG_DEBUG, __METHOD__ . " ok got $wanted items"); + return $notices; + } + + common_log(LOG_DEBUG, __METHOD__ . " got $notices->N of $wanted items, getting more"); + // There were deleted notices, we'll need to look for more. + assert($notices instanceof ArrayWrapper); + $items = $notices->_items; + $remainder = array_slice($ids, $limit); + + while (count($items) < $wanted && count($remainder) > 0) { + $notice = Notice::staticGet(array_shift($remainder)); + if ($notice) { + common_log(LOG_DEBUG, __METHOD__ . " got another one"); + $items[] = $notice; + } else { + common_log(LOG_DEBUG, __METHOD__ . " skipping another deleted one"); + } + } + return new ArrayWrapper($items); + } } diff --git a/classes/User.php b/classes/User.php index 0ab816b57e..72c3f39e94 100644 --- a/classes/User.php +++ b/classes/User.php @@ -502,28 +502,22 @@ class User extends Memcached_DataObject function noticesWithFriends($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, false); - return Notice::getStreamByIds($ids); + return Inbox::streamNotices($this->id, $offset, $limit, $since_id, $before_id, $since, false); } function noticeInbox($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, true); - return Notice::getStreamByIds($ids); + return Inbox::streamNotices($this->id, $offset, $limit, $since_id, $before_id, $since, true); } function friendsTimeline($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, false); - - return Notice::getStreamByIds($ids); + return Inbox::streamNotices($this->id, $offset, $limit, $since_id, $before_id, $since, false); } function ownFriendsTimeline($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, true); - - return Notice::getStreamByIds($ids); + return Inbox::streamNotices($this->id, $offset, $limit, $since_id, $before_id, $since, true); } function blowFavesCache() From bb16898b1c73073e0442de72f2af133a3bd39713 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Thu, 4 Feb 2010 14:50:20 -0800 Subject: [PATCH 11/18] Inbox::streamNotices() with deletion compensation: inbox paging should more or less work with deleted items now. No change in efficiency for the common case where nothing's deleted: does the same bulk fetch of just the notices we think we'll need as before, then if we turned up short keeps checking one by one until we've filled up to our $limit. This can leave us with overlap between pages, but we already have that when new messages come in between clicks; seems to be the lesser of evils versus not getting a 'before' button. More permanent fix for that will be to switch timeline paging in the UI to use notice IDs. --- classes/Inbox.php | 66 ++++++++++++++++++++++++++++++++++++++++++++--- classes/User.php | 14 +++------- 2 files changed, 67 insertions(+), 13 deletions(-) diff --git a/classes/Inbox.php b/classes/Inbox.php index 26b27d2b58..0294107990 100644 --- a/classes/Inbox.php +++ b/classes/Inbox.php @@ -32,6 +32,7 @@ require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; class Inbox extends Memcached_DataObject { const BOXCAR = 128; + const MAX_NOTICES = 1024; ###START_AUTOCODE /* the code below is auto generated do not remove the above tag */ @@ -81,7 +82,7 @@ class Inbox extends Memcached_DataObject $ni->selectAdd(); $ni->selectAdd('notice_id'); $ni->orderBy('notice_id DESC'); - $ni->limit(0, 1024); + $ni->limit(0, self::MAX_NOTICES); if ($ni->find()) { while($ni->fetch()) { @@ -115,9 +116,11 @@ class Inbox extends Memcached_DataObject $result = $inbox->query(sprintf('UPDATE inbox '. 'set notice_ids = concat(cast(0x%08x as binary(4)), '. - 'substr(notice_ids, 1, 4092)) '. + 'substr(notice_ids, 1, %d)) '. 'WHERE user_id = %d', - $notice_id, $user_id)); + $notice_id, + 4 * (self::MAX_NOTICES - 1), + $user_id)); if ($result) { self::blow('inbox:user_id:%d', $user_id); @@ -173,4 +176,61 @@ class Inbox extends Memcached_DataObject return $ids; } + + /** + * Wrapper for Inbox::stream() and Notice::getStreamByIds() returning + * additional items up to the limit if we were short due to deleted + * notices still being listed in the inbox. + * + * The fast path (when no items are deleted) should be just as fast; the + * offset parameter is applied *before* lookups for maximum efficiency. + * + * This means offset-based paging may show duplicates, but similar behavior + * already exists when new notices are posted between page views, so we + * think people will be ok with this until id-based paging is introduced + * to the user interface. + * + * @param int $user_id + * @param int $offset skip past the most recent N notices (after since_id checks) + * @param int $limit + * @param mixed $since_id return only notices after but not including this id + * @param mixed $max_id return only notices up to and including this id + * @param mixed $since obsolete/ignored + * @param mixed $own ignored? + * @return array of Notice objects + * + * @todo consider repacking the inbox when this happens? + */ + function streamNotices($user_id, $offset, $limit, $since_id, $max_id, $since, $own=false) + { + $ids = self::stream($user_id, $offset, self::MAX_NOTICES, $since_id, $max_id, $since, $own); + + // Do a bulk lookup for the first $limit items + // Fast path when nothing's deleted. + $firstChunk = array_slice($ids, 0, $limit); + $notices = Notice::getStreamByIds($firstChunk); + + $wanted = count($firstChunk); // raw entry count in the inbox up to our $limit + if ($notices->N >= $wanted) { + common_log(LOG_DEBUG, __METHOD__ . " ok got $wanted items"); + return $notices; + } + + common_log(LOG_DEBUG, __METHOD__ . " got $notices->N of $wanted items, getting more"); + // There were deleted notices, we'll need to look for more. + assert($notices instanceof ArrayWrapper); + $items = $notices->_items; + $remainder = array_slice($ids, $limit); + + while (count($items) < $wanted && count($remainder) > 0) { + $notice = Notice::staticGet(array_shift($remainder)); + if ($notice) { + common_log(LOG_DEBUG, __METHOD__ . " got another one"); + $items[] = $notice; + } else { + common_log(LOG_DEBUG, __METHOD__ . " skipping another deleted one"); + } + } + return new ArrayWrapper($items); + } } diff --git a/classes/User.php b/classes/User.php index 0ab816b57e..72c3f39e94 100644 --- a/classes/User.php +++ b/classes/User.php @@ -502,28 +502,22 @@ class User extends Memcached_DataObject function noticesWithFriends($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, false); - return Notice::getStreamByIds($ids); + return Inbox::streamNotices($this->id, $offset, $limit, $since_id, $before_id, $since, false); } function noticeInbox($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, true); - return Notice::getStreamByIds($ids); + return Inbox::streamNotices($this->id, $offset, $limit, $since_id, $before_id, $since, true); } function friendsTimeline($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, false); - - return Notice::getStreamByIds($ids); + return Inbox::streamNotices($this->id, $offset, $limit, $since_id, $before_id, $since, false); } function ownFriendsTimeline($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, true); - - return Notice::getStreamByIds($ids); + return Inbox::streamNotices($this->id, $offset, $limit, $since_id, $before_id, $since, true); } function blowFavesCache() From 4502bea9a86fe5992eb9b359d90f0c1f004998c1 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Thu, 4 Feb 2010 15:16:27 -0800 Subject: [PATCH 12/18] drop debug messages from inbox deletion fix --- classes/Inbox.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/classes/Inbox.php b/classes/Inbox.php index 0294107990..be62611a16 100644 --- a/classes/Inbox.php +++ b/classes/Inbox.php @@ -212,11 +212,9 @@ class Inbox extends Memcached_DataObject $wanted = count($firstChunk); // raw entry count in the inbox up to our $limit if ($notices->N >= $wanted) { - common_log(LOG_DEBUG, __METHOD__ . " ok got $wanted items"); return $notices; } - common_log(LOG_DEBUG, __METHOD__ . " got $notices->N of $wanted items, getting more"); // There were deleted notices, we'll need to look for more. assert($notices instanceof ArrayWrapper); $items = $notices->_items; @@ -225,10 +223,8 @@ class Inbox extends Memcached_DataObject while (count($items) < $wanted && count($remainder) > 0) { $notice = Notice::staticGet(array_shift($remainder)); if ($notice) { - common_log(LOG_DEBUG, __METHOD__ . " got another one"); $items[] = $notice; } else { - common_log(LOG_DEBUG, __METHOD__ . " skipping another deleted one"); } } return new ArrayWrapper($items); From 5e0cc07b0e687cf0d28d57ae80e5024b7e711fbd Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Fri, 5 Feb 2010 01:13:23 +0000 Subject: [PATCH 13/18] Fix issue with OAuth request parameters being parsed/stored twice when calling /api/account/verify_credentials.:format --- actions/apiaccountverifycredentials.php | 33 ++++++++++++++----------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/actions/apiaccountverifycredentials.php b/actions/apiaccountverifycredentials.php index 1095d51626..ea61a32059 100644 --- a/actions/apiaccountverifycredentials.php +++ b/actions/apiaccountverifycredentials.php @@ -66,18 +66,21 @@ class ApiAccountVerifyCredentialsAction extends ApiAuthAction { parent::handle($args); - switch ($this->format) { - case 'xml': - case 'json': - $args['id'] = $this->auth_user->id; - $action_obj = new ApiUserShowAction(); - if ($action_obj->prepare($args)) { - $action_obj->handle($args); - } - break; - default: - header('Content-Type: text/html; charset=utf-8'); - print 'Authorized'; + if (!in_array($this->format, array('xml', 'json'))) { + $this->clientError(_('API method not found.'), $code = 404); + return; + } + + $twitter_user = $this->twitterUserArray($this->auth_user->getProfile(), true); + + if ($this->format == 'xml') { + $this->initDocument('xml'); + $this->showTwitterXmlUser($twitter_user); + $this->endDocument('xml'); + } elseif ($this->format == 'json') { + $this->initDocument('json'); + $this->showJsonObjects($twitter_user); + $this->endDocument('json'); } } @@ -86,14 +89,14 @@ class ApiAccountVerifyCredentialsAction extends ApiAuthAction * Is this action read only? * * @param array $args other arguments - * + * * @return boolean true * **/ - + function isReadOnly($args) { return true; } - + } From 82f11190734c203ed6b2fd4a07cb9460f25b2183 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Fri, 5 Feb 2010 01:24:21 +0000 Subject: [PATCH 14/18] OAuth app name should not be null --- classes/statusnet.ini | 2 +- db/statusnet.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/classes/statusnet.ini b/classes/statusnet.ini index 4ace4407b1..2c09033f67 100644 --- a/classes/statusnet.ini +++ b/classes/statusnet.ini @@ -353,7 +353,7 @@ notice_id = K id = 129 owner = 129 consumer_key = 130 -name = 2 +name = 130 description = 2 icon = 130 source_url = 2 diff --git a/db/statusnet.sql b/db/statusnet.sql index 8946f4d7e2..3434648016 100644 --- a/db/statusnet.sql +++ b/db/statusnet.sql @@ -214,7 +214,7 @@ create table oauth_application ( id integer auto_increment primary key comment 'unique identifier', owner integer not null comment 'owner of the application' references profile (id), consumer_key varchar(255) not null comment 'application consumer key' references consumer (consumer_key), - name varchar(255) unique key comment 'name of the application', + name varchar(255) not null unique key comment 'name of the application', description varchar(255) comment 'description of the application', icon varchar(255) not null comment 'application icon', source_url varchar(255) comment 'application homepage - used for source link', From 10dfcde0b2099a169ccd3af0ecfbf2de9da551d6 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Fri, 5 Feb 2010 01:38:29 +0000 Subject: [PATCH 15/18] Actually store the timestamp on each nonce --- lib/oauthstore.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/oauthstore.php b/lib/oauthstore.php index b30fb49d57..eabe37f9fa 100644 --- a/lib/oauthstore.php +++ b/lib/oauthstore.php @@ -65,7 +65,7 @@ class StatusNetOAuthDataStore extends OAuthDataStore { $n = new Nonce(); $n->consumer_key = $consumer->key; - $n->ts = $timestamp; + $n->ts = common_sql_date($timestamp); $n->nonce = $nonce; if ($n->find(true)) { return true; @@ -362,7 +362,6 @@ class StatusNetOAuthDataStore extends OAuthDataStore array('is_local' => Notice::REMOTE_OMB, 'uri' => $omb_notice->getIdentifierURI())); - } /** From 875e1a70ce231b6b07765210328656abb353ad5b Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 5 Feb 2010 09:47:56 -0800 Subject: [PATCH 16/18] Don't spew warnings on usage of MEMCACHE_COMPRESSED constant when memcache PHP extension is not present. Switched to a locally-defined Cache::COMPRESSED, translating that to MEMCACHE_COMPRESSED in the plugin. --- classes/Memcached_DataObject.php | 2 +- lib/cache.php | 4 +++- plugins/MemcachePlugin.php | 18 ++++++++++++++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index ab65c30ce2..dfd06b57e5 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -363,7 +363,7 @@ class Memcached_DataObject extends DB_DataObject $cached[] = clone($inst); } $inst->free(); - $c->set($ckey, $cached, MEMCACHE_COMPRESSED, $expiry); + $c->set($ckey, $cached, Cache::COMPRESSED, $expiry); return new ArrayWrapper($cached); } diff --git a/lib/cache.php b/lib/cache.php index 635c96ad4c..df6fc36493 100644 --- a/lib/cache.php +++ b/lib/cache.php @@ -47,6 +47,8 @@ class Cache var $_items = array(); static $_inst = null; + const COMPRESSED = 1; + /** * Singleton constructor * @@ -133,7 +135,7 @@ class Cache * * @param string $key The key to use for lookups * @param string $value The value to store - * @param integer $flag Flags to use, mostly ignored + * @param integer $flag Flags to use, may include Cache::COMPRESSED * @param integer $expiry Expiry value, mostly ignored * * @return boolean success flag diff --git a/plugins/MemcachePlugin.php b/plugins/MemcachePlugin.php index 2bc4b892bd..c5e74fb416 100644 --- a/plugins/MemcachePlugin.php +++ b/plugins/MemcachePlugin.php @@ -102,7 +102,7 @@ class MemcachePlugin extends Plugin * * @param string &$key in; Key to use for lookups * @param mixed &$value in; Value to associate - * @param integer &$flag in; Flag (passed through to Memcache) + * @param integer &$flag in; Flag empty or Cache::COMPRESSED * @param integer &$expiry in; Expiry (passed through to Memcache) * @param boolean &$success out; Whether the set was successful * @@ -115,7 +115,7 @@ class MemcachePlugin extends Plugin if ($expiry === null) { $expiry = $this->defaultExpiry; } - $success = $this->_conn->set($key, $value, $flag, $expiry); + $success = $this->_conn->set($key, $value, $this->flag(intval($flag)), $expiry); Event::handle('EndCacheSet', array($key, $value, $flag, $expiry)); return false; @@ -197,6 +197,20 @@ class MemcachePlugin extends Plugin } } + /** + * Translate general flags to Memcached-specific flags + * @param int $flag + * @return int + */ + protected function flag($flag) + { + $out = 0; + if ($flag & Cache::COMPRESSED == Cache::COMPRESSED) { + $out |= MEMCACHE_COMPRESSED; + } + return $out; + } + function onPluginVersion(&$versions) { $versions[] = array('name' => 'Memcache', From 558934d1ddc1c1163c6a142bfe1c4232496b25d6 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Fri, 5 Feb 2010 21:39:29 -0800 Subject: [PATCH 17/18] Store Twitter screen_name, not name, for foreign_user.nickname when saving Twitter user. --- plugins/TwitterBridge/twitterauthorization.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/TwitterBridge/twitterauthorization.php b/plugins/TwitterBridge/twitterauthorization.php index b2657ff61f..dbef438a4b 100644 --- a/plugins/TwitterBridge/twitterauthorization.php +++ b/plugins/TwitterBridge/twitterauthorization.php @@ -219,7 +219,7 @@ class TwitterauthorizationAction extends Action $user = common_current_user(); $this->saveForeignLink($user->id, $twitter_user->id, $atok); - save_twitter_user($twitter_user->id, $twitter_user->name); + save_twitter_user($twitter_user->id, $twitter_user->screen_name); } else { From 70abea3ac4034cbbfc68cdc8288fc7e2d1cea17c Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Sat, 6 Feb 2010 06:46:00 +0000 Subject: [PATCH 18/18] Delete old Twitter user record when user changes screen name instead of updating. Simpler. --- plugins/TwitterBridge/twitter.php | 54 +++++-------------------------- 1 file changed, 8 insertions(+), 46 deletions(-) diff --git a/plugins/TwitterBridge/twitter.php b/plugins/TwitterBridge/twitter.php index 33dfb788bf..de30d9ebf1 100644 --- a/plugins/TwitterBridge/twitter.php +++ b/plugins/TwitterBridge/twitter.php @@ -26,38 +26,6 @@ define('TWITTER_SERVICE', 1); // Twitter is foreign_service ID 1 require_once INSTALLDIR . '/plugins/TwitterBridge/twitterbasicauthclient.php'; require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; -function updateTwitter_user($twitter_id, $screen_name) -{ - $uri = 'http://twitter.com/' . $screen_name; - $fuser = new Foreign_user(); - - $fuser->query('BEGIN'); - - // Dropping down to SQL because regular DB_DataObject udpate stuff doesn't seem - // to work so good with tables that have multiple column primary keys - - // Any time we update the uri for a forein user we have to make sure there - // are no dupe entries first -- unique constraint on the uri column - - $qry = 'UPDATE foreign_user set uri = \'\' WHERE uri = '; - $qry .= '\'' . $uri . '\'' . ' AND service = ' . TWITTER_SERVICE; - - $fuser->query($qry); - - // Update the user - - $qry = 'UPDATE foreign_user SET nickname = '; - $qry .= '\'' . $screen_name . '\'' . ', uri = \'' . $uri . '\' '; - $qry .= 'WHERE id = ' . $twitter_id . ' AND service = ' . TWITTER_SERVICE; - - $fuser->query('COMMIT'); - - $fuser->free(); - unset($fuser); - - return true; -} - function add_twitter_user($twitter_id, $screen_name) { @@ -105,7 +73,6 @@ function add_twitter_user($twitter_id, $screen_name) // Creates or Updates a Twitter user function save_twitter_user($twitter_id, $screen_name) { - // Check to see whether the Twitter user is already in the system, // and update its screen name and uri if so. @@ -115,25 +82,20 @@ function save_twitter_user($twitter_id, $screen_name) $result = true; - // Only update if Twitter screen name has changed + // Delete old record if Twitter user changed screen name if ($fuser->nickname != $screen_name) { - $result = updateTwitter_user($twitter_id, $screen_name); - - common_debug('Twitter bridge - Updated nickname (and URI) for Twitter user ' . - "$fuser->id to $screen_name, was $fuser->nickname"); + $oldname = $fuser->nickname; + $fuser->delete(); + common_log(LOG_INFO, sprintf('Twitter bridge - Updated nickname (and URI) ' . + 'for Twitter user %1$d - %2$s, was %3$s.', + $fuser->id, + $screen_name, + $oldname)); } - return $result; - - } else { return add_twitter_user($twitter_id, $screen_name); } - - $fuser->free(); - unset($fuser); - - return true; } function is_twitter_bound($notice, $flink) {