From 1e8ea1eb460b163176c4d7d1e7dffa500024ef91 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 16:09:03 -0400 Subject: [PATCH 01/12] Make the tag stream use ID mechanism --- classes/Notice.php | 5 +--- classes/Notice_tag.php | 59 +++++++++++++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index 6a7522bd32..2bb466155a 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -328,10 +328,7 @@ class Notice extends Memcached_DataObject $tag->notice_id = $this->id; if ($tag->find()) { while ($tag->fetch()) { - $cache->delete(common_cache_key('notice_tag:notice_stream:' . $tag->tag)); - if ($blowLast) { - $cache->delete(common_cache_key('notice_tag:notice_stream:' . $tag->tag . ';last')); - } + $tag->blowCache($blowLast); } } $tag->free(); diff --git a/classes/Notice_tag.php b/classes/Notice_tag.php index f2247299a4..e5b7722430 100644 --- a/classes/Notice_tag.php +++ b/classes/Notice_tag.php @@ -37,21 +37,62 @@ class Notice_tag extends Memcached_DataObject ###END_AUTOCODE static function getStream($tag, $offset=0, $limit=20) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN notice_tag ON notice.id = notice_tag.notice_id ' . - "WHERE notice_tag.tag = '%s' "; - return Notice::getStream(sprintf($qry, $tag), - 'notice_tag:notice_stream:' . common_keyize($tag), - $offset, $limit); + $ids = Notice::stream(array('Notice_tag', '_streamDirect'), + array($tag), + 'notice_tag:notice_ids:' . common_keyize($tag), + $offset, $limit); + + return Notice::getStreamByIds($ids); } - function blowCache() + function _streamDirect($tag, $offset, $limit, $since_id, $before_id, $since) + { + $nt = new Notice_tag(); + + $nt->tag = $tag; + + $nt->selectAdd(); + $nt->selectAdd('notice_id'); + + if ($since_id != 0) { + $nt->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $nt->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $nt->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $nt->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $nt->limit($offset, $limit); + } + + $ids = array(); + + if ($nt->find()) { + while ($nt->fetch()) { + $ids[] = $nt->notice_id; + } + } + + return $ids; + } + + function blowCache($blowLast=false) { $cache = common_memcache(); if ($cache) { - $cache->delete(common_cache_key('notice_tag:notice_stream:' . $this->tag)); + $idkey = common_cache_key('notice_tag:notice_ids:' . common_keyize($this->tag)); + $cache->delete($idkey); + if ($blowLast) { + $cache->delete($idkey.';last'); + } } } From aee641ee1e311fb0af0f9f6d75ca7fae2c7d8477 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 29 Apr 2009 20:45:33 -0400 Subject: [PATCH 02/12] make replies use new query format --- classes/Notice.php | 4 ++-- classes/Reply.php | 47 ++++++++++++++++++++++++++++++++++++++++++++-- classes/User.php | 10 +++------- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index 2bb466155a..808631f4dc 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -380,9 +380,9 @@ class Notice extends Memcached_DataObject $reply->notice_id = $this->id; if ($reply->find()) { while ($reply->fetch()) { - $cache->delete(common_cache_key('user:replies:'.$reply->profile_id)); + $cache->delete(common_cache_key('reply:stream:'.$reply->profile_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:replies:'.$reply->profile_id.';last')); + $cache->delete(common_cache_key('reply:stream:'.$reply->profile_id.';last')); } } } diff --git a/classes/Reply.php b/classes/Reply.php index af86aaf878..4439053b44 100644 --- a/classes/Reply.php +++ b/classes/Reply.php @@ -4,7 +4,7 @@ */ require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; -class Reply extends Memcached_DataObject +class Reply extends Memcached_DataObject { ###START_AUTOCODE /* the code below is auto generated do not remove the above tag */ @@ -13,7 +13,7 @@ class Reply extends Memcached_DataObject public $notice_id; // int(4) primary_key not_null public $profile_id; // int(4) primary_key not_null public $modified; // timestamp() not_null default_CURRENT_TIMESTAMP - public $replied_id; // int(4) + public $replied_id; // int(4) /* Static get */ function staticGet($k,$v=null) @@ -21,4 +21,47 @@ class Reply extends Memcached_DataObject /* the code above is auto generated do not remove the tag below */ ###END_AUTOCODE + + function stream($user_id, $offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) + { + $ids = Notice::stream(array('Reply', '_streamDirect'), + array($user_id), + 'reply:stream:' . $user_id, + $offset, $limit, $since_id, $before_id, $since); + return $ids; + } + + function _streamDirect($user_id, $offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) + { + $reply = new Reply(); + $reply->profile_id = $user_id; + + if ($since_id != 0) { + $reply->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $reply->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $reply->whereAdd('modified > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $reply->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $reply->limit($offset, $limit); + } + + $ids = array(); + + if ($reply->find()) { + while ($reply->fetch()) { + $ids[] = $reply->notice_id; + } + } + + return $ids; + } } diff --git a/classes/User.php b/classes/User.php index ce7ea1464f..b76e45c330 100644 --- a/classes/User.php +++ b/classes/User.php @@ -401,13 +401,9 @@ class User extends Memcached_DataObject function getReplies($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN reply ON notice.id = reply.notice_id ' . - 'WHERE reply.profile_id = %d '; - return Notice::getStream(sprintf($qry, $this->id), - 'user:replies:'.$this->id, - $offset, $limit, $since_id, $before_id, null, $since); + $ids = Reply::stream($this->id, $offset, $limit, $since_id, $before_id, $since); + common_debug("Ids = " . implode(',', $ids)); + return Notice::getStreamByIds($ids); } function getNotices($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) From a86a0e91a5acb5ea894a3d066f9adf3b1ef305ae Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 1 May 2009 08:00:37 -0700 Subject: [PATCH 03/12] add favor, reply, delete buttons for cometed notices --- plugins/Comet/CometPlugin.php | 16 ++++++- plugins/Comet/updatetimeline.js | 74 +++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/plugins/Comet/CometPlugin.php b/plugins/Comet/CometPlugin.php index 2e0bb40a46..48ac9dcad6 100644 --- a/plugins/Comet/CometPlugin.php +++ b/plugins/Comet/CometPlugin.php @@ -82,8 +82,22 @@ class CometPlugin extends Plugin ' '); } + $user = common_current_user(); + + if (!empty($user->id)) { + $user_id = $user->id; + } else { + $user_id = 0; + } + + $replyurl = common_local_url('newnotice'); + $favorurl = common_local_url('favor'); + // FIXME: need to find a better way to pass this pattern in + $deleteurl = common_local_url('deletenotice', + array('notice' => '0000000000')); + $action->elementStart('script', array('type' => 'text/javascript')); - $action->raw("$(document).ready(function() { updater.init(\"$this->server\", \"$timeline\");});"); + $action->raw("$(document).ready(function() { updater.init(\"$this->server\", \"$timeline\", $user_id, \"$replyurl\", \"$favorurl\", \"$deleteurl\"); });"); $action->elementEnd('script'); return true; diff --git a/plugins/Comet/updatetimeline.js b/plugins/Comet/updatetimeline.js index de750baba3..e89b3bddf7 100644 --- a/plugins/Comet/updatetimeline.js +++ b/plugins/Comet/updatetimeline.js @@ -3,14 +3,26 @@ var updater = function() { + var _server; + var _timeline; + var _userid; + var _replyurl; + var _favorurl; + var _deleteurl; var _cometd; return { - init: function(server, timeline) + init: function(server, timeline, userid, replyurl, favorurl, deleteurl) { _cometd = $.cometd; // Uses the default Comet object _cometd.setLogLevel('debug'); _cometd.init(server); + _server = server; + _timeline = timeline; + _userid = userid; + _favorurl = favorurl; + _replyurl = replyurl; + _deleteurl = deleteurl; _cometd.subscribe(timeline, receive); $(window).unload(leave); } @@ -34,7 +46,7 @@ var updater = function() var noticeItem = makeNoticeItem(message.data); $("#notices_primary .notices").prepend(noticeItem, true); $("#notices_primary .notice:first").css({display:"none"}); - $("#notices_primary .notice:first").fadeIn(2500); + $("#notices_primary .notice:first").fadeIn(1000); NoticeHover(); NoticeReply(); } @@ -68,10 +80,64 @@ var updater = function() "
"+data['source']+"
"+ ""+ ""+ - "
"+ - "
"+ + "
"; + + if (_userid != 0) { + var input = $("form#form_notice fieldset input#token"); + var session_key = input.val(); + ni = ni+makeFavoriteForm(data['id'], session_key); + ni = ni+makeReplyLink(data['id'], data['user']['screen_name']); + if (_userid == data['user']['id']) { + ni = ni+makeDeleteLink(data['id']); + } + } + + ni = ni+"
"+ ""; return ni; } + + function makeFavoriteForm(id, session_key) + { + var ff; + + ff = "
"+ + "
"+ + "Favor this notice"+ // XXX: i18n + ""+ + ""+ + ""+ + "
"+ + "
"; + return ff; + } + + function makeReplyLink(id, nickname) + { + var rl; + rl = "
"+ + "
Reply to this notice
"+ + "
"+ + "Reply "+id+""+ + ""+ + "
"+ + "
"; + return rl; + } + + function makeDeleteLink(id) + { + var dl, delurl; + delurl = _deleteurl.replace("0000000000", id); + + dl = "
"+ + "
Delete this notice
"+ + "
"+ + "Delete"+ + "
"+ + "
"; + + return dl; + } }(); From 5affe093aba97a0e4ac559b685a240d568929ffb Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 1 May 2009 08:39:47 -0700 Subject: [PATCH 04/12] add in_reply_to link and make HTML in source work correctly --- plugins/Comet/CometPlugin.php | 9 +++++++++ plugins/Comet/updatetimeline.js | 19 +++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/plugins/Comet/CometPlugin.php b/plugins/Comet/CometPlugin.php index 48ac9dcad6..0f2fcd701d 100644 --- a/plugins/Comet/CometPlugin.php +++ b/plugins/Comet/CometPlugin.php @@ -158,6 +158,15 @@ class CometPlugin extends Plugin $arr = $act->twitter_status_array($notice, true); $arr['url'] = $notice->bestUrl(); $arr['html'] = htmlspecialchars($notice->rendered); + $arr['source'] = htmlspecialchars($arr['source']); + + if (!empty($notice->reply_to)) { + $reply_to = Notice::staticGet('id', $notice->reply_to); + if (!empty($reply_to)) { + $arr['in_reply_to_status_url'] = $reply_to->bestUrl(); + } + $reply_to = null; + } $profile = $notice->getProfile(); $arr['user']['profile_url'] = $profile->profileurl; diff --git a/plugins/Comet/updatetimeline.js b/plugins/Comet/updatetimeline.js index e89b3bddf7..170949e9ba 100644 --- a/plugins/Comet/updatetimeline.js +++ b/plugins/Comet/updatetimeline.js @@ -54,7 +54,8 @@ var updater = function() function makeNoticeItem(data) { user = data['user']; - html = data['html'].replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>'); + html = data['html'].replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"'); + source = data['source'].replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"'); ni = "
  • "+ "
    "+ @@ -77,9 +78,19 @@ var updater = function() ""+ "
    "+ "
    From
    "+ - "
    "+data['source']+"
    "+ - "
    "+ - "
    "+ + "
    "+source+"
    "+ // may have a link, I think + ""; + + if (data['in_reply_to_status_id']) { + ni = ni+"
    "+ + "
    To
    "+ + "
    "+ + "in reply to"+ + "
    "+ + "
    "; + } + + ni = ni+""+ "
    "; if (_userid != 0) { From b12e72ae312488caf7cb1e1a396eb05dd38326a9 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 1 May 2009 09:42:38 -0700 Subject: [PATCH 05/12] optionally add a username/password on server side for Comet --- plugins/Comet/CometPlugin.php | 8 +++++--- plugins/Comet/bayeux.class.inc.php | 9 ++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/plugins/Comet/CometPlugin.php b/plugins/Comet/CometPlugin.php index 0f2fcd701d..45251c66f0 100644 --- a/plugins/Comet/CometPlugin.php +++ b/plugins/Comet/CometPlugin.php @@ -45,9 +45,11 @@ class CometPlugin extends Plugin { var $server = null; - function __construct($server=null) + function __construct($server=null, $username=null, $password=null) { - $this->server = $server; + $this->server = $server; + $this->username = $username; + $this->password = $password; parent::__construct(); } @@ -131,7 +133,7 @@ class CometPlugin extends Plugin $json = $this->noticeAsJson($notice); // Bayeux? Comet? Huh? These terms confuse me - $bay = new Bayeux($this->server); + $bay = new Bayeux($this->server, $this->user, $this->password); foreach ($timelines as $timeline) { $this->log(LOG_INFO, "Posting notice $notice->id to '$timeline'."); diff --git a/plugins/Comet/bayeux.class.inc.php b/plugins/Comet/bayeux.class.inc.php index 785d3e3935..39ad8a8fc6 100644 --- a/plugins/Comet/bayeux.class.inc.php +++ b/plugins/Comet/bayeux.class.inc.php @@ -26,9 +26,12 @@ class Bayeux private $oCurl = ''; private $nNextId = 0; + private $sUser = ''; + private $sPassword = ''; + public $sUrl = ''; - function __construct($sUrl) + function __construct($sUrl, $sUser='', $sPassword='') { $this->sUrl = $sUrl; @@ -43,6 +46,10 @@ class Bayeux curl_setopt($this->oCurl, CURLOPT_POST, 1); curl_setopt($this->oCurl, CURLOPT_RETURNTRANSFER,1); + if (!is_null($sUser) && mb_strlen($sUser) > 0) { + curl_setopt($this->oCurl, CURLOPT_USERPWD,"$sUser:$sPassword"); + } + $this->handShake(); } From 3328ec545c36fc2408cdb9d048effe24feafe218 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 1 May 2009 11:27:57 -0700 Subject: [PATCH 06/12] make profile notice getting use ids --- classes/Notice.php | 6 +++--- classes/Profile.php | 51 ++++++++++++++++++++++++++++++++++++++------- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index 808631f4dc..eceed325b0 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -363,10 +363,10 @@ class Notice extends Memcached_DataObject { if ($this->is_local) { $cache = common_memcache(); - if ($cache) { - $cache->delete(common_cache_key('profile:notices:'.$this->profile_id)); + if (!empty($cache)) { + $cache->delete(common_cache_key('profile:notice_ids:'.$this->profile_id)); if ($blowLast) { - $cache->delete(common_cache_key('profile:notices:'.$this->profile_id.';last')); + $cache->delete(common_cache_key('profile:notice_ids:'.$this->profile_id.';last')); } } } diff --git a/classes/Profile.php b/classes/Profile.php index f3bfe299cf..ae5641d79d 100644 --- a/classes/Profile.php +++ b/classes/Profile.php @@ -155,14 +155,51 @@ class Profile extends Memcached_DataObject function getNotices($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0) { - $qry = - 'SELECT * ' . - 'FROM notice ' . - 'WHERE profile_id = %d '; + // XXX: I'm not sure this is going to be any faster. It probably isn't. + $ids = Notice::stream(array($this, '_streamDirect'), + array(), + 'profile:notice_ids:' . $this->id, + $offset, $limit, $since_id, $before_id); - return Notice::getStream(sprintf($qry, $this->id), - 'profile:notices:'.$this->id, - $offset, $limit, $since_id, $before_id); + return Notice::getStreamByIds($ids); + } + + function _streamDirect($offset, $limit, $since_id, $before_id, $since) + { + $notice = new Notice(); + + $notice->profile_id = $this->id; + + $notice->selectAdd(); + $notice->selectAdd('id'); + + if ($since_id != 0) { + $notice->whereAdd('id > ' . $since_id); + } + + if ($before_id != 0) { + $notice->whereAdd('id < ' . $before_id); + } + + if (!is_null($since)) { + $notice->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $notice->orderBy('id DESC'); + + if (!is_null($offset)) { + $notice->limit($offset, $limit); + } + + $ids = array(); + + if ($notice->find()) { + while ($notice->fetch()) { + $ids[] = $notice->id; + } + } + + return $ids; } function isMember($group) From 021b520a11d3449a1476182e1ad117582999d364 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 1 May 2009 11:38:50 -0700 Subject: [PATCH 07/12] Make user group stream use IDs --- classes/Notice.php | 4 ++-- classes/User_group.php | 53 +++++++++++++++++++++++++++++++++++------- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index eceed325b0..c036e6e9e5 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -299,9 +299,9 @@ class Notice extends Memcached_DataObject $group_inbox->notice_id = $this->id; if ($group_inbox->find()) { while ($group_inbox->fetch()) { - $cache->delete(common_cache_key('group:notices:'.$group_inbox->group_id)); + $cache->delete(common_cache_key('user_group:notice_ids:' . $group_inbox->group_id)); if ($blowLast) { - $cache->delete(common_cache_key('group:notices:'.$group_inbox->group_id.';last')); + $cache->delete(common_cache_key('user_group:notice_ids:' . $group_inbox->group_id.';last')); } $member = new Group_member(); $member->group_id = $group_inbox->group_id; diff --git a/classes/User_group.php b/classes/User_group.php index d152f9d567..7cc31e7026 100755 --- a/classes/User_group.php +++ b/classes/User_group.php @@ -50,13 +50,50 @@ class User_group extends Memcached_DataObject function getNotices($offset, $limit) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN group_inbox ON notice.id = group_inbox.notice_id ' . - 'WHERE group_inbox.group_id = %d '; - return Notice::getStream(sprintf($qry, $this->id), - 'group:notices:'.$this->id, - $offset, $limit); + $ids = Notice::stream(array($this, '_streamDirect'), + array(), + 'user_group:notice_ids:' . $this->id, + $offset, $limit); + + return Notice::getStreamByIds($ids); + } + + function _streamDirect($offset, $limit, $since_id, $before_id, $since) + { + $inbox = new Group_inbox(); + + $inbox->group_id = $this->id; + + $inbox->selectAdd(); + $inbox->selectAdd('notice_id'); + + if ($since_id != 0) { + $inbox->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $inbox->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + $inbox->orderBy('notice_id DESC'); + + if (!is_null($offset)) { + $inbox->limit($offset, $limit); + } + + $ids = array(); + + if ($inbox->find()) { + while ($inbox->fetch()) { + $ids[] = $inbox->notice_id; + } + } + + return $ids; } function allowedNickname($nickname) @@ -91,7 +128,7 @@ class User_group extends Memcached_DataObject function setOriginal($filename) { $imagefile = new ImageFile($this->id, Avatar::path($filename)); - + $orig = clone($this); $this->original_logo = Avatar::url($filename); $this->homepage_logo = Avatar::url($imagefile->resize(AVATAR_PROFILE_SIZE)); From 5314d9b2cfaef3f2fd0ead262e18d1776fd99c8d Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 1 May 2009 12:01:28 -0700 Subject: [PATCH 08/12] make faves work with ids --- classes/Fave.php | 53 ++++++++++++++++++++++++++++++++++++++++++++-- classes/Notice.php | 4 ++-- classes/User.php | 38 +++++++++++++++------------------ 3 files changed, 70 insertions(+), 25 deletions(-) diff --git a/classes/Fave.php b/classes/Fave.php index 24df5938c2..915b4572ff 100644 --- a/classes/Fave.php +++ b/classes/Fave.php @@ -4,7 +4,7 @@ */ require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; -class Fave extends Memcached_DataObject +class Fave extends Memcached_DataObject { ###START_AUTOCODE /* the code below is auto generated do not remove the above tag */ @@ -31,9 +31,58 @@ class Fave extends Memcached_DataObject } return $fave; } - + function &pkeyGet($kv) { return Memcached_DataObject::pkeyGet('Fave', $kv); } + + function stream($user_id, $offset=0, $limit=NOTICES_PER_PAGE) + { + $ids = Notice::stream(array('Fave', '_streamDirect'), + array($user_id), + 'fave:ids_by_user:'.$user_id, + $offset, $limit); + return $ids; + } + + function _streamDirect($user_id, $offset, $limit, $since_id, $before_id, $since) + { + $fav = new Fave(); + + $fav->user_id = $user_id; + + $fav->selectAdd(); + $fav->selectAdd('notice_id'); + + if ($since_id != 0) { + $fav->whereAdd('notice_id > ' . $since_id); + } + + if ($before_id != 0) { + $fav->whereAdd('notice_id < ' . $before_id); + } + + if (!is_null($since)) { + $fav->whereAdd('modified > \'' . date('Y-m-d H:i:s', $since) . '\''); + } + + // NOTE: we sort by fave time, not by notice time! + + $fav->orderBy('modified DESC'); + + if (!is_null($offset)) { + $fav->limit($offset, $limit); + } + + $ids = array(); + + if ($fav->find()) { + while ($fav->fetch()) { + $ids[] = $fav->notice_id; + } + } + + return $ids; + } } diff --git a/classes/Notice.php b/classes/Notice.php index c036e6e9e5..771a4e715f 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -412,9 +412,9 @@ class Notice extends Memcached_DataObject $fave->notice_id = $this->id; if ($fave->find()) { while ($fave->fetch()) { - $cache->delete(common_cache_key('user:faves:'.$fave->user_id)); + $cache->delete(common_cache_key('fave:ids_by_user:'.$fave->user_id)); if ($blowLast) { - $cache->delete(common_cache_key('user:faves:'.$fave->user_id.';last')); + $cache->delete(common_cache_key('fave:ids_by_user:'.$fave->user_id.';last')); } } } diff --git a/classes/User.php b/classes/User.php index b76e45c330..b5ac7b2206 100644 --- a/classes/User.php +++ b/classes/User.php @@ -349,30 +349,31 @@ class User extends Memcached_DataObject $cache = common_memcache(); // XXX: Kind of a hack. + if ($cache) { // This is the stream of favorite notices, in rev chron // order. This forces it into cache. - $faves = $this->favoriteNotices(0, NOTICE_CACHE_WINDOW); - $cnt = 0; - while ($faves->fetch()) { - if ($faves->id < $notice->id) { - // If we passed it, it's not a fave - return false; - } else if ($faves->id == $notice->id) { - // If it matches a cached notice, then it's a fave - return true; - } - $cnt++; + + $ids = Fave::stream($this->id, 0, NOTICE_CACHE_WINDOW); + + // If it's in the list, then it's a fave + + if (in_array($notice->id, $ids)) { + return true; } + // If we're not past the end of the cache window, // then the cache has all available faves, so this one // is not a fave. - if ($cnt < NOTICE_CACHE_WINDOW) { + + if (count($ids) < NOTICE_CACHE_WINDOW) { return false; } + // Otherwise, cache doesn't have all faves; // fall through to the default } + $fave = Fave::pkeyGet(array('user_id' => $this->id, 'notice_id' => $notice->id)); return ((is_null($fave)) ? false : true); @@ -418,13 +419,8 @@ class User extends Memcached_DataObject function favoriteNotices($offset=0, $limit=NOTICES_PER_PAGE) { - $qry = - 'SELECT notice.* ' . - 'FROM notice JOIN fave ON notice.id = fave.notice_id ' . - 'WHERE fave.user_id = %d '; - return Notice::getStream(sprintf($qry, $this->id), - 'user:faves:'.$this->id, - $offset, $limit); + $ids = Fave::stream($this->id, $offset, $limit); + return Notice::getStreamByIds($ids); } function noticesWithFriends($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) @@ -459,8 +455,8 @@ class User extends Memcached_DataObject if ($cache) { // Faves don't happen chronologically, so we need to blow // ;last cache, too - $cache->delete(common_cache_key('user:faves:'.$this->id)); - $cache->delete(common_cache_key('user:faves:'.$this->id).';last'); + $cache->delete(common_cache_key('fave:ids_by_user:'.$this->id)); + $cache->delete(common_cache_key('fave:ids_by_user:'.$this->id.';last')); } } From 6a12598695637e7ebdc613977c797413957cc464 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sun, 3 May 2009 21:36:03 -0700 Subject: [PATCH 09/12] add pingvine notice source --- db/notice_source.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/db/notice_source.sql b/db/notice_source.sql index ce44f32354..17720028dc 100644 --- a/db/notice_source.sql +++ b/db/notice_source.sql @@ -24,6 +24,7 @@ VALUES ('peoplebrowsr', 'PeopleBrowsr', 'http://www.peoplebrowsr.com/', now()), ('Pikchur','Pikchur','http://www.pikchur.com/', now()), ('Ping.fm','Ping.fm','http://ping.fm/', now()), + ('pingvine','PingVine','http://pingvine.com/', now()), ('pocketwit','PockeTwit','http://code.google.com/p/pocketwit/', now()), ('posty','Posty','http://spreadingfunkyness.com/posty/', now()), ('royalewithcheese','Royale With Cheese','http://p.hellyeah.org/', now()), From 3e7b1e69e3e97ac007465376b62084f10bcf97ca Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Tue, 28 Apr 2009 17:08:20 -0700 Subject: [PATCH 10/12] Added dirty dates to Foreign_link --- classes/Foreign_link.php | 2 ++ classes/laconica.ini | 2 ++ db/laconica.sql | 2 ++ 3 files changed, 6 insertions(+) diff --git a/classes/Foreign_link.php b/classes/Foreign_link.php index afc0e21804..af2b3f189d 100644 --- a/classes/Foreign_link.php +++ b/classes/Foreign_link.php @@ -17,6 +17,8 @@ class Foreign_link extends Memcached_DataObject public $noticesync; // tinyint(1) not_null default_1 public $friendsync; // tinyint(1) not_null default_2 public $profilesync; // tinyint(1) not_null default_1 + public $last_noticesync; // datetime() + public $last_friendsync; // datetime() public $created; // datetime() not_null public $modified; // timestamp() not_null default_CURRENT_TIMESTAMP diff --git a/classes/laconica.ini b/classes/laconica.ini index 529454d99b..c054195884 100755 --- a/classes/laconica.ini +++ b/classes/laconica.ini @@ -55,6 +55,8 @@ credentials = 2 noticesync = 145 friendsync = 145 profilesync = 145 +last_noticesync = 14 +last_friendsync = 14 created = 142 modified = 384 diff --git a/db/laconica.sql b/db/laconica.sql index 5b57494d98..c9730098e3 100644 --- a/db/laconica.sql +++ b/db/laconica.sql @@ -289,6 +289,8 @@ create table foreign_link ( noticesync tinyint not null default 1 comment 'notice synchronization, bit 1 = sync outgoing, bit 2 = sync incoming, bit 3 = filter local replies', friendsync tinyint not null default 2 comment 'friend synchronization, bit 1 = sync outgoing, bit 2 = sync incoming', profilesync tinyint not null default 1 comment 'profile synchronization, bit 1 = sync outgoing, bit 2 = sync incoming', + last_noticesync datetime default null comment 'last time notices were imported', + last_friendsync datetime default null comment 'last time friends were imported', created datetime not null comment 'date this record was created', modified timestamp comment 'date this record was modified', From 11e0db8c2cec18337fd960ccda055dd14d89f9d7 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Thu, 7 May 2009 18:22:14 -0700 Subject: [PATCH 11/12] Twitter friends sync now does 25 users at a time and uses last_friendsync field to prioritize --- actions/twittersettings.php | 4 +++- scripts/synctwitterfriends.php | 38 ++++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/actions/twittersettings.php b/actions/twittersettings.php index 45725d3ff4..0b98eef591 100644 --- a/actions/twittersettings.php +++ b/actions/twittersettings.php @@ -261,7 +261,7 @@ class TwittersettingsAction extends ConnectSettingsAction 'alt' => ($other->fullname) ? $other->fullname : $other->nickname)); - + $this->element('span', 'fn nickname', $other->nickname); $this->elementEnd('a'); $this->elementEnd('li'); @@ -375,6 +375,8 @@ class TwittersettingsAction extends ConnectSettingsAction if ($friendsync) { save_twitter_friends($user, $twit_user->id, $screen_name, $password); + $flink->last_friendsync = common_sql_now(); + $flink->update(); } $this->showForm(_('Twitter settings saved.'), true); diff --git a/scripts/synctwitterfriends.php b/scripts/synctwitterfriends.php index 794301f0f0..bd08ba58d6 100755 --- a/scripts/synctwitterfriends.php +++ b/scripts/synctwitterfriends.php @@ -32,8 +32,25 @@ define('LACONICA', true); require_once(INSTALLDIR . '/lib/common.php'); +// Make a lockfile +$lockfilename = lockFilename(); +if (!($lockfile = @fopen($lockfilename, "w"))) { + print "Already running... exiting.\n"; + exit(1); +} + +// Obtain an exlcusive lock on file (will fail if script is already going) +if (!@flock( $lockfile, LOCK_EX | LOCK_NB, &$wouldblock) || $wouldblock) { + // Script already running - abort + @fclose($lockfile); + print "Already running... exiting.\n"; + exit(1); +} + $flink = new Foreign_link(); $flink->service = 1; // Twitter +$flink->orderBy('last_friendsync'); +$flink->limit(25); // sync this many users during this run $cnt = $flink->find(); print "Updating Twitter friends subscriptions for $cnt users.\n"; @@ -60,8 +77,11 @@ while ($flink->fetch()) { continue; } - $result = save_twitter_friends($user, $fuser->id, - $fuser->nickname, $flink->credentials); + save_twitter_friends($user, $fuser->id, $fuser->nickname, $flink->credentials); + + $flink->last_friendsync = common_sql_now(); + $flink->update(); + if (defined('SCRIPT_DEBUG')) { print "\nDONE\n"; } else { @@ -70,4 +90,18 @@ while ($flink->fetch()) { } } +function lockFilename() +{ + $piddir = common_config('daemon', 'piddir'); + if (!$piddir) { + $piddir = '/var/run'; + } + + return $piddir . '/synctwitterfriends.lock'; +} + +// Cleanup +fclose($lockfile); +unlink($lockfilename); + exit(0); From 9a8095079dc602c7f2b74e48237445d682844de6 Mon Sep 17 00:00:00 2001 From: CiaranG Date: Fri, 8 May 2009 08:14:50 +0100 Subject: [PATCH 12/12] PostgreSQL - added dirty dates to Foreign_link - see 3e7b1e69e3e97ac007465376b62084f10bcf97ca --- db/laconica_pg.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/laconica_pg.sql b/db/laconica_pg.sql index f879d7936f..a27a616f24 100644 --- a/db/laconica_pg.sql +++ b/db/laconica_pg.sql @@ -291,6 +291,8 @@ create table foreign_link ( noticesync int not null default 1 /* comment 'notice synchronisation, bit 1 = sync outgoing, bit 2 = sync incoming, bit 3 = filter local replies' */, friendsync int not null default 2 /* comment 'friend synchronisation, bit 1 = sync outgoing, bit 2 = sync incoming */, profilesync int not null default 1 /* comment 'profile synchronization, bit 1 = sync outgoing, bit 2 = sync incoming' */, + last_noticesync timestamp default null /* comment 'last time notices were imported' */, + last_friendsync timestamp default null /* comment 'last time friends were imported' */, created timestamp not null default CURRENT_TIMESTAMP /* comment 'date this record was created' */, modified timestamp /* comment 'date this record was modified' */,