From 7c84c355878353ae0aeefb9f6b3adbc673f758b6 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 12:09:02 -0800 Subject: [PATCH 01/18] Notice::getAsTimestamp() static function to look up the timestamp for a given notice, even if it's been deleted. To be used for converting since_id/max_id processing to use timestamp sorting internally. --- classes/Notice.php | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/classes/Notice.php b/classes/Notice.php index a067cd3741..079f56dcf9 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -1978,4 +1978,26 @@ class Notice extends Memcached_DataObject $d = new DateTime($dateStr, new DateTimeZone('UTC')); return $d->format(DATE_W3C); } + + /** + * Look up the creation timestamp for a given notice ID, even + * if it's been deleted. + * + * @param int $id + * @return mixed string recorded creation timestamp, or false if can't be found + */ + public static function getAsTimestamp($id) + { + $notice = Notice::staticGet('id', $id); + if ($notice) { + return $notice->created; + } else { + $deleted = Deleted_notice::staticGet('id', $id); + if ($deleted) { + return $deleted->created; + } else { + return false; + } + } + } } From 5de86f0ccc0208c2fc1a3924e4142bcac9679893 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 12:38:38 -0800 Subject: [PATCH 02/18] Initial switch of public timeline stream to use timestamps for internal sorting --- classes/Notice.php | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index 079f56dcf9..b5a1b45971 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -654,7 +654,7 @@ class Notice extends Memcached_DataObject $notice->selectAdd(); // clears it $notice->selectAdd('id'); - $notice->orderBy('id DESC'); + $notice->orderBy('created DESC, id DESC'); if (!is_null($offset)) { $notice->limit($offset, $limit); @@ -668,12 +668,14 @@ class Notice extends Memcached_DataObject $notice->whereAdd('is_local !='. Notice::GATEWAY); } - if ($since_id != 0) { - $notice->whereAdd('id > ' . $since_id); + $since = Notice::getAsTimestamp($since_id); + if ($since) { + $notice->whereAdd(sprintf("(created = '%s' and id > %d) or (created > '%s')", $since, $since_id, $since)); } - if ($max_id != 0) { - $notice->whereAdd('id <= ' . $max_id); + $max = Notice::getAsTimestamp($max_id); + if ($max) { + $notice->whereAdd(sprintf("(created < '%s') or (created = '%s' and id <= %d)", $max, $max, $max_id)); } $ids = array(); @@ -1988,16 +1990,20 @@ class Notice extends Memcached_DataObject */ public static function getAsTimestamp($id) { + if (!$id) { + return false; + } + $notice = Notice::staticGet('id', $id); if ($notice) { return $notice->created; - } else { - $deleted = Deleted_notice::staticGet('id', $id); - if ($deleted) { - return $deleted->created; - } else { - return false; - } } + + $deleted = Deleted_notice::staticGet('id', $id); + if ($deleted) { + return $deleted->created; + } + + return false; } } From 53dd2583fcf5d026f682b0966e87dbdd35fc19a8 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 12:47:50 -0800 Subject: [PATCH 03/18] Switch public timeline to new sorting; new index notice_created_id_is_local_idx http://status.net/wiki/Sorting_changes --- db/096to097.sql | 3 +++ db/statusnet.sql | 6 ++++++ 2 files changed, 9 insertions(+) create mode 100644 db/096to097.sql diff --git a/db/096to097.sql b/db/096to097.sql new file mode 100644 index 0000000000..53f4e97c9c --- /dev/null +++ b/db/096to097.sql @@ -0,0 +1,3 @@ +-- Add indexes for sorting changes in 0.9.7 +-- Allows sorting public timeline by timestamp efficiently +alter table notice add index notice_created_id_is_local_idx (created,id,is_local); diff --git a/db/statusnet.sql b/db/statusnet.sql index ac48e6253a..b372305d00 100644 --- a/db/statusnet.sql +++ b/db/statusnet.sql @@ -131,7 +131,13 @@ create table notice ( location_ns integer comment 'namespace for location', repeat_of integer comment 'notice this is a repeat of' references notice (id), + -- For public timeline... + index notice_created_id_is_local_idx (created,id,is_local), + + -- For profile timelines... index notice_profile_id_idx (profile_id,created,id), + + -- Are these enough? index notice_conversation_idx (conversation), index notice_created_idx (created), index notice_replyto_idx (reply_to), From 9e8bbff8ac3825dc789bcd19b1751fe24017a789 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 13:03:18 -0800 Subject: [PATCH 04/18] Notice::whereSinceId() and Notice::whereMaxId() encapsulate logic for building where clauses for since_id/max_id parameters. Can override the field names from 'id' and 'created'. --- classes/Notice.php | 48 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index b5a1b45971..14a91977df 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -668,14 +668,14 @@ class Notice extends Memcached_DataObject $notice->whereAdd('is_local !='. Notice::GATEWAY); } - $since = Notice::getAsTimestamp($since_id); + $since = Notice::whereSinceId($since_id); if ($since) { - $notice->whereAdd(sprintf("(created = '%s' and id > %d) or (created > '%s')", $since, $since_id, $since)); + $notice->whereAdd($since); } - $max = Notice::getAsTimestamp($max_id); + $max = Notice::whereMaxId($max_id); if ($max) { - $notice->whereAdd(sprintf("(created < '%s') or (created = '%s' and id <= %d)", $max, $max, $max_id)); + $notice->whereAdd($max); } $ids = array(); @@ -2006,4 +2006,44 @@ class Notice extends Memcached_DataObject return false; } + + /** + * Build an SQL 'where' fragment for timestamp-based sorting from a since_id + * parameter, matching notices posted after the given one (exclusive). + * + * If the referenced notice can't be found, will return false. + * + * @param int $id + * @param string $idField + * @param string $createdField + * @return mixed string or false if no match + */ + public static function whereSinceId($id, $idField='id', $createdField='created') + { + $since = Notice::getAsTimestamp($id); + if ($since) { + return sprintf("($createdField = '%s' and $idField > %d) or ($createdField > '%s')", $since, $id, $since); + } + return false; + } + + /** + * Build an SQL 'where' fragment for timestamp-based sorting from a max_id + * parameter, matching notices posted before the given one (inclusive). + * + * If the referenced notice can't be found, will return false. + * + * @param int $id + * @param string $idField + * @param string $createdField + * @return mixed string or false if no match + */ + public static function whereMaxId($id, $idField='id', $createdField='created') + { + $max = Notice::getAsTimestamp($id); + if ($max) { + return sprintf("($createdField < '%s') or ($createdField = '%s' and $idField <= %d)", $max, $max, $id); + } + return false; + } } From 4cd3a0756bdfba4589dbf1efeab9a2a509e9d566 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 13:20:38 -0800 Subject: [PATCH 05/18] Update notice sorting for profile streams; extract more common code to Notice::addSinceId() and Notice::addMaxId() --- classes/Notice.php | 49 +++++++++++++++++++++++++++++++-------- classes/Profile.php | 56 ++++++++------------------------------------- 2 files changed, 50 insertions(+), 55 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index 14a91977df..ef5fba0631 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -668,15 +668,8 @@ class Notice extends Memcached_DataObject $notice->whereAdd('is_local !='. Notice::GATEWAY); } - $since = Notice::whereSinceId($since_id); - if ($since) { - $notice->whereAdd($since); - } - - $max = Notice::whereMaxId($max_id); - if ($max) { - $notice->whereAdd($max); - } + Notice::addWhereSinceId($notice, $since_id); + Notice::addWhereMaxId($notice, $max_id); $ids = array(); @@ -2027,6 +2020,25 @@ class Notice extends Memcached_DataObject return false; } + /** + * Build an SQL 'where' fragment for timestamp-based sorting from a since_id + * parameter, matching notices posted after the given one (exclusive), and + * if necessary add it to the data object's query. + * + * @param DB_DataObject $obj + * @param int $id + * @param string $idField + * @param string $createdField + * @return mixed string or false if no match + */ + public static function addWhereSinceId(DB_DataObject $obj, $id, $idField='id', $createdField='created') + { + $since = self::whereSinceId($id); + if ($since) { + $obj->whereAdd($since); + } + } + /** * Build an SQL 'where' fragment for timestamp-based sorting from a max_id * parameter, matching notices posted before the given one (inclusive). @@ -2046,4 +2058,23 @@ class Notice extends Memcached_DataObject } return false; } + + /** + * Build an SQL 'where' fragment for timestamp-based sorting from a max_id + * parameter, matching notices posted before the given one (inclusive), and + * if necessary add it to the data object's query. + * + * @param DB_DataObject $obj + * @param int $id + * @param string $idField + * @param string $createdField + * @return mixed string or false if no match + */ + public static function addWhereMaxId(DB_DataObject $obj, $id, $idField='id', $createdField='created') + { + $max = self::whereMaxId($id); + if ($max) { + $obj->whereAdd($max); + } + } } diff --git a/classes/Profile.php b/classes/Profile.php index 332d51e203..62789a4899 100644 --- a/classes/Profile.php +++ b/classes/Profile.php @@ -252,58 +252,22 @@ class Profile extends Memcached_DataObject { $notice = new Notice(); - // Temporary hack until notice_profile_id_idx is updated - // to (profile_id, id) instead of (profile_id, created, id). - // It's been falling back to PRIMARY instead, which is really - // very inefficient for a profile that hasn't posted in a few - // months. Even though forcing the index will cause a filesort, - // it's usually going to be better. - if (common_config('db', 'type') == 'mysql') { - $index = ''; - $query = - "select id from notice force index (notice_profile_id_idx) ". - "where profile_id=" . $notice->escape($this->id); + $notice->profile_id = $this->id; - if ($since_id != 0) { - $query .= " and id > $since_id"; - } + $notice->selectAdd(); + $notice->selectAdd('id'); - if ($max_id != 0) { - $query .= " and id <= $max_id"; - } + Notice::addWhereSinceId($notice, $since_id); + Notice::addWhereMaxId($notice, $max_id); - $query .= ' order by id DESC'; + $notice->orderBy('created DESC, id DESC'); - if (!is_null($offset)) { - $query .= " LIMIT $limit OFFSET $offset"; - } - - $notice->query($query); - } else { - $index = ''; - - $notice->profile_id = $this->id; - - $notice->selectAdd(); - $notice->selectAdd('id'); - - if ($since_id != 0) { - $notice->whereAdd('id > ' . $since_id); - } - - if ($max_id != 0) { - $notice->whereAdd('id <= ' . $max_id); - } - - $notice->orderBy('id DESC'); - - if (!is_null($offset)) { - $notice->limit($offset, $limit); - } - - $notice->find(); + if (!is_null($offset)) { + $notice->limit($offset, $limit); } + $notice->find(); + $ids = array(); while ($notice->fetch()) { From 4adf551f9fe73209bf44afb2960667bb62d66c6f Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 13:45:40 -0800 Subject: [PATCH 06/18] Update sorting for user tagged timelines (indexing was bad before and remains bad -- we need some DB changes to make this one nice) --- classes/Profile.php | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/classes/Profile.php b/classes/Profile.php index 62789a4899..fe1a070bdf 100644 --- a/classes/Profile.php +++ b/classes/Profile.php @@ -215,26 +215,29 @@ class Profile extends Memcached_DataObject function _streamTaggedDirect($tag, $offset, $limit, $since_id, $max_id) { // XXX It would be nice to do this without a join + // (necessary to do it efficiently on accounts with long history) $notice = new Notice(); $query = "select id from notice join notice_tag on id=notice_id where tag='". $notice->escape($tag) . - "' and profile_id=" . $notice->escape($this->id); + "' and profile_id=" . intval($this->id); - if ($since_id != 0) { - $query .= " and id > $since_id"; + $since = Notice::whereSinceId($since_id, 'id', 'notice.created'); + if ($since) { + $query .= " and ($since)"; } - if ($max_id != 0) { - $query .= " and id <= $max_id"; + $max = Notice::whereMaxId($max_id, 'id', 'notice.created'); + if ($max) { + $query .= " and ($max)"; } - $query .= ' order by id DESC'; + $query .= ' order by notice.created DESC, id DESC'; if (!is_null($offset)) { - $query .= " LIMIT $limit OFFSET $offset"; + $query .= " LIMIT " . intval($limit) . " OFFSET " . intval($offset); } $notice->query($query); From 33daace6cb7bf607b94ae684389d5577c290f026 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 14:32:06 -0800 Subject: [PATCH 07/18] add fixme for since_id/max_id on fave streaming (?) --- classes/Fave.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/classes/Fave.php b/classes/Fave.php index 3aa23e7b4e..4a9cfaae06 100644 --- a/classes/Fave.php +++ b/classes/Fave.php @@ -85,6 +85,19 @@ class Fave extends Memcached_DataObject return $ids; } + /** + * Note that the sorting for this is by order of *fave* not order of *notice*. + * + * @fixme add since_id, max_id support? + * + * @param $user_id + * @param $own + * @param $offset + * @param $limit + * @param $since_id + * @param $max_id + * @return + */ function _streamDirect($user_id, $own, $offset, $limit, $since_id, $max_id) { $fav = new Fave(); From 00a5a5342ab0c44d59697cded81f6e96ba49c42c Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 14:37:46 -0800 Subject: [PATCH 08/18] Update sorting for tag-filtered public timeline: needs notice_tag_tag_created_notice_id_idx index added to notice_tag --- classes/Notice_tag.php | 11 +++-------- db/096to097.sql | 4 ++++ db/statusnet.sql | 5 ++++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/classes/Notice_tag.php b/classes/Notice_tag.php index 6eada70224..bb67c8f819 100644 --- a/classes/Notice_tag.php +++ b/classes/Notice_tag.php @@ -55,15 +55,10 @@ class Notice_tag extends Memcached_DataObject $nt->selectAdd(); $nt->selectAdd('notice_id'); - if ($since_id != 0) { - $nt->whereAdd('notice_id > ' . $since_id); - } + Notice::addWhereSinceId($nt, $since_id, 'notice_id'); + Notice::addWhereMaxId($nt, $max_id, 'notice_id'); - if ($max_id != 0) { - $nt->whereAdd('notice_id <= ' . $max_id); - } - - $nt->orderBy('notice_id DESC'); + $nt->orderBy('created DESC, notice_id DESC'); if (!is_null($offset)) { $nt->limit($offset, $limit); diff --git a/db/096to097.sql b/db/096to097.sql index 53f4e97c9c..38e4e958b8 100644 --- a/db/096to097.sql +++ b/db/096to097.sql @@ -1,3 +1,7 @@ -- Add indexes for sorting changes in 0.9.7 + -- Allows sorting public timeline by timestamp efficiently alter table notice add index notice_created_id_is_local_idx (created,id,is_local); + +-- Allows sorting tag-filtered public timeline by timestamp efficiently +alter table notice_tag add index notice_tag_tag_created_notice_id_idx (tag, created, notice_id); diff --git a/db/statusnet.sql b/db/statusnet.sql index b372305d00..76a821bafe 100644 --- a/db/statusnet.sql +++ b/db/statusnet.sql @@ -307,7 +307,10 @@ create table notice_tag ( constraint primary key (tag, notice_id), index notice_tag_created_idx (created), - index notice_tag_notice_id_idx (notice_id) + index notice_tag_notice_id_idx (notice_id), + + -- For sorting tag-filtered public timeline + index notice_tag_tag_created_notice_id_idx (tag, created, notice_id) ) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin; /* Synching with foreign services */ From 3ddfa4de931f4eb3083ac877898b5ee8b03a82f1 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 14:43:45 -0800 Subject: [PATCH 09/18] Update sorting on reply/mentions timeline: added reply_profile_id_modified_notice_id_idx index to reply table --- classes/Reply.php | 11 +++-------- db/096to097.sql | 3 +++ db/statusnet.sql | 5 ++++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/classes/Reply.php b/classes/Reply.php index da8a4f685b..371c16cf48 100644 --- a/classes/Reply.php +++ b/classes/Reply.php @@ -50,15 +50,10 @@ class Reply extends Memcached_DataObject $reply = new Reply(); $reply->profile_id = $user_id; - if ($since_id != 0) { - $reply->whereAdd('notice_id > ' . $since_id); - } + Notice::addWhereSinceId($reply, $since_id, 'notice_id', 'modified'); + Notice::addWhereMaxId($reply, $max_id, 'notice_id', 'modified'); - if ($max_id != 0) { - $reply->whereAdd('notice_id <= ' . $max_id); - } - - $reply->orderBy('notice_id DESC'); + $reply->orderBy('modified DESC, notice_id DESC'); if (!is_null($offset)) { $reply->limit($offset, $limit); diff --git a/db/096to097.sql b/db/096to097.sql index 38e4e958b8..4171e95897 100644 --- a/db/096to097.sql +++ b/db/096to097.sql @@ -5,3 +5,6 @@ alter table notice add index notice_created_id_is_local_idx (created,id,is_local -- Allows sorting tag-filtered public timeline by timestamp efficiently alter table notice_tag add index notice_tag_tag_created_notice_id_idx (tag, created, notice_id); + +-- Needed for sorting reply/mentions timelines +alter table reply add index reply_profile_id_modified_notice_id_idx (profile_id, modified, notice_id); diff --git a/db/statusnet.sql b/db/statusnet.sql index 76a821bafe..dfc46f79e2 100644 --- a/db/statusnet.sql +++ b/db/statusnet.sql @@ -162,7 +162,10 @@ create table reply ( constraint primary key (notice_id, profile_id), index reply_notice_id_idx (notice_id), index reply_profile_id_idx (profile_id), - index reply_replied_id_idx (replied_id) + index reply_replied_id_idx (replied_id), + + -- Needed for sorting reply/mentions timelines + index reply_profile_id_modified_notice_id_idx (profile_id, modified, notice_id) ) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin; From 66474586af58c4e505117f0fed7382831ec008b1 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 14:51:37 -0800 Subject: [PATCH 10/18] Update sorting for group inbox timelines; adds group_inbox_group_id_created_notice_id_idx index to group_inbox table --- classes/User_group.php | 11 +++-------- db/096to097.sql | 3 +++ db/statusnet.sql | 5 ++++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/classes/User_group.php b/classes/User_group.php index 60217e960e..cffc786458 100644 --- a/classes/User_group.php +++ b/classes/User_group.php @@ -100,15 +100,10 @@ class User_group extends Memcached_DataObject $inbox->selectAdd(); $inbox->selectAdd('notice_id'); - if ($since_id != 0) { - $inbox->whereAdd('notice_id > ' . $since_id); - } + Notice::addWhereSinceId($inbox, $since_id, 'notice_id'); + Notice::addWhereMaxId($inbox, $max_id, 'notice_id'); - if ($max_id != 0) { - $inbox->whereAdd('notice_id <= ' . $max_id); - } - - $inbox->orderBy('notice_id DESC'); + $inbox->orderBy('created DESC, notice_id DESC'); if (!is_null($offset)) { $inbox->limit($offset, $limit); diff --git a/db/096to097.sql b/db/096to097.sql index 4171e95897..c3f1fb4253 100644 --- a/db/096to097.sql +++ b/db/096to097.sql @@ -8,3 +8,6 @@ alter table notice_tag add index notice_tag_tag_created_notice_id_idx (tag, crea -- Needed for sorting reply/mentions timelines alter table reply add index reply_profile_id_modified_notice_id_idx (profile_id, modified, notice_id); + +-- Needed for sorting group messages by timestamp +alter table group_inbox add index group_inbox_group_id_created_notice_id_idx (group_id, created, notice_id); diff --git a/db/statusnet.sql b/db/statusnet.sql index dfc46f79e2..5898abf463 100644 --- a/db/statusnet.sql +++ b/db/statusnet.sql @@ -480,7 +480,10 @@ create table group_inbox ( constraint primary key (group_id, notice_id), index group_inbox_created_idx (created), - index group_inbox_notice_id_idx (notice_id) + index group_inbox_notice_id_idx (notice_id), + + -- Needed for sorting group messages by timestamp + index group_inbox_group_id_created_notice_id_idx (group_id, created, notice_id) ) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin; From 04aa8bd70f0c3f84af349b8a801ea73753077760 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 15:04:10 -0800 Subject: [PATCH 11/18] work around borkage in statuses/repeats -- tries to check an offset var that's not there. use the limit var which is there instead --- classes/Notice.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index ef5fba0631..c58705c4b8 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -1692,8 +1692,8 @@ class Notice extends Memcached_DataObject $notice->orderBy('created'); // NB: asc! - if (!is_null($offset)) { - $notice->limit($offset, $limit); + if (!is_null($limit)) { + $notice->limit(0, $limit); } $ids = array(); From 1b90ed564a19dcb3d24d0c0620ce0623773fe4d0 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 15:13:09 -0800 Subject: [PATCH 12/18] Update sorting on api/statuses/retweets: adds notice_repeat_of_created_id_idx index to replace notice_repeatof_idx --- classes/Notice.php | 2 +- db/096to097.sql | 7 +++++-- db/statusnet.sql | 4 ++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index c58705c4b8..ea69a5beda 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -1690,7 +1690,7 @@ class Notice extends Memcached_DataObject $notice->repeat_of = $this->id; - $notice->orderBy('created'); // NB: asc! + $notice->orderBy('created, id'); // NB: asc! if (!is_null($limit)) { $notice->limit(0, $limit); diff --git a/db/096to097.sql b/db/096to097.sql index c3f1fb4253..5947538da8 100644 --- a/db/096to097.sql +++ b/db/096to097.sql @@ -1,7 +1,10 @@ -- Add indexes for sorting changes in 0.9.7 --- Allows sorting public timeline by timestamp efficiently -alter table notice add index notice_created_id_is_local_idx (created,id,is_local); +-- Allows sorting public timeline and api/statuses/repeats by timestamp efficiently +alter table notice + add index notice_created_id_is_local_idx (created,id,is_local), + drop index notice_repeatof_idx, + add index notice_repeat_of_created_id_idx (repeat_of, created, id); -- Allows sorting tag-filtered public timeline by timestamp efficiently alter table notice_tag add index notice_tag_tag_created_notice_id_idx (tag, created, notice_id); diff --git a/db/statusnet.sql b/db/statusnet.sql index 5898abf463..9624edd6f0 100644 --- a/db/statusnet.sql +++ b/db/statusnet.sql @@ -137,11 +137,15 @@ create table notice ( -- For profile timelines... index notice_profile_id_idx (profile_id,created,id), + -- For api/statuses/repeats... + index notice_repeat_of_created_id_idx (repeat_of, created, id), + -- Are these enough? index notice_conversation_idx (conversation), index notice_created_idx (created), index notice_replyto_idx (reply_to), index notice_repeatof_idx (repeat_of), + FULLTEXT(content) ) ENGINE=MyISAM CHARACTER SET utf8 COLLATE utf8_general_ci; From b80151275a8659e241b66e8f8454541d236f2f0e Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 15:25:19 -0800 Subject: [PATCH 13/18] Update sorting on api/statuses/retweets_of_me; was and remains poorly indexed, but will use updated sorting method. --- classes/User.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/classes/User.php b/classes/User.php index 1b1b971ec7..edbd6bb2e3 100644 --- a/classes/User.php +++ b/classes/User.php @@ -800,17 +800,17 @@ class User extends Memcached_DataObject 'FROM notice original JOIN notice rept ON original.id = rept.repeat_of ' . 'WHERE original.profile_id = ' . $this->id . ' '; - if ($since_id != 0) { - $qry .= 'AND original.id > ' . $since_id . ' '; + $since = Notice::whereSinceId($since_id, 'original.id', 'original.created'); + if ($since) { + $qry .= "AND ($since) "; } - if ($max_id != 0) { - $qry .= 'AND original.id <= ' . $max_id . ' '; + $max = Notice::whereMaxId($max_id, 'original.id', 'original.created'); + if ($max) { + $qry .= "AND ($max) "; } - // NOTE: we sort by fave time, not by notice time! - - $qry .= 'ORDER BY original.id DESC '; + $qry .= 'ORDER BY original.created, original.id DESC '; if (!is_null($offset)) { $qry .= "LIMIT $limit OFFSET $offset"; From 71151b3bc0e1b1d4d560bf296eb0a572485389e8 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 15:28:55 -0800 Subject: [PATCH 14/18] Update sorting for User::repeatedByMe() -- currently unused. Likely not ideally indexed yet. --- classes/User.php | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/classes/User.php b/classes/User.php index edbd6bb2e3..47bf8a6177 100644 --- a/classes/User.php +++ b/classes/User.php @@ -755,19 +755,14 @@ class User extends Memcached_DataObject $notice->profile_id = $this->id; $notice->whereAdd('repeat_of IS NOT NULL'); - $notice->orderBy('id DESC'); + $notice->orderBy('created DESC, id DESC'); if (!is_null($offset)) { $notice->limit($offset, $limit); } - if ($since_id != 0) { - $notice->whereAdd('id > ' . $since_id); - } - - if ($max_id != 0) { - $notice->whereAdd('id <= ' . $max_id); - } + Notice::addWhereSinceId($notice, $since_id); + Notice::addWhereMaxId($notice, $max_id); $ids = array(); From fb65d5901d586a13886b81d84de8959b67b6aa9e Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 16:08:37 -0800 Subject: [PATCH 15/18] Update sorting for conversation views: adds notice_conversation_created_id_idx index on notice, replacing more limited notice_conversation_idx --- classes/Notice.php | 11 +++-------- db/096to097.sql | 8 ++++++-- db/statusnet.sql | 8 ++++---- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index ea69a5beda..629b7089de 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -704,19 +704,14 @@ class Notice extends Memcached_DataObject $notice->conversation = $id; - $notice->orderBy('id DESC'); + $notice->orderBy('created DESC, id DESC'); if (!is_null($offset)) { $notice->limit($offset, $limit); } - if ($since_id != 0) { - $notice->whereAdd('id > ' . $since_id); - } - - if ($max_id != 0) { - $notice->whereAdd('id <= ' . $max_id); - } + Notice::addWhereSinceId($notice, $since_id); + Notice::addWhereMaxId($notice, $max_id); $ids = array(); diff --git a/db/096to097.sql b/db/096to097.sql index 5947538da8..88cbea4578 100644 --- a/db/096to097.sql +++ b/db/096to097.sql @@ -1,10 +1,14 @@ -- Add indexes for sorting changes in 0.9.7 --- Allows sorting public timeline and api/statuses/repeats by timestamp efficiently +-- Allows sorting public timeline, api/statuses/repeats, and conversations by timestamp efficiently alter table notice add index notice_created_id_is_local_idx (created,id,is_local), + + add index notice_repeat_of_created_id_idx (repeat_of, created, id), drop index notice_repeatof_idx, - add index notice_repeat_of_created_id_idx (repeat_of, created, id); + + add index notice_conversation_created_id_idx (conversation, created, id), + drop index notice_conversation_idx; -- Allows sorting tag-filtered public timeline by timestamp efficiently alter table notice_tag add index notice_tag_tag_created_notice_id_idx (tag, created, notice_id); diff --git a/db/statusnet.sql b/db/statusnet.sql index 9624edd6f0..8b38f9ffe3 100644 --- a/db/statusnet.sql +++ b/db/statusnet.sql @@ -140,11 +140,11 @@ create table notice ( -- For api/statuses/repeats... index notice_repeat_of_created_id_idx (repeat_of, created, id), - -- Are these enough? - index notice_conversation_idx (conversation), - index notice_created_idx (created), + -- For conversation views + index notice_conversation_created_id_idx (conversation, created, id), + + -- Are these needed/used? index notice_replyto_idx (reply_to), - index notice_repeatof_idx (repeat_of), FULLTEXT(content) ) ENGINE=MyISAM CHARACTER SET utf8 COLLATE utf8_general_ci; From 146d6b8b735472a2ae4c2f8d2653243492651399 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 16:12:44 -0800 Subject: [PATCH 16/18] Sorting index fix for role lookups: adds profile_role_role_created_profile_id_idx index on profile_role --- db/096to097.sql | 3 +++ db/statusnet.sql | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/db/096to097.sql b/db/096to097.sql index 88cbea4578..875077e8b8 100644 --- a/db/096to097.sql +++ b/db/096to097.sql @@ -18,3 +18,6 @@ alter table reply add index reply_profile_id_modified_notice_id_idx (profile_id, -- Needed for sorting group messages by timestamp alter table group_inbox add index group_inbox_group_id_created_notice_id_idx (group_id, created, notice_id); + +-- Helps make some reverse role lookups more efficient if there's a lot of assigned accounts +alter table profile_role add index profile_role_role_created_profile_id_idx (role, created, profile_id); diff --git a/db/statusnet.sql b/db/statusnet.sql index 8b38f9ffe3..0c1697a7e9 100644 --- a/db/statusnet.sql +++ b/db/statusnet.sql @@ -627,7 +627,8 @@ create table profile_role ( role varchar(32) not null comment 'string representing the role', created datetime not null comment 'date the role was granted', - constraint primary key (profile_id, role) + constraint primary key (profile_id, role), + index profile_role_role_created_profile_id_idx (role, created, profile_id) ) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin; From c0669969f28f957d13ad39cdd5ec01509ed47aac Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 16:21:33 -0800 Subject: [PATCH 17/18] fix typo in showstream --- actions/showstream.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/showstream.php b/actions/showstream.php index 5a22bdf288..8a67d3fc9c 100644 --- a/actions/showstream.php +++ b/actions/showstream.php @@ -317,7 +317,7 @@ class ProfileNoticeListItem extends DoFollowListItem 'class' => 'url'); if (!empty($this->profile->fullname)) { - $attrs['title'] = $this->getFancyName(); + $attrs['title'] = $this->profile->getFancyName(); } $this->out->elementStart('span', 'repeat'); From 5300d657cc881a1e8dba59498c90098930029d68 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 17 Dec 2010 16:22:26 -0800 Subject: [PATCH 18/18] Sort indexing fix for profile sidebar: add group_member_profile_id_created_idx to group_member table, streamlines sorting of your group memberships in the sidebar --- db/096to097.sql | 3 +++ db/statusnet.sql | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/db/096to097.sql b/db/096to097.sql index 875077e8b8..209a3a8811 100644 --- a/db/096to097.sql +++ b/db/096to097.sql @@ -21,3 +21,6 @@ alter table group_inbox add index group_inbox_group_id_created_notice_id_idx (gr -- Helps make some reverse role lookups more efficient if there's a lot of assigned accounts alter table profile_role add index profile_role_role_created_profile_id_idx (role, created, profile_id); + +-- Fix for sorting a user's group memberships by order joined +alter table group_member add index group_member_profile_id_created_idx (profile_id, created); diff --git a/db/statusnet.sql b/db/statusnet.sql index 0c1697a7e9..4a24d016a2 100644 --- a/db/statusnet.sql +++ b/db/statusnet.sql @@ -463,7 +463,10 @@ create table group_member ( constraint primary key (group_id, profile_id), index group_member_profile_id_idx (profile_id), - index group_member_created_idx (created) + index group_member_created_idx (created), + + -- To pull up a list of someone's groups in order joined + index group_member_profile_id_created_idx (profile_id, created) ) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin;