From 55ba858e8cb4eac0fa60fb78f8e8c4813be065a9 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 31 Dec 2009 12:38:58 -1000 Subject: [PATCH 1/4] Script to update the location ID for users Since we added locations to the database, some users may have location strings in their profiles but not structured locations. This script updates the locations for single users or for all users. --- scripts/updatelocation.php | 125 +++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 scripts/updatelocation.php diff --git a/scripts/updatelocation.php b/scripts/updatelocation.php new file mode 100644 index 0000000000..4110660ab0 --- /dev/null +++ b/scripts/updatelocation.php @@ -0,0 +1,125 @@ +#!/usr/bin/env php +. + */ + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); + +$shortoptions = 'i:n:af'; +$longoptions = array('id=', 'nickname=', 'all', 'force'); + +$helptext = <<find()) { + while ($user->fetch()) { + updateLocation($user); + } + } + } else { + show_help(); + exit(1); + } +} catch (Exception $e) { + print $e->getMessage()."\n"; + exit(1); +} + +function updateLocation($user) +{ + $profile = $user->getProfile(); + + if (empty($profile)) { + throw new Exception("User has no profile: " . $user->nickname); + } + + if (empty($profile->location)) { + if (have_option('v', 'verbose')) { + print "No location string for '".$user->nickname."'\n"; + } + return; + } + + if (!empty($profile->location_id) && !have_option('f', 'force')) { + if (have_option('v', 'verbose')) { + print "Location ID already set for '".$user->nickname."'\n"; + } + return; + } + + $loc = Location::fromName($profile->location); + + if (empty($loc)) { + if (have_option('v', 'verbose')) { + print "No structured location for string '".$profile->location."' for user '".$user->nickname."'\n"; + } + return; + } else { + $orig = clone($profile); + + $profile->lat = $loc->lat; + $profile->lon = $loc->lon; + $profile->location_id = $loc->location_id; + $profile->location_ns = $loc->location_ns; + + $result = $profile->update($orig); + + if (!$result) { + common_log_db_error($profile, 'UPDATE', __FILE__); + } + + if (!have_option('q', 'quiet')) { + print "Location ID " . $profile->location_id . " set for user " . $user->nickname . "\n"; + } + } + + $profile->free(); + unset($loc); + unset($profile); + + return; +} From af95005bc481d6f8f84a780bdc062426e22f3a03 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 13:01:17 -0800 Subject: [PATCH 2/4] Ticket 2141: bugs with weighted popularity lists across year boundary. Consolidated several separate implementations of the same weighting algorithm into common_sql_weight() and fixed some bugs... For MySQL, now using timestampdiff() instead of subtraction for the comparison, so we get sane results when the year doesn't match, and utc_timestamp() rather than now() so we don't get negative ages for recent items with local server timezone. Unknown whether the same problems affect PostgreSQL, but note that it lacks the timestampdiff() SQL function. --- actions/favorited.php | 8 ++------ actions/publictagcloud.php | 15 ++++++++------- lib/grouptagcloudsection.php | 7 +------ lib/personaltagcloudsection.php | 11 +++-------- lib/popularnoticesection.php | 6 +++--- lib/util.php | 20 ++++++++++++++++++++ 6 files changed, 37 insertions(+), 30 deletions(-) diff --git a/actions/favorited.php b/actions/favorited.php index 150b67b0b0..9ffa5b8445 100644 --- a/actions/favorited.php +++ b/actions/favorited.php @@ -185,11 +185,7 @@ class FavoritedAction extends Action function showContent() { - if (common_config('db', 'type') == 'pgsql') { - $weightexpr='sum(exp(-extract(epoch from (now() - fave.modified)) / %s))'; - } else { - $weightexpr='sum(exp(-(now() - fave.modified) / %s))'; - } + $weightexpr = common_sql_weight('fave.modified', common_config('popular', 'dropoff')); $qry = 'SELECT notice.*, '. $weightexpr . ' as weight ' . @@ -207,7 +203,7 @@ class FavoritedAction extends Action } $notice = Memcached_DataObject::cachedQuery('Notice', - sprintf($qry, common_config('popular', 'dropoff')), + $qry, 600); $nl = new NoticeList($notice, $this); diff --git a/actions/publictagcloud.php b/actions/publictagcloud.php index e7f6ee36c7..9e4478dbb1 100644 --- a/actions/publictagcloud.php +++ b/actions/publictagcloud.php @@ -105,12 +105,8 @@ class PublictagcloudAction extends Action #Add the aggregated columns... $tags->selectAdd('max(notice_id) as last_notice_id'); - if(common_config('db','type')=='pgsql') { - $calc='sum(exp(-extract(epoch from (now()-created))/%s)) as weight'; - } else { - $calc='sum(exp(-(now() - created)/%s)) as weight'; - } - $tags->selectAdd(sprintf($calc, common_config('tag', 'dropoff'))); + $calc = common_sql_weight('created', common_config('tag', 'dropoff')); + $tags->selectAdd($calc . ' as weight'); $tags->groupBy('tag'); $tags->orderBy('weight DESC'); @@ -136,7 +132,12 @@ class PublictagcloudAction extends Action $this->elementStart('dd'); $this->elementStart('ul', 'tags xoxo tag-cloud'); foreach ($tw as $tag => $weight) { - $this->showTag($tag, $weight, $weight/$sum); + if ($sum) { + $weightedSum = $weight/$sum; + } else { + $weightedSum = 0.5; + } + $this->showTag($tag, $weight, $weightedSum); } $this->elementEnd('ul'); $this->elementEnd('dd'); diff --git a/lib/grouptagcloudsection.php b/lib/grouptagcloudsection.php index 091cf48457..14ceda0850 100644 --- a/lib/grouptagcloudsection.php +++ b/lib/grouptagcloudsection.php @@ -58,11 +58,7 @@ class GroupTagCloudSection extends TagCloudSection function getTags() { - if (common_config('db', 'type') == 'pgsql') { - $weightexpr='sum(exp(-extract(epoch from (now() - notice_tag.created)) / %s))'; - } else { - $weightexpr='sum(exp(-(now() - notice_tag.created) / %s))'; - } + $weightexpr = common_sql_weight('notice_tag.created', common_config('tag', 'dropoff')); $names = $this->group->getAliases(); @@ -99,7 +95,6 @@ class GroupTagCloudSection extends TagCloudSection $tag = Memcached_DataObject::cachedQuery('Notice_tag', sprintf($qry, - common_config('tag', 'dropoff'), $this->group->id, $namestring), 3600); diff --git a/lib/personaltagcloudsection.php b/lib/personaltagcloudsection.php index 0b29d58ca6..091425f926 100644 --- a/lib/personaltagcloudsection.php +++ b/lib/personaltagcloudsection.php @@ -58,13 +58,9 @@ class PersonalTagCloudSection extends TagCloudSection function getTags() { - if (common_config('db', 'type') == 'pgsql') { - $weightexpr='sum(exp(-extract(epoch from (now() - notice_tag.created)) / %s))'; - } else { - $weightexpr='sum(exp(-(now() - notice_tag.created) / %s))'; - } - - $qry = 'SELECT notice_tag.tag, '. + $weightexpr = common_sql_weight('notice_tag.created', common_config('tag', 'dropoff')); + + $qry = 'SELECT notice_tag.tag, '. $weightexpr . ' as weight ' . 'FROM notice_tag JOIN notice ' . 'ON notice_tag.notice_id = notice.id ' . @@ -83,7 +79,6 @@ class PersonalTagCloudSection extends TagCloudSection $tag = Memcached_DataObject::cachedQuery('Notice_tag', sprintf($qry, - common_config('tag', 'dropoff'), $this->user->id), 3600); return $tag; diff --git a/lib/popularnoticesection.php b/lib/popularnoticesection.php index 9fbc9d2ddd..fbf9a60ab8 100644 --- a/lib/popularnoticesection.php +++ b/lib/popularnoticesection.php @@ -48,17 +48,17 @@ class PopularNoticeSection extends NoticeSection { function getNotices() { + // @fixme there should be a common func for this if (common_config('db', 'type') == 'pgsql') { - $weightexpr='sum(exp(-extract(epoch from (now() - fave.modified)) / %s))'; if (!empty($this->out->tag)) { $tag = pg_escape_string($this->out->tag); } } else { - $weightexpr='sum(exp(-(now() - fave.modified) / %s))'; if (!empty($this->out->tag)) { $tag = mysql_escape_string($this->out->tag); } } + $weightexpr = common_sql_weight('fave.modified', common_config('popular', 'dropoff')); $qry = "SELECT notice.*, $weightexpr as weight "; if(isset($tag)) { $qry .= 'FROM notice_tag, notice JOIN fave ON notice.id = fave.notice_id ' . @@ -78,7 +78,7 @@ class PopularNoticeSection extends NoticeSection $qry .= ' LIMIT ' . $limit . ' OFFSET ' . $offset; $notice = Memcached_DataObject::cachedQuery('Notice', - sprintf($qry, common_config('popular', 'dropoff')), + $qry, 1200); return $notice; } diff --git a/lib/util.php b/lib/util.php index ed81aeba16..fdcc87677b 100644 --- a/lib/util.php +++ b/lib/util.php @@ -908,6 +908,26 @@ function common_sql_date($datetime) return strftime('%Y-%m-%d %H:%M:%S', $datetime); } +/** + * Return an SQL fragment to calculate an age-based weight from a given + * timestamp or datetime column. + * + * @param string $column name of field we're comparing against current time + * @param integer $dropoff divisor for age in seconds before exponentiation + * @return string SQL fragment + */ +function common_sql_weight($column, $dropoff) +{ + if (common_config('db', 'type') == 'pgsql') { + // PostgreSQL doesn't support timestampdiff function. + // @fixme will this use the right time zone? + // @fixme does this handle cross-year subtraction correctly? + return "sum(exp(-extract(epoch from (now() - $column)) / $dropoff))"; + } else { + return "sum(exp(timestampdiff(second, utc_timestamp(), $column) / $dropoff))"; + } +} + function common_redirect($url, $code=307) { static $status = array(301 => "Moved Permanently", From 8f02379f6e05e4bf1bff69d6d0fcf3d90a4d1181 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 14:37:39 -0800 Subject: [PATCH 3/4] Revert "Take Memcached_DataObject destructor back out to check whether it might be causing some under-the-hood problems." This reverts commit 89cca01259d71f3da961ef64def3647f86a01567. --- classes/Memcached_DataObject.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index b43cb0b56f..d89a9421e6 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -23,6 +23,20 @@ require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; class Memcached_DataObject extends DB_DataObject { + /** + * Destructor to free global memory resources associated with + * this data object when it's unset or goes out of scope. + * DB_DataObject doesn't do this yet by itself. + */ + + function __destruct() + { + $this->free(); + if (method_exists('DB_DataObject', '__destruct')) { + parent::__destruct(); + } + } + function &staticGet($cls, $k, $v=null) { if (is_null($v)) { From 78214c4e067616828478d600d500b34b6fcb9061 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 14:38:56 -0800 Subject: [PATCH 4/4] Exclude process-specific link & result cache references from serialized Memcached_Data_Object instances. Should fix seemingly-random bugs due to destructor free()ing local resources by mistake. cherry-pick from 0.9.x --- classes/Memcached_DataObject.php | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index d89a9421e6..ca360d4111 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -37,6 +37,51 @@ class Memcached_DataObject extends DB_DataObject } } + /** + * Magic function called at serialize() time. + * + * We use this to drop a couple process-specific references + * from DB_DataObject which can cause trouble in future + * processes. + * + * @return array of variable names to include in serialization. + */ + function __sleep() + { + $vars = array_keys(get_object_vars($this)); + $skip = array('_DB_resultid', '_link_loaded'); + return array_diff($vars, $skip); + } + + /** + * Magic function called at unserialize() time. + * + * Clean out some process-specific variables which might + * be floating around from a previous process's cached + * objects. + * + * Old cached objects may still have them. + */ + function __wakeup() + { + // Refers to global state info from a previous process. + // Clear this out so we don't accidentally break global + // state in *this* process. + $this->_DB_resultid = null; + + // We don't have any local DBO refs, so clear these out. + $this->_link_loaded = false; + } + + /** + * Wrapper for DB_DataObject's static lookup using memcached + * as backing instead of an in-process cache array. + * + * @param string $cls classname of object type to load + * @param mixed $k key field name, or value for primary key + * @param mixed $v key field value, or leave out for primary key lookup + * @return mixed Memcached_DataObject subtype or false + */ function &staticGet($cls, $k, $v=null) { if (is_null($v)) {