From 4f6052d8d8bd3d64d395e7849a31183371040eb8 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 2 Feb 2010 05:53:05 -0800 Subject: [PATCH 1/9] Apply xopher's fix to add 'sitetype' parameter to setup_status_network.sh, exposed to the email --- scripts/setup_status_network.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/setup_status_network.sh b/scripts/setup_status_network.sh index 4ad808011c..bacf3c3e7f 100755 --- a/scripts/setup_status_network.sh +++ b/scripts/setup_status_network.sh @@ -13,6 +13,11 @@ export sitename="$2" export tags="$3" export email="$4" export fullname="$5" +export sitetype="$6" + +if [ "$sitetype" == '' ]; then + sitetype='single-user' +fi # Fixme: if this is changed later we need to update profile URLs # for the created user. @@ -71,6 +76,7 @@ then sed "s/\$nickname/$nickname/" | \ sed "s/\$sitename/$sitename/" | \ sed "s/\$userpass/$userpass/" | \ + sed "s/\$sitetype/$sitetype/" | \ php $PHPBASE/scripts/sendemail.php \ -s"$server" \ -n"$nickname" \ From 51c3606715573d8cea3c79ff7bc989a4ba86acc5 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 2 Feb 2010 09:30:15 -0800 Subject: [PATCH 2/9] Fix regression breaking file attachments. This is what I get for rushing fixes and not properly testing them. :P --- classes/File.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/File.php b/classes/File.php index 6dd9e0c06e..ee418a8024 100644 --- a/classes/File.php +++ b/classes/File.php @@ -181,7 +181,7 @@ class File extends Memcached_DataObject */ static function validFilename($filename) { - return preg_match('^/[A-Za-z0-9._-]+$/', $filename); + return preg_match('/^[A-Za-z0-9._-]+$/', $filename); } /** From 6215b9788c8d1f523d4d173ecf40f185bd024bca Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 3 Feb 2010 11:53:31 -0500 Subject: [PATCH 3/9] add nickname as unique key for user_group table --- classes/statusnet.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/classes/statusnet.ini b/classes/statusnet.ini index 6203650a69..e28424ce2a 100644 --- a/classes/statusnet.ini +++ b/classes/statusnet.ini @@ -585,6 +585,7 @@ modified = 384 [user_group__keys] id = N +nickname = U [user_openid] canonical = 130 From 644a9163831938518ed3473c7a6d74251d254e8b Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 3 Feb 2010 11:55:54 -0500 Subject: [PATCH 4/9] change 'sitetype' to 'siteplan' to use consistent language --- scripts/setup_status_network.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/setup_status_network.sh b/scripts/setup_status_network.sh index bacf3c3e7f..89d15415f9 100755 --- a/scripts/setup_status_network.sh +++ b/scripts/setup_status_network.sh @@ -13,10 +13,10 @@ export sitename="$2" export tags="$3" export email="$4" export fullname="$5" -export sitetype="$6" +export siteplan="$6" -if [ "$sitetype" == '' ]; then - sitetype='single-user' +if [ "$siteplan" == '' ]; then + siteplan='single-user' fi # Fixme: if this is changed later we need to update profile URLs @@ -76,7 +76,7 @@ then sed "s/\$nickname/$nickname/" | \ sed "s/\$sitename/$sitename/" | \ sed "s/\$userpass/$userpass/" | \ - sed "s/\$sitetype/$sitetype/" | \ + sed "s/\$siteplan/$siteplan/" | \ php $PHPBASE/scripts/sendemail.php \ -s"$server" \ -n"$nickname" \ From aee73d87ee33d1743a74fbe87f964bfc46ba1873 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 3 Feb 2010 12:13:07 -0500 Subject: [PATCH 5/9] showcache.php column flag was conflicting with default flag --- scripts/showcache.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/showcache.php b/scripts/showcache.php index f179795728..93b57a484b 100644 --- a/scripts/showcache.php +++ b/scripts/showcache.php @@ -20,14 +20,14 @@ define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); -$shortoptions = "t:c:v:k:"; +$shortoptions = "t:l:v:k:"; $helptext = << shows the cached object based on the args -t table Table to look up - -c column Column to look up, default "id" + -l column Column to look up, default "id" -v value Value to look up -k key Key to look up; other args are ignored @@ -44,7 +44,7 @@ if (!empty($karg)) { if (empty($table)) { die("No table or key specified\n"); } - $column = get_option_value('c'); + $column = get_option_value('l'); if (empty($column)) { $column = 'id'; } From 0e0beea5aa75e58de251752b2720d4ea6a8c89f7 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 3 Feb 2010 12:13:20 -0500 Subject: [PATCH 6/9] clearcache.php column flag was conflicting with default flag --- scripts/clearcache.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/clearcache.php b/scripts/clearcache.php index 702c1e3d67..0fb75daa03 100644 --- a/scripts/clearcache.php +++ b/scripts/clearcache.php @@ -20,35 +20,35 @@ define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); -$shortoptions = "t:c:v:k:"; +$shortoptions = "t:l:v:k:"; -$helptext = << clears the cached object based on the args -t table Table to look up - -c column Column to look up, default "id" + -l column Column to look up, default "id" -v value Value to look up -k key Key to look up; other args are ignored -ENDOFHELP; +END_OF_CLEARCACHE_HELP; require_once INSTALLDIR.'/scripts/commandline.inc'; -$karg = get_option_value('k'); +$karg = get_option_value('k', 'key'); if (!empty($karg)) { $k = common_cache_key($karg); } else { - $table = get_option_value('t'); + $table = get_option_value('t', 'table'); if (empty($table)) { die("No table or key specified\n"); } - $column = get_option_value('c'); + $column = get_option_value('l', 'column'); if (empty($column)) { $column = 'id'; } - $value = get_option_value('v'); + $value = get_option_value('v', 'value'); $k = Memcached_DataObject::cacheKey($table, $column, $value); } From dbeb388adefec94c3bb34e99305a004d845b719c Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 3 Feb 2010 12:31:25 -0500 Subject: [PATCH 7/9] clear cache for Profile::hasRepeated() at Notice::insert() time --- classes/Notice.php | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/classes/Notice.php b/classes/Notice.php index 42878d94f1..f9f3863579 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -1468,4 +1468,25 @@ class Notice extends Memcached_DataObject $handler->handle($this); } } + + function insert() + { + $result = parent::insert(); + + if ($result) { + // Profile::hasRepeated() abuses pkeyGet(), so we + // have to clear manually + if (!empty($this->repeat_of)) { + $c = self::memcache(); + if (!empty($c)) { + $ck = self::multicacheKey('Notice', + array('profile_id' => $this->profile_id, + 'repeat_of' => $this->repeat_of)); + $c->delete($ck); + } + } + } + + return $result; + } } From 7a7e2162dd7eed59e60d9360d8692abc111d940c Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 3 Feb 2010 14:58:29 -0500 Subject: [PATCH 8/9] Script to update profile URLs --- scripts/updateprofileurl.php | 99 ++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 scripts/updateprofileurl.php diff --git a/scripts/updateprofileurl.php b/scripts/updateprofileurl.php new file mode 100644 index 0000000000..2fc6828e60 --- /dev/null +++ b/scripts/updateprofileurl.php @@ -0,0 +1,99 @@ +#!/usr/bin/env php +. + */ + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); + +$shortoptions = 'i:n:a'; +$longoptions = array('id=', 'nickname=', 'all'); + +$helptext = <<find()) { + while ($user->fetch()) { + updateProfileURL($user); + } + } + } else { + show_help(); + exit(1); + } +} catch (Exception $e) { + print $e->getMessage()."\n"; + exit(1); +} + +function updateProfileURL($user) +{ + $profile = $user->getProfile(); + + if (empty($profile)) { + throw new Exception("Can't find profile for user $user->nickname ($user->id)"); + } + + $orig = clone($profile); + + $profile->profileurl = common_profile_url($user->nickname); + + if (!have_option('q', 'quiet')) { + print "Updating profile url for $user->nickname ($user->id) ". + "from $orig->profileurl to $profile->profileurl..."; + } + + $result = $profile->update($orig); + + if (!$result) { + print "FAIL.\n"; + common_log_db_error($profile, 'UPDATE', __FILE__); + throw new Exception("Can't update profile for user $user->nickname ($user->id)"); + } + + common_broadcast_profile($profile); + + print "OK.\n"; +} From 5a1cbdc6f1e32be7b8430924a1125422d8457584 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Thu, 4 Feb 2010 11:06:01 -0800 Subject: [PATCH 9/9] Add time-based cutoffs for public tag cloud, favorited lists to speed up those queries. Defaulting to only looking at last 90 days of activity, can be adjusted up or down. $config['tag']['cutoff'] = 86400 * 90; $config['popular']['cutoff'] = 86400 * 90; Per-user and per-group tag clouds do not use the cutoff (and it doesn't help with indexing on them). --- actions/favorited.php | 3 +++ actions/publictagcloud.php | 3 +++ lib/default.php | 6 ++++-- lib/grouptagcloudsection.php | 1 + lib/personaltagcloudsection.php | 1 + lib/popularnoticesection.php | 7 +++++-- 6 files changed, 17 insertions(+), 4 deletions(-) diff --git a/actions/favorited.php b/actions/favorited.php index 9ffa5b8445..d8980440d1 100644 --- a/actions/favorited.php +++ b/actions/favorited.php @@ -186,10 +186,13 @@ class FavoritedAction extends Action function showContent() { $weightexpr = common_sql_weight('fave.modified', common_config('popular', 'dropoff')); + $cutoff = sprintf("fave.modified > '%s'", + common_sql_date(time() - common_config('popular', 'cutoff'))); $qry = 'SELECT notice.*, '. $weightexpr . ' as weight ' . 'FROM notice JOIN fave ON notice.id = fave.notice_id ' . + "WHERE $cutoff " . 'GROUP BY id,profile_id,uri,content,rendered,url,created,notice.modified,reply_to,is_local,source,notice.conversation ' . 'ORDER BY weight DESC'; diff --git a/actions/publictagcloud.php b/actions/publictagcloud.php index 9e4478dbb1..9993b2d3fd 100644 --- a/actions/publictagcloud.php +++ b/actions/publictagcloud.php @@ -106,7 +106,10 @@ class PublictagcloudAction extends Action #Add the aggregated columns... $tags->selectAdd('max(notice_id) as last_notice_id'); $calc = common_sql_weight('created', common_config('tag', 'dropoff')); + $cutoff = sprintf("notice_tag.created > '%s'", + common_sql_date(time() - common_config('tag', 'cutoff'))); $tags->selectAdd($calc . ' as weight'); + $tags->addWhere($cutoff); $tags->groupBy('tag'); $tags->orderBy('weight DESC'); diff --git a/lib/default.php b/lib/default.php index 2bedc4bf08..485a08ba44 100644 --- a/lib/default.php +++ b/lib/default.php @@ -144,9 +144,11 @@ $default = 'invite' => array('enabled' => true), 'tag' => - array('dropoff' => 864000.0), + array('dropoff' => 864000.0, # controls weighting based on age + 'cutoff' => 86400 * 90), # only look at notices posted in last 90 days 'popular' => - array('dropoff' => 864000.0), + array('dropoff' => 864000.0, # controls weighting based on age + 'cutoff' => 86400 * 90), # only look at notices favorited in last 90 days 'daemon' => array('piddir' => '/var/run', 'user' => false, diff --git a/lib/grouptagcloudsection.php b/lib/grouptagcloudsection.php index 14ceda0850..f1106cc7bf 100644 --- a/lib/grouptagcloudsection.php +++ b/lib/grouptagcloudsection.php @@ -59,6 +59,7 @@ class GroupTagCloudSection extends TagCloudSection function getTags() { $weightexpr = common_sql_weight('notice_tag.created', common_config('tag', 'dropoff')); + // @fixme should we use the cutoff too? Doesn't help with indexing per-group. $names = $this->group->getAliases(); diff --git a/lib/personaltagcloudsection.php b/lib/personaltagcloudsection.php index 091425f926..5ea3f188db 100644 --- a/lib/personaltagcloudsection.php +++ b/lib/personaltagcloudsection.php @@ -59,6 +59,7 @@ class PersonalTagCloudSection extends TagCloudSection function getTags() { $weightexpr = common_sql_weight('notice_tag.created', common_config('tag', 'dropoff')); + // @fixme should we use the cutoff too? Doesn't help with indexing per-user. $qry = 'SELECT notice_tag.tag, '. $weightexpr . ' as weight ' . diff --git a/lib/popularnoticesection.php b/lib/popularnoticesection.php index fbf9a60ab8..296ddbbb50 100644 --- a/lib/popularnoticesection.php +++ b/lib/popularnoticesection.php @@ -59,12 +59,15 @@ class PopularNoticeSection extends NoticeSection } } $weightexpr = common_sql_weight('fave.modified', common_config('popular', 'dropoff')); + $cutoff = sprintf("fave.modified > '%s'", + common_sql_date(time() - common_config('popular', 'cutoff'))); $qry = "SELECT notice.*, $weightexpr as weight "; if(isset($tag)) { $qry .= 'FROM notice_tag, notice JOIN fave ON notice.id = fave.notice_id ' . - "WHERE notice.id = notice_tag.notice_id and '$tag' = notice_tag.tag"; + "WHERE $cutoff and notice.id = notice_tag.notice_id and '$tag' = notice_tag.tag"; } else { - $qry .= 'FROM notice JOIN fave ON notice.id = fave.notice_id'; + $qry .= 'FROM notice JOIN fave ON notice.id = fave.notice_id ' . + "WHERE $cutoff"; } $qry .= ' GROUP BY notice.id,notice.profile_id,notice.content,notice.uri,' . 'notice.rendered,notice.url,notice.created,notice.modified,' .