GNUsocial/gnu-social
4
0
Fork 1
Code Issues Releases Wiki Activity

[DATABASE] Some query improvements

Browse Source 
Make common_sql_weight employ standard SQL functions for the timestamp
difference in seconds.
Also replace UTC_TIMESTAMP in the MariaDB-specific part with CURRENT_TIMESTAMP
as it is the only occurence and GNU social sets UTC as a default timezone.

In a delete_orphan_files.php script simplify the main query considerably.

In clean_profiles.php stop using COUNT as if it is ANY, that is unnecessary
punishment for the database. Instead implement the anti-join with a
left outer join.

In Autocomplete and Activitypub_profile use joins instead of a WHERE OR
anti-pattern for the semi-joins.

In lib/ui/galleryaction.php replace a CROSS JOIN with an INNER JOIN.

In actions/sup.php remove a redundant subquery: WHERE is applied before
grouping either way.
This commit is contained in:
Alexei Sorokin 2020-06-10 16:52:00 +03:00
parent ef17f3ea7d
commit 194976135f
9 changed files with 187 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
actions/sup.php
View File

@ -65,17 +65,17 @@ class SupAction extends Action
$divider = common_sql_date(time() - $seconds); $divider = common_sql_date(time() - $seconds);
$notice->query('SELECT profile_id, max(id) AS max_id ' . $notice->selectAdd();
'FROM ( ' . $notice->selectAdd('profile_id, MAX(id) AS max_id');
'SELECT profile_id, id FROM notice ' . $notice->whereAdd("created > TIMESTAMP '{$divider}'");
"WHERE created > TIMESTAMP '" . $divider . "' " . $notice->groupBy('profile_id');
') AS latest ' .
'GROUP BY profile_id');
$updates = array(); $updates = [];
while ($notice->fetch()) { if ($notice->find()) {
$updates[] = array($notice->profile_id, $notice->max_id); while ($notice->fetch()) {
$updates[] = [$notice->profile_id, $notice->max_id];
}
} }
return $updates; return $updates;

15
lib/ui/galleryaction.php
View File

@ -107,13 +107,14 @@ class GalleryAction extends ProfileAction
public function getTags($lst, $usr) public function getTags($lst, $usr)
{ {
$profile_tag = new Notice_tag(); $profile_tag = new Notice_tag();
$profile_tag->query('SELECT DISTINCT(tag) ' . $profile_tag->query(
'FROM profile_tag, subscription ' . <<<END
'WHERE tagger = ' . $this->target->id . ' ' . SELECT DISTINCT tag FROM profile_tag INNER JOIN subscription
'AND ' . $usr . ' = ' . $this->target->id . ' ' . ON tagger = {$usr} AND {$lst} = tagged
'AND ' . $lst . ' = tagged ' . WHERE tagger = {$this->target->id} AND tagger <> tagged;
'AND tagger <> tagged'); END
$tags = array(); );
$tags = [];
while ($profile_tag->fetch()) { while ($profile_tag->fetch()) {
$tags[] = $profile_tag->tag; $tags[] = $profile_tag->tag;
} }

14
lib/util/util.php
View File

@ -1647,14 +1647,16 @@ function common_sql_date($datetime)
*/ */
function common_sql_weight($column, $dropoff) function common_sql_weight($column, $dropoff)
{ {
if (common_config('db', 'type') == 'pgsql') { if (common_config('db', 'type') !== 'mysql') {
// PostgreSQL doesn't support timestampdiff function. $expr = sprintf(
// @fixme will this use the right time zone? '(((EXTRACT(DAY %1$s) * 24 + EXTRACT(HOUR %1$s)) * 60 + '
// @fixme does this handle cross-year subtraction correctly? . 'EXTRACT(MINUTE %1$s)) * 60 + EXTRACT(SECOND %1$s))',
return "sum(exp(-extract(epoch from (now() - $column)) / $dropoff))"; "FROM ({$column} - CURRENT_TIMESTAMP)"
);
} else { } else {
return "sum(exp(timestampdiff(second, utc_timestamp(), $column) / $dropoff))"; $expr = "timestampdiff(SECOND, CURRENT_TIMESTAMP, {$column})";
} }
return "SUM(EXP({$expr} / {$dropoff}))";
} }
function common_redirect($url, $code=307) function common_redirect($url, $code=307)

20
plugins/ActivityPub/classes/Activitypub_profile.php
View File

@ -557,9 +557,15 @@ class Activitypub_profile extends Managed_DataObject
$user_table = common_database_tablename('user'); $user_table = common_database_tablename('user');
$sub = new Subscription(); $sub = new Subscription();
$sub->subscribed = $profile->id; $sub->subscribed = $profile->id;
$sub->_join .= "\n" . <<<END
INNER JOIN (
SELECT id AS subscriber FROM {$user_table}
UNION ALL
SELECT profile_id FROM activitypub_profile
) AS t1 USING (subscriber)
END;
$sub->whereAdd('subscriber <> subscribed'); $sub->whereAdd('subscriber <> subscribed');
$sub->whereAdd("subscriber IN (SELECT id FROM {$user_table} UNION SELECT profile_id AS id FROM activitypub_profile)"); $cnt = $sub->count('DISTINCT subscriber');
$cnt = $sub->count('distinct subscriber');
self::cacheSet(sprintf('activitypub_profile:subscriberCount:%d', $profile->id), $cnt); self::cacheSet(sprintf('activitypub_profile:subscriberCount:%d', $profile->id), $cnt);
@ -585,9 +591,15 @@ class Activitypub_profile extends Managed_DataObject
$user_table = common_database_tablename('user'); $user_table = common_database_tablename('user');
$sub = new Subscription(); $sub = new Subscription();
$sub->subscriber = $profile->id; $sub->subscriber = $profile->id;
$sub->_join .= "\n" . <<<END
INNER JOIN (
SELECT id AS subscribed FROM {$user_table}
UNION ALL
SELECT profile_id FROM activitypub_profile
) AS t1 USING (subscribed)
END;
$sub->whereAdd('subscriber <> subscribed'); $sub->whereAdd('subscriber <> subscribed');
$sub->whereAdd("subscribed IN (SELECT id FROM {$user_table} UNION SELECT profile_id AS id FROM activitypub_profile)"); $cnt = $sub->count('DISTINCT subscribed');
$cnt = $sub->count('distinct subscribed');
self::cacheSet(sprintf('activitypub_profile:subscriptionCount:%d', $profile->id), $cnt); self::cacheSet(sprintf('activitypub_profile:subscriptionCount:%d', $profile->id), $cnt);

128
plugins/Autocomplete/actions/autocomplete.php
View File

@ -1,47 +1,43 @@
<?php <?php
// This file is part of GNU social - https://www.gnu.org/software/social
//
// GNU social is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// GNU social is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with GNU social. If not, see <http://www.gnu.org/licenses/>.
/** /**
* StatusNet, the distributed open-source microblogging tool
*
* List profiles and groups for autocompletion * List profiles and groups for autocompletion
* *
* PHP version 5
*
* LICENCE: This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @category Plugin * @category Plugin
* @package StatusNet * @package GNUsocial
* @author Craig Andrews <candrews@integralblue.com> * @author Craig Andrews <candrews@integralblue.com>
* @author Mikael Nordfeldth <mmn@hethane.se> * @author Mikael Nordfeldth <mmn@hethane.se>
* @copyright 2008-2009 StatusNet, Inc. * @copyright 2008-2009 StatusNet, Inc.
* @copyright 2009-2013 Free Software Foundation, Inc http://www.fsf.org * @copyright 2009-2013 Free Software Foundation, Inc http://www.fsf.org
* @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 * @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later
* @link http://status.net/
*/ */
if (!defined('GNUSOCIAL')) { exit(1); } defined('GNUSOCIAL') || die();
/** /**
* List users for autocompletion * List users for autocompletion
* *
* This is the form for adding a new g * This is the form for adding a new g
* *
* @category Plugin * @category Plugin
* @package StatusNet * @package GNUsocial
* @author Craig Andrews <candrews@integralblue.com> * @author Craig Andrews <candrews@integralblue.com>
* @author Mikael Nordfeldth <mmn@hethane.se> * @author Mikael Nordfeldth <mmn@hethane.se>
* @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 * @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later
* @link http://status.net/
*/ */
class AutocompleteAction extends Action class AutocompleteAction extends Action
{ {
@ -57,14 +53,14 @@ class AutocompleteAction extends Action
* *
* @return int last-modified date as unix timestamp * @return int last-modified date as unix timestamp
*/ */
function lastModified() public function lastModified()
{ {
$max=0; $max = 0;
foreach($this->profiles as $profile){ foreach ($this->profiles as $profile) {
$max = max($max, strtotime($profile->modified)); $max = max($max, strtotime($profile->modified));
} }
foreach($this->groups as $group){ foreach ($this->groups as $group) {
$max = max($max,strtotime($group->modified)); $max = max($max, strtotime($group->modified));
} }
// but maybe this file has been modified after that and could // but maybe this file has been modified after that and could
// respond differently // respond differently
@ -80,11 +76,13 @@ class AutocompleteAction extends Action
* *
* @return string etag * @return string etag
*/ */
function etag() public function etag()
{ {
return '"' . implode(':', array($this->arg('action'), return '"' . implode(':', array($this->arg('action'),
common_user_cache_hash(), common_user_cache_hash(),
crc32($this->arg('term')), //the actual string can have funny characters in we don't want showing up in the etag // the actual string can have funny characters in we don't want
// showing up in the etag
crc32($this->arg('term')),
$this->arg('limit'), $this->arg('limit'),
$this->lastModified())) . '"'; $this->lastModified())) . '"';
} }
@ -96,39 +94,55 @@ class AutocompleteAction extends Action
parent::prepare($args); parent::prepare($args);
$this->groups=array(); $this->groups = [];
$this->profiles=array(); $this->profiles = [];
$term = $this->arg('term'); $term = $this->arg('term');
$limit = $this->arg('limit'); $limit = $this->arg('limit');
if($limit > 200) $limit=200; //prevent DOS attacks // prevent DOS attacks
if(substr($term,0,1)=='@'){ if ($limit > 200) {
$limit = 200;
}
if (substr($term, 0, 1) === '@') {
//profile search //profile search
$term=substr($term,1); $term = substr($term, 1);
$user_table = common_database_tablename('user'); $user_table = common_database_tablename('user');
$profile = new Profile(); $profile = new Profile();
$profile->_join .= sprintf(
"\n" . <<<'END'
LEFT JOIN (
SELECT id FROM %s
UNION ALL
SELECT subscribed AS id FROM subscription WHERE subscriber = %d
) AS t1 USING (id)
END,
$user_table,
$this->scoped->id
);
$profile->whereAdd('t1.id IS NOT NULL');
$profile->whereAdd('nickname LIKE \'' . trim($profile->escape($term), '\'') . '%\'');
$profile->limit($limit); $profile->limit($limit);
$profile->whereAdd('nickname like \'' . trim($profile->escape($term), '\'') . '%\'');
$profile->whereAdd(sprintf('id in (SELECT id FROM %s) OR '
. 'id in (SELECT subscribed from subscription'
. ' where subscriber = %d)', $user_table, $this->scoped->id));
if ($profile->find()) { if ($profile->find()) {
while($profile->fetch()) { while ($profile->fetch()) {
$this->profiles[]=clone($profile); $this->profiles[] = clone($profile);
} }
} }
} }
if(substr($term,0,1)=='!'){ if (substr($term, 0, 1) === '!') {
//group search //group search
$term=substr($term,1); $term = substr($term, 1);
$group = new User_group(); $group = new User_group();
$group->limit($limit);
$group->whereAdd('nickname like \'' . trim($group->escape($term), '\'') . '%\'');
//Can't post to groups we're not subscribed to...: //Can't post to groups we're not subscribed to...:
$group->whereAdd(sprintf('id in (SELECT group_id FROM group_member' $group->whereAdd(sprintf(
. ' WHERE profile_id = %d)', $this->scoped->id)); 'id IN (SELECT group_id FROM group_member WHERE profile_id = %d)',
if($group->find()){ $this->scoped->id
while($group->fetch()) { ));
$this->groups[]=clone($group); $group->whereAdd('nickname LIKE \'' . trim($group->escape($term), '\'') . '%\'');
$group->limit($limit);
if ($group->find()) {
while ($group->fetch()) {
$this->groups[] = clone($group);
} }
} }
} }
@ -140,7 +154,7 @@ class AutocompleteAction extends Action
parent::handle(); parent::handle();
$results = array(); $results = array();
foreach($this->profiles as $profile){ foreach ($this->profiles as $profile) {
$avatarUrl = $profile->avatarUrl(AVATAR_MINI_SIZE); $avatarUrl = $profile->avatarUrl(AVATAR_MINI_SIZE);
$acct = $profile->getAcctUri(); $acct = $profile->getAcctUri();
$identifier = explode(':', $profile->getAcctUri(), 2)[1]; $identifier = explode(':', $profile->getAcctUri(), 2)[1];
@ -153,7 +167,7 @@ class AutocompleteAction extends Action
'type' => 'user' 'type' => 'user'
); );
} }
foreach($this->groups as $group){ foreach ($this->groups as $group) {
$profile = $group->getProfile(); $profile = $group->getProfile();
// sigh.... encapsulate this upstream! // sigh.... encapsulate this upstream!
if ($group->mini_logo) { if ($group->mini_logo) {
@ -181,7 +195,7 @@ class AutocompleteAction extends Action
* *
* @return boolean is read only action? * @return boolean is read only action?
*/ */
function isReadOnly($args) public function isReadOnly($args)
{ {
return true; return true;
} }

3
plugins/Nodeinfo/actions/nodeinfo_2_0.php
View File

@ -338,7 +338,8 @@ class Nodeinfo_2_0Action extends Action
UNION ALL UNION ALL
SELECT id FROM {$userTable} WHERE {$userTable}.created >= CURRENT_DATE - INTERVAL '{$days}' DAY SELECT id FROM {$userTable} WHERE {$userTable}.created >= CURRENT_DATE - INTERVAL '{$days}' DAY
) AS source ) AS source
WHERE profile_id NOT IN (SELECT profile_id FROM profile_role WHERE role = 'silenced') LEFT JOIN profile_role USING (profile_id)
WHERE profile_role.profile_id IS NULL OR profile_role.role <> 'silenced';
END; END;
$activeUsersCount = new DB_DataObject(); $activeUsersCount = new DB_DataObject();

20
plugins/OStatus/scripts/update-profile.php
View File

@ -102,14 +102,18 @@ if ($feedurl != $oprofile->feeduri || $salmonuri != $oprofile->salmonuri) {
print "\n"; print "\n";
print "Updating...\n"; print "Updating...\n";
// @fixme update keys :P // @fixme update keys :P
#$orig = clone($oprofile); //$orig = clone($oprofile);
#$oprofile->feeduri = $feedurl; //$oprofile->feeduri = $feedurl;
#$oprofile->salmonuri = $salmonuri; //$oprofile->salmonuri = $salmonuri;
#$ok = $oprofile->update($orig); //$ok = $oprofile->update($orig);
$ok = $oprofile->query('UPDATE ostatus_profile SET ' . $ok = $oprofile->query(
'feeduri=\'' . $oprofile->escape($feedurl) . '\',' . <<<END
'salmonuri=\'' . $oprofile->escape($salmonuri) . '\' ' . UPDATE ostatus_profile
'WHERE uri=\'' . $oprofile->escape($uri) . '\''); SET feeduri = '{$oprofile->escape($feedurl)}',
salmonuri = '{$oprofile->escape($salmonuri)}'
WHERE uri = '{$oprofile->escape($uri)}'
END
);
if (!$ok) { if (!$ok) {
print "Failed to update profile record...\n"; print "Failed to update profile record...\n";

62
scripts/clean_profiles.php
View File

@ -1,33 +1,30 @@
#!/usr/bin/env php #!/usr/bin/env php
<?php <?php
/* // This file is part of GNU social - https://www.gnu.org/software/social
* StatusNet - a distributed open-source microblogging tool //
* Copyright (C) 2008, 2009, StatusNet, Inc. // GNU social is free software: you can redistribute it and/or modify
* // it under the terms of the GNU Affero General Public License as published by
* This program is free software: you can redistribute it and/or modify // the Free Software Foundation, either version 3 of the License, or
* it under the terms of the GNU Affero General Public License as published by // (at your option) any later version.
* the Free Software Foundation, either version 3 of the License, or //
* (at your option) any later version. // GNU social is distributed in the hope that it will be useful,
* // but WITHOUT ANY WARRANTY; without even the implied warranty of
* This program is distributed in the hope that it will be useful, // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* but WITHOUT ANY WARRANTY; without even the implied warranty of // GNU Affero General Public License for more details.
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
* GNU Affero General Public License for more details. // You should have received a copy of the GNU Affero General Public License
* // along with GNU social. If not, see <http://www.gnu.org/licenses/>.
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
define('INSTALLDIR', dirname(__DIR__)); define('INSTALLDIR', dirname(__DIR__));
define('PUBLICDIR', INSTALLDIR . DIRECTORY_SEPARATOR . 'public'); define('PUBLICDIR', INSTALLDIR . DIRECTORY_SEPARATOR . 'public');
$shortoptions = 'y'; $shortoptions = 'y';
$longoptions = array('yes'); $longoptions = ['yes'];
$helptext = <<<END_OF_HELP $helptext = <<<END_OF_HELP
clean_profiles.php [options] clean_profiles.php [options]
Deletes all profile table entries where the profile does not occur in the Deletes all profile table entries where the profile does not occur in the
notice table, is not a group and is not a local user. Very MySQL specific I think. notice table, is not a group and is not a local user.
WARNING: This has not been tested thoroughly. Maybe we've missed a table to compare somewhere. WARNING: This has not been tested thoroughly. Maybe we've missed a table to compare somewhere.
@ -35,7 +32,7 @@ WARNING: This has not been tested thoroughly. Maybe we've missed a table to comp
END_OF_HELP; END_OF_HELP;
require_once INSTALLDIR.'/scripts/commandline.inc'; require_once INSTALLDIR . '/scripts/commandline.inc';
if (!have_option('y', 'yes')) { if (!have_option('y', 'yes')) {
print "About to delete profiles that we think are useless to save. Are you sure? [y/N] "; print "About to delete profiles that we think are useless to save. Are you sure? [y/N] ";
@ -46,14 +43,27 @@ if (!have_option('y', 'yes')) {
} }
} }
print "Deleting"; echo 'Deleting';
$user_table = common_database_tablename('user'); $user_table = common_database_tablename('user');
$profile = new Profile(); $profile = new Profile();
$profile->query('SELECT * FROM profile WHERE ' . $profile->query(
'NOT (SELECT COUNT(*) FROM notice WHERE profile_id=profile.id) ' . <<<END
"AND NOT (SELECT COUNT(*) FROM {$user_table} WHERE user.id=profile.id) " . SELECT profile.*
'AND NOT (SELECT COUNT(*) FROM user_group WHERE user_group.profile_id=profile.id) ' . FROM profile
'AND NOT (SELECT COUNT(*) FROM subscription WHERE subscriber=profile.id OR subscribed=profile.id) '); LEFT JOIN (
SELECT profile_id AS id FROM notice
UNION ALL
SELECT id FROM {$user_table}
UNION ALL
SELECT profile_id AS id FROM user_group
UNION ALL
SELECT subscriber FROM subscription
UNION ALL
SELECT subscribed FROM subscription
) AS t1 USING (id)
WHERE t1.id IS NULL
END
);
while ($profile->fetch()) { while ($profile->fetch()) {
echo ' '.$profile->getID().':'.$profile->getNickname(); echo ' '.$profile->getID().':'.$profile->getNickname();
$profile->delete(); $profile->delete();

47
scripts/delete_orphan_files.php
View File

@ -1,21 +1,23 @@
#!/usr/bin/env php #!/usr/bin/env php
<?php <?php
/* // This file is part of GNU social - https://www.gnu.org/software/social
* StatusNet - a distributed open-source microblogging tool //
* Copyright (C) 2008, 2009, StatusNet, Inc. // GNU social is free software: you can redistribute it and/or modify
* // it under the terms of the GNU Affero General Public License as published by
* This program is free software: you can redistribute it and/or modify // the Free Software Foundation, either version 3 of the License, or
* it under the terms of the GNU Affero General Public License as published by // (at your option) any later version.
* the Free Software Foundation, either version 3 of the License, or //
* (at your option) any later version. // GNU social is distributed in the hope that it will be useful,
* // but WITHOUT ANY WARRANTY; without even the implied warranty of
* This program is distributed in the hope that it will be useful, // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* but WITHOUT ANY WARRANTY; without even the implied warranty of // GNU Affero General Public License for more details.
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
* GNU Affero General Public License for more details. // You should have received a copy of the GNU Affero General Public License
* // along with GNU social. If not, see <http://www.gnu.org/licenses/>.
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. /**
* @copyright 2008, 2009 StatusNet, Inc.
* @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later
*/ */
define('INSTALLDIR', dirname(__DIR__)); define('INSTALLDIR', dirname(__DIR__));
@ -43,12 +45,13 @@ require_once INSTALLDIR.'/scripts/commandline.inc';
print "Finding File entries that are not related to a Notice (or the notice has been deleted)..."; print "Finding File entries that are not related to a Notice (or the notice has been deleted)...";
$file = new File(); $file = new File();
$sql = 'SELECT file.* FROM file'. $sql = <<<'END'
' LEFT JOIN file_to_post ON file_to_post.file_id=file.id'. SELECT file.*
' WHERE'. FROM file_to_post
' NOT EXISTS (SELECT file_to_post.file_id FROM file_to_post WHERE file.id=file_to_post.file_id)'. INNER JOIN notice ON file_to_post.post_id = notice.id
' OR NOT EXISTS (SELECT notice.id FROM notice WHERE notice.id=file_to_post.post_id)'. RIGHT JOIN file ON file_to_post.file_id = file.id
' GROUP BY file.id;'; WHERE file_to_post.file_id IS NULL;
END;
if ($file->query($sql) !== false) { if ($file->query($sql) !== false) {
print " {$file->N} found.\n"; print " {$file->N} found.\n";