GNUsocial/gnu-social
4
0
Fork 1
Code Issues Releases Wiki Activity

[DATABASE] Some query improvements

Browse Source 
Make common_sql_weight employ standard SQL functions for the timestamp
difference in seconds.
Also replace UTC_TIMESTAMP in the MariaDB-specific part with CURRENT_TIMESTAMP
as it is the only occurence and GNU social sets UTC as a default timezone.

In a delete_orphan_files.php script simplify the main query considerably.

In clean_profiles.php stop using COUNT as if it is ANY, that is unnecessary
punishment for the database. Instead implement the anti-join with a
left outer join.

In Autocomplete and Activitypub_profile use joins instead of a WHERE OR
anti-pattern for the semi-joins.

In lib/ui/galleryaction.php replace a CROSS JOIN with an INNER JOIN.

In actions/sup.php remove a redundant subquery: WHERE is applied before
grouping either way.
This commit is contained in:
Alexei Sorokin 2020-06-10 16:52:00 +03:00
parent ef17f3ea7d
commit 194976135f
9 changed files with 187 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
actions/sup.php
View File

@ -65,17 +65,17 @@ class SupAction extends Action
$divider = common_sql_date(time() - $seconds);
$notice->query('SELECT profile_id, max(id) AS max_id ' .
'FROM ( ' .
'SELECT profile_id, id FROM notice ' .
"WHERE created > TIMESTAMP '" . $divider . "' " .
') AS latest ' .
'GROUP BY profile_id');
$notice->selectAdd();
$notice->selectAdd('profile_id, MAX(id) AS max_id');
$notice->whereAdd("created > TIMESTAMP '{$divider}'");
$notice->groupBy('profile_id');
$updates = array();
$updates = [];
while ($notice->fetch()) {
$updates[] = array($notice->profile_id, $notice->max_id);
if ($notice->find()) {
while ($notice->fetch()) {
$updates[] = [$notice->profile_id, $notice->max_id];
}
}
return $updates;

15
lib/ui/galleryaction.php
View File

@ -107,13 +107,14 @@ class GalleryAction extends ProfileAction
public function getTags($lst, $usr)
{
$profile_tag = new Notice_tag();
$profile_tag->query('SELECT DISTINCT(tag) ' .
'FROM profile_tag, subscription ' .
'WHERE tagger = ' . $this->target->id . ' ' .
'AND ' . $usr . ' = ' . $this->target->id . ' ' .
'AND ' . $lst . ' = tagged ' .
'AND tagger <> tagged');
$tags = array();
$profile_tag->query(
<<<END
SELECT DISTINCT tag FROM profile_tag INNER JOIN subscription
ON tagger = {$usr} AND {$lst} = tagged
WHERE tagger = {$this->target->id} AND tagger <> tagged;
END
);
$tags = [];
while ($profile_tag->fetch()) {
$tags[] = $profile_tag->tag;
}

14
lib/util/util.php
View File

@ -1647,14 +1647,16 @@ function common_sql_date($datetime)
*/
function common_sql_weight($column, $dropoff)
{
if (common_config('db', 'type') == 'pgsql') {
// PostgreSQL doesn't support timestampdiff function.
// @fixme will this use the right time zone?
// @fixme does this handle cross-year subtraction correctly?
return "sum(exp(-extract(epoch from (now() - $column)) / $dropoff))";
if (common_config('db', 'type') !== 'mysql') {
$expr = sprintf(
'(((EXTRACT(DAY %1$s) * 24 + EXTRACT(HOUR %1$s)) * 60 + '
. 'EXTRACT(MINUTE %1$s)) * 60 + EXTRACT(SECOND %1$s))',
"FROM ({$column} - CURRENT_TIMESTAMP)"
);
} else {
return "sum(exp(timestampdiff(second, utc_timestamp(), $column) / $dropoff))";
$expr = "timestampdiff(SECOND, CURRENT_TIMESTAMP, {$column})";
}
return "SUM(EXP({$expr} / {$dropoff}))";
}
function common_redirect($url, $code=307)

20
plugins/ActivityPub/classes/Activitypub_profile.php
View File

@ -557,9 +557,15 @@ class Activitypub_profile extends Managed_DataObject
$user_table = common_database_tablename('user');
$sub = new Subscription();
$sub->subscribed = $profile->id;
$sub->_join .= "\n" . <<<END
INNER JOIN (
SELECT id AS subscriber FROM {$user_table}
UNION ALL
SELECT profile_id FROM activitypub_profile
) AS t1 USING (subscriber)
END;
$sub->whereAdd('subscriber <> subscribed');
$sub->whereAdd("subscriber IN (SELECT id FROM {$user_table} UNION SELECT profile_id AS id FROM activitypub_profile)");
$cnt = $sub->count('distinct subscriber');
$cnt = $sub->count('DISTINCT subscriber');
self::cacheSet(sprintf('activitypub_profile:subscriberCount:%d', $profile->id), $cnt);
@ -585,9 +591,15 @@ class Activitypub_profile extends Managed_DataObject
$user_table = common_database_tablename('user');
$sub = new Subscription();
$sub->subscriber = $profile->id;
$sub->_join .= "\n" . <<<END
INNER JOIN (
SELECT id AS subscribed FROM {$user_table}
UNION ALL
SELECT profile_id FROM activitypub_profile
) AS t1 USING (subscribed)
END;
$sub->whereAdd('subscriber <> subscribed');
$sub->whereAdd("subscribed IN (SELECT id FROM {$user_table} UNION SELECT profile_id AS id FROM activitypub_profile)");
$cnt = $sub->count('distinct subscribed');
$cnt = $sub->count('DISTINCT subscribed');
self::cacheSet(sprintf('activitypub_profile:subscriptionCount:%d', $profile->id), $cnt);

128
plugins/Autocomplete/actions/autocomplete.php
View File

@ -1,47 +1,43 @@
<?php
// This file is part of GNU social - https://www.gnu.org/software/social
//
// GNU social is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// GNU social is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with GNU social. If not, see <http://www.gnu.org/licenses/>.
/**
* StatusNet, the distributed open-source microblogging tool
*
* List profiles and groups for autocompletion
*
* PHP version 5
*
* LICENCE: This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @category Plugin
* @package StatusNet
* @package GNUsocial
* @author Craig Andrews <candrews@integralblue.com>
* @author Mikael Nordfeldth <mmn@hethane.se>
* @copyright 2008-2009 StatusNet, Inc.
* @copyright 2009-2013 Free Software Foundation, Inc http://www.fsf.org
* @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
* @link http://status.net/
* @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later
*/
if (!defined('GNUSOCIAL')) { exit(1); }
defined('GNUSOCIAL') || die();
/**
* List users for autocompletion
*
* This is the form for adding a new g
*
* @category Plugin
* @package StatusNet
* @author Craig Andrews <candrews@integralblue.com>
* @author Mikael Nordfeldth <mmn@hethane.se>
* @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
* @link http://status.net/
* @category Plugin
* @package GNUsocial
* @author Craig Andrews <candrews@integralblue.com>
* @author Mikael Nordfeldth <mmn@hethane.se>
* @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later
*/
class AutocompleteAction extends Action
{
@ -57,14 +53,14 @@ class AutocompleteAction extends Action
*
* @return int last-modified date as unix timestamp
*/
function lastModified()
public function lastModified()
{
$max=0;
foreach($this->profiles as $profile){
$max = 0;
foreach ($this->profiles as $profile) {
$max = max($max, strtotime($profile->modified));
}
foreach($this->groups as $group){
$max = max($max,strtotime($group->modified));
foreach ($this->groups as $group) {
$max = max($max, strtotime($group->modified));
}
// but maybe this file has been modified after that and could
// respond differently
@ -80,11 +76,13 @@ class AutocompleteAction extends Action
*
* @return string etag
*/
function etag()
public function etag()
{
return '"' . implode(':', array($this->arg('action'),
common_user_cache_hash(),
crc32($this->arg('term')), //the actual string can have funny characters in we don't want showing up in the etag
// the actual string can have funny characters in we don't want
// showing up in the etag
crc32($this->arg('term')),
$this->arg('limit'),
$this->lastModified())) . '"';
}
@ -96,39 +94,55 @@ class AutocompleteAction extends Action
parent::prepare($args);
$this->groups=array();
$this->profiles=array();
$this->groups = [];
$this->profiles = [];
$term = $this->arg('term');
$limit = $this->arg('limit');
if($limit > 200) $limit=200; //prevent DOS attacks
if(substr($term,0,1)=='@'){
// prevent DOS attacks
if ($limit > 200) {
$limit = 200;
}
if (substr($term, 0, 1) === '@') {
//profile search
$term=substr($term,1);
$term = substr($term, 1);
$user_table = common_database_tablename('user');
$profile = new Profile();
$profile->_join .= sprintf(
"\n" . <<<'END'
LEFT JOIN (
SELECT id FROM %s
UNION ALL
SELECT subscribed AS id FROM subscription WHERE subscriber = %d
) AS t1 USING (id)
END,
$user_table,
$this->scoped->id
);
$profile->whereAdd('t1.id IS NOT NULL');
$profile->whereAdd('nickname LIKE \'' . trim($profile->escape($term), '\'') . '%\'');
$profile->limit($limit);
$profile->whereAdd('nickname like \'' . trim($profile->escape($term), '\'') . '%\'');
$profile->whereAdd(sprintf('id in (SELECT id FROM %s) OR '
. 'id in (SELECT subscribed from subscription'
. ' where subscriber = %d)', $user_table, $this->scoped->id));
if ($profile->find()) {
while($profile->fetch()) {
$this->profiles[]=clone($profile);
while ($profile->fetch()) {
$this->profiles[] = clone($profile);
}
}
}
if(substr($term,0,1)=='!'){
if (substr($term, 0, 1) === '!') {
//group search
$term=substr($term,1);
$term = substr($term, 1);
$group = new User_group();
$group->limit($limit);
$group->whereAdd('nickname like \'' . trim($group->escape($term), '\'') . '%\'');
//Can't post to groups we're not subscribed to...:
$group->whereAdd(sprintf('id in (SELECT group_id FROM group_member'
. ' WHERE profile_id = %d)', $this->scoped->id));
if($group->find()){
while($group->fetch()) {
$this->groups[]=clone($group);
$group->whereAdd(sprintf(
'id IN (SELECT group_id FROM group_member WHERE profile_id = %d)',
$this->scoped->id
));
$group->whereAdd('nickname LIKE \'' . trim($group->escape($term), '\'') . '%\'');
$group->limit($limit);
if ($group->find()) {
while ($group->fetch()) {
$this->groups[] = clone($group);
}
}
}
@ -140,7 +154,7 @@ class AutocompleteAction extends Action
parent::handle();
$results = array();
foreach($this->profiles as $profile){
foreach ($this->profiles as $profile) {
$avatarUrl = $profile->avatarUrl(AVATAR_MINI_SIZE);
$acct = $profile->getAcctUri();
$identifier = explode(':', $profile->getAcctUri(), 2)[1];
@ -153,7 +167,7 @@ class AutocompleteAction extends Action
'type' => 'user'
);
}
foreach($this->groups as $group){
foreach ($this->groups as $group) {
$profile = $group->getProfile();
// sigh.... encapsulate this upstream!
if ($group->mini_logo) {
@ -181,7 +195,7 @@ class AutocompleteAction extends Action
*
* @return boolean is read only action?
*/
function isReadOnly($args)
public function isReadOnly($args)
{
return true;
}

3
plugins/Nodeinfo/actions/nodeinfo_2_0.php
View File

@ -338,7 +338,8 @@ class Nodeinfo_2_0Action extends Action
UNION ALL
SELECT id FROM {$userTable} WHERE {$userTable}.created >= CURRENT_DATE - INTERVAL '{$days}' DAY
) AS source
WHERE profile_id NOT IN (SELECT profile_id FROM profile_role WHERE role = 'silenced')
LEFT JOIN profile_role USING (profile_id)
WHERE profile_role.profile_id IS NULL OR profile_role.role <> 'silenced';
END;
$activeUsersCount = new DB_DataObject();

20
plugins/OStatus/scripts/update-profile.php
View File

@ -102,14 +102,18 @@ if ($feedurl != $oprofile->feeduri || $salmonuri != $oprofile->salmonuri) {
print "\n";
print "Updating...\n";
// @fixme update keys :P
#$orig = clone($oprofile);
#$oprofile->feeduri = $feedurl;
#$oprofile->salmonuri = $salmonuri;
#$ok = $oprofile->update($orig);
$ok = $oprofile->query('UPDATE ostatus_profile SET ' .
'feeduri=\'' . $oprofile->escape($feedurl) . '\',' .
'salmonuri=\'' . $oprofile->escape($salmonuri) . '\' ' .
'WHERE uri=\'' . $oprofile->escape($uri) . '\'');
//$orig = clone($oprofile);
//$oprofile->feeduri = $feedurl;
//$oprofile->salmonuri = $salmonuri;
//$ok = $oprofile->update($orig);
$ok = $oprofile->query(
<<<END
UPDATE ostatus_profile
SET feeduri = '{$oprofile->escape($feedurl)}',
salmonuri = '{$oprofile->escape($salmonuri)}'
WHERE uri = '{$oprofile->escape($uri)}'
END
);
if (!$ok) {
print "Failed to update profile record...\n";

62
scripts/clean_profiles.php
View File

@ -1,33 +1,30 @@
#!/usr/bin/env php
<?php
/*
* StatusNet - a distributed open-source microblogging tool
* Copyright (C) 2008, 2009, StatusNet, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// This file is part of GNU social - https://www.gnu.org/software/social
//
// GNU social is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// GNU social is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with GNU social. If not, see <http://www.gnu.org/licenses/>.
define('INSTALLDIR', dirname(__DIR__));
define('PUBLICDIR', INSTALLDIR . DIRECTORY_SEPARATOR . 'public');
$shortoptions = 'y';
$longoptions = array('yes');
$longoptions = ['yes'];
$helptext = <<<END_OF_HELP
clean_profiles.php [options]
Deletes all profile table entries where the profile does not occur in the
notice table, is not a group and is not a local user. Very MySQL specific I think.
notice table, is not a group and is not a local user.
WARNING: This has not been tested thoroughly. Maybe we've missed a table to compare somewhere.
@ -35,7 +32,7 @@ WARNING: This has not been tested thoroughly. Maybe we've missed a table to comp
END_OF_HELP;
require_once INSTALLDIR.'/scripts/commandline.inc';
require_once INSTALLDIR . '/scripts/commandline.inc';
if (!have_option('y', 'yes')) {
print "About to delete profiles that we think are useless to save. Are you sure? [y/N] ";
@ -46,14 +43,27 @@ if (!have_option('y', 'yes')) {
}
}
print "Deleting";
echo 'Deleting';
$user_table = common_database_tablename('user');
$profile = new Profile();
$profile->query('SELECT * FROM profile WHERE ' .
'NOT (SELECT COUNT(*) FROM notice WHERE profile_id=profile.id) ' .
"AND NOT (SELECT COUNT(*) FROM {$user_table} WHERE user.id=profile.id) " .
'AND NOT (SELECT COUNT(*) FROM user_group WHERE user_group.profile_id=profile.id) ' .
'AND NOT (SELECT COUNT(*) FROM subscription WHERE subscriber=profile.id OR subscribed=profile.id) ');
$profile->query(
<<<END
SELECT profile.*
FROM profile
LEFT JOIN (
SELECT profile_id AS id FROM notice
UNION ALL
SELECT id FROM {$user_table}
UNION ALL
SELECT profile_id AS id FROM user_group
UNION ALL
SELECT subscriber FROM subscription
UNION ALL
SELECT subscribed FROM subscription
) AS t1 USING (id)
WHERE t1.id IS NULL
END
);
while ($profile->fetch()) {
echo ' '.$profile->getID().':'.$profile->getNickname();
$profile->delete();

47
scripts/delete_orphan_files.php
View File

@ -1,21 +1,23 @@
#!/usr/bin/env php
<?php
/*
* StatusNet - a distributed open-source microblogging tool
* Copyright (C) 2008, 2009, StatusNet, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
// This file is part of GNU social - https://www.gnu.org/software/social
//
// GNU social is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// GNU social is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with GNU social. If not, see <http://www.gnu.org/licenses/>.
/**
* @copyright 2008, 2009 StatusNet, Inc.
* @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later
*/
define('INSTALLDIR', dirname(__DIR__));
@ -43,12 +45,13 @@ require_once INSTALLDIR.'/scripts/commandline.inc';
print "Finding File entries that are not related to a Notice (or the notice has been deleted)...";
$file = new File();
$sql = 'SELECT file.* FROM file'.
' LEFT JOIN file_to_post ON file_to_post.file_id=file.id'.
' WHERE'.
' NOT EXISTS (SELECT file_to_post.file_id FROM file_to_post WHERE file.id=file_to_post.file_id)'.
' OR NOT EXISTS (SELECT notice.id FROM notice WHERE notice.id=file_to_post.post_id)'.
' GROUP BY file.id;';
$sql = <<<'END'
SELECT file.*
FROM file_to_post
INNER JOIN notice ON file_to_post.post_id = notice.id
RIGHT JOIN file ON file_to_post.file_id = file.id
WHERE file_to_post.file_id IS NULL;
END;
if ($file->query($sql) !== false) {
print " {$file->N} found.\n";