[DATABASE] Some query improvements

Make common_sql_weight employ standard SQL functions for the timestamp difference in seconds. Also replace UTC_TIMESTAMP in the MariaDB-specific part with CURRENT_TIMESTAMP as it is the only occurence and GNU social sets UTC as a default timezone. In a delete_orphan_files.php script simplify the main query considerably. In clean_profiles.php stop using COUNT as if it is ANY, that is unnecessary punishment for the database. Instead implement the anti-join with a left outer join. In Autocomplete and Activitypub_profile use joins instead of a WHERE OR anti-pattern for the semi-joins. In lib/ui/galleryaction.php replace a CROSS JOIN with an INNER JOIN. In actions/sup.php remove a redundant subquery: WHERE is applied before grouping either way.
2020-06-10 16:52:00 +03:00 · 2020-06-10 16:52:00 +03:00 · 194976135f
commit 194976135f
parent ef17f3ea7d
9 changed files with 187 additions and 140 deletions
--- a/actions/sup.php
+++ b/actions/sup.php
@ -65,17 +65,17 @@ class SupAction extends Action

        $divider = common_sql_date(time() - $seconds);

-        $notice->query('SELECT profile_id, max(id) AS max_id ' .
-                       'FROM ( ' .
-                       'SELECT profile_id, id FROM notice ' .
-                       "WHERE created > TIMESTAMP '" . $divider . "' " .
-                       ') AS latest ' .
-                       'GROUP BY profile_id');
+        $notice->selectAdd();
+        $notice->selectAdd('profile_id, MAX(id) AS max_id');
+        $notice->whereAdd("created > TIMESTAMP '{$divider}'");
+        $notice->groupBy('profile_id');

-        $updates = array();
+        $updates = [];

-        while ($notice->fetch()) {
-            $updates[] = array($notice->profile_id, $notice->max_id);
+        if ($notice->find()) {
+            while ($notice->fetch()) {
+                $updates[] = [$notice->profile_id, $notice->max_id];
+            }
        }

        return $updates;
--- a/lib/ui/galleryaction.php
+++ b/lib/ui/galleryaction.php
@ -107,13 +107,14 @@ class GalleryAction extends ProfileAction
    public function getTags($lst, $usr)
    {
        $profile_tag = new Notice_tag();
-        $profile_tag->query('SELECT DISTINCT(tag) ' .
-                            'FROM profile_tag, subscription ' .
-                            'WHERE tagger = ' . $this->target->id . ' ' .
-                            'AND ' . $usr . ' = ' . $this->target->id . ' ' .
-                            'AND ' . $lst . ' = tagged ' .
-                            'AND tagger <> tagged');
-        $tags = array();
+        $profile_tag->query(
+            <<<END
+            SELECT DISTINCT tag FROM profile_tag INNER JOIN subscription
+            ON tagger = {$usr} AND {$lst} = tagged
+            WHERE tagger = {$this->target->id} AND tagger <> tagged;
+            END
+        );
+        $tags = [];
        while ($profile_tag->fetch()) {
            $tags[] = $profile_tag->tag;
        }
--- a/lib/util/util.php
+++ b/lib/util/util.php
@ -1647,14 +1647,16 @@ function common_sql_date($datetime)
 */
 function common_sql_weight($column, $dropoff)
 {
-    if (common_config('db', 'type') == 'pgsql') {
-        // PostgreSQL doesn't support timestampdiff function.
-        // @fixme will this use the right time zone?
-        // @fixme does this handle cross-year subtraction correctly?
-        return "sum(exp(-extract(epoch from (now() - $column)) / $dropoff))";
+    if (common_config('db', 'type') !== 'mysql') {
+        $expr = sprintf(
+            '(((EXTRACT(DAY %1$s) * 24 + EXTRACT(HOUR %1$s)) * 60 + '
+            . 'EXTRACT(MINUTE %1$s)) * 60 + EXTRACT(SECOND %1$s))',
+            "FROM ({$column} - CURRENT_TIMESTAMP)"
+        );
    } else {
-        return "sum(exp(timestampdiff(second, utc_timestamp(), $column) / $dropoff))";
+        $expr = "timestampdiff(SECOND, CURRENT_TIMESTAMP, {$column})";
    }
+    return "SUM(EXP({$expr} / {$dropoff}))";
 }

 function common_redirect($url, $code=307)
--- a/plugins/ActivityPub/classes/Activitypub_profile.php
+++ b/plugins/ActivityPub/classes/Activitypub_profile.php
@ -557,9 +557,15 @@ class Activitypub_profile extends Managed_DataObject
        $user_table = common_database_tablename('user');
        $sub = new Subscription();
        $sub->subscribed = $profile->id;
+        $sub->_join .= "\n" . <<<END
+            INNER JOIN (
+              SELECT id AS subscriber FROM {$user_table}
+              UNION ALL
+              SELECT profile_id FROM activitypub_profile
+            ) AS t1 USING (subscriber)
+            END;
        $sub->whereAdd('subscriber <> subscribed');
-        $sub->whereAdd("subscriber IN (SELECT id FROM {$user_table} UNION SELECT profile_id AS id FROM activitypub_profile)");
-        $cnt = $sub->count('distinct subscriber');
+        $cnt = $sub->count('DISTINCT subscriber');

        self::cacheSet(sprintf('activitypub_profile:subscriberCount:%d', $profile->id), $cnt);

@ -585,9 +591,15 @@ class Activitypub_profile extends Managed_DataObject
        $user_table = common_database_tablename('user');
        $sub = new Subscription();
        $sub->subscriber = $profile->id;
+        $sub->_join .= "\n" . <<<END
+            INNER JOIN (
+              SELECT id AS subscribed FROM {$user_table}
+              UNION ALL
+              SELECT profile_id FROM activitypub_profile
+            ) AS t1 USING (subscribed)
+            END;
        $sub->whereAdd('subscriber <> subscribed');
-        $sub->whereAdd("subscribed IN (SELECT id FROM {$user_table} UNION SELECT profile_id AS id FROM activitypub_profile)");
-        $cnt = $sub->count('distinct subscribed');
+        $cnt = $sub->count('DISTINCT subscribed');

        self::cacheSet(sprintf('activitypub_profile:subscriptionCount:%d', $profile->id), $cnt);

--- a/plugins/Autocomplete/actions/autocomplete.php
+++ b/plugins/Autocomplete/actions/autocomplete.php
@ -1,47 +1,43 @@
 <?php
+// This file is part of GNU social - https://www.gnu.org/software/social
+//
+// GNU social is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// GNU social is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with GNU social.  If not, see <http://www.gnu.org/licenses/>.
+
 /**
- * StatusNet, the distributed open-source microblogging tool
- *
 * List profiles and groups for autocompletion
 *
- * PHP version 5
- *
- * LICENCE: This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
 * @category  Plugin
- * @package   StatusNet
+ * @package   GNUsocial
 * @author    Craig Andrews <candrews@integralblue.com>
 * @author    Mikael Nordfeldth <mmn@hethane.se>
 * @copyright 2008-2009 StatusNet, Inc.
 * @copyright 2009-2013 Free Software Foundation, Inc http://www.fsf.org
- * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
- * @link      http://status.net/
+ * @license   https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later
 */

-if (!defined('GNUSOCIAL')) { exit(1); }
+defined('GNUSOCIAL') || die();

 /**
 * List users for autocompletion
 *
 * This is the form for adding a new g
 *
- * @category Plugin
- * @package  StatusNet
- * @author   Craig Andrews <candrews@integralblue.com>
- * @author   Mikael Nordfeldth <mmn@hethane.se>
- * @license  http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
- * @link     http://status.net/
+ * @category  Plugin
+ * @package   GNUsocial
+ * @author    Craig Andrews <candrews@integralblue.com>
+ * @author    Mikael Nordfeldth <mmn@hethane.se>
+ * @license   https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later
 */
 class AutocompleteAction extends Action
 {
@ -57,14 +53,14 @@ class AutocompleteAction extends Action
     *
     * @return int last-modified date as unix timestamp
     */
-    function lastModified()
+    public function lastModified()
    {
-        $max=0;
-        foreach($this->profiles as $profile){
+        $max = 0;
+        foreach ($this->profiles as $profile) {
            $max = max($max, strtotime($profile->modified));
        }
-        foreach($this->groups as $group){
-            $max = max($max,strtotime($group->modified));
+        foreach ($this->groups as $group) {
+            $max = max($max, strtotime($group->modified));
        }
        // but maybe this file has been modified after that and could
        // respond differently
@ -80,11 +76,13 @@ class AutocompleteAction extends Action
     *
     * @return string etag
     */
-    function etag()
+    public function etag()
    {
        return '"' . implode(':', array($this->arg('action'),
            common_user_cache_hash(),
-            crc32($this->arg('term')), //the actual string can have funny characters in we don't want showing up in the etag
+            // the actual string can have funny characters in we don't want
+            // showing up in the etag
+            crc32($this->arg('term')),
            $this->arg('limit'),
            $this->lastModified())) . '"';
    }
@ -96,39 +94,55 @@ class AutocompleteAction extends Action

        parent::prepare($args);

-        $this->groups=array();
-        $this->profiles=array();
+        $this->groups = [];
+        $this->profiles = [];
        $term = $this->arg('term');
        $limit = $this->arg('limit');
-        if($limit > 200) $limit=200; //prevent DOS attacks
-        if(substr($term,0,1)=='@'){
+        // prevent DOS attacks
+        if ($limit > 200) {
+            $limit = 200;
+        }
+        if (substr($term, 0, 1) === '@') {
            //profile search
-            $term=substr($term,1);
+            $term = substr($term, 1);
            $user_table = common_database_tablename('user');
            $profile = new Profile();
+            $profile->_join .= sprintf(
+                "\n" . <<<'END'
+                LEFT JOIN (
+                  SELECT id FROM %s
+                  UNION ALL
+                  SELECT subscribed AS id FROM subscription WHERE subscriber = %d
+                ) AS t1 USING (id)
+                END,
+                $user_table,
+                $this->scoped->id
+            );
+            $profile->whereAdd('t1.id IS NOT NULL');
+            $profile->whereAdd('nickname LIKE \'' . trim($profile->escape($term), '\'') . '%\'');
            $profile->limit($limit);
-            $profile->whereAdd('nickname like \'' . trim($profile->escape($term), '\'') . '%\'');
-            $profile->whereAdd(sprintf('id in (SELECT id FROM %s) OR '
-                               . 'id in (SELECT subscribed from subscription'
-                               . ' where subscriber = %d)', $user_table, $this->scoped->id));
+
            if ($profile->find()) {
-                while($profile->fetch()) {
-                    $this->profiles[]=clone($profile);
+                while ($profile->fetch()) {
+                    $this->profiles[] = clone($profile);
                }
            }
        }
-        if(substr($term,0,1)=='!'){
+        if (substr($term, 0, 1) === '!') {
            //group search
-            $term=substr($term,1);
+            $term = substr($term, 1);
            $group = new User_group();
-            $group->limit($limit);
-            $group->whereAdd('nickname like \'' . trim($group->escape($term), '\'') . '%\'');
            //Can't post to groups we're not subscribed to...:
-            $group->whereAdd(sprintf('id in (SELECT group_id FROM group_member'
-                             . ' WHERE profile_id = %d)', $this->scoped->id));
-            if($group->find()){
-                while($group->fetch()) {
-                    $this->groups[]=clone($group);
+            $group->whereAdd(sprintf(
+                'id IN (SELECT group_id FROM group_member WHERE profile_id = %d)',
+                $this->scoped->id
+            ));
+            $group->whereAdd('nickname LIKE \'' . trim($group->escape($term), '\'') . '%\'');
+            $group->limit($limit);
+
+            if ($group->find()) {
+                while ($group->fetch()) {
+                    $this->groups[] = clone($group);
                }
            }
        }
@ -140,7 +154,7 @@ class AutocompleteAction extends Action
        parent::handle();

        $results = array();
-        foreach($this->profiles as $profile){
+        foreach ($this->profiles as $profile) {
            $avatarUrl = $profile->avatarUrl(AVATAR_MINI_SIZE);
            $acct = $profile->getAcctUri();
            $identifier = explode(':', $profile->getAcctUri(), 2)[1];
@ -153,7 +167,7 @@ class AutocompleteAction extends Action
                'type' => 'user'
            );
        }
-        foreach($this->groups as $group){
+        foreach ($this->groups as $group) {
            $profile = $group->getProfile();
            // sigh.... encapsulate this upstream!
            if ($group->mini_logo) {
@ -181,7 +195,7 @@ class AutocompleteAction extends Action
     *
     * @return boolean is read only action?
     */
-    function isReadOnly($args)
+    public function isReadOnly($args)
    {
        return true;
    }
--- a/plugins/Nodeinfo/actions/nodeinfo_2_0.php
+++ b/plugins/Nodeinfo/actions/nodeinfo_2_0.php
@ -338,7 +338,8 @@ class Nodeinfo_2_0Action extends Action
                UNION ALL
                SELECT id FROM {$userTable} WHERE {$userTable}.created >= CURRENT_DATE - INTERVAL '{$days}' DAY
              ) AS source
-              WHERE profile_id NOT IN (SELECT profile_id FROM profile_role WHERE role = 'silenced')
+              LEFT JOIN profile_role USING (profile_id)
+              WHERE profile_role.profile_id IS NULL OR profile_role.role <> 'silenced';
            END;

        $activeUsersCount = new DB_DataObject();
--- a/plugins/OStatus/scripts/update-profile.php
+++ b/plugins/OStatus/scripts/update-profile.php
@ -102,14 +102,18 @@ if ($feedurl != $oprofile->feeduri || $salmonuri != $oprofile->salmonuri) {
    print "\n";
    print "Updating...\n";
    // @fixme update keys :P
-    #$orig = clone($oprofile);
-    #$oprofile->feeduri = $feedurl;
-    #$oprofile->salmonuri = $salmonuri;
-    #$ok = $oprofile->update($orig);
-    $ok = $oprofile->query('UPDATE ostatus_profile SET ' .
-        'feeduri=\'' . $oprofile->escape($feedurl) . '\',' .
-        'salmonuri=\'' . $oprofile->escape($salmonuri) . '\' ' .
-        'WHERE uri=\'' . $oprofile->escape($uri) . '\'');
+    //$orig = clone($oprofile);
+    //$oprofile->feeduri = $feedurl;
+    //$oprofile->salmonuri = $salmonuri;
+    //$ok = $oprofile->update($orig);
+    $ok = $oprofile->query(
+        <<<END
+        UPDATE ostatus_profile
+        SET feeduri   = '{$oprofile->escape($feedurl)}',
+            salmonuri = '{$oprofile->escape($salmonuri)}'
+        WHERE uri = '{$oprofile->escape($uri)}'
+        END
+    );

    if (!$ok) {
        print "Failed to update profile record...\n";
--- a/scripts/clean_profiles.php
+++ b/scripts/clean_profiles.php
@ -1,33 +1,30 @@
 #!/usr/bin/env php
 <?php
-/*
- * StatusNet - a distributed open-source microblogging tool
- * Copyright (C) 2008, 2009, StatusNet, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
+// This file is part of GNU social - https://www.gnu.org/software/social
+//
+// GNU social is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// GNU social is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with GNU social.  If not, see <http://www.gnu.org/licenses/>.

 define('INSTALLDIR', dirname(__DIR__));
 define('PUBLICDIR', INSTALLDIR . DIRECTORY_SEPARATOR . 'public');

 $shortoptions = 'y';
-$longoptions = array('yes');
+$longoptions = ['yes'];

 $helptext = <<<END_OF_HELP
 clean_profiles.php [options]
 Deletes all profile table entries where the profile does not occur in the
-notice table, is not a group and is not a local user. Very MySQL specific I think.
+notice table, is not a group and is not a local user.

 WARNING: This has not been tested thoroughly. Maybe we've missed a table to compare somewhere.

@ -35,7 +32,7 @@ WARNING: This has not been tested thoroughly. Maybe we've missed a table to comp

 END_OF_HELP;

-require_once INSTALLDIR.'/scripts/commandline.inc';
+require_once INSTALLDIR . '/scripts/commandline.inc';

 if (!have_option('y', 'yes')) {
    print "About to delete profiles that we think are useless to save. Are you sure? [y/N] ";
@ -46,14 +43,27 @@ if (!have_option('y', 'yes')) {
    }
 }

-print "Deleting";
+echo 'Deleting';
 $user_table = common_database_tablename('user');
 $profile = new Profile();
-$profile->query('SELECT * FROM profile WHERE ' .
-                'NOT (SELECT COUNT(*) FROM notice WHERE profile_id=profile.id) ' .
-                "AND NOT (SELECT COUNT(*) FROM {$user_table} WHERE user.id=profile.id) " .
-                'AND NOT (SELECT COUNT(*) FROM user_group WHERE user_group.profile_id=profile.id) ' .
-                'AND NOT (SELECT COUNT(*) FROM subscription WHERE subscriber=profile.id OR subscribed=profile.id) ');
+$profile->query(
+    <<<END
+    SELECT profile.*
+      FROM profile
+      LEFT JOIN (
+        SELECT profile_id AS id FROM notice
+        UNION ALL
+        SELECT id FROM {$user_table}
+        UNION ALL
+        SELECT profile_id AS id FROM user_group
+        UNION ALL
+        SELECT subscriber FROM subscription
+        UNION ALL
+        SELECT subscribed FROM subscription
+      ) AS t1 USING (id)
+      WHERE t1.id IS NULL
+    END
+);
 while ($profile->fetch()) {
    echo ' '.$profile->getID().':'.$profile->getNickname();
    $profile->delete();
--- a/scripts/delete_orphan_files.php
+++ b/scripts/delete_orphan_files.php
@ -1,21 +1,23 @@
 #!/usr/bin/env php
 <?php
-/*
- * StatusNet - a distributed open-source microblogging tool
- * Copyright (C) 2008, 2009, StatusNet, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// This file is part of GNU social - https://www.gnu.org/software/social
+//
+// GNU social is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// GNU social is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with GNU social.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * @copyright 2008, 2009 StatusNet, Inc.
+ * @license   https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later
 */

 define('INSTALLDIR', dirname(__DIR__));
@ -43,12 +45,13 @@ require_once INSTALLDIR.'/scripts/commandline.inc';

 print "Finding File entries that are not related to a Notice (or the notice has been deleted)...";
 $file = new File();
-$sql = 'SELECT file.* FROM file'.
-        ' LEFT JOIN file_to_post ON file_to_post.file_id=file.id'.
-        ' WHERE'.
-            ' NOT EXISTS (SELECT file_to_post.file_id FROM file_to_post WHERE file.id=file_to_post.file_id)'.
-            ' OR NOT EXISTS (SELECT notice.id FROM notice WHERE notice.id=file_to_post.post_id)'.
-            ' GROUP BY file.id;';
+$sql = <<<'END'
+    SELECT file.*
+      FROM file_to_post
+      INNER JOIN notice ON file_to_post.post_id = notice.id
+      RIGHT JOIN file ON file_to_post.file_id = file.id
+      WHERE file_to_post.file_id IS NULL;
+    END;

 if ($file->query($sql) !== false) {
    print " {$file->N} found.\n";