From 318d2cb6cab2cf07aefaf6e373e4a239ed372a04 Mon Sep 17 00:00:00 2001 From: Alexei Sorokin Date: Wed, 16 Sep 2020 19:34:49 +0300 Subject: [PATCH] Set the character set before making a connection Ideally the character set should be set with the connection, and so this is exactly what's being done now. And now the character set code is attempted to be generalised. --- classes/Memcached_DataObject.php | 6 --- lib/database/mysqlschema.php | 52 ++++++++++++++------- lib/database/schema.php | 10 ++++ lib/util/gnusocial.php | 34 +++++++++----- lib/util/installer.php | 42 +++++++++++------ lib/util/util.php | 9 ++++ plugins/OpenID/lib/openiddbconn.php | 5 ++ plugins/OpenID/openid.php | 3 ++ plugins/SphinxSearch/scripts/gen_config.php | 11 +++-- 9 files changed, 118 insertions(+), 54 deletions(-) diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index c64cb8b4b1..6cb60e3802 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -863,9 +863,6 @@ class Memcached_DataObject extends Safe_DataObject return $string; } - // We overload so that 'SET NAMES "utf8mb4"' is called for - // each connection - public function _connect() { global $_DB_DATAOBJECT, $_PEAR; @@ -908,13 +905,10 @@ class Memcached_DataObject extends Safe_DataObject if ($result && !$exists) { // Required to make timestamp values usefully comparable. - // And set the character set to UTF-8. if (common_config('db', 'type') !== 'mysql') { parent::_query("SET TIME ZONE INTERVAL '+00:00' HOUR TO MINUTE"); - parent::_query("SET NAMES 'UTF8'"); } else { parent::_query("SET time_zone = '+0:00'"); - parent::_query("SET NAMES 'utf8mb4'"); } } diff --git a/lib/database/mysqlschema.php b/lib/database/mysqlschema.php index 0d22668261..41235f89f5 100644 --- a/lib/database/mysqlschema.php +++ b/lib/database/mysqlschema.php @@ -361,16 +361,36 @@ class MysqlSchema extends Schema * * @param string $name * @param array $def - * @return string; * + * @return string */ public function endCreateTable($name, array $def) { - $engine = $this->preferredEngine($def); - return ") ENGINE=$engine CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"; + $engine = self::storageEngine($def); + $charset = self::charset(); + return ") ENGINE '{$engine}' " + . "DEFAULT CHARACTER SET '{$charset}' " + . "DEFAULT COLLATE '{$charset}_bin'"; } - public function preferredEngine($def) + /** + * Returns the character set of choice for MariaDB. + * Overrides default standard "UTF8". + * + * @return string + */ + public static function charset(): string + { + return 'utf8mb4'; + } + + /** + * Returns the storage engine of choice for the supplied definition. + * + * @param array $def + * @return string + */ + protected static function storageEngine(array $def): string { return 'InnoDB'; } @@ -432,18 +452,18 @@ class MysqlSchema extends Schema */ public function appendAlterExtras(array &$phrase, $tableName, array $def) { - // Check for table properties: make sure we're using a sane - // engine type and collation. - // @fixme make the default engine configurable? + // Check for table properties: make sure we are using sane + // storage engine, character set and collation. $oldProps = $this->getTableProperties($tableName, ['ENGINE', 'TABLE_COLLATION']); - $engine = $this->preferredEngine($def); - if (strtolower($oldProps['ENGINE']) != strtolower($engine)) { - $phrase[] = "ENGINE=$engine"; + $engine = self::storageEngine($def); + $charset = self::charset(); + if (mb_strtolower($oldProps['ENGINE']) !== mb_strtolower($engine)) { + $phrase[] = "ENGINE '{$engine}'"; } - if (strtolower($oldProps['TABLE_COLLATION']) != 'utf8mb4_bin') { - $phrase[] = 'CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin'; - $phrase[] = 'DEFAULT CHARACTER SET = utf8mb4'; - $phrase[] = 'DEFAULT COLLATE = utf8mb4_bin'; + if (strtolower($oldProps['TABLE_COLLATION']) !== "{$charset}_bin") { + $phrase[] = "CONVERT TO CHARACTER SET '{$charset} COLLATE '{$charset}_bin'"; + $phrase[] = "DEFAULT CHARACTER SET '{$charset}'"; + $phrase[] = "DEFAULT COLLATE '{$charset}_bin'"; } } @@ -573,11 +593,11 @@ class MysqlSchema extends Schema foreach ($column['enum'] as &$val) { $vals[] = "'{$val}'"; } - return 'enum(' . implode(',', $vals) . ')'; + return 'ENUM(' . implode(',', $vals) . ')'; } elseif ($this->isStringType($column)) { $col = parent::typeAndSize($name, $column); if (!empty($column['collate'])) { - $col .= ' COLLATE ' . $column['collate']; + $col .= " COLLATE '{$column['collate']}'"; } return $col; } else { diff --git a/lib/database/schema.php b/lib/database/schema.php index dc4fddff2a..9d010c4a56 100644 --- a/lib/database/schema.php +++ b/lib/database/schema.php @@ -456,6 +456,16 @@ class Schema return true; } + /** + * Returns the character set of choice for this DBMS (in practice, UTF-8). + * + * @return string + */ + public static function charset(): string + { + return 'UTF8'; + } + /** * Modifies a column in the schema. * diff --git a/lib/util/gnusocial.php b/lib/util/gnusocial.php index 72597d2c6d..2467f75188 100644 --- a/lib/util/gnusocial.php +++ b/lib/util/gnusocial.php @@ -54,7 +54,6 @@ class GNUsocial $moduleclass = "{$name}Module"; if (!class_exists($moduleclass)) { - $files = [ "modules/{$moduleclass}.php", "modules/{$name}/{$moduleclass}.php" @@ -120,7 +119,6 @@ class GNUsocial $moduleclass = "{$name}Plugin"; if (!class_exists($moduleclass)) { - $files = [ "local/plugins/{$moduleclass}.php", "local/plugins/{$name}/{$moduleclass}.php", @@ -466,18 +464,30 @@ class GNUsocial } if (!self::$have_config) { - throw new NoConfigException("No configuration file found.", - $config_files); + throw new NoConfigException( + 'No configuration file found.', + $config_files + ); } + $dsn = $config['db']['database']; + // Check for database server; must exist! - - if (empty($config['db']['database'])) { - throw new ServerException("No database server for this site."); + if (empty($dsn)) { + throw new ServerException('No database server for this site.'); } + + $database = MDB2::parseDSN($dsn); + + if (empty($database['phptype'])) { + throw new ServerException('Database server for this site is invalid.'); + } + + $database['charset'] = common_database_charset(); + $config['db']['database'] = $database; } - static function fillConfigVoids() + public static function fillConfigVoids() { // special cases on empty configuration options if (!common_config('thumbnail', 'dir')) { @@ -492,7 +502,7 @@ class GNUsocial * Might make changes to the filesystem, to created dirs, but will * not make database changes. */ - static function verifyLoadedConfig() + public static function verifyLoadedConfig() { $mkdirs = []; @@ -533,7 +543,7 @@ class GNUsocial * @return boolean true if we're running with HTTPS; else false */ - static function isHTTPS() + public static function isHTTPS() { if (common_config('site', 'sslproxy')) { return true; @@ -551,7 +561,7 @@ class GNUsocial /** * Can we use HTTPS? Then do! Only return false if it's not configured ("never"). */ - static function useHTTPS() + public static function useHTTPS() { return self::isHTTPS() || common_config('site', 'ssl') != 'never'; } @@ -561,7 +571,7 @@ class NoConfigException extends Exception { public $configFiles; - function __construct($msg, $configFiles) + public function __construct($msg, $configFiles) { parent::__construct($msg); $this->configFiles = $configFiles; diff --git a/lib/util/installer.php b/lib/util/installer.php index dfe121ba80..faac3a2c55 100644 --- a/lib/util/installer.php +++ b/lib/util/installer.php @@ -67,11 +67,13 @@ abstract class Installer 'name' => 'MariaDB 10.3+', 'check_module' => 'mysqli', 'scheme' => 'mysqli', // DSN prefix for MDB2 + 'charset' => 'utf8mb4', ], 'pgsql' => [ 'name' => 'PostgreSQL 11+', 'check_module' => 'pgsql', 'scheme' => 'pgsql', // DSN prefix for MDB2 + 'charset' => 'UTF8', ] ]; @@ -296,14 +298,12 @@ abstract class Installer $dsn = "{$scheme}://{$this->username}{$auth}@{$this->host}/{$this->database}"; $this->updateStatus('Checking database...'); - $conn = $this->connectDatabase($dsn); + $charset = self::$dbModules[$this->dbtype]['charset']; + $conn = $this->connectDatabase($dsn, $charset); - $charset = ($this->dbtype !== 'mysql') ? 'UTF8' : 'utf8mb4'; $server_charset = $this->getDatabaseCharset($conn, $this->dbtype); // Ensure the database server character set is UTF-8. - $conn->exec("SET NAMES '{$charset}'"); - if ($server_charset !== $charset) { $this->updateStatus( 'GNU social requires the "' . $charset . '" character set. ' @@ -346,11 +346,17 @@ abstract class Installer * Open a connection to the database. * * @param string $dsn + * @param string $charset * @return MDB2_Driver_Common * @throws Exception */ - protected function connectDatabase(string $dsn) + protected function connectDatabase(string $dsn, string $charset) { + $dsn = MDB2::parseDSN($dsn); + + // Ensure the database client character set is UTF-8. + $dsn['charset'] = $charset; + $conn = MDB2::connect($dsn); if (MDB2::isError($conn)) { throw new Exception( @@ -375,25 +381,31 @@ abstract class Installer switch ($dbtype) { case 'pgsql': $res = $conn->query('SHOW server_encoding'); - - if (MDB2::isError($res)) { - throw new Exception($res->getMessage()); - } - $ret = $res->fetchOne(); break; case 'mysql': - $res = $conn->query( - "SHOW SESSION VARIABLES LIKE 'character_set_database'" + $stmt = $conn->prepare( + <<getMessage()); + if (MDB2::isError($stmt)) { + return null; } - [, $ret] = $res->fetchRow(MDB2_FETCHMODE_ORDERED); + $res = $stmt->execute([$database]); break; default: throw new Exception('Unknown DB type selected.'); } + if (MDB2::isError($res)) { + throw new Exception($res->getMessage()); + } + + $ret = $res->fetchOne(); if (MDB2::isError($ret)) { throw new Exception($ret->getMessage()); } diff --git a/lib/util/util.php b/lib/util/util.php index 4dadc027c3..161e1eade3 100644 --- a/lib/util/util.php +++ b/lib/util/util.php @@ -2390,6 +2390,15 @@ function common_compatible_license($from, $to) return ($from == $to); } +/** + * returns the character set name used for the current DBMS + */ +function common_database_charset(): string +{ + $schema = Schema::get(); + return $schema->charset(); +} + /** * returns a quoted table name */ diff --git a/plugins/OpenID/lib/openiddbconn.php b/plugins/OpenID/lib/openiddbconn.php index 514ad6d38e..475f46a30b 100644 --- a/plugins/OpenID/lib/openiddbconn.php +++ b/plugins/OpenID/lib/openiddbconn.php @@ -49,6 +49,11 @@ class SQLStore_DB_Connection extends Auth_OpenID_DatabaseConnection } $dsn['new_link'] = true; + if (array_key_exists('charset', $options)) { + $dsn['charset'] = $options['charset']; + unset($options['charset']); + } + // To create a new Database connection is an absolute must, because // php-openid code delays its transactions commitment. // Is a must because our Internal Session Handler uses the database diff --git a/plugins/OpenID/openid.php b/plugins/OpenID/openid.php index ff06feccbe..795ae3f627 100644 --- a/plugins/OpenID/openid.php +++ b/plugins/OpenID/openid.php @@ -45,6 +45,9 @@ function oid_store() if (!is_array($options)) { $options = []; } + // A special case, carry the charset. + $options['charset'] = common_database_charset(); + $dbconn = new SQLStore_DB_Connection($dsn, $options); switch (common_config('db', 'type')) { diff --git a/plugins/SphinxSearch/scripts/gen_config.php b/plugins/SphinxSearch/scripts/gen_config.php index 192efb9af1..3d0148da0c 100755 --- a/plugins/SphinxSearch/scripts/gen_config.php +++ b/plugins/SphinxSearch/scripts/gen_config.php @@ -39,11 +39,11 @@ END_OF_TRIM_HELP; require_once INSTALLDIR . '/scripts/commandline.inc'; require dirname(__FILE__) . '/sphinx-utils.php'; - +$base = $base ?? '/usr/local'; $timestamp = date('r'); -print <<sitename} @@ -122,7 +123,7 @@ source {$sn->dbname}_src_{$table} sql_user = {$sn->dbuser} sql_pass = {$sn->dbpass} sql_db = {$sn->dbname} - sql_query_pre = SET NAMES utf8; + sql_query_pre = SET NAMES '{$charset}'; sql_query = {$query} sql_query_info = {$query_info} sql_attr_timestamp = created_ts