From 698228acb9a6262f2fb4e0c82118249aa94cc0f8 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Sun, 26 Apr 2009 13:16:59 -0400 Subject: [PATCH 01/11] Revert "Revert "run 'set names' after each connection to deal with UTF8 correctly"" This reverts commit 86770ccde7914219a0a572ced6dd21fa65566e1d. --- classes/Memcached_DataObject.php | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 5f71f716b3..877bbf2e0f 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -227,4 +227,20 @@ class Memcached_DataObject extends DB_DataObject $c->set($ckey, $cached, MEMCACHE_COMPRESSED, $expiry); return new ArrayWrapper($cached); } + + // We overload so that 'SET NAMES "utf8"' is called for + // each connection + + function _connect() + { + global $_DB_DATAOBJECT; + $exists = !empty($this->_database_dsn_md5) && + isset($_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]); + $result = parent::_connect(); + if (!$exists) { + $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]; + $DB->query('SET NAMES "utf8"'); + } + return $result; + } } From 18b130456287a24ac9024b57dc7a38d1dcaa498f Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 25 May 2009 14:23:00 -0700 Subject: [PATCH 02/11] Add EventBox to notice sources --- db/notice_source.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/db/notice_source.sql b/db/notice_source.sql index 7c31a9af49..221b1ee016 100644 --- a/db/notice_source.sql +++ b/db/notice_source.sql @@ -7,6 +7,7 @@ VALUES ('cliqset', 'Cliqset', 'http://www.cliqset.com/', now()), ('deskbar','Deskbar-Applet','http://www.gnome.org/projects/deskbar-applet/', now()), ('Do','Gnome Do','http://do.davebsd.com/wiki/index.php?title=Microblog_Plugin', now()), + ('eventbox','EventBox','http://thecosmicmachine.com/eventbox/ ', now()), ('Facebook','Facebook','http://apps.facebook.com/identica/', now()), ('feed2omb','feed2omb','http://projects.ciarang.com/p/feed2omb/', now()), ('gravity', 'Gravity', 'http://mobileways.de/gravity', now()), From 5f2cf2553cbe97b9983c8c5d0a85abb00b1a8375 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Tue, 26 May 2009 02:23:19 +0000 Subject: [PATCH 03/11] Ticket #1196 - Fixed: Warning on invite tab in Facebook App --- actions/facebookinvite.php | 65 +++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/actions/facebookinvite.php b/actions/facebookinvite.php index 1302064ad7..2207580f77 100644 --- a/actions/facebookinvite.php +++ b/actions/facebookinvite.php @@ -17,7 +17,9 @@ * along with this program. If not, see . */ -if (!defined('LACONICA')) { exit(1); } +if (!defined('LACONICA')) { + exit(1); +} require_once(INSTALLDIR.'/lib/facebookaction.php'); @@ -67,7 +69,7 @@ class FacebookinviteAction extends FacebookAction function showSuccessContent() { - $this->element('h2', null, sprintf(_('Thanks for inviting your friends to use %s'), + $this->element('h2', null, sprintf(_('Thanks for inviting your friends to use %s'), common_config('site', 'name'))); $this->element('p', null, _('Invitations have been sent to the following users:')); @@ -89,16 +91,6 @@ class FacebookinviteAction extends FacebookAction function showFormContent() { - - // Get a list of users who are already using the app for exclusion - $exclude_ids = $this->facebook->api_client->friends_getAppUsers(); - $exclude_ids_csv = null; - - // fbml needs these as a csv string, not an array - if ($exclude_ids) { - $exclude_ids_csv = implode(',', $exclude_ids); - } - $content = sprintf(_('You have been invited to %s'), common_config('site', 'name')) . htmlentities(''); @@ -109,36 +101,43 @@ class FacebookinviteAction extends FacebookAction 'content' => $content)); $this->hidden('invite', 'true'); $actiontext = sprintf(_('Invite your friends to use %s'), common_config('site', 'name')); - - $multi_params = array('showborder' => 'false'); + + $multi_params = array('showborder' => 'false'); $multi_params['actiontext'] = $actiontext; - - if ($exclude_ids_csv) { + $multi_params['bypass'] = 'cancel'; + + // Get a list of users who are already using the app for exclusion + $exclude_ids = $this->facebook->api_client->friends_getAppUsers(); + $exclude_ids_csv = null; + + // fbml needs these as a csv string, not an array + if ($exclude_ids) { + $exclude_ids_csv = implode(',', $exclude_ids); $multi_params['exclude_ids'] = $exclude_ids_csv; } - $multi_params['bypass'] = 'cancel'; - $this->element('fb:multi-friend-selector', $multi_params); - $this->elementEnd('fb:request-form'); - $this->element('h2', null, sprintf(_('Friends already using %s:'), - common_config('site', 'name'))); - $this->elementStart('ul', array('id' => 'facebook-friends')); - - foreach ($exclude_ids as $friend) { - $this->elementStart('li'); - $this->element('fb:profile-pic', array('uid' => $friend, 'size' => 'square')); - $this->element('fb:name', array('uid' => $friend, - 'capitalize' => 'true')); - $this->elementEnd('li'); - } + if ($exclude_ids) { - $this->elementEnd("ul"); + $this->element('h2', null, sprintf(_('Friends already using %s:'), + common_config('site', 'name'))); + $this->elementStart('ul', array('id' => 'facebook-friends')); + + foreach ($exclude_ids as $friend) { + $this->elementStart('li'); + $this->element('fb:profile-pic', array('uid' => $friend, 'size' => 'square')); + $this->element('fb:name', array('uid' => $friend, + 'capitalize' => 'true')); + $this->elementEnd('li'); + } + + $this->elementEnd("ul"); + } } - - function title() + + function title() { return sprintf(_('Send invitations')); } From a1f83b293394af34782f74640c3d29f800bdc0bf Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 26 May 2009 15:22:12 -0400 Subject: [PATCH 04/11] added qtwitter --- db/notice_source.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/db/notice_source.sql b/db/notice_source.sql index 7c31a9af49..e7ccadde3a 100644 --- a/db/notice_source.sql +++ b/db/notice_source.sql @@ -28,6 +28,7 @@ VALUES ('pingvine','PingVine','http://pingvine.com/', now()), ('pocketwit','PockeTwit','http://code.google.com/p/pocketwit/', now()), ('posty','Posty','http://spreadingfunkyness.com/posty/', now()), + ('qtwitter','qTwitter','http://qtwitter.ayoy.net/', now()), ('royalewithcheese','Royale With Cheese','http://p.hellyeah.org/', now()), ('rssdent','rssdent','http://github.com/zcopley/rssdent/tree/master', now()), ('rygh.no','rygh.no','http://rygh.no/', now()), From 74b08bff53b614a0d3da56e848940cd3e9ca70ce Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 26 May 2009 15:27:13 -0400 Subject: [PATCH 05/11] Added AgentSolo.com --- db/notice_source.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/db/notice_source.sql b/db/notice_source.sql index a22967b1b1..ac73d3d13b 100644 --- a/db/notice_source.sql +++ b/db/notice_source.sql @@ -2,6 +2,7 @@ INSERT INTO notice_source (code, name, url, created) VALUES ('adium', 'Adium', 'http://www.adiumx.com/', now()), + ('AgentSolo.com','AgentSolo.com','http://www.agentsolo.com/', now()), ('betwittered','BeTwittered','http://www.32hours.com/betwitteredinfo/', now()), ('bti','bti','http://gregkh.github.com/bti/', now()), ('cliqset', 'Cliqset', 'http://www.cliqset.com/', now()), From b2c28faf526e55ddc97d42692048e644b004e1f8 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Tue, 26 May 2009 21:14:42 +0000 Subject: [PATCH 06/11] Removed silly "dented:" status prefix from Facebook App --- actions/facebookhome.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/actions/facebookhome.php b/actions/facebookhome.php index 5946e6c984..4c2b26355c 100644 --- a/actions/facebookhome.php +++ b/actions/facebookhome.php @@ -138,9 +138,6 @@ class FacebookhomeAction extends FacebookAction function setDefaults() { - // A default prefix string for notices - $this->facebook->api_client->data_setUserPreference( - FACEBOOK_NOTICE_PREFIX, 'dented: '); $this->facebook->api_client->data_setUserPreference( FACEBOOK_PROMPTED_UPDATE_PREF, 'false'); } From 2b81dcb25329c496c7c4f2c99334b79ba0421c1e Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Fri, 15 May 2009 21:44:58 +0000 Subject: [PATCH 07/11] Better err handling when trying to get Facebook client --- lib/facebookutil.php | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/facebookutil.php b/lib/facebookutil.php index ec39872732..242d2e06f2 100644 --- a/lib/facebookutil.php +++ b/lib/facebookutil.php @@ -27,9 +27,21 @@ define("FACEBOOK_PROMPTED_UPDATE_PREF", 2); function getFacebook() { + static $facebook = null; + $apikey = common_config('facebook', 'apikey'); $secret = common_config('facebook', 'secret'); - return new Facebook($apikey, $secret); + + if ($facebook === null) { + $facebook = new Facebook($apikey, $secret); + } + + if (!$facebook) { + common_log(LOG_ERR, 'Could not make new Facebook client obj!', + __FILE__); + } + + return $facebook; } function updateProfileBox($facebook, $flink, $notice) { @@ -92,7 +104,6 @@ function isFacebookBound($notice, $flink) { } - function facebookBroadcastNotice($notice) { $facebook = getFacebook(); From d068680e1913a32eba0f5b0e1e746ac66ec638d3 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 27 May 2009 14:57:45 -0400 Subject: [PATCH 08/11] optionally use SET NAMES for utf8 to DB --- README | 4 ++++ classes/Memcached_DataObject.php | 4 +++- lib/common.php | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README b/README index 7b2dcacc50..c23d4e464a 100644 --- a/README +++ b/README @@ -906,6 +906,10 @@ mirror: you can set this to an array of DSNs, like the above and adding the slaves to this array. Note that if you want some requests to go to the 'database' (master) server, you'll need to include it in this array, too. +utf8: whether to talk to the database in UTF-8 mode. This is the default + with new installations, but older sites may want to turn it off + until they get their databases fixed up. See "UTF-8 database" + above for details. syslog ------ diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 877bbf2e0f..52ad4100fc 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -239,7 +239,9 @@ class Memcached_DataObject extends DB_DataObject $result = parent::_connect(); if (!$exists) { $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5]; - $DB->query('SET NAMES "utf8"'); + if (common_config('db', 'utf8')) { + $DB->query('SET NAMES "utf8"'); + } } return $result; } diff --git a/lib/common.php b/lib/common.php index f983c4d168..8f95c23615 100644 --- a/lib/common.php +++ b/lib/common.php @@ -167,6 +167,7 @@ $config['db'] = 'require_prefix' => 'classes/', 'class_prefix' => '', 'mirror' => null, + 'utf8' => true, 'db_driver' => 'DB', # XXX: JanRain libs only work with DB 'quote_identifiers' => false, 'type' => 'mysql' ); From 497d1bfe0283598021f2be2bd6664ac8bbab25ce Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 27 May 2009 19:12:37 -0400 Subject: [PATCH 09/11] Add fixup_utf8 to fixup problems with UTF-8 in a database Fixes up an old database to store strings in UTF-8. Hopefully works! --- scripts/fixup_utf8.php | 131 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 scripts/fixup_utf8.php diff --git a/scripts/fixup_utf8.php b/scripts/fixup_utf8.php new file mode 100644 index 0000000000..0763c72c91 --- /dev/null +++ b/scripts/fixup_utf8.php @@ -0,0 +1,131 @@ +#!/usr/bin/env php +. + */ + +# Abort if called from a web server +if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) { + print "This script must be run from the command line\n"; + exit(1); +} + +ini_set("max_execution_time", "0"); +ini_set("max_input_time", "0"); +set_time_limit(0); +mb_internal_encoding('UTF-8'); + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); +define('LACONICA', true); + +require_once(INSTALLDIR . '/lib/common.php'); +require_once('DB.php'); + +function main() { + + $dbl = doConnect('latin1'); + + if (empty($dbl)) { + return; + } + + $dbu = doConnect('utf8'); + + if (empty($dbu)) { + return; + } + + // Do a separate DB connection + + $sth = $dbu->prepare("UPDATE notice SET content = UNHEX(?), rendered = UNHEX(?) WHERE id = ?"); + + if (PEAR::isError($sth)) { + echo "ERROR: " . $sth->getMessage() . "\n"; + return; + } + + $rn = $dbl->query('SELECT id, content, rendered FROM notice ' . + 'WHERE LENGTH(content) != CHAR_LENGTH(content)'); + + if (PEAR::isError($rn)) { + echo "ERROR: " . $rn->getMessage() . "\n"; + return; + } + + echo "Number of rows: " . $rn->numRows() . "\n"; + + $notice = array(); + + while (DB_OK == $rn->fetchInto($notice)) { + + $id = ($notice[0])+0; + $content = bin2hex($notice[1]); + $rendered = bin2hex($notice[2]); + + echo "$id..."; + + $result =& $dbu->execute($sth, array($content, $rendered, $id)); + + if (PEAR::isError($result)) { + echo "ERROR: " . $result->getMessage() . "\n"; + continue; + } + + $cnt = $dbu->affectedRows(); + + if ($cnt != 1) { + echo "ERROR: 0 rows affected\n"; + continue; + } + + $notice = Notice::staticGet('id', $id); + $notice->decache(); + + echo "OK\n"; + } +} + +function doConnect($charset) +{ + $db = DB::connect(common_config('db', 'database'), + array('persistent' => false)); + + if (PEAR::isError($db)) { + echo "ERROR: " . $db->getMessage() . "\n"; + return NULL; + } + + $result = $db->query("SET NAMES $charset"); + + if (PEAR::isError($result)) { + echo "ERROR: " . $result->getMessage() . "\n"; + $db->disconnect(); + return NULL; + } + + $result = $db->autoCommit(true); + + if (PEAR::isError($result)) { + echo "ERROR: " . $result->getMessage() . "\n"; + $db->disconnect(); + return NULL; + } + + return $db; +} + +main(); From 121cb6afb1822517e215eef23a5cb87796ecea35 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 28 May 2009 16:35:34 -0400 Subject: [PATCH 10/11] document utf8 problems and solutions --- README | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README b/README index c23d4e464a..4649eb79ce 100644 --- a/README +++ b/README @@ -690,6 +690,13 @@ to users on a remote site. (Or not... it's not well tested.) The Upgrading ========= +IMPORTANT NOTE: Laconica 0.7.4 introduced a fix for some +incorrectly-stored international characters ("UTF-8"). For new +installations, it will now store non-ASCII characters correctly. +However, older installations will have the incorrect storage, and will +consequently show up "wrong" in browsers. See below for how to deal +with this situation. + If you've been using Laconica 0.6, 0.5 or lower, or if you've been tracking the "git" version of the software, you will probably want to upgrade and keep your existing data. There is no automated upgrade @@ -779,6 +786,29 @@ problem. 3. When fixup_inboxes is finished, you can set the enabled flag to 'true'. +UTF-8 Database +-------------- + +Laconica 0.7.4 introduced a fix for some incorrectly-stored +international characters ("UTF-8"). This fix is not +backwards-compatible; installations from before 0.7.4 will show +non-ASCII characters of old notices incorrectly. This section explains +what to do. + +0. You can disable the new behaviour by setting the 'db''utf8' config + option to "false". You should only do this until you're ready to + convert your DB to the new format. +1. When you're ready to convert, you can run the fixup_utf8.php script + in the scripts/ subdirectory. If you've had the "new behaviour" + enabled (probably a good idea), you can give the ID of the first + "new" notice as a parameter, and only notices before that one will + be converted. Notices are converted in reverse chronological order, + so the most recent (and visible) ones will be converted first. The + script should work whether or not you have the 'db''utf8' config + option enabled. +2. When you're ready, set $config['db']['utf8'] to true, so that + new notices will be stored correctly. + Configuration options ===================== From daf845dbe67a909e2bc9e039d3c2ccc560345b4d Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 28 May 2009 16:36:07 -0400 Subject: [PATCH 11/11] take an argument for fixup_utf8 --- scripts/fixup_utf8.php | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/fixup_utf8.php b/scripts/fixup_utf8.php index 0763c72c91..e5021ff343 100644 --- a/scripts/fixup_utf8.php +++ b/scripts/fixup_utf8.php @@ -35,7 +35,7 @@ define('LACONICA', true); require_once(INSTALLDIR . '/lib/common.php'); require_once('DB.php'); -function main() { +function fixup_utf8($id) { $dbl = doConnect('latin1'); @@ -58,8 +58,16 @@ function main() { return; } - $rn = $dbl->query('SELECT id, content, rendered FROM notice ' . - 'WHERE LENGTH(content) != CHAR_LENGTH(content)'); + $sql = 'SELECT id, content, rendered FROM notice ' . + 'WHERE LENGTH(content) != CHAR_LENGTH(content)'; + + if (!empty($id)) { + $sql .= ' AND id < ' . $id; + } + + $sql .= ' ORDER BY id DESC'; + + $rn = $dbl->query($sql); if (PEAR::isError($rn)) { echo "ERROR: " . $rn->getMessage() . "\n"; @@ -128,4 +136,6 @@ function doConnect($charset) return $db; } -main(); +$id = ($argc > 1) ? $argv[1] : null; + +fixup_utf8($id);