From c201baffbfbf812ecba504e6829dd9e9d17a4bac Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Tue, 16 Feb 2010 06:12:08 +0000 Subject: [PATCH 1/4] Upgrade Twitter bridge to use OAuth 1.0a. It's more secure, and allows us to automatically send in a callback url instead of having to manually configure one for each StatusNet instance. --- lib/oauthclient.php | 88 ++++++++++++++----- .../TwitterBridge/twitterauthorization.php | 10 +-- plugins/TwitterBridge/twitteroauthclient.php | 28 ++++++ 3 files changed, 98 insertions(+), 28 deletions(-) diff --git a/lib/oauthclient.php b/lib/oauthclient.php index b22fd78974..bc7587183b 100644 --- a/lib/oauthclient.php +++ b/lib/oauthclient.php @@ -90,20 +90,47 @@ class OAuthClient /** * Gets a request token from the given url * - * @param string $url OAuth endpoint for grabbing request tokens + * @param string $url OAuth endpoint for grabbing request tokens + * @param string $callback authorized request token callback * * @return OAuthToken $token the request token */ - function getRequestToken($url) + function getRequestToken($url, $callback = null) { - $response = $this->oAuthGet($url); + $params = null; + + if (!is_null($callback)) { + $params['oauth_callback'] = $callback; + } + + $response = $this->oAuthGet($url, $params); + $arr = array(); parse_str($response, $arr); - if (isset($arr['oauth_token']) && isset($arr['oauth_token_secret'])) { - $token = new OAuthToken($arr['oauth_token'], @$arr['oauth_token_secret']); + + $token = $arr['oauth_token']; + $secret = $arr['oauth_token_secret']; + $confirm = $arr['oauth_callback_confirmed']; + + if (isset($token) && isset($secret)) { + + $token = new OAuthToken($token, $secret); + + if (isset($confirm)) { + if ($confirm == 'true') { + common_debug('Twitter bridge - callback confirmed.'); + return $token; + } else { + throw new OAuthClientException( + 'Callback was not confirmed by Twitter.' + ); + } + } return $token; } else { - throw new OAuthClientException(); + throw new OAuthClientException( + 'Could not get a request token from Twitter.' + ); } } @@ -113,49 +140,64 @@ class OAuthClient * * @param string $url endpoint for authorizing request tokens * @param OAuthToken $request_token the request token to be authorized - * @param string $oauth_callback optional callback url * * @return string $authorize_url the url to redirect to */ - function getAuthorizeLink($url, $request_token, $oauth_callback = null) + function getAuthorizeLink($url, $request_token) { $authorize_url = $url . '?oauth_token=' . $request_token->key; - if (isset($oauth_callback)) { - $authorize_url .= '&oauth_callback=' . urlencode($oauth_callback); - } - return $authorize_url; } /** * Fetches an access token * - * @param string $url OAuth endpoint for exchanging authorized request tokens - * for access tokens + * @param string $url OAuth endpoint for exchanging authorized request tokens + * for access tokens + * @param string $verifier 1.0a verifier * * @return OAuthToken $token the access token */ - function getAccessToken($url) + function getAccessToken($url, $verifier = null) { - $response = $this->oAuthPost($url); - parse_str($response); - $token = new OAuthToken($oauth_token, $oauth_token_secret); - return $token; + $params = array(); + + if (!is_null($verifier)) { + $params['oauth_verifier'] = $verifier; + } + + $response = $this->oAuthPost($url, $params); + + $arr = array(); + parse_str($response, $arr); + + $token = $arr['oauth_token']; + $secret = $arr['oauth_token_secret']; + + if (isset($token) && isset($secret)) { + $token = new OAuthToken($token, $secret); + return $token; + } else { + throw new OAuthClientException( + 'Could not get a access token from Twitter.' + ); + } } /** - * Use HTTP GET to make a signed OAuth request + * Use HTTP GET to make a signed OAuth requesta * - * @param string $url OAuth endpoint + * @param string $url OAuth request token endpoint + * @param array $params additional parameters * * @return mixed the request */ - function oAuthGet($url) + function oAuthGet($url, $params = null) { $request = OAuthRequest::from_consumer_and_token($this->consumer, - $this->token, 'GET', $url, null); + $this->token, 'GET', $url, $params); $request->sign_request($this->sha1_method, $this->consumer, $this->token); diff --git a/plugins/TwitterBridge/twitterauthorization.php b/plugins/TwitterBridge/twitterauthorization.php index 6822d33dd1..c154932bbc 100644 --- a/plugins/TwitterBridge/twitterauthorization.php +++ b/plugins/TwitterBridge/twitterauthorization.php @@ -56,6 +56,7 @@ class TwitterauthorizationAction extends Action var $tw_fields = null; var $access_token = null; var $signin = null; + var $verifier = null; /** * Initialize class members. Looks for 'oauth_token' parameter. @@ -70,6 +71,7 @@ class TwitterauthorizationAction extends Action $this->signin = $this->boolean('signin'); $this->oauth_token = $this->arg('oauth_token'); + $this->verifier = $this->arg('oauth_verifier'); return true; } @@ -160,8 +162,7 @@ class TwitterauthorizationAction extends Action // Get a new request token and authorize it $client = new TwitterOAuthClient(); - $req_tok = - $client->getRequestToken(TwitterOAuthClient::$requestTokenURL); + $req_tok = $client->getRequestToken(); // Sock the request token away in the session temporarily @@ -171,7 +172,7 @@ class TwitterauthorizationAction extends Action $auth_link = $client->getAuthorizeLink($req_tok, $this->signin); } catch (OAuthClientException $e) { - $msg = sprintf('OAuth client cURL error - code: %1s, msg: %2s', + $msg = sprintf('OAuth client error - code: %1s, msg: %2s', $e->getCode(), $e->getMessage()); $this->serverError(_m('Couldn\'t link your Twitter account.')); } @@ -187,7 +188,6 @@ class TwitterauthorizationAction extends Action */ function saveAccessToken() { - // Check to make sure Twitter returned the same request // token we sent them @@ -204,7 +204,7 @@ class TwitterauthorizationAction extends Action // Exchange the request token for an access token - $atok = $client->getAccessToken(TwitterOAuthClient::$accessTokenURL); + $atok = $client->getAccessToken($this->verifier); // Test the access token and get the user's Twitter info diff --git a/plugins/TwitterBridge/twitteroauthclient.php b/plugins/TwitterBridge/twitteroauthclient.php index 277e7ab409..ba45b533dc 100644 --- a/plugins/TwitterBridge/twitteroauthclient.php +++ b/plugins/TwitterBridge/twitteroauthclient.php @@ -91,6 +91,19 @@ class TwitterOAuthClient extends OAuthClient } } + /** + * Gets a request token from Twitter + * + * @return OAuthToken $token the request token + */ + function getRequestToken() + { + return parent::getRequestToken( + self::$requestTokenURL, + common_local_url('twitterauthorization') + ); + } + /** * Builds a link to Twitter's endpoint for authorizing a request token * @@ -107,6 +120,21 @@ class TwitterOAuthClient extends OAuthClient common_local_url('twitterauthorization')); } + /** + * Fetches an access token from Twitter + * + * @param string $verifier 1.0a verifier + * + * @return OAuthToken $token the access token + */ + function getAccessToken($verifier = null) + { + return parent::getAccessToken( + self::$accessTokenURL, + $verifier + ); + } + /** * Calls Twitter's /account/verify_credentials API method * From 05c50499c31b80d2a1c41e5256fae96cc06252ad Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Wed, 17 Feb 2010 19:24:38 +0000 Subject: [PATCH 2/4] Better logging for Twitter bridge account linking process --- .../TwitterBridge/twitterauthorization.php | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/plugins/TwitterBridge/twitterauthorization.php b/plugins/TwitterBridge/twitterauthorization.php index c154932bbc..8bfdacee91 100644 --- a/plugins/TwitterBridge/twitterauthorization.php +++ b/plugins/TwitterBridge/twitterauthorization.php @@ -131,8 +131,7 @@ class TwitterauthorizationAction extends Action } else if ($this->arg('connect')) { $this->connectNewUser(); } else { - common_debug('Twitter Connect Plugin - ' . - print_r($this->args, true)); + common_debug('Twitter bridge - ' . print_r($this->args, true)); $this->showForm(_('Something weird happened.'), $this->trimmed('newname')); } @@ -172,9 +171,15 @@ class TwitterauthorizationAction extends Action $auth_link = $client->getAuthorizeLink($req_tok, $this->signin); } catch (OAuthClientException $e) { - $msg = sprintf('OAuth client error - code: %1s, msg: %2s', - $e->getCode(), $e->getMessage()); - $this->serverError(_m('Couldn\'t link your Twitter account.')); + $msg = sprintf( + 'OAuth client error - code: %1s, msg: %2s', + $e->getCode(), + $e->getMessage() + ); + common_log(LOG_INFO, 'Twitter bridge - ' . $msg); + $this->serverError( + _m('Couldn\'t link your Twitter account: ') . $e->getMessage() + ); } common_redirect($auth_link); @@ -192,7 +197,9 @@ class TwitterauthorizationAction extends Action // token we sent them if ($_SESSION['twitter_request_token'] != $this->oauth_token) { - $this->serverError(_m('Couldn\'t link your Twitter account.')); + $this->serverError( + _m('Couldn\'t link your Twitter account: oauth_token mismatch.') + ); } $twitter_user = null; @@ -212,9 +219,15 @@ class TwitterauthorizationAction extends Action $twitter_user = $client->verifyCredentials(); } catch (OAuthClientException $e) { - $msg = sprintf('OAuth client error - code: %1$s, msg: %2$s', - $e->getCode(), $e->getMessage()); - $this->serverError(_m('Couldn\'t link your Twitter account.')); + $msg = sprintf( + 'OAuth client error - code: %1$s, msg: %2$s', + $e->getCode(), + $e->getMessage() + ); + common_log(LOG_INFO, 'Twitter bridge - ' . $msg); + $this->serverError( + _m('Couldn\'t link your Twitter account: ') . $e-getMessage() + ); } if (common_logged_in()) { From 73ba26efe3d9d97c478a507d351ac92d28d82655 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Wed, 17 Feb 2010 20:53:16 +0000 Subject: [PATCH 3/4] Twitter bridge - fix for Ticket #2192 --- plugins/TwitterBridge/twitter.php | 14 +++++++++----- plugins/TwitterBridge/twitterauthorization.php | 6 +++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/plugins/TwitterBridge/twitter.php b/plugins/TwitterBridge/twitter.php index e5afde62ca..ceb83b037f 100644 --- a/plugins/TwitterBridge/twitter.php +++ b/plugins/TwitterBridge/twitter.php @@ -1,7 +1,7 @@ delete(); - if ($result != false) { - common_log(LOG_INFO, - "Twitter bridge - removed old Twitter user: $screen_name ($twitter_id)."); + if (!empty($luser)) { + $result = $luser->delete(); + if ($result != false) { + common_log( + LOG_INFO, + "Twitter bridge - removed old Twitter user: $screen_name ($twitter_id)." + ); + } } $fuser = new Foreign_user(); diff --git a/plugins/TwitterBridge/twitterauthorization.php b/plugins/TwitterBridge/twitterauthorization.php index 8bfdacee91..cabf69d7a8 100644 --- a/plugins/TwitterBridge/twitterauthorization.php +++ b/plugins/TwitterBridge/twitterauthorization.php @@ -178,7 +178,7 @@ class TwitterauthorizationAction extends Action ); common_log(LOG_INFO, 'Twitter bridge - ' . $msg); $this->serverError( - _m('Couldn\'t link your Twitter account: ') . $e->getMessage() + _m('Couldn\'t link your Twitter account.') ); } @@ -226,7 +226,7 @@ class TwitterauthorizationAction extends Action ); common_log(LOG_INFO, 'Twitter bridge - ' . $msg); $this->serverError( - _m('Couldn\'t link your Twitter account: ') . $e-getMessage() + _m('Couldn\'t link your Twitter account.') ); } @@ -292,7 +292,7 @@ class TwitterauthorizationAction extends Action if (empty($flink_id)) { common_log_db_error($flink, 'INSERT', __FILE__); - $this->serverError(_('Couldn\'t link your Twitter account.')); + $this->serverError(_('Couldn\'t link your Twitter account.')); } return $flink_id; From ce6be4f83624d8c39a93d2b54567cc2f33580812 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Wed, 17 Feb 2010 16:49:00 -0800 Subject: [PATCH 4/4] Queues: redid the breakout control model so we can start up and subscribe to queues without running through the complete site list, which is ok at 1k sites but too slow at 10k. All breakout queues that we're going to need to listen to now need to be explicitly listed in $config['queue']['breakout']. Until XMPP is moved to component model, this setting will let the individual processes work with their own queues: $config['queue']['breakout'][] = 'xmpp/xmppout/' . $config['site']['nickname']; --- lib/default.php | 11 ++-- lib/iomaster.php | 23 +++----- lib/stompqueuemanager.php | 117 ++++++++++++++++++++------------------ scripts/queuedaemon.php | 3 +- 4 files changed, 76 insertions(+), 78 deletions(-) diff --git a/lib/default.php b/lib/default.php index a74cccae12..c969c3b337 100644 --- a/lib/default.php +++ b/lib/default.php @@ -91,10 +91,13 @@ $default = 'spawndelay' => 1, // Wait at least N seconds between (re)spawns of child processes to avoid slamming the queue server with subscription startup 'debug_memory' => false, // true to spit memory usage to log 'inboxes' => true, // true to do inbox distribution & output queueing from in background via 'distrib' queue - 'breakout' => array('*' => 'shared'), // set global or per-handler queue breakout - // 'shared': use a shared queue for all sites - // 'handler': share each/this handler over multiple sites - // 'site': break out for each/this handler on this site + 'breakout' => array(), // List queue specifiers to break out when using Stomp queue. + // Default will share all queues for all sites within each group. + // Specify as / or //, + // using nickname identifier as site. + // + // 'main/distrib' separate "distrib" queue covering all sites + // 'xmpp/xmppout/mysite' separate "xmppout" queue covering just 'mysite' 'max_retries' => 10, // drop messages after N failed attempts to process (Stomp) 'dead_letter_dir' => false, // set to directory to save dropped messages into (Stomp) ), diff --git a/lib/iomaster.php b/lib/iomaster.php index 54e2dfe841..d20837ba54 100644 --- a/lib/iomaster.php +++ b/lib/iomaster.php @@ -55,27 +55,18 @@ abstract class IoMaster if ($multiSite !== null) { $this->multiSite = $multiSite; } - if ($this->multiSite) { - $this->sites = StatusNet::findAllSites(); - } else { - $this->sites = array(StatusNet::currentSite()); - } - if (empty($this->sites)) { - throw new Exception("Empty status_network table, cannot init"); - } - - foreach ($this->sites as $site) { - StatusNet::switchSite($site); - $this->initManagers(); - } + $this->initManagers(); } /** - * Initialize IoManagers for the currently configured site - * which are appropriate to this instance. + * Initialize IoManagers which are appropriate to this instance; + * pass class names or instances into $this->instantiate(). * - * Pass class names into $this->instantiate() + * If setup and configuration may vary between sites in multi-site + * mode, it's the subclass's responsibility to set them up here. + * + * Switching site configurations is an acceptable side effect. */ abstract function initManagers(); diff --git a/lib/stompqueuemanager.php b/lib/stompqueuemanager.php index bfeeb23b7f..9af8b2f482 100644 --- a/lib/stompqueuemanager.php +++ b/lib/stompqueuemanager.php @@ -63,7 +63,7 @@ class StompQueueManager extends QueueManager $this->password = common_config('queue', 'stomp_password'); $this->base = common_config('queue', 'queue_basename'); $this->control = common_config('queue', 'control_channel'); - $this->subscriptions = array($this->control => $this->control); + $this->breakout = common_config('queue', 'breakout'); } /** @@ -75,28 +75,6 @@ class StompQueueManager extends QueueManager return IoManager::INSTANCE_PER_PROCESS; } - /** - * Record queue subscriptions we'll need to handle the current site. - */ - public function addSite() - { - $this->sites[] = StatusNet::currentSite(); - - // Set up handlers active for this site... - $this->initialize(); - - foreach ($this->activeGroups as $group) { - if (isset($this->groups[$group])) { - // Actual queues may be broken out or consolidated... - // Subscribe to all the target queues we'll need. - foreach ($this->groups[$group] as $transport => $class) { - $target = $this->queueName($transport); - $this->subscriptions[$target] = $target; - } - } - } - } - /** * Optional; ping any running queue handler daemons with a notification * such as announcing a new site to handle or requesting clean shutdown. @@ -166,14 +144,15 @@ class StompQueueManager extends QueueManager $con = $this->cons[$idx]; $host = $con->getServer(); - $result = $con->send($this->queueName($queue), $msg, $props); + $target = $this->queueName($queue); + $result = $con->send($target, $msg, $props); if (!$result) { - $this->_log(LOG_ERR, "Error sending $rep to $queue queue on $host"); + $this->_log(LOG_ERR, "Error sending $rep to $queue queue on $host $target"); return false; } - $this->_log(LOG_DEBUG, "complete remote queueing $rep for $queue on $host"); + $this->_log(LOG_DEBUG, "complete remote queueing $rep for $queue on $host $target"); $this->stats('enqueued', $queue); return true; } @@ -432,11 +411,42 @@ class StompQueueManager extends QueueManager protected function doSubscribe(LiberalStomp $con) { $host = $con->getServer(); - foreach ($this->subscriptions as $queue) { - $this->_log(LOG_INFO, "Subscribing to $queue on $host"); - $con->subscribe($queue); + foreach ($this->subscriptions() as $sub) { + $this->_log(LOG_INFO, "Subscribing to $sub on $host"); + $con->subscribe($sub); } } + + /** + * Grab a full list of stomp-side queue subscriptions. + * Will include: + * - control broadcast channel + * - shared group queues for active groups + * - per-handler and per-site breakouts from $config['queue']['breakout'] + * that are rooted in the active groups. + * + * @return array of strings + */ + protected function subscriptions() + { + $subs = array(); + $subs[] = $this->control; + + foreach ($this->activeGroups as $group) { + $subs[] = $this->base . $group; + } + + foreach ($this->breakout as $spec) { + $parts = explode('/', $spec); + if (count($parts) < 2 || count($parts) > 3) { + common_log(LOG_ERR, "Bad queue breakout specifier $spec"); + } + if (in_array($parts[0], $this->activeGroups)) { + $subs[] = $this->base . $spec; + } + } + return array_unique($subs); + } /** * Handle and acknowledge an event that's come in through a queue. @@ -612,32 +622,26 @@ class StompQueueManager extends QueueManager } /** - * Set us up with queue subscriptions for a new site added at runtime, + * (Re)load runtime configuration for a given site by nickname, * triggered by a broadcast to the 'statusnet-control' topic. * + * Configuration changes in database should update, but config + * files might not. + * * @param array $frame Stomp frame * @return bool true to continue; false to stop further processing. */ protected function updateSiteConfig($nickname) { - if (empty($this->sites)) { - if ($nickname == common_config('site', 'nickname')) { - StatusNet::init(common_config('site', 'server')); - } else { - $this->_log(LOG_INFO, "Ignoring update ping for other site $nickname"); + $sn = Status_network::staticGet($nickname); + if ($sn) { + $this->switchSite($nickname); + if (!in_array($nickname, $this->sites)) { + $this->addSite(); } + $this->stats('siteupdate'); } else { - $sn = Status_network::staticGet($nickname); - if ($sn) { - $this->switchSite($nickname); - if (!in_array($nickname, $this->sites)) { - $this->addSite(); - } - // @fixme update subscriptions, if applicable - $this->stats('siteupdate'); - } else { - $this->_log(LOG_ERR, "Ignoring ping for unrecognized new site $nickname"); - } + $this->_log(LOG_ERR, "Ignoring ping for unrecognized new site $nickname"); } } @@ -646,24 +650,25 @@ class StompQueueManager extends QueueManager * group name for this queue to give eg: * * /queue/statusnet/main + * /queue/statusnet/main/distrib + * /queue/statusnet/xmpp/xmppout/site01 * * @param string $queue * @return string */ protected function queueName($queue) { - $base = common_config('queue', 'queue_basename'); $group = $this->queueGroup($queue); - $breakout = $this->breakoutMode($queue); - if ($breakout == 'shared') { - return $base . "$group"; - } else if ($breakout == 'handler') { - return $base . "$group/$queue"; - } else if ($breakout == 'site') { - $site = StatusNet::currentSite(); - return $base . "$group/$queue/$site"; + $site = StatusNet::currentSite(); + + $specs = array("$group/$queue/$site", + "$group/$queue"); + foreach ($specs as $spec) { + if (in_array($spec, $this->breakout)) { + return $this->base . $spec; + } } - throw Exception("Unrecognized queue breakout mode '$breakout' for '$queue'"); + return $this->base . $group; } /** diff --git a/scripts/queuedaemon.php b/scripts/queuedaemon.php index d372d898fa..6dba16f953 100755 --- a/scripts/queuedaemon.php +++ b/scripts/queuedaemon.php @@ -126,8 +126,7 @@ class QueueDaemon extends SpawningDaemon class QueueMaster extends IoMaster { /** - * Initialize IoManagers for the currently configured site - * which are appropriate to this instance. + * Initialize IoManagers which are appropriate to this instance. */ function initManagers() {