Merge branch 'testing' of gitorious.org:statusnet/mainline into 0.9.x

Conflicts:
	plugins/OpenID/openid.php

actions/apistatusesdestroy.php

@@ -58,7 +58,7 @@ require_once INSTALLDIR . '/lib/apiauth.php';
 
 class ApiStatusesDestroyAction extends ApiAuthAction
 {
-    var $status                = null;
+    var $status = null;
 
     /**
      * Take arguments for running
@@ -121,18 +121,11 @@ class ApiStatusesDestroyAction extends ApiAuthAction
              $replies->get('notice_id', $this->notice_id);
              $replies->delete();
              $this->notice->delete();
-
+	     $this->showNotice();
-             if ($this->format == 'xml') {
-                 $this->showSingleXmlStatus($this->notice);
-             } elseif ($this->format == 'json') {
-                 $this->show_single_json_status($this->notice);
-             }
          } else {
              $this->clientError(_('You may not delete another user\'s status.'),
                  403, $this->format);
          }
-
-        $this->showNotice();
     }
 
     /**

classes/Inbox.php

@@ -115,9 +115,12 @@ class Inbox extends Memcached_DataObject
      */
     static function insertNotice($user_id, $notice_id)
     {
-        $inbox = DB_DataObject::staticGet('inbox', 'user_id', $user_id);
+		// Going straight to the DB rather than trusting our caching
-
+		// during an update. Note: not using DB_DataObject::staticGet,
-        if (empty($inbox)) {
+		// which is unsafe to use directly (in-process caching causes
+		// memory leaks, which accumulate in queue processes).
+        $inbox = new Inbox();
+        if (!$inbox->get('user_id', $user_id)) {
             $inbox = Inbox::initialize($user_id);
         }
 

classes/Notice.php

@@ -1865,4 +1865,16 @@ class Notice extends Memcached_DataObject
         return $ns;
     }
 
+    /**
+     * Determine whether the notice was locally created
+     *
+     * @return boolean locality
+     */
+
+    public function isLocal()
+    {
+        return ($this->is_local == Notice::LOCAL_PUBLIC ||
+                $this->is_local == Notice::LOCAL_NONPUBLIC);
+    }
+
 }

classes/Status_network.php

@@ -149,21 +149,15 @@ class Status_network extends Safe_DataObject
         $this->decache(); # while we still have the values!
         return parent::delete();
     }
-
+    
     /**
      * @param string $servername hostname
-     * @param string $pathname URL base path
      * @param string $wildcard hostname suffix to match wildcard config
+     * @return mixed Status_network or null
      */
-    static function setupSite($servername, $pathname, $wildcard)
+    static function getFromHostname($servername, $wildcard)
     {
-        global $config;
-
         $sn = null;
-
-        // XXX I18N, probably not crucial for hostnames
-        // XXX This probably needs a tune up
-
         if (0 == strncasecmp(strrev($wildcard), strrev($servername), strlen($wildcard))) {
             // special case for exact match
             if (0 == strcasecmp($servername, $wildcard)) {
@@ -182,6 +176,23 @@ class Status_network extends Safe_DataObject
                 }
             }
         }
+        return $sn;
+    }
+
+    /**
+     * @param string $servername hostname
+     * @param string $pathname URL base path
+     * @param string $wildcard hostname suffix to match wildcard config
+     */
+    static function setupSite($servername, $pathname, $wildcard)
+    {
+        global $config;
+
+        $sn = null;
+
+        // XXX I18N, probably not crucial for hostnames
+        // XXX This probably needs a tune up
+        $sn = self::getFromHostname($servername, $wildcard);
 
         if (!empty($sn)) {
 

db/notice_source.sql

@@ -9,6 +9,7 @@ VALUES
     ('bti','bti','http://gregkh.github.com/bti/', now()),
     ('choqok', 'Choqok', 'http://choqok.gnufolks.org/', now()),
     ('cliqset', 'Cliqset', 'http://www.cliqset.com/', now()),
+    ('DarterosStatus', 'Darteros Status', 'http://www.darteros.com/doc/Darteros_Status', now()),
     ('deskbar','Deskbar-Applet','http://www.gnome.org/projects/deskbar-applet/', now()),
     ('Do','Gnome Do','http://do.davebsd.com/wiki/index.php?title=Microblog_Plugin', now()),
     ('drupal','Drupal','http://drupal.org/', now()),

lib/language.php

@@ -61,7 +61,7 @@ if (!function_exists('dpgettext')) {
      * Not currently exposed in PHP's gettext module; implemented to be compat
      * with gettext.h's macros.
      *
-     * @param string $domain domain identifier, or null for default domain
+     * @param string $domain domain identifier
      * @param string $context context identifier, should be some key like "menu|file"
      * @param string $msgid English source text
      * @return string original or translated message
@@ -106,7 +106,7 @@ if (!function_exists('dnpgettext')) {
      * Not currently exposed in PHP's gettext module; implemented to be compat
      * with gettext.h's macros.
      *
-     * @param string $domain domain identifier, or null for default domain
+     * @param string $domain domain identifier
      * @param string $context context identifier, should be some key like "menu|file"
      * @param string $msg singular English source text
      * @param string $plural plural English source text
@@ -180,7 +180,11 @@ function _m($msg/*, ...*/)
 }
 
 /**
- * Looks for which plugin we've been called from to set the gettext domain.
+ * Looks for which plugin we've been called from to set the gettext domain;
+ * if not in a plugin subdirectory, we'll use the default 'statusnet'.
+ *
+ * Note: we can't return null for default domain since most of the PHP gettext
+ * wrapper functions turn null into "" before passing to the backend library.
  *
  * @param array $backtrace debug_backtrace() output
  * @return string
@@ -208,8 +212,8 @@ function _mdomain($backtrace)
         }
         $plug = strpos($path, '/plugins/');
         if ($plug === false) {
-            // We're not in a plugin; return null for the default domain.
+            // We're not in a plugin; return default domain.
-            return null;
+            return 'statusnet';
         } else {
             $cut = $plug + 9;
             $cut2 = strpos($path, '/', $cut);
@@ -218,7 +222,7 @@ function _mdomain($backtrace)
             } else {
                 // We might be running directly from the plugins dir?
                 // If so, there's no place to store locale info.
-                return null;
+                return 'statusnet';
             }
         }
         $cached[$path] = $final;

lib/liberalstomp.php

@@ -147,5 +147,30 @@ class LiberalStomp extends Stomp
         }
         return $frame;
     }
-}
+
+    /**
+     * Write frame to server
+     *
+     * @param StompFrame $stompFrame
+     */
+    protected function _writeFrame (StompFrame $stompFrame)
+    {
+        if (!is_resource($this->_socket)) {
+            require_once 'Stomp/Exception.php';
+            throw new StompException('Socket connection hasn\'t been established');
+        }
+
+        $data = $stompFrame->__toString();
+
+        // Make sure the socket's in a writable state; if not, wait a bit.
+        stream_set_blocking($this->_socket, 1);
+
+        $r = fwrite($this->_socket, $data, strlen($data));
+        stream_set_blocking($this->_socket, 0);
+        if ($r === false || $r == 0) {
+            $this->_reconnect();
+            $this->_writeFrame($stompFrame);
+        }
+    }
+ }
 

lib/stompqueuemanager.php

@@ -115,11 +115,12 @@ class StompQueueManager extends QueueManager
      *
      * @param mixed $object
      * @param string $queue
+     * @param string $siteNickname optional override to drop into another site's queue
      *
      * @return boolean true on success
      * @throws StompException on connection or send error
      */
-    public function enqueue($object, $queue)
+    public function enqueue($object, $queue, $siteNickname=null)
     {
         $this->_connect();
         if (common_config('queue', 'stomp_enqueue_on')) {
@@ -134,7 +135,7 @@ class StompQueueManager extends QueueManager
         } else {
             $idx = $this->defaultIdx;
         }
-        return $this->_doEnqueue($object, $queue, $idx);
+        return $this->_doEnqueue($object, $queue, $idx, $siteNickname);
     }
 
     /**
@@ -144,10 +145,10 @@ class StompQueueManager extends QueueManager
      * @return boolean true on success
      * @throws StompException on connection or send error
      */
-    protected function _doEnqueue($object, $queue, $idx)
+    protected function _doEnqueue($object, $queue, $idx, $siteNickname=null)
     {
         $rep = $this->logrep($object);
-        $envelope = array('site' => common_config('site', 'nickname'),
+        $envelope = array('site' => $siteNickname ? $siteNickname : common_config('site', 'nickname'),
                           'handler' => $queue,
                           'payload' => $this->encode($object));
         $msg = serialize($envelope);

lib/util.php

@@ -1249,9 +1249,8 @@ function common_enqueue_notice($notice)
         $transports[] = 'jabber';
     }
 
-    // @fixme move these checks into QueueManager and/or individual handlers
+    // We can skip these for gatewayed notices.
-    if ($notice->is_local == Notice::LOCAL_PUBLIC ||
+    if ($notice->isLocal()) {
-        $notice->is_local == Notice::LOCAL_NONPUBLIC) {
         $transports = array_merge($transports, $localTransports);
         if ($xmpp) {
             $transports[] = 'public';

plugins/Facebook/FacebookPlugin.php

@@ -585,7 +585,7 @@ class FacebookPlugin extends Plugin
 
     function onStartEnqueueNotice($notice, &$transports)
     {
-        if (self::hasKeys()) {
+        if (self::hasKeys() && $notice->isLocal()) {
             array_push($transports, 'facebook');
         }
         return true;

plugins/Mapstraction/MapstractionPlugin.php

@@ -125,8 +125,8 @@ class MapstractionPlugin extends Plugin
             $action->script('http://tile.cloudmade.com/wml/0.2/web-maps-lite.js');
             break;
         case 'google':
-            $action->script(sprintf('http://maps.google.com/maps?file=api&v=2&sensor=false&key=%s',
+            $action->script(sprintf('http://maps.google.com/maps?file=api&v=2&sensor=false&key=%s',
-                                    $this->apikey));
+                                    urlencode($this->apikey)));
             break;
         case 'microsoft':
             $action->script('http://dev.virtualearth.net/mapcontrol/mapcontrol.ashx?v=6');
@@ -137,7 +137,7 @@ class MapstractionPlugin extends Plugin
             break;
         case 'yahoo':
             $action->script(sprintf('http://api.maps.yahoo.com/ajaxymap?v=3.8&appid=%s',
-                                    $this->apikey));
+                                    urlencode($this->apikey)));
             break;
         case 'geocommons': // don't support this yet
         default:

plugins/Mapstraction/usermap.js

@@ -104,7 +104,7 @@ function showMapstraction(element, notices) {
           pt = new mxn.LatLonPoint(lat, lon);
           mkr = new mxn.Marker(pt);
 
-          mkr.setIcon(n['user']['profile_image_url']);
+          mkr.setIcon(n['user']['profile_image_url'], [24, 24]);
           mkr.setInfoBubble('<a href="'+ n['user']['profile_url'] + '">' + n['user']['screen_name'] + '</a>' + ' ' + n['html'] +
                             '<br/><a href="'+ n['url'] + '">'+ n['created_at'] + '</a>');
 

plugins/Meteor/MeteorPlugin.php

@@ -50,6 +50,7 @@ class MeteorPlugin extends RealtimePlugin
     public $controlport   = null;
     public $controlserver = null;
     public $channelbase   = null;
+    public $persistent    = true;
     protected $_socket    = null;
 
     function __construct($webserver=null, $webport=4670, $controlport=4671, $controlserver=null, $channelbase='')
@@ -102,8 +103,14 @@ class MeteorPlugin extends RealtimePlugin
     function _connect()
     {
         $controlserver = (empty($this->controlserver)) ? $this->webserver : $this->controlserver;
+
+        $errno = $errstr = null;
+        $timeout = 5;
+        $flags = STREAM_CLIENT_CONNECT;
+        if ($this->persistent) $flags |= STREAM_CLIENT_PERSISTENT;
+
         // May throw an exception.
-        $this->_socket = stream_socket_client("tcp://{$controlserver}:{$this->controlport}");
+        $this->_socket = stream_socket_client("tcp://{$controlserver}:{$this->controlport}", $errno, $errstr, $timeout, $flags);
         if (!$this->_socket) {
             throw new Exception("Couldn't connect to {$controlserver} on {$this->controlport}");
         }
@@ -124,8 +131,10 @@ class MeteorPlugin extends RealtimePlugin
 
     function _disconnect()
     {
-        $cnt = fwrite($this->_socket, "QUIT\n");
+        if (!$this->persistent) {
-        @fclose($this->_socket);
+            $cnt = fwrite($this->_socket, "QUIT\n");
+            @fclose($this->_socket);
+        }
     }
 
     // Meteord flips out with default '/' separator

plugins/OStatus/OStatusPlugin.php

@@ -87,6 +87,8 @@ class OStatusPlugin extends Plugin
 
         // Outgoing from our internal PuSH hub
         $qm->connect('hubconf', 'HubConfQueueHandler');
+        $qm->connect('hubprep', 'HubPrepQueueHandler');
+
         $qm->connect('hubout', 'HubOutQueueHandler');
 
         // Outgoing Salmon replies (when we don't need a return value)
@@ -102,8 +104,10 @@ class OStatusPlugin extends Plugin
      */
     function onStartEnqueueNotice($notice, &$transports)
     {
-        // put our transport first, in case there's any conflict (like OMB)
+        if ($notice->isLocal()) {
-        array_unshift($transports, 'ostatus');
+            // put our transport first, in case there's any conflict (like OMB)
+            array_unshift($transports, 'ostatus');
+        }
         return true;
     }
 

plugins/OStatus/classes/HubSub.php

@@ -260,6 +260,37 @@ class HubSub extends Memcached_DataObject
             $retries = intval(common_config('ostatus', 'hub_retries'));
         }
 
+        if (common_config('ostatus', 'local_push_bypass')) {
+            // If target is a local site, bypass the web server and drop the
+            // item directly into the target's input queue.
+            $url = parse_url($this->callback);
+            $wildcard = common_config('ostatus', 'local_wildcard');
+            $site = Status_network::getFromHostname($url['host'], $wildcard);
+
+            if ($site) {
+                if ($this->secret) {
+                    $hmac = 'sha1=' . hash_hmac('sha1', $atom, $this->secret);
+                } else {
+                    $hmac = '';
+                }
+
+                // Hack: at the moment we stick the subscription ID in the callback
+                // URL so we don't have to look inside the Atom to route the subscription.
+                // For now this means we need to extract that from the target URL
+                // so we can include it in the data.
+                $parts = explode('/', $url['path']);
+                $subId = intval(array_pop($parts));
+
+                $data = array('feedsub_id' => $subId,
+                              'post' => $atom,
+                              'hmac' => $hmac);
+                common_log(LOG_DEBUG, "Cross-site PuSH bypass enqueueing straight to $site->nickname feed $subId");
+                $qm = QueueManager::get();
+                $qm->enqueue($data, 'pushin', $site->nickname);
+                return;
+            }
+        }
+
         // We dare not clone() as when the clone is discarded it'll
         // destroy the result data for the parent query.
         // @fixme use clone() again when it's safe to copy an
@@ -273,6 +304,26 @@ class HubSub extends Memcached_DataObject
         $qm->enqueue($data, 'hubout');
     }
 
+    /**
+     * Queue up a large batch of pushes to multiple subscribers
+     * for this same topic update.
+     * 
+     * If queues are disabled, this will run immediately.
+     * 
+     * @param string $atom well-formed Atom feed
+     * @param array $pushCallbacks list of callback URLs
+     */
+    function bulkDistribute($atom, $pushCallbacks)
+    {
+        $data = array('atom' => $atom,
+                      'topic' => $this->topic,
+                      'pushCallbacks' => $pushCallbacks);
+        common_log(LOG_INFO, "Queuing PuSH batch: $this->topic to " .
+                             count($pushCallbacks) . " sites");
+        $qm = QueueManager::get();
+        $qm->enqueue($data, 'hubprep');
+    }
+
     /**
      * Send a 'fat ping' to the subscriber's callback endpoint
      * containing the given Atom feed chunk.

plugins/OStatus/lib/ostatusqueuehandler.php

@@ -25,6 +25,18 @@
  */
 class OStatusQueueHandler extends QueueHandler
 {
+    // If we have more than this many subscribing sites on a single feed,
+    // break up the PuSH distribution into smaller batches which will be
+    // rolled into the queue progressively. This reduces disruption to
+    // other, shorter activities being enqueued while we work.
+    const MAX_UNBATCHED = 50;
+
+    // Each batch (a 'hubprep' entry) will have this many items.
+    // Selected to provide a balance between queue packet size
+    // and number of batches that will end up getting processed.
+    // For 20,000 target sites, 1000 should work acceptably.
+    const BATCH_SIZE = 1000;
+
     function transport()
     {
         return 'ostatus';
@@ -147,14 +159,31 @@ class OStatusQueueHandler extends QueueHandler
 
     /**
      * Queue up direct feed update pushes to subscribers on our internal hub.
+     * If there are a large number of subscriber sites, intermediate bulk
+     * distribution triggers may be queued.
+     * 
      * @param string $atom update feed, containing only new/changed items
      * @param HubSub $sub open query of subscribers
      */
     function pushFeedInternal($atom, $sub)
     {
         common_log(LOG_INFO, "Preparing $sub->N PuSH distribution(s) for $sub->topic");
+        $n = 0;
+        $batch = array();
         while ($sub->fetch()) {
-            $sub->distribute($atom);
+            $n++;
+            if ($n < self::MAX_UNBATCHED) {
+                $sub->distribute($atom);
+            } else {
+                $batch[] = $sub->callback;
+                if (count($batch) >= self::BATCH_SIZE) {
+                    $sub->bulkDistribute($atom, $batch);
+                    $batch = array();
+                }
+            }
+        }
+        if (count($batch) >= 0) {
+            $sub->bulkDistribute($atom, $batch);
         }
     }
 

plugins/OpenID/openid.php

@@ -145,9 +145,11 @@ function oid_authenticate($openid_url, $returnto, $immediate=false)
 
     // Handle failure status return values.
     if (!$auth_request) {
+        common_log(LOG_ERR, __METHOD__ . ": mystery fail contacting $openid_url");
         // TRANS: OpenID plugin message. Given when an OpenID is not valid.
         return _m('Not a valid OpenID.');
     } else if (Auth_OpenID::isFailure($auth_request)) {
+        common_log(LOG_ERR, __METHOD__ . ": OpenID fail to $openid_url: $auth_request->message");
         // TRANS: OpenID plugin server error. Given when the OpenID authentication request fails.
         // TRANS: %s is the failure message.
         return sprintf(_m('OpenID failure: %s'), $auth_request->message);

plugins/OpenID/openidadminpanel.php

@@ -91,6 +91,7 @@ class OpenidadminpanelAction extends AdminPanelAction
         );
 
         static $booleans = array(
+            'openid' => array('append_username'),
             'site' => array('openidonly')
         );
 
@@ -222,6 +223,15 @@ class OpenIDAdminPanelForm extends AdminForm
         );
         $this->unli();
 
+        $this->li();
+        $this->out->checkbox(
+            'append_username', _m('Append a username to base URL'),
+            (bool) $this->value('append_username', 'openid'),
+            _m('Login form will show the base URL and prompt for a username to add at the end. Use when OpenID provider URL should be the profile page for individual users.'),
+            'true'
+        );
+        $this->unli();
+
         $this->li();
         $this->input(
             'required_team',

plugins/OpenID/openidlogin.php

@@ -33,6 +33,9 @@ class OpenidloginAction extends Action
             $provider = common_config('openid', 'trusted_provider');
             if ($provider) {
                 $openid_url = $provider;
+                if (common_config('openid', 'append_username')) {
+                    $openid_url .= $this->trimmed('openid_username');
+                }
             } else {
                 $openid_url = $this->trimmed('openid_url');
             }
@@ -100,7 +103,15 @@ class OpenidloginAction extends Action
     function showScripts()
     {
         parent::showScripts();
-        $this->autofocus('openid_url');
+        if (common_config('openid', 'trusted_provider')) {
+            if (common_config('openid', 'append_username')) {
+                $this->autofocus('openid_username');
+            } else {
+                $this->autofocus('rememberme');
+            }
+        } else {
+            $this->autofocus('openid_url');
+        }
     }
 
     function title()
@@ -130,10 +141,17 @@ class OpenidloginAction extends Action
         $this->elementStart('ul', 'form_data');
         $this->elementStart('li');
         $provider = common_config('openid', 'trusted_provider');
+        $appendUsername = common_config('openid', 'append_username');
         if ($provider) {
             $this->element('label', array(), _m('OpenID provider'));
             $this->element('span', array(), $provider);
+            if ($appendUsername) {
+                $this->element('input', array('id' => 'openid_username',
+                                              'name' => 'openid_username',
+                                              'style' => 'float: none'));
+            }
             $this->element('p', 'form_guide',
+                           ($appendUsername ? _m('Enter your username.') . ' ' : '') .
                            _m('You will be sent to the provider\'s site for authentication.'));
             $this->hidden('openid_url', $provider);
         } else {

plugins/RSSCloud/RSSCloudPlugin.php

@@ -192,24 +192,12 @@ class RSSCloudPlugin extends Plugin
 
     function onStartEnqueueNotice($notice, &$transports)
     {
-        array_push($transports, 'rsscloud');
+        if ($notice->isLocal()) {
+            array_push($transports, 'rsscloud');
+        }
         return true;
     }
 
-    /**
-     * Determine whether the notice was locally created
-     *
-     * @param Notice $notice the notice in question
-     *
-     * @return boolean locality
-     */
-
-    function _isLocal($notice)
-    {
-        return ($notice->is_local == Notice::LOCAL_PUBLIC ||
-                $notice->is_local == Notice::LOCAL_NONPUBLIC);
-    }
-
     /**
      * Create the rsscloud_subscription table if it's not
      * already in the DB

plugins/TwitterBridge/TwitterBridgePlugin.php

@@ -221,7 +221,7 @@ class TwitterBridgePlugin extends Plugin
      */
     function onStartEnqueueNotice($notice, &$transports)
     {
-        if (self::hasKeys()) {
+        if (self::hasKeys() && $notice->isLocal()) {
             // Avoid a possible loop
             if ($notice->source != 'twitter') {
                 array_push($transports, 'twitter');