Merge branch 'testing' of gitorious.org:statusnet/mainline into testing

* 'testing' of gitorious.org:statusnet/mainline:
  OStatus remote sending test cases. Doesn't actually run within PHPUnit right now, must be run from command line -- specify base URLs to two StatusNet sites that will be able to communicate with each other.
  Math_BigInteger doesn't correctly handle serialization/deserialization for a value of 0, which can end up spewing notices to output and otherwise intefering with Salmon signature setup and verification when using memcached.
  Log backtraces for non-ClientException exceptions caught at the top-level handler.
  Confirm there's actually user and domain portions of acct string before assigning things from output of explode(); avoids notice message when invalid input passed to main/xrd
  Fixing HTTP Header LRDD parsing (sites in subdirectories need this)
  Replace the "give up and dump object" attachment view fallback with a client-side redirect to the target URL, which will at least be useful.
  ignore unrecognized object types
  Pull <atom:author> info as well as <activity:actor> when we have an old-style ActivityStreams feed. This fixes subscription setup for Cliqset feeds, which currently have a bogus activity:actor/atom:id but a good atom:author/atom:uri
  Accept 'tag' and other non-http id URIs in Ostatus_profile::getActivityObjectProfileURI().

index.php

@@ -324,10 +324,10 @@ function main()
             $cac = new ClientErrorAction($cex->getMessage(), $cex->getCode());
             $cac->showPage();
         } catch (ServerException $sex) { // snort snort guffaw
-            $sac = new ServerErrorAction($sex->getMessage(), $sex->getCode());
+            $sac = new ServerErrorAction($sex->getMessage(), $sex->getCode(), $sex);
             $sac->showPage();
         } catch (Exception $ex) {
-            $sac = new ServerErrorAction($ex->getMessage());
+            $sac = new ServerErrorAction($ex->getMessage(), 500, $ex);
             $sac->showPage();
         }
     }

lib/activityobject.php

@@ -156,7 +156,11 @@ class ActivityObject
     {
         $this->type  = self::PERSON; // XXX: is this fair?
         $this->title = $this->_childContent($element, self::NAME);
-        $this->id    = $this->_childContent($element, self::URI);
+
+        $id = $this->_childContent($element, self::URI);
+        if (ActivityUtils::validateUri($id)) {
+            $this->id = $id;
+        }
 
         if (empty($this->id)) {
             $email = $this->_childContent($element, self::EMAIL);
@@ -169,6 +173,15 @@ class ActivityObject
 
     private function _fromAtomEntry($element)
     {
+        if ($element->localName == 'actor') {
+            // Old-fashioned <activity:actor>...
+            // First pull anything from <author>, then we'll add on top.
+            $author = ActivityUtils::child($element->parentNode, 'author');
+            if ($author) {
+                $this->_fromAuthor($author);
+            }
+        }
+
         $this->type = $this->_childContent($element, Activity::OBJECTTYPE,
                                            Activity::SPEC);
 
@@ -176,7 +189,11 @@ class ActivityObject
             $this->type = ActivityObject::NOTE;
         }
 
-        $this->id      = $this->_childContent($element, self::ID);
+        $id = $this->_childContent($element, self::ID);
+        if (ActivityUtils::validateUri($id)) {
+            $this->id = $id;
+        }
+
         $this->summary = ActivityUtils::childHtmlContent($element, self::SUMMARY);
         $this->content = ActivityUtils::getContent($element);
 

lib/activityutils.php

@@ -240,4 +240,26 @@ class ActivityUtils
             throw new ClientException(_("Can't handle embedded Base64 content yet."));
         }
     }
+
+    /**
+     * Is this a valid URI for remote profile/notice identification?
+     * Does not have to be a resolvable URL.
+     * @param string $uri
+     * @return boolean
+     */
+    static function validateUri($uri)
+    {
+        if (Validate::uri($uri)) {
+            return true;
+        }
+
+        // Possibly an upstream bug; tag: URIs aren't validated properly
+        // unless you explicitly ask for them. All other schemes are accepted
+        // for basic URI validation without asking.
+        if (Validate::uri($uri, array('allowed_scheme' => array('tag')))) {
+            return true;
+        }
+
+        return false;
+    }
 }

lib/attachmentlist.php

@@ -306,7 +306,7 @@ class Attachment extends AttachmentListItem
     function showRepresentation() {
         if (empty($this->oembed->type)) {
             if (empty($this->attachment->mimetype)) {
-                $this->out->element('pre', null, 'oh well... not sure how to handle the following: ' . print_r($this->attachment, true));
+                $this->showFallback();
             } else {
                 switch ($this->attachment->mimetype) {
                 case 'image/gif':
@@ -332,6 +332,8 @@ class Attachment extends AttachmentListItem
                     $this->out->element('param', array('name' => 'autoStart', 'value' => 1));
                     $this->out->elementEnd('object');
                     break;
+                default:
+                    $this->showFallback();
                 }
             }
         } else {
@@ -354,9 +356,23 @@ class Attachment extends AttachmentListItem
                 break;
 
             default:
-                $this->out->element('pre', null, 'oh well... not sure how to handle the following oembed: ' . print_r($this->oembed, true));
+                $this->showFallback();
             }
         }
     }
+
+    function showFallback()
+    {
+        // If we don't know how to display an attachment inline, we probably
+        // shouldn't have gotten to this point.
+        //
+        // But, here we are... displaying details on a file or remote URL
+        // either on the main view or in an ajax-loaded lightbox. As a lesser
+        // of several evils, we'll try redirecting to the actual target via
+        // client-side JS.
+
+        common_log(LOG_ERR, "Empty or unknown type for file id {$this->attachment->id}; falling back to client-side redirect.");
+        $this->out->raw('<script>window.location = ' . json_encode($this->attachment->url) . ';</script>');
+    }
 }
 

lib/servererroraction.php

@@ -62,15 +62,18 @@ class ServerErrorAction extends ErrorAction
                            504 => 'Gateway Timeout',
                            505 => 'HTTP Version Not Supported');
 
-    function __construct($message='Error', $code=500)
+    function __construct($message='Error', $code=500, $ex=null)
     {
         parent::__construct($message, $code);
 
         $this->default = 500;
 
         // Server errors must be logged.
-
+        $log = "ServerErrorAction: $code $message";
-        common_log(LOG_ERR, "ServerErrorAction: $code $message");
+        if ($ex) {
+            $log .= "\n" . $ex->getTraceAsString();
+        }
+        common_log(LOG_ERR, $log);
     }
 
     // XXX: Should these error actions even be invokable via URI?

plugins/OStatus/actions/userxrd.php

@@ -35,9 +35,13 @@ class UserxrdAction extends XrdAction
         $this->uri = Discovery::normalize($this->uri);
         
         if (Discovery::isWebfinger($this->uri)) {
-            list($nick, $domain) = explode('@', substr(urldecode($this->uri), 5));
+            $parts = explode('@', substr(urldecode($this->uri), 5));
-            $nick = common_canonical_nickname($nick);
+            if (count($parts) == 2) {
-            $this->user = User::staticGet('nickname', $nick);
+                list($nick, $domain) = $parts;
+                // @fixme confirm the domain too
+                $nick = common_canonical_nickname($nick);
+                $this->user = User::staticGet('nickname', $nick);
+            }
         } else {
             $this->user = User::staticGet('uri', $this->uri);
         }

plugins/OStatus/classes/Magicsig.php

@@ -27,8 +27,6 @@
  * @link      http://status.net/
  */
 
-require_once 'Crypt/RSA.php';
-
 class Magicsig extends Memcached_DataObject
 {
 
@@ -102,16 +100,16 @@ class Magicsig extends Memcached_DataObject
 
     public function generate($user_id)
     {
-        $rsa = new Crypt_RSA();
+        $rsa = new SafeCrypt_RSA();
         
         $keypair = $rsa->createKey();
 
         $rsa->loadKey($keypair['privatekey']);
 
-        $this->privateKey = new Crypt_RSA();
+        $this->privateKey = new SafeCrypt_RSA();
         $this->privateKey->loadKey($keypair['privatekey']);
 
-        $this->publicKey = new Crypt_RSA();
+        $this->publicKey = new SafeCrypt_RSA();
         $this->publicKey->loadKey($keypair['publickey']);
         
         $this->user_id = $user_id;
@@ -163,7 +161,7 @@ class Magicsig extends Memcached_DataObject
     {
         common_log(LOG_DEBUG, "Adding ".$type." key: (".$mod .', '. $exp .")");
 
-        $rsa = new Crypt_RSA();
+        $rsa = new SafeCrypt_RSA();
         $rsa->signatureMode = CRYPT_RSA_SIGNATURE_PKCS1;
         $rsa->setHash('sha256');
         $rsa->modulus = new Math_BigInteger(base64_url_decode($mod), 256);

plugins/OStatus/classes/Ostatus_profile.php

@@ -442,6 +442,17 @@ class Ostatus_profile extends Memcached_DataObject
     {
         $activity = new Activity($entry, $feed);
 
+        switch ($activity->object->type) {
+        case ActivityObject::ARTICLE:
+        case ActivityObject::BLOGENTRY:
+        case ActivityObject::NOTE:
+        case ActivityObject::STATUS:
+        case ActivityObject::COMMENT:
+            break;
+        default:
+            throw new ClientException("Can't handle that kind of post.");
+        }
+
         if ($activity->verb == ActivityVerb::POST) {
             $this->processPost($activity, $source);
         } else {
@@ -1140,35 +1151,45 @@ class Ostatus_profile extends Memcached_DataObject
     /**
      * @param Activity $activity
      * @return mixed matching Ostatus_profile or false if none known
+     * @throws ServerException if feed info invalid
      */
     public static function getActorProfile($activity)
     {
         return self::getActivityObjectProfile($activity->actor);
     }
 
+    /**
+     * @param ActivityObject $activity
+     * @return mixed matching Ostatus_profile or false if none known
+     * @throws ServerException if feed info invalid
+     */
     protected static function getActivityObjectProfile($object)
     {
         $uri = self::getActivityObjectProfileURI($object);
         return Ostatus_profile::staticGet('uri', $uri);
     }
 
-    protected static function getActorProfileURI($activity)
-    {
-        return self::getActivityObjectProfileURI($activity->actor);
-    }
-
     /**
-     * @param Activity $activity
+     * Get the identifier URI for the remote entity described
+     * by this ActivityObject. This URI is *not* guaranteed to be
+     * a resolvable HTTP/HTTPS URL.
+     *
+     * @param ActivityObject $object
      * @return string
-     * @throws ServerException
+     * @throws ServerException if feed info invalid
      */
     protected static function getActivityObjectProfileURI($object)
     {
-        $opts = array('allowed_schemes' => array('http', 'https'));
+        if ($object->id) {
-        if ($object->id && Validate::uri($object->id, $opts)) {
+            if (ActivityUtils::validateUri($object->id)) {
-            return $object->id;
+                return $object->id;
+            }
         }
-        if ($object->link && Validate::uri($object->link, $opts)) {
+
+        // If the id is missing or invalid (we've seen feeds mistakenly listing
+        // things like local usernames in that field) then we'll use the profile
+        // page link, if valid.
+        if ($object->link && common_valid_http_url($object->link)) {
             return $object->link;
         }
         throw new ServerException("No author ID URI found");

plugins/OStatus/lib/discovery.php

@@ -195,7 +195,7 @@ class Discovery_LRDD_Link_Header implements Discovery_LRDD
             //            return false;
         }
 
-        return Discovery_LRDD_Link_Header::parseHeader($link_header);
+        return array(Discovery_LRDD_Link_Header::parseHeader($link_header));
     }
 
     protected static function parseHeader($header)

plugins/OStatus/lib/linkheader.php

@@ -11,7 +11,7 @@ class LinkHeader
         preg_match('/^<[^>]+>/', $str, $uri_reference);
         //if (empty($uri_reference)) return;
 
-        $this->uri = trim($uri_reference[0], '<>');
+        $this->href = trim($uri_reference[0], '<>');
         $this->rel = array();
         $this->type = null;
 

plugins/OStatus/lib/safecrypt_rsa.php

@@ -0,0 +1,18 @@
+<?php
+
+require_once 'Crypt/RSA.php';
+
+/**
+ * Crypt_RSA stores a Math_BigInteger with value 0, which triggers a bug
+ * in Math_BigInteger's wakeup function which spews notices to log or output.
+ * This wrapper replaces it with a version that survives serialization.
+ */
+class SafeCrypt_RSA extends Crypt_RSA
+{
+    function __construct()
+    {
+        parent::__construct();
+        $this->zero = new SafeMath_BigInteger();
+    }
+}
+

plugins/OStatus/lib/safemath_biginteger.php

@@ -0,0 +1,20 @@
+<?php
+
+require_once 'Math/BigInteger.php';
+
+/**
+ * Crypt_RSA stores a Math_BigInteger with value 0, which triggers a bug
+ * in Math_BigInteger's wakeup function which spews notices to log or output.
+ * This wrapper replaces it with a version that survives serialization.
+ */
+class SafeMath_BigInteger extends Math_BigInteger
+{
+    function __wakeup()
+    {
+        if ($this->hex == '') {
+            $this->hex = '0';
+        }
+        parent::__wakeup();
+    }
+}
+

plugins/OStatus/tests/remote-tests.php

@@ -0,0 +1,392 @@
+<?php
+
+if (php_sapi_name() != 'cli') {
+    die('not for web');
+}
+
+define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__)))));
+set_include_path(INSTALLDIR . '/extlib' . PATH_SEPARATOR . get_include_path());
+
+require_once 'PEAR.php';
+require_once 'Net/URL2.php';
+require_once 'HTTP/Request2.php';
+
+
+// ostatus test script, client-side :)
+
+class TestBase
+{
+    function log($str)
+    {
+        $args = func_get_args();
+        array_shift($args);
+
+        $msg = vsprintf($str, $args);
+        print $msg . "\n";
+    }
+
+    function assertEqual($a, $b)
+    {
+        if ($a != $b) {
+            throw new Exception("Failed to assert equality: expected $a, got $b");
+        }
+        return true;
+    }
+
+    function assertNotEqual($a, $b)
+    {
+        if ($a == $b) {
+            throw new Exception("Failed to assert inequality: expected not $a, got $b");
+        }
+        return true;
+    }
+}
+
+class OStatusTester extends TestBase
+{
+    /**
+     * @param string $a base URL of test site A (eg http://localhost/mublog)
+     * @param string $b base URL of test site B (eg http://localhost/mublog2)
+     */
+    function __construct($a, $b) {
+        $this->a = $a;
+        $this->b = $b;
+
+        $base = 'test' . mt_rand(1, 1000000);
+        $this->pub = new SNTestClient($this->a, 'pub' . $base, 'pw-' . mt_rand(1, 1000000));
+        $this->sub = new SNTestClient($this->b, 'sub' . $base, 'pw-' . mt_rand(1, 1000000));
+    }
+
+    function run()
+    {
+        $this->setup();
+        $this->testLocalPost();
+        $this->testMentionUrl();
+        $this->log("DONE!");
+    }
+
+    function setup()
+    {
+        $this->pub->register();
+        $this->pub->assertRegistered();
+
+        $this->sub->register();
+        $this->sub->assertRegistered();
+    }
+
+    function testLocalPost()
+    {
+        $post = $this->pub->post("Local post, no subscribers yet.");
+        $this->assertNotEqual('', $post);
+
+        $post = $this->sub->post("Local post, no subscriptions yet.");
+        $this->assertNotEqual('', $post);
+    }
+
+    /**
+     * pub posts: @b/sub
+     */
+    function testMentionUrl()
+    {
+        $bits = parse_url($this->b);
+        $base = $bits['host'];
+        if (isset($bits['path'])) {
+            $base .= $bits['path'];
+        }
+        $name = $this->sub->username;
+
+        $post = $this->pub->post("@$base/$name should have this in home and replies");
+        $this->sub->assertReceived($post);
+    }
+}
+
+class SNTestClient extends TestBase
+{
+    function __construct($base, $username, $password)
+    {
+        $this->basepath = $base;
+        $this->username = $username;
+        $this->password = $password;
+
+        $this->fullname = ucfirst($username) . ' Smith';
+        $this->homepage = 'http://example.org/' . $username;
+        $this->bio = 'Stub account for OStatus tests.';
+        $this->location = 'Montreal, QC';
+    }
+
+    /**
+     * Make a low-level web hit to this site, with authentication.
+     * @param string $path URL fragment for something under the base path
+     * @param array $params POST parameters to send
+     * @param boolean $auth whether to include auth data
+     * @return string
+     * @throws Exception on low-level error conditions
+     */
+    protected function hit($path, $params=array(), $auth=false, $cookies=array())
+    {
+        $url = $this->basepath . '/' . $path;
+
+        $http = new HTTP_Request2($url, 'POST');
+        if ($auth) {
+            $http->setAuth($this->username, $this->password, HTTP_Request2::AUTH_BASIC);
+        }
+        foreach ($cookies as $name => $val) {
+            $http->addCookie($name, $val);
+        }
+        $http->addPostParameter($params);
+        $response = $http->send();
+
+        $code = $response->getStatus();
+        if ($code < '200' || $code >= '400') {
+            throw new Exception("Failed API hit to $url: $code\n" . $response->getBody());
+        }
+
+        return $response;
+    }
+
+    /**
+     * Make a hit to a web form, without authentication but with a session.
+     * @param string $path URL fragment relative to site base
+     * @param string $form id of web form to pull initial parameters from
+     * @param array $params POST parameters, will be merged with defaults in form
+     */
+    protected function web($path, $form, $params=array())
+    {
+        $url = $this->basepath . '/' . $path;
+        $http = new HTTP_Request2($url, 'GET');
+        $response = $http->send();
+
+        $dom = $this->checkWeb($url, 'GET', $response);
+        $cookies = array();
+        foreach ($response->getCookies() as $cookie) {
+            // @fixme check for expirations etc
+            $cookies[$cookie['name']] = $cookie['value'];
+        }
+
+        $form = $dom->getElementById($form);
+        if (!$form) {
+            throw new Exception("Form $form not found on $url");
+        }
+        $inputs = $form->getElementsByTagName('input');
+        foreach ($inputs as $item) {
+            $type = $item->getAttribute('type');
+            if ($type != 'check') {
+                $name = $item->getAttribute('name');
+                $val = $item->getAttribute('value');
+                if ($name && $val && !isset($params[$name])) {
+                    $params[$name] = $val;
+                }
+            }
+        }
+
+        $response = $this->hit($path, $params, false, $cookies);
+        $dom = $this->checkWeb($url, 'POST', $response);
+
+        return $dom;
+    }
+
+    protected function checkWeb($url, $method, $response)
+    {
+        $dom = new DOMDocument();
+        if (!$dom->loadHTML($response->getBody())) {
+            throw new Exception("Invalid HTML from $method to $url");
+        }
+
+        $xpath = new DOMXPath($dom);
+        $error = $xpath->query('//p[@class="error"]');
+        if ($error && $error->length) {
+            throw new Exception("Error on $method to $url: " .
+                                $error->item(0)->textContent);
+        }
+
+        return $dom;
+    }
+
+    /**
+     * Make an API hit to this site, with authentication.
+     * @param string $path URL fragment for something under 'api' folder
+     * @param string $style one of 'json', 'xml', or 'atom'
+     * @param array $params POST parameters to send
+     * @return mixed associative array for JSON, DOMDocument for XML/Atom
+     * @throws Exception on low-level error conditions
+     */
+    protected function api($path, $style, $params=array())
+    {
+        $response = $this->hit("api/$path.$style", $params, true);
+        $body = $response->getBody();
+        if ($style == 'json') {
+            $data = json_decode($body, true);
+            if ($data !== null) {
+                if (!empty($data['error'])) {
+                    throw new Exception("JSON API returned error: " . $data['error']);
+                }
+                return $data;
+            } else {
+                throw new Exception("Bogus JSON data from $path:\n$body");
+            }
+        } else if ($style == 'xml' || $style == 'atom') {
+            $dom = new DOMDocument();
+            if ($dom->loadXML($body)) {
+                return $dom;
+            } else {
+                throw new Exception("Bogus XML data from $path:\n$body");
+            }
+        } else {
+            throw new Exception("API needs to be JSON, XML, or Atom");
+        }
+    }
+
+    /**
+     * Register the account.
+     *
+     * Unfortunately there's not an API method for registering, so we fake it.
+     */
+    function register()
+    {
+        $this->log("Registering user %s on %s",
+                   $this->username,
+                   $this->basepath);
+        $ret = $this->web('main/register', 'form_register',
+            array('nickname' => $this->username,
+                  'password' => $this->password,
+                  'confirm' => $this->password,
+                  'fullname' => $this->fullname,
+                  'homepage' => $this->homepage,
+                  'bio' => $this->bio,
+                  'license' => 1,
+                  'submit' => 'Register'));
+    }
+
+    /**
+     * Check that the account has been registered and can be used.
+     * On failure, throws a test failure exception.
+     */
+    function assertRegistered()
+    {
+        $this->log("Confirming %s is registered on %s",
+                   $this->username,
+                   $this->basepath);
+        $data = $this->api('account/verify_credentials', 'json');
+        $this->assertEqual($this->username, $data['screen_name']);
+        $this->assertEqual($this->fullname, $data['name']);
+        $this->assertEqual($this->homepage, $data['url']);
+        $this->assertEqual($this->bio, $data['description']);
+    }
+
+    /**
+     * Post a given message from this account
+     * @param string $message
+     * @return string URL/URI of notice
+     * @todo reply, location options
+     */
+    function post($message)
+    {
+        $this->log("Posting notice as %s on %s: %s",
+                   $this->username,
+                   $this->basepath,
+                   $message);
+        $data = $this->api('statuses/update', 'json',
+            array('status' => $message));
+
+        $url = $this->basepath . '/notice/' . $data['id'];
+        return $url;
+    }
+
+    /**
+     * Check that this account has received the notice.
+     * @param string $notice_uri URI for the notice to check for
+     */
+    function assertReceived($notice_uri)
+    {
+        $timeout = 5;
+        $tries = 6;
+        while ($tries) {
+            $ok = $this->checkReceived($notice_uri);
+            if ($ok) {
+                return true;
+            }
+            $tries--;
+            if ($tries) {
+                $this->log("Didn't see it yet, waiting $timeout seconds");
+                sleep($timeout);
+            }
+        }
+        throw new Exception("Message $notice_uri not received by $this->username");
+    }
+
+    /**
+     * Pull the user's home timeline to check if a notice with the given
+     * source URL has been received recently.
+     * If we don't see it, we'll try a couple more times up to 10 seconds.
+     *
+     * @param string $notice_uri
+     */
+    function checkReceived($notice_uri)
+    {
+        $this->log("Checking if %s on %s received notice %s",
+                   $this->username,
+                   $this->basepath,
+                   $notice_uri);
+        $params = array();
+        $dom = $this->api('statuses/home_timeline', 'atom', $params);
+
+        $xml = simplexml_import_dom($dom);
+        if (!$xml->entry) {
+            return false;
+        }
+        if (is_array($xml->entry)) {
+            $entries = $xml->entry;
+        } else {
+            $entries = array($xml->entry);
+        }
+        foreach ($entries as $entry) {
+            if ($entry->id == $notice_uri) {
+                $this->log("found it $notice_uri");
+                return true;
+            }
+            //$this->log("nope... " . $entry->id);
+        }
+        return false;
+    }
+
+    /**
+     * Check that this account is subscribed to the given profile.
+     * @param string $profile_uri URI for the profile to check for
+     */
+    function assertHasSubscription($profile_uri)
+    {
+        throw new Exception('tbi');
+    }
+
+    /**
+     * Check that this account is subscribed to by the given profile.
+     * @param string $profile_uri URI for the profile to check for
+     */
+    function assertHasSubscriber($profile_uri)
+    {
+        throw new Exception('tbi');
+    }
+
+}
+
+$args = array_slice($_SERVER['argv'], 1);
+if (count($args) < 2) {
+    print <<<END_HELP
+remote-tests.php <url1> <url2>
+  url1: base URL of a StatusNet instance
+  url2: base URL of another StatusNet instance
+
+This will register user accounts on the two given StatusNet instances
+and run some tests to confirm that OStatus subscription and posting
+between the two sites works correctly.
+
+END_HELP;
+exit(1);
+}
+
+$a = $args[0];
+$b = $args[1];
+
+$tester = new OStatusTester($a, $b);
+$tester->run();
+