From 324fada5ea80ece851a2481127a5cbd5732e171f Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 20 Apr 2010 18:49:07 +0200
Subject: [PATCH 01/35] initial work on yammer importer code

---
 plugins/YammerImport/YammerImportPlugin.php |  77 ++++++++++
 plugins/YammerImport/yamdump.php            | 157 ++++++++++++++++++++
 2 files changed, 234 insertions(+)
 create mode 100644 plugins/YammerImport/YammerImportPlugin.php
 create mode 100644 plugins/YammerImport/yamdump.php

diff --git a/plugins/YammerImport/YammerImportPlugin.php b/plugins/YammerImport/YammerImportPlugin.php
new file mode 100644
index 0000000000..02923493fb
--- /dev/null
+++ b/plugins/YammerImport/YammerImportPlugin.php
@@ -0,0 +1,77 @@
+<?php
+/*
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @package YammerImportPlugin
+ * @maintainer Brion Vibber <brion@status.net>
+ */
+
+if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); }
+
+set_include_path(get_include_path() . PATH_SEPARATOR . dirname(__FILE__) . '/extlib/');
+
+class YammerImportPlugin extends Plugin
+{
+    /**
+     * Hook for RouterInitialized event.
+     *
+     * @param Net_URL_Mapper $m path-to-action mapper
+     * @return boolean hook return
+     */
+    function onRouterInitialized($m)
+    {
+        $m->connect('admin/import/yammer',
+                    array('action' => 'importyammer'));
+        return true;
+    }
+
+    /**
+     * Set up queue handlers for import processing
+     * @param QueueManager $qm
+     * @return boolean hook return
+     */
+    function onEndInitializeQueueManager(QueueManager $qm)
+    {
+        $qm->connect('importym', 'ImportYmQueueHandler');
+
+        return true;
+    }
+
+    /**
+     * Automatically load the actions and libraries used by the plugin
+     *
+     * @param Class $cls the class
+     *
+     * @return boolean hook return
+     *
+     */
+    function onAutoload($cls)
+    {
+        $base = dirname(__FILE__);
+        $lower = strtolower($cls);
+        switch ($cls) {
+        case 'yammerimqueuehandler':
+        case 'importyammeraction':
+            require_once $base . $lower . '.php';
+            return false;
+        default:
+            return true;
+        }
+    }
+}
diff --git a/plugins/YammerImport/yamdump.php b/plugins/YammerImport/yamdump.php
new file mode 100644
index 0000000000..60be81ca5d
--- /dev/null
+++ b/plugins/YammerImport/yamdump.php
@@ -0,0 +1,157 @@
+<?php
+
+if (php_sapi_name() != 'cli') {
+    die('no');
+}
+
+define('INSTALLDIR', dirname(dirname(dirname(__FILE__))));
+
+require INSTALLDIR . "/scripts/commandline.inc";
+
+class SN_YammerClient
+{
+    protected $apiBase = "https://www.yammer.com";
+    protected $consumerKey, $consumerSecret;
+    protected $token, $tokenSecret;
+
+    public function __construct($consumerKey, $consumerSecret, $token=null, $tokenSecret=null)
+    {
+        $this->consumerKey = $consumerKey;
+        $this->consumerSecret = $consumerSecret;
+        $this->token = $token;
+        $this->tokenSecret = $tokenSecret;
+    }
+
+    /**
+     * Make an HTTP hit with OAuth headers and return the response body on success.
+     *
+     * @param string $path URL chunk for the API method
+     * @param array $params
+     * @return array
+     *
+     * @throws Exception for HTTP error
+     */
+    protected function fetch($path, $params=array())
+    {
+        $url = $this->apiBase . '/' . $path;
+        if ($params) {
+            $url .= '?' . http_build_query($params, null, '&');
+        }
+        $headers = array('Authorization: ' . $this->authHeader());
+        var_dump($headers);
+
+        $client = HTTPClient::start();
+        $response = $client->get($url, $headers);
+
+        if ($response->isOk()) {
+            return $response->getBody();
+        } else {
+            throw new Exception("Yammer API returned HTTP code " . $response->getStatus() . ': ' . $response->getBody());
+        }
+    }
+
+    /**
+     * Hit the main Yammer API point and decode returned JSON data.
+     *
+     * @param string $method
+     * @param array $params
+     * @return array from JSON data
+     *
+     * @throws Exception for HTTP error or bad JSON return
+     */
+    protected function api($method, $params=array())
+    {
+        $body = $this->fetch("api/v1/$method.json", $params);
+        $data = json_decode($body, true);
+        if (!$data) {
+            throw new Exception("Invalid JSON response from Yammer API");
+        }
+        return $data;
+    }
+
+    /**
+     * Build an Authorization header value from the keys we have available.
+     */
+    protected function authHeader()
+    {
+        // token
+        // token_secret
+        $params = array('realm' => '',
+                        'oauth_consumer_key' => $this->consumerKey,
+                        'oauth_signature_method' => 'PLAINTEXT',
+                        'oauth_timestamp' => time(),
+                        'oauth_nonce' => time(),
+                        'oauth_version' => '1.0');
+        if ($this->token) {
+            $params['oauth_token'] = $this->token;
+        }
+        if ($this->tokenSecret) {
+            $params['oauth_signature'] = $this->consumerSecret . '&' . $this->tokenSecret;
+        } else {
+            $params['oauth_signature'] = $this->consumerSecret . '&';
+        }
+        if ($this->verifier) {
+            $params['oauth_verifier'] = $this->verifier;
+        }
+        $parts = array_map(array($this, 'authHeaderChunk'), array_keys($params), array_values($params));
+        return 'OAuth ' . implode(', ', $parts);
+    }
+
+    /**
+     * @param string $key
+     * @param string $val
+     */
+    protected function authHeaderChunk($key, $val)
+    {
+        return urlencode($key) . '="' . urlencode($val) . '"';
+    }
+
+    /**
+     * @return array of oauth return data; should contain nice things
+     */
+    public function requestToken()
+    {
+        if ($this->token || $this->tokenSecret) {
+            throw new Exception("Requesting a token, but already set up with a token");
+        }
+        $data = $this->fetch('oauth/request_token');
+        $arr = array();
+        parse_str($data, $arr);
+        return $arr;
+    }
+
+    /**
+     * @return array of oauth return data; should contain nice things
+     */
+    public function accessToken($verifier)
+    {
+        $this->verifier = $verifier;
+        $data = $this->fetch('oauth/access_token');
+        $this->verifier = null;
+        $arr = array();
+        parse_str($data, $arr);
+        return $arr;
+    }
+
+    /**
+     * Give the URL to send users to to authorize a new app setup
+     *
+     * @param string $token as returned from accessToken()
+     * @return string URL
+     */
+    public function authorizeUrl($token)
+    {
+        return $this->apiBase . '/oauth/authorize?oauth_token=' . urlencode($token);
+    }
+
+    public function messages($params)
+    {
+        return $this->api('messages', $params);
+    }
+}
+
+
+// temp stuff
+require 'yam-config.php';
+$yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
+var_dump($yam->messages());
\ No newline at end of file

From 025184ce75b1e6a16fe6facfcd52f3da3a121c3d Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 13:29:44 -0700
Subject: [PATCH 02/35] Split SN_YammerClient out to own class file

---
 plugins/YammerImport/sn_yammerclient.php | 165 +++++++++++++++++++++++
 1 file changed, 165 insertions(+)
 create mode 100644 plugins/YammerImport/sn_yammerclient.php

diff --git a/plugins/YammerImport/sn_yammerclient.php b/plugins/YammerImport/sn_yammerclient.php
new file mode 100644
index 0000000000..c77fc4ce36
--- /dev/null
+++ b/plugins/YammerImport/sn_yammerclient.php
@@ -0,0 +1,165 @@
+<?php
+/*
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Basic client class for Yammer's OAuth/JSON API.
+ * 
+ * @package YammerImportPlugin
+ * @author Brion Vibber <brion@status.net>
+ */
+class SN_YammerClient
+{
+    protected $apiBase = "https://www.yammer.com";
+    protected $consumerKey, $consumerSecret;
+    protected $token, $tokenSecret;
+
+    public function __construct($consumerKey, $consumerSecret, $token=null, $tokenSecret=null)
+    {
+        $this->consumerKey = $consumerKey;
+        $this->consumerSecret = $consumerSecret;
+        $this->token = $token;
+        $this->tokenSecret = $tokenSecret;
+    }
+
+    /**
+     * Make an HTTP hit with OAuth headers and return the response body on success.
+     *
+     * @param string $path URL chunk for the API method
+     * @param array $params
+     * @return array
+     *
+     * @throws Exception for HTTP error
+     */
+    protected function fetch($path, $params=array())
+    {
+        $url = $this->apiBase . '/' . $path;
+        if ($params) {
+            $url .= '?' . http_build_query($params, null, '&');
+        }
+        $headers = array('Authorization: ' . $this->authHeader());
+
+        $client = HTTPClient::start();
+        $response = $client->get($url, $headers);
+
+        if ($response->isOk()) {
+            return $response->getBody();
+        } else {
+            throw new Exception("Yammer API returned HTTP code " . $response->getStatus() . ': ' . $response->getBody());
+        }
+    }
+
+    /**
+     * Hit the main Yammer API point and decode returned JSON data.
+     *
+     * @param string $method
+     * @param array $params
+     * @return array from JSON data
+     *
+     * @throws Exception for HTTP error or bad JSON return
+     */
+    protected function api($method, $params=array())
+    {
+        $body = $this->fetch("api/v1/$method.json", $params);
+        $data = json_decode($body, true);
+        if (!$data) {
+            throw new Exception("Invalid JSON response from Yammer API");
+        }
+        return $data;
+    }
+
+    /**
+     * Build an Authorization header value from the keys we have available.
+     */
+    protected function authHeader()
+    {
+        // token
+        // token_secret
+        $params = array('realm' => '',
+                        'oauth_consumer_key' => $this->consumerKey,
+                        'oauth_signature_method' => 'PLAINTEXT',
+                        'oauth_timestamp' => time(),
+                        'oauth_nonce' => time(),
+                        'oauth_version' => '1.0');
+        if ($this->token) {
+            $params['oauth_token'] = $this->token;
+        }
+        if ($this->tokenSecret) {
+            $params['oauth_signature'] = $this->consumerSecret . '&' . $this->tokenSecret;
+        } else {
+            $params['oauth_signature'] = $this->consumerSecret . '&';
+        }
+        if ($this->verifier) {
+            $params['oauth_verifier'] = $this->verifier;
+        }
+        $parts = array_map(array($this, 'authHeaderChunk'), array_keys($params), array_values($params));
+        return 'OAuth ' . implode(', ', $parts);
+    }
+
+    /**
+     * @param string $key
+     * @param string $val
+     */
+    protected function authHeaderChunk($key, $val)
+    {
+        return urlencode($key) . '="' . urlencode($val) . '"';
+    }
+
+    /**
+     * @return array of oauth return data; should contain nice things
+     */
+    public function requestToken()
+    {
+        if ($this->token || $this->tokenSecret) {
+            throw new Exception("Requesting a token, but already set up with a token");
+        }
+        $data = $this->fetch('oauth/request_token');
+        $arr = array();
+        parse_str($data, $arr);
+        return $arr;
+    }
+
+    /**
+     * @return array of oauth return data; should contain nice things
+     */
+    public function accessToken($verifier)
+    {
+        $this->verifier = $verifier;
+        $data = $this->fetch('oauth/access_token');
+        $this->verifier = null;
+        $arr = array();
+        parse_str($data, $arr);
+        return $arr;
+    }
+
+    /**
+     * Give the URL to send users to to authorize a new app setup
+     *
+     * @param string $token as returned from accessToken()
+     * @return string URL
+     */
+    public function authorizeUrl($token)
+    {
+        return $this->apiBase . '/oauth/authorize?oauth_token=' . urlencode($token);
+    }
+
+    public function messages($params)
+    {
+        return $this->api('messages', $params);
+    }
+}

From 14a3697a619ae22034aad4e6cb8d11a0ad7ff623 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 13:56:30 -0700
Subject: [PATCH 03/35] Beginning stub of Yammer message->notice import

---
 plugins/YammerImport/YammerImportPlugin.php |   6 +-
 plugins/YammerImport/yamdump.php            | 162 +++-----------------
 plugins/YammerImport/yammerimporter.php     |  68 ++++++++
 3 files changed, 90 insertions(+), 146 deletions(-)
 create mode 100644 plugins/YammerImport/yammerimporter.php

diff --git a/plugins/YammerImport/YammerImportPlugin.php b/plugins/YammerImport/YammerImportPlugin.php
index 02923493fb..a3520d8a86 100644
--- a/plugins/YammerImport/YammerImportPlugin.php
+++ b/plugins/YammerImport/YammerImportPlugin.php
@@ -65,10 +65,12 @@ class YammerImportPlugin extends Plugin
     {
         $base = dirname(__FILE__);
         $lower = strtolower($cls);
-        switch ($cls) {
+        switch ($lower) {
+        case 'sn_yammerclient':
+        case 'yammerimporter':
         case 'yammerimqueuehandler':
         case 'importyammeraction':
-            require_once $base . $lower . '.php';
+            require_once "$base/$lower.php";
             return false;
         default:
             return true;
diff --git a/plugins/YammerImport/yamdump.php b/plugins/YammerImport/yamdump.php
index 60be81ca5d..953b7d1a62 100644
--- a/plugins/YammerImport/yamdump.php
+++ b/plugins/YammerImport/yamdump.php
@@ -8,150 +8,24 @@ define('INSTALLDIR', dirname(dirname(dirname(__FILE__))));
 
 require INSTALLDIR . "/scripts/commandline.inc";
 
-class SN_YammerClient
-{
-    protected $apiBase = "https://www.yammer.com";
-    protected $consumerKey, $consumerSecret;
-    protected $token, $tokenSecret;
-
-    public function __construct($consumerKey, $consumerSecret, $token=null, $tokenSecret=null)
-    {
-        $this->consumerKey = $consumerKey;
-        $this->consumerSecret = $consumerSecret;
-        $this->token = $token;
-        $this->tokenSecret = $tokenSecret;
-    }
-
-    /**
-     * Make an HTTP hit with OAuth headers and return the response body on success.
-     *
-     * @param string $path URL chunk for the API method
-     * @param array $params
-     * @return array
-     *
-     * @throws Exception for HTTP error
-     */
-    protected function fetch($path, $params=array())
-    {
-        $url = $this->apiBase . '/' . $path;
-        if ($params) {
-            $url .= '?' . http_build_query($params, null, '&');
-        }
-        $headers = array('Authorization: ' . $this->authHeader());
-        var_dump($headers);
-
-        $client = HTTPClient::start();
-        $response = $client->get($url, $headers);
-
-        if ($response->isOk()) {
-            return $response->getBody();
-        } else {
-            throw new Exception("Yammer API returned HTTP code " . $response->getStatus() . ': ' . $response->getBody());
-        }
-    }
-
-    /**
-     * Hit the main Yammer API point and decode returned JSON data.
-     *
-     * @param string $method
-     * @param array $params
-     * @return array from JSON data
-     *
-     * @throws Exception for HTTP error or bad JSON return
-     */
-    protected function api($method, $params=array())
-    {
-        $body = $this->fetch("api/v1/$method.json", $params);
-        $data = json_decode($body, true);
-        if (!$data) {
-            throw new Exception("Invalid JSON response from Yammer API");
-        }
-        return $data;
-    }
-
-    /**
-     * Build an Authorization header value from the keys we have available.
-     */
-    protected function authHeader()
-    {
-        // token
-        // token_secret
-        $params = array('realm' => '',
-                        'oauth_consumer_key' => $this->consumerKey,
-                        'oauth_signature_method' => 'PLAINTEXT',
-                        'oauth_timestamp' => time(),
-                        'oauth_nonce' => time(),
-                        'oauth_version' => '1.0');
-        if ($this->token) {
-            $params['oauth_token'] = $this->token;
-        }
-        if ($this->tokenSecret) {
-            $params['oauth_signature'] = $this->consumerSecret . '&' . $this->tokenSecret;
-        } else {
-            $params['oauth_signature'] = $this->consumerSecret . '&';
-        }
-        if ($this->verifier) {
-            $params['oauth_verifier'] = $this->verifier;
-        }
-        $parts = array_map(array($this, 'authHeaderChunk'), array_keys($params), array_values($params));
-        return 'OAuth ' . implode(', ', $parts);
-    }
-
-    /**
-     * @param string $key
-     * @param string $val
-     */
-    protected function authHeaderChunk($key, $val)
-    {
-        return urlencode($key) . '="' . urlencode($val) . '"';
-    }
-
-    /**
-     * @return array of oauth return data; should contain nice things
-     */
-    public function requestToken()
-    {
-        if ($this->token || $this->tokenSecret) {
-            throw new Exception("Requesting a token, but already set up with a token");
-        }
-        $data = $this->fetch('oauth/request_token');
-        $arr = array();
-        parse_str($data, $arr);
-        return $arr;
-    }
-
-    /**
-     * @return array of oauth return data; should contain nice things
-     */
-    public function accessToken($verifier)
-    {
-        $this->verifier = $verifier;
-        $data = $this->fetch('oauth/access_token');
-        $this->verifier = null;
-        $arr = array();
-        parse_str($data, $arr);
-        return $arr;
-    }
-
-    /**
-     * Give the URL to send users to to authorize a new app setup
-     *
-     * @param string $token as returned from accessToken()
-     * @return string URL
-     */
-    public function authorizeUrl($token)
-    {
-        return $this->apiBase . '/oauth/authorize?oauth_token=' . urlencode($token);
-    }
-
-    public function messages($params)
-    {
-        return $this->api('messages', $params);
-    }
-}
-
-
 // temp stuff
 require 'yam-config.php';
 $yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
-var_dump($yam->messages());
\ No newline at end of file
+$imp = new YammerImporter();
+
+$data = $yam->messages();
+/*
+  ["messages"]=>
+  ["meta"]=> // followed_user_ids, current_user_id, etc
+  ["threaded_extended"]=> // empty!
+  ["references"]=> // lists the users, threads, replied messages, tags
+*/
+
+// 1) we're getting messages in descending order, but we'll want to process ascending
+// 2) we'll need to pull out all those referenced items too?
+// 3) do we need to page over or anything?
+
+foreach ($data['messages'] as $message) {
+    $notice = $imp->messageToNotice($message);
+    var_dump($notice);
+}
diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
new file mode 100644
index 0000000000..b322c9b64e
--- /dev/null
+++ b/plugins/YammerImport/yammerimporter.php
@@ -0,0 +1,68 @@
+<?php
+/*
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Basic client class for Yammer's OAuth/JSON API.
+ *
+ * @package YammerImportPlugin
+ * @author Brion Vibber <brion@status.net>
+ */
+class YammerImporter
+{
+    function messageToNotice($message)
+    {
+        $messageId = $message['id'];
+        $messageUrl = $message['url'];
+
+        $profile = $this->findImportedProfile($message['sender_id']);
+        $content = $message['body']['plain'];
+        $source = 'yammer';
+        $options = array();
+
+        if ($message['replied_to_id']) {
+            $replyto = $this->findImportedNotice($message['replied_to_id']);
+            if ($replyto) {
+                $options['replyto'] = $replyto;
+            }
+        }
+        $options['created'] = common_sql_date(strtotime($message['created_at']));
+
+        // Parse/save rendered text?
+        // Save liked info?
+        // @todo attachments?
+
+        return array('orig_id' => $messageId,
+                     'profile' => $profile,
+                     'content' => $content,
+                     'source' => $source,
+                     'options' => $options);
+    }
+
+    function findImportedProfile($userId)
+    {
+        // @fixme
+        return $userId;
+    }
+
+    function findImportedNotice($messageId)
+    {
+        // @fixme
+        return $messageId;
+    }
+}
\ No newline at end of file

From 05af14e1ca785b65723935486bb236fd6352758e Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 14:56:20 -0700
Subject: [PATCH 04/35] YammerImport: initial processing code for users,
 groups, and messages

---
 plugins/YammerImport/yamdump.php        |  33 ++++-
 plugins/YammerImport/yammerimporter.php | 169 ++++++++++++++++++++++--
 2 files changed, 189 insertions(+), 13 deletions(-)

diff --git a/plugins/YammerImport/yamdump.php b/plugins/YammerImport/yamdump.php
index 953b7d1a62..ad739760a8 100644
--- a/plugins/YammerImport/yamdump.php
+++ b/plugins/YammerImport/yamdump.php
@@ -25,7 +25,36 @@ $data = $yam->messages();
 // 2) we'll need to pull out all those referenced items too?
 // 3) do we need to page over or anything?
 
-foreach ($data['messages'] as $message) {
-    $notice = $imp->messageToNotice($message);
+// 20 qualifying messages per hit...
+// use older_than to grab more
+// (better if we can go in reverse though!)
+// meta: The older-available element indicates whether messages older than those shown are available to be fetched. See the older_than parameter mentioned above.
+
+foreach ($data['references'] as $item) {
+    if ($item['type'] == 'user') {
+        $user = $imp->prepUser($item);
+        var_dump($user);
+    } else if ($item['type'] == 'group') {
+        $group = $imp->prepGroup($item);
+        var_dump($group);
+    } else if ($item['type'] == 'tag') {
+        // could need these if we work from the parsed message text
+        // otherwise, the #blarf in orig text is fine.
+    } else if ($item['type'] == 'thread') {
+        // Shouldn't need thread info; we'll reconstruct conversations
+        // from the reply-to chains.
+    } else if ($item['type'] == 'message') {
+        // If we're processing everything, then we don't need the refs here.
+    } else {
+        echo "(skipping unknown ref: " . $item['type'] . ")\n";
+    }
+}
+
+// Process in reverse chron order...
+// @fixme follow paging
+$messages = $data['messages'];
+array_reverse($messages);
+foreach ($messages as $message) {
+    $notice = $imp->prepNotice($message);
     var_dump($notice);
 }
diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
index b322c9b64e..7710d41b52 100644
--- a/plugins/YammerImport/yammerimporter.php
+++ b/plugins/YammerImport/yammerimporter.php
@@ -25,29 +25,160 @@
  */
 class YammerImporter
 {
-    function messageToNotice($message)
-    {
-        $messageId = $message['id'];
-        $messageUrl = $message['url'];
 
-        $profile = $this->findImportedProfile($message['sender_id']);
-        $content = $message['body']['plain'];
+    /**
+     * Load or create an imported profile from Yammer data.
+     * 
+     * @param object $item loaded JSON data for Yammer importer
+     * @return Profile
+     */
+    function importUserProfile($item)
+    {
+        $data = $this->prepUser($item);
+
+        $profileId = $this->findImportedProfile($data['orig_id']);
+        if ($profileId) {
+            return Profile::staticGet('id', $profileId);
+        } else {
+            $user = User::register($data['options']);
+            // @fixme set avatar!
+            return $user->getProfile();
+        }
+    }
+
+    /**
+     * Load or create an imported group from Yammer data.
+     *
+     * @param object $item loaded JSON data for Yammer importer
+     * @return User_group
+     */
+    function importGroup($item)
+    {
+        $data = $this->prepGroup($item);
+
+        $groupId = $this->findImportedGroup($data['orig_id']);
+        if ($groupId) {
+            return User_group::staticGet('id', $groupId);
+        } else {
+            $group = User_group::register($data['options']);
+            // @fixme set avatar!
+            return $group;
+        }
+    }
+
+    /**
+     * Load or create an imported notice from Yammer data.
+     *
+     * @param object $item loaded JSON data for Yammer importer
+     * @return Notice
+     */
+    function importNotice($item)
+    {
+        $data = $this->prepNotice($item);
+
+        $noticeId = $this->findImportedNotice($data['orig_id']);
+        if ($noticeId) {
+            return Notice::staticGet('id', $noticeId);
+        } else {
+            $notice = Notice::saveNew($data['profile'],
+                                      $data['content'],
+                                      $data['source'],
+                                      $data['options']);
+            // @fixme attachments?
+            return $notice;
+        }
+    }
+
+    function prepUser($item)
+    {
+        if ($item['type'] != 'user') {
+            throw new Exception('Wrong item type sent to Yammer user import processing.');
+        }
+
+        $origId = $item['id'];
+        $origUrl = $item['url'];
+
+        // @fixme check username rules?
+
+        $options['nickname'] = $item['name'];
+        $options['fullname'] = trim($item['full_name']);
+
+        // We don't appear to have full bio avail here!
+        $options['bio'] = $item['job_title'];
+
+        // What about other data like emails?
+
+        // Avatar... this will be the "_small" variant.
+        // Remove that (pre-extension) suffix to get the orig-size image.
+        $avatar = $item['mugshot_url'];
+
+        // Warning: we don't have following state for other users?
+
+        return array('orig_id' => $origId,
+                     'orig_url' => $origUrl,
+                     'avatar' => $avatar,
+                     'options' => $options);
+
+    }
+
+    function prepGroup($item)
+    {
+        if ($item['type'] != 'group') {
+            throw new Exception('Wrong item type sent to Yammer group import processing.');
+        }
+
+        $origId = $item['id'];
+        $origUrl = $item['url'];
+
+        $privacy = $item['privacy']; // Warning! only public groups in SN so far
+
+        $options['nickname'] = $item['name'];
+        $options['fullname'] = $item['full_name'];
+        $options['created'] = $this->timestamp($item['created_at']);
+
+        $avatar = $item['mugshot_url']; // as with user profiles...
+
+
+        $options['mainpage'] = common_local_url('showgroup',
+                                   array('nickname' => $options['nickname']));
+
+        // @fixme what about admin user for the group?
+        // bio? homepage etc? aliases?
+
+        $options['local'] = true;
+        return array('orig_id' => $origId,
+                     'orig_url' => $origUrl,
+                     'options' => $options);
+    }
+
+    function prepNotice($item)
+    {
+        if (isset($item['type']) && $item['type'] != 'message') {
+            throw new Exception('Wrong item type sent to Yammer message import processing.');
+        }
+
+        $origId = $item['id'];
+        $origUrl = $item['url'];
+
+        $profile = $this->findImportedProfile($item['sender_id']);
+        $content = $item['body']['plain'];
         $source = 'yammer';
         $options = array();
 
-        if ($message['replied_to_id']) {
-            $replyto = $this->findImportedNotice($message['replied_to_id']);
+        if ($item['replied_to_id']) {
+            $replyto = $this->findImportedNotice($item['replied_to_id']);
             if ($replyto) {
                 $options['replyto'] = $replyto;
             }
         }
-        $options['created'] = common_sql_date(strtotime($message['created_at']));
+        $options['created'] = $this->timestamp($item['created_at']);
 
         // Parse/save rendered text?
         // Save liked info?
         // @todo attachments?
 
-        return array('orig_id' => $messageId,
+        return array('orig_id' => $origId,
+                     'orig_url' => $origUrl,
                      'profile' => $profile,
                      'content' => $content,
                      'source' => $source,
@@ -60,9 +191,25 @@ class YammerImporter
         return $userId;
     }
 
+    function findImportedGroup($groupId)
+    {
+        // @fixme
+        return $groupId;
+    }
+
     function findImportedNotice($messageId)
     {
         // @fixme
         return $messageId;
     }
-}
\ No newline at end of file
+
+    /**
+     * Normalize timestamp format.
+     * @param string $ts
+     * @return string
+     */
+    function timestamp($ts)
+    {
+        return common_sql_date(strtotime($ts));
+    }
+}

From 9b1b9b711b79663ece7c306225342b9f9d750cff Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 15:24:14 -0700
Subject: [PATCH 05/35] Poking around at import funcs...

---
 plugins/YammerImport/yammerimporter.php | 75 ++++++++++++++++++++-----
 1 file changed, 62 insertions(+), 13 deletions(-)

diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
index 7710d41b52..2c8d09b9b8 100644
--- a/plugins/YammerImport/yammerimporter.php
+++ b/plugins/YammerImport/yammerimporter.php
@@ -26,22 +26,27 @@
 class YammerImporter
 {
 
+    protected $users=array();
+    protected $groups=array();
+    protected $notices=array();
+
     /**
      * Load or create an imported profile from Yammer data.
      * 
      * @param object $item loaded JSON data for Yammer importer
      * @return Profile
      */
-    function importUserProfile($item)
+    function importUser($item)
     {
         $data = $this->prepUser($item);
 
-        $profileId = $this->findImportedProfile($data['orig_id']);
+        $profileId = $this->findImportedUser($data['orig_id']);
         if ($profileId) {
             return Profile::staticGet('id', $profileId);
         } else {
             $user = User::register($data['options']);
             // @fixme set avatar!
+            $this->recordImportedUser($data['orig_id'], $user->id);
             return $user->getProfile();
         }
     }
@@ -62,6 +67,7 @@ class YammerImporter
         } else {
             $group = User_group::register($data['options']);
             // @fixme set avatar!
+            $this->recordImportedGroup($data['orig_id'], $group->id);
             return $group;
         }
     }
@@ -85,10 +91,17 @@ class YammerImporter
                                       $data['source'],
                                       $data['options']);
             // @fixme attachments?
+            $this->recordImportedNotice($data['orig_id'], $notice->id);
             return $notice;
         }
     }
 
+    /**
+     * Pull relevant info out of a Yammer data record for a user import.
+     *
+     * @param array $item
+     * @return array
+     */
     function prepUser($item)
     {
         if ($item['type'] != 'user') {
@@ -121,6 +134,12 @@ class YammerImporter
 
     }
 
+    /**
+     * Pull relevant info out of a Yammer data record for a group import.
+     *
+     * @param array $item
+     * @return array
+     */
     function prepGroup($item)
     {
         if ($item['type'] != 'group') {
@@ -151,6 +170,12 @@ class YammerImporter
                      'options' => $options);
     }
 
+    /**
+     * Pull relevant info out of a Yammer data record for a notice import.
+     *
+     * @param array $item
+     * @return array
+     */
     function prepNotice($item)
     {
         if (isset($item['type']) && $item['type'] != 'message') {
@@ -160,7 +185,7 @@ class YammerImporter
         $origId = $item['id'];
         $origUrl = $item['url'];
 
-        $profile = $this->findImportedProfile($item['sender_id']);
+        $profile = $this->findImportedUser($item['sender_id']);
         $content = $item['body']['plain'];
         $source = 'yammer';
         $options = array();
@@ -185,22 +210,46 @@ class YammerImporter
                      'options' => $options);
     }
 
-    function findImportedProfile($userId)
+    private function findImportedUser($origId)
     {
-        // @fixme
-        return $userId;
+        if (isset($this->users[$origId])) {
+            return $this->users[$origId];
+        } else {
+            return false;
+        }
     }
 
-    function findImportedGroup($groupId)
+    private function findImportedGroup($origId)
     {
-        // @fixme
-        return $groupId;
+        if (isset($this->groups[$origId])) {
+            return $this->groups[$origId];
+        } else {
+            return false;
+        }
     }
 
-    function findImportedNotice($messageId)
+    private function findImportedNotice($origId)
     {
-        // @fixme
-        return $messageId;
+        if (isset($this->notices[$origId])) {
+            return $this->notices[$origId];
+        } else {
+            return false;
+        }
+    }
+
+    private function recordImportedUser($origId, $userId)
+    {
+        $this->users[$origId] = $userId;
+    }
+
+    private function recordImportedGroup($origId, $groupId)
+    {
+        $this->groups[$origId] = $groupId;
+    }
+
+    private function recordImportedNotice($origId, $noticeId)
+    {
+        $this->notices[$origId] = $noticeId;
     }
 
     /**
@@ -208,7 +257,7 @@ class YammerImporter
      * @param string $ts
      * @return string
      */
-    function timestamp($ts)
+    private function timestamp($ts)
     {
         return common_sql_date(strtotime($ts));
     }

From 5b9efbb5014307c6ed3e4cf9a6e87ceb4331c223 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 15:29:04 -0700
Subject: [PATCH 06/35] fix notices in SN_YammerClient

---
 plugins/YammerImport/sn_yammerclient.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/YammerImport/sn_yammerclient.php b/plugins/YammerImport/sn_yammerclient.php
index c77fc4ce36..21caa7b7ca 100644
--- a/plugins/YammerImport/sn_yammerclient.php
+++ b/plugins/YammerImport/sn_yammerclient.php
@@ -27,7 +27,7 @@ class SN_YammerClient
 {
     protected $apiBase = "https://www.yammer.com";
     protected $consumerKey, $consumerSecret;
-    protected $token, $tokenSecret;
+    protected $token, $tokenSecret, $verifier;
 
     public function __construct($consumerKey, $consumerSecret, $token=null, $tokenSecret=null)
     {
@@ -158,7 +158,7 @@ class SN_YammerClient
         return $this->apiBase . '/oauth/authorize?oauth_token=' . urlencode($token);
     }
 
-    public function messages($params)
+    public function messages($params=array())
     {
         return $this->api('messages', $params);
     }

From 3e2cf3876db1cdb546f9bc9c16b2f28ceb693107 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 15:54:39 -0700
Subject: [PATCH 07/35] Initial semi-working yammer import :D

* no avatars
* no details of user accounts or their auth info
* no group memberships or subscriptions
* no attachments
* will probably esplode if >20 messages in your network *whistle innocently*
---
 plugins/YammerImport/yammer-import.php | 59 ++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 plugins/YammerImport/yammer-import.php

diff --git a/plugins/YammerImport/yammer-import.php b/plugins/YammerImport/yammer-import.php
new file mode 100644
index 0000000000..d6ec975132
--- /dev/null
+++ b/plugins/YammerImport/yammer-import.php
@@ -0,0 +1,59 @@
+<?php
+
+if (php_sapi_name() != 'cli') {
+    die('no');
+}
+
+define('INSTALLDIR', dirname(dirname(dirname(__FILE__))));
+
+require INSTALLDIR . "/scripts/commandline.inc";
+
+// temp stuff
+require 'yam-config.php';
+$yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
+$imp = new YammerImporter();
+
+$data = $yam->messages();
+/*
+  ["messages"]=>
+  ["meta"]=> // followed_user_ids, current_user_id, etc
+  ["threaded_extended"]=> // empty!
+  ["references"]=> // lists the users, threads, replied messages, tags
+*/
+
+// 1) we're getting messages in descending order, but we'll want to process ascending
+// 2) we'll need to pull out all those referenced items too?
+// 3) do we need to page over or anything?
+
+// 20 qualifying messages per hit...
+// use older_than to grab more
+// (better if we can go in reverse though!)
+// meta: The older-available element indicates whether messages older than those shown are available to be fetched. See the older_than parameter mentioned above.
+foreach ($data['references'] as $item) {
+    if ($item['type'] == 'user') {
+        $user = $imp->importUser($item);
+        echo "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)\n";
+    } else if ($item['type'] == 'group') {
+        $group = $imp->importGroup($item);
+        echo "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)\n";
+    } else if ($item['type'] == 'tag') {
+        // could need these if we work from the parsed message text
+        // otherwise, the #blarf in orig text is fine.
+    } else if ($item['type'] == 'thread') {
+        // Shouldn't need thread info; we'll reconstruct conversations
+        // from the reply-to chains.
+    } else if ($item['type'] == 'message') {
+        // If we're processing everything, then we don't need the refs here.
+    } else {
+        echo "(skipping unknown ref: " . $item['type'] . ")\n";
+    }
+}
+
+// Process in reverse chron order...
+// @fixme follow paging
+$messages = $data['messages'];
+$messages = array_reverse($messages);
+foreach ($messages as $item) {
+    $notice = $imp->importNotice($item);
+    echo "Imported Yammer notice " . $item['id'] . " as $notice->id\n";
+}

From 8091c4d2913b9bbfa24235ad8d263ae324e56765 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 16:10:44 -0700
Subject: [PATCH 08/35] Avatars for Yammer import

---
 plugins/YammerImport/yammerimporter.php | 61 +++++++++++++++++++++++--
 1 file changed, 57 insertions(+), 4 deletions(-)

diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
index 2c8d09b9b8..15970d0729 100644
--- a/plugins/YammerImport/yammerimporter.php
+++ b/plugins/YammerImport/yammerimporter.php
@@ -45,9 +45,14 @@ class YammerImporter
             return Profile::staticGet('id', $profileId);
         } else {
             $user = User::register($data['options']);
-            // @fixme set avatar!
-            $this->recordImportedUser($data['orig_id'], $user->id);
-            return $user->getProfile();
+            $profile = $user->getProfile();
+            try {
+                $this->saveAvatar($data['avatar'], $profile);
+            } catch (Exception $e) {
+                common_log(LOG_ERROR, "Error importing Yammer avatar: " . $e->getMessage());
+            }
+            $this->recordImportedUser($data['orig_id'], $profile->id);
+            return $profile;
         }
     }
 
@@ -66,7 +71,11 @@ class YammerImporter
             return User_group::staticGet('id', $groupId);
         } else {
             $group = User_group::register($data['options']);
-            // @fixme set avatar!
+            try {
+                $this->saveAvatar($data['avatar'], $group);
+            } catch (Exception $e) {
+                common_log(LOG_ERROR, "Error importing Yammer avatar: " . $e->getMessage());
+            }
             $this->recordImportedGroup($data['orig_id'], $group->id);
             return $group;
         }
@@ -261,4 +270,48 @@ class YammerImporter
     {
         return common_sql_date(strtotime($ts));
     }
+
+    /**
+     * Download and update given avatar image
+     *
+     * @param string $url
+     * @param mixed $dest either a Profile or User_group object
+     * @throws Exception in various failure cases
+     */
+    private function saveAvatar($url, $dest)
+    {
+        // Yammer API data mostly gives us the small variant.
+        // Try hitting the source image if we can!
+        // @fixme no guarantee of this URL scheme I think.
+        $url = preg_replace('/_small(\..*?)$/', '$1', $url);
+
+        if (!common_valid_http_url($url)) {
+            throw new ServerException(sprintf(_m("Invalid avatar URL %s."), $url));
+        }
+
+        // @fixme this should be better encapsulated
+        // ripped from oauthstore.php (for old OMB client)
+        $temp_filename = tempnam(sys_get_temp_dir(), 'listener_avatar');
+        if (!copy($url, $temp_filename)) {
+            throw new ServerException(sprintf(_m("Unable to fetch avatar from %s."), $url));
+        }
+
+        $id = $dest->id;
+        // @fixme should we be using different ids?
+        $imagefile = new ImageFile($id, $temp_filename);
+        $filename = Avatar::filename($id,
+                                     image_type_to_extension($imagefile->type),
+                                     null,
+                                     common_timestamp());
+        rename($temp_filename, Avatar::path($filename));
+        // @fixme hardcoded chmod is lame, but seems to be necessary to
+        // keep from accidentally saving images from command-line (queues)
+        // that can't be read from web server, which causes hard-to-notice
+        // problems later on:
+        //
+        // http://status.net/open-source/issues/2663
+        chmod(Avatar::path($filename), 0644);
+
+        $dest->setOriginal($filename);
+    }
 }

From 0ff28ac8e07994267b3e7b83fd82690e33ba222b Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 16:19:02 -0700
Subject: [PATCH 09/35] Fix for replies in Yammer import

---
 plugins/YammerImport/yammerimporter.php | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
index 15970d0729..ae0037ffee 100644
--- a/plugins/YammerImport/yammerimporter.php
+++ b/plugins/YammerImport/yammerimporter.php
@@ -200,9 +200,9 @@ class YammerImporter
         $options = array();
 
         if ($item['replied_to_id']) {
-            $replyto = $this->findImportedNotice($item['replied_to_id']);
-            if ($replyto) {
-                $options['replyto'] = $replyto;
+            $replyTo = $this->findImportedNotice($item['replied_to_id']);
+            if ($replyTo) {
+                $options['reply_to'] = $replyTo;
             }
         }
         $options['created'] = $this->timestamp($item['created_at']);

From 9be9d2f72013a451820e3e3e9e7905466ddb8857 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 16:27:10 -0700
Subject: [PATCH 10/35] Full dump of input data in yamdump also for my
 reference...

---
 plugins/YammerImport/yamdump.php | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/plugins/YammerImport/yamdump.php b/plugins/YammerImport/yamdump.php
index ad739760a8..96127dd176 100644
--- a/plugins/YammerImport/yamdump.php
+++ b/plugins/YammerImport/yamdump.php
@@ -14,6 +14,8 @@ $yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
 $imp = new YammerImporter();
 
 $data = $yam->messages();
+var_dump($data);
+
 /*
   ["messages"]=>
   ["meta"]=> // followed_user_ids, current_user_id, etc

From 47cf29b2a2fd82b0045dad7686633200479a6b37 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 16:27:26 -0700
Subject: [PATCH 11/35] Copy favorites in Yammer importer

---
 plugins/YammerImport/yammerimporter.php | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
index ae0037ffee..08cbbf790b 100644
--- a/plugins/YammerImport/yammerimporter.php
+++ b/plugins/YammerImport/yammerimporter.php
@@ -99,6 +99,12 @@ class YammerImporter
                                       $data['content'],
                                       $data['source'],
                                       $data['options']);
+            foreach ($data['faves'] as $nickname) {
+                $user = User::staticGet('nickname', $nickname);
+                if ($user) {
+                    Fave::addNew($user->getProfile(), $notice);
+                }
+            }
             // @fixme attachments?
             $this->recordImportedNotice($data['orig_id'], $notice->id);
             return $notice;
@@ -207,8 +213,13 @@ class YammerImporter
         }
         $options['created'] = $this->timestamp($item['created_at']);
 
+        $faves = array();
+        foreach ($item['liked_by']['names'] as $liker) {
+            // "permalink" is the username. wtf?
+            $faves[] = $liker['permalink'];
+        }
+
         // Parse/save rendered text?
-        // Save liked info?
         // @todo attachments?
 
         return array('orig_id' => $origId,
@@ -216,7 +227,8 @@ class YammerImporter
                      'profile' => $profile,
                      'content' => $content,
                      'source' => $source,
-                     'options' => $options);
+                     'options' => $options,
+                     'faves' => $faves);
     }
 
     private function findImportedUser($origId)

From ed3d9a11bfc1718ba88f359b393bb41ad9c80497 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 17:08:40 -0700
Subject: [PATCH 12/35] Image file attachment support for Yammer import

---
 plugins/YammerImport/sn_yammerclient.php | 51 ++++++++++++----
 plugins/YammerImport/yamdump.php         |  2 +-
 plugins/YammerImport/yammer-import.php   |  2 +-
 plugins/YammerImport/yammerimporter.php  | 78 +++++++++++++++++++++---
 4 files changed, 112 insertions(+), 21 deletions(-)

diff --git a/plugins/YammerImport/sn_yammerclient.php b/plugins/YammerImport/sn_yammerclient.php
index 21caa7b7ca..f7382abae1 100644
--- a/plugins/YammerImport/sn_yammerclient.php
+++ b/plugins/YammerImport/sn_yammerclient.php
@@ -38,25 +38,34 @@ class SN_YammerClient
     }
 
     /**
-     * Make an HTTP hit with OAuth headers and return the response body on success.
+     * Make an HTTP GET request with OAuth headers and return an HTTPResponse
+     * with the returned body and codes.
      *
-     * @param string $path URL chunk for the API method
-     * @param array $params
-     * @return array
+     * @param string $url
+     * @return HTTPResponse
      *
-     * @throws Exception for HTTP error
+     * @throws Exception on low-level network error
      */
-    protected function fetch($path, $params=array())
+    protected function httpGet($url)
     {
-        $url = $this->apiBase . '/' . $path;
-        if ($params) {
-            $url .= '?' . http_build_query($params, null, '&');
-        }
         $headers = array('Authorization: ' . $this->authHeader());
 
         $client = HTTPClient::start();
-        $response = $client->get($url, $headers);
+        return $client->get($url, $headers);
+    }
 
+    /**
+     * Make an HTTP GET request with OAuth headers and return the response body
+     * on success.
+     *
+     * @param string $url
+     * @return string
+     *
+     * @throws Exception on low-level network or HTTP error
+     */
+    public function fetchUrl($url)
+    {
+        $response = $this->httpGet($url);
         if ($response->isOk()) {
             return $response->getBody();
         } else {
@@ -64,6 +73,24 @@ class SN_YammerClient
         }
     }
 
+    /**
+     * Make an HTTP hit with OAuth headers and return the response body on success.
+     *
+     * @param string $path URL chunk for the API method
+     * @param array $params
+     * @return string
+     *
+     * @throws Exception on low-level network or HTTP error
+     */
+    protected function fetchApi($path, $params=array())
+    {
+        $url = $this->apiBase . '/' . $path;
+        if ($params) {
+            $url .= '?' . http_build_query($params, null, '&');
+        }
+        return $this->fetchUrl($url);
+    }
+
     /**
      * Hit the main Yammer API point and decode returned JSON data.
      *
@@ -75,7 +102,7 @@ class SN_YammerClient
      */
     protected function api($method, $params=array())
     {
-        $body = $this->fetch("api/v1/$method.json", $params);
+        $body = $this->fetchApi("api/v1/$method.json", $params);
         $data = json_decode($body, true);
         if (!$data) {
             throw new Exception("Invalid JSON response from Yammer API");
diff --git a/plugins/YammerImport/yamdump.php b/plugins/YammerImport/yamdump.php
index 96127dd176..ef39275981 100644
--- a/plugins/YammerImport/yamdump.php
+++ b/plugins/YammerImport/yamdump.php
@@ -11,7 +11,7 @@ require INSTALLDIR . "/scripts/commandline.inc";
 // temp stuff
 require 'yam-config.php';
 $yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
-$imp = new YammerImporter();
+$imp = new YammerImporter($yam);
 
 $data = $yam->messages();
 var_dump($data);
diff --git a/plugins/YammerImport/yammer-import.php b/plugins/YammerImport/yammer-import.php
index d6ec975132..da99c48e90 100644
--- a/plugins/YammerImport/yammer-import.php
+++ b/plugins/YammerImport/yammer-import.php
@@ -11,7 +11,7 @@ require INSTALLDIR . "/scripts/commandline.inc";
 // temp stuff
 require 'yam-config.php';
 $yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
-$imp = new YammerImporter();
+$imp = new YammerImporter($yam);
 
 $data = $yam->messages();
 /*
diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
index 08cbbf790b..48bb5dda29 100644
--- a/plugins/YammerImport/yammerimporter.php
+++ b/plugins/YammerImport/yammerimporter.php
@@ -25,11 +25,16 @@
  */
 class YammerImporter
 {
-
+    protected $client;
     protected $users=array();
     protected $groups=array();
     protected $notices=array();
 
+    function __construct(SN_YammerClient $client)
+    {
+        $this->client = $client;
+    }
+
     /**
      * Load or create an imported profile from Yammer data.
      * 
@@ -95,17 +100,39 @@ class YammerImporter
         if ($noticeId) {
             return Notice::staticGet('id', $noticeId);
         } else {
-            $notice = Notice::saveNew($data['profile'],
-                                      $data['content'],
+            $content = $data['content'];
+            $user = User::staticGet($data['profile']);
+
+            // Fetch file attachments and add the URLs...
+            $uploads = array();
+            foreach ($data['attachments'] as $url) {
+                try {
+                    $upload = $this->saveAttachment($url, $user);
+                    $content .= ' ' . $upload->shortUrl();
+                    $uploads[] = $upload;
+                } catch (Exception $e) {
+                    common_log(LOG_ERROR, "Error importing Yammer attachment: " . $e->getMessage());
+                }
+            }
+
+            // Here's the meat! Actually save the dang ol' notice.
+            $notice = Notice::saveNew($user->id,
+                                      $content,
                                       $data['source'],
                                       $data['options']);
+
+            // Save "likes" as favorites...
             foreach ($data['faves'] as $nickname) {
                 $user = User::staticGet('nickname', $nickname);
                 if ($user) {
                     Fave::addNew($user->getProfile(), $notice);
                 }
             }
-            // @fixme attachments?
+
+            // And finally attach the upload records...
+            foreach ($uploads as $upload) {
+                $upload->attachToNotice($notice);
+            }
             $this->recordImportedNotice($data['orig_id'], $notice->id);
             return $notice;
         }
@@ -219,8 +246,14 @@ class YammerImporter
             $faves[] = $liker['permalink'];
         }
 
-        // Parse/save rendered text?
-        // @todo attachments?
+        $attachments = array();
+        foreach ($item['attachments'] as $attach) {
+            if ($attach['type'] == 'image') {
+                $attachments[] = $attach['image']['url'];
+            } else {
+                common_log(LOG_WARNING, "Unrecognized Yammer attachment type: " . $attach['type']);
+            }
+        }
 
         return array('orig_id' => $origId,
                      'orig_url' => $origUrl,
@@ -228,7 +261,8 @@ class YammerImporter
                      'content' => $content,
                      'source' => $source,
                      'options' => $options,
-                     'faves' => $faves);
+                     'faves' => $faves,
+                     'attachments' => $attachments);
     }
 
     private function findImportedUser($origId)
@@ -326,4 +360,34 @@ class YammerImporter
 
         $dest->setOriginal($filename);
     }
+
+    /**
+     * Fetch an attachment from Yammer and save it into our system.
+     * Unlike avatars, the attachment URLs are guarded by authentication,
+     * so we need to run the HTTP hit through our OAuth API client.
+     *
+     * @param string $url
+     * @param User $user
+     * @return MediaFile
+     *
+     * @throws Exception on low-level network or HTTP error
+     */
+    private function saveAttachment($url, User $user)
+    {
+        // Fetch the attachment...
+        // WARNING: file must fit in memory here :(
+        $body = $this->client->fetchUrl($url);
+
+        // Save to a temporary file and shove it into our file-attachment space...
+        $temp = tmpfile();
+        fwrite($temp, $body);
+        try {
+            $upload = MediaFile::fromFileHandle($temp, $user);
+            fclose($temp);
+            return $upload;
+        } catch (Exception $e) {
+            fclose($temp);
+            throw $e;
+        }
+    }
 }

From db5a4ce70df4528c0efc7eef71aec6d783f5423f Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 17:25:02 -0700
Subject: [PATCH 13/35] Pull group descriptions in Yammer import

---
 plugins/YammerImport/yammerimporter.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
index 48bb5dda29..fb88fc5069 100644
--- a/plugins/YammerImport/yammerimporter.php
+++ b/plugins/YammerImport/yammerimporter.php
@@ -195,6 +195,7 @@ class YammerImporter
 
         $options['nickname'] = $item['name'];
         $options['fullname'] = $item['full_name'];
+        $options['description'] = $item['description'];
         $options['created'] = $this->timestamp($item['created_at']);
 
         $avatar = $item['mugshot_url']; // as with user profiles...

From 9652e77376ae6b16496e93085a7b573e242e96e1 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 17:35:32 -0700
Subject: [PATCH 14/35] Yammer import: mark group posts with the proper group
 inbox (should we append a !foo or leave them as is, as current?)

---
 plugins/YammerImport/yammerimporter.php | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
index fb88fc5069..bb6db73528 100644
--- a/plugins/YammerImport/yammerimporter.php
+++ b/plugins/YammerImport/yammerimporter.php
@@ -241,6 +241,13 @@ class YammerImporter
         }
         $options['created'] = $this->timestamp($item['created_at']);
 
+        if ($item['group_id']) {
+            $groupId = $this->findImportedGroup($item['group_id']);
+            if ($groupId) {
+                $options['groups'] = array($groupId);
+            }
+        }
+
         $faves = array();
         foreach ($item['liked_by']['names'] as $liker) {
             // "permalink" is the username. wtf?

From da87d4334ab7ffb1dd76fc58b8ac6384597dd1e7 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 18:15:32 -0700
Subject: [PATCH 15/35] Fetch more user data in Yammer imports, including the
 primary email address (preconfirmed, so we can do stuff like tell people to
 reset their passwords and log in!) and some bio info.

---
 plugins/YammerImport/sn_yammerclient.php | 28 ++++++++++
 plugins/YammerImport/yamdump.php         | 48 ++++-------------
 plugins/YammerImport/yammer-import.php   | 42 +++++----------
 plugins/YammerImport/yammerimporter.php  | 68 +++++++++++++++++++++---
 4 files changed, 112 insertions(+), 74 deletions(-)

diff --git a/plugins/YammerImport/sn_yammerclient.php b/plugins/YammerImport/sn_yammerclient.php
index f7382abae1..0f244ced6a 100644
--- a/plugins/YammerImport/sn_yammerclient.php
+++ b/plugins/YammerImport/sn_yammerclient.php
@@ -185,8 +185,36 @@ class SN_YammerClient
         return $this->apiBase . '/oauth/authorize?oauth_token=' . urlencode($token);
     }
 
+    /**
+     * High-level API hit: fetch all messages in the network (up to 20 at a time).
+     * Return data is the full JSON array returned, including meta and references
+     * sections.
+     *
+     * The matching messages themselves will be in the 'messages' item within.
+     *
+     * @param array $options optional set of additional params for the request.
+     * @return array
+     *
+     * @throws Exception on low-level or HTTP error
+     */
     public function messages($params=array())
     {
         return $this->api('messages', $params);
     }
+
+    /**
+     * High-level API hit: fetch all users in the network (up to 50 at a time).
+     * Return data is the full JSON array returned, listing user items.
+     *
+     * The matching messages themselves will be in the 'users' item within.
+     *
+     * @param array $options optional set of additional params for the request.
+     * @return array of JSON-sourced user data arrays
+     *
+     * @throws Exception on low-level or HTTP error
+     */
+    public function users($params=array())
+    {
+        return $this->api('users', $params);
+    }
 }
diff --git a/plugins/YammerImport/yamdump.php b/plugins/YammerImport/yamdump.php
index ef39275981..809baa1223 100644
--- a/plugins/YammerImport/yamdump.php
+++ b/plugins/YammerImport/yamdump.php
@@ -13,50 +13,22 @@ require 'yam-config.php';
 $yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
 $imp = new YammerImporter($yam);
 
-$data = $yam->messages();
+$data = $yam->users();
 var_dump($data);
-
-/*
-  ["messages"]=>
-  ["meta"]=> // followed_user_ids, current_user_id, etc
-  ["threaded_extended"]=> // empty!
-  ["references"]=> // lists the users, threads, replied messages, tags
-*/
-
-// 1) we're getting messages in descending order, but we'll want to process ascending
-// 2) we'll need to pull out all those referenced items too?
-// 3) do we need to page over or anything?
-
-// 20 qualifying messages per hit...
-// use older_than to grab more
-// (better if we can go in reverse though!)
-// meta: The older-available element indicates whether messages older than those shown are available to be fetched. See the older_than parameter mentioned above.
-
-foreach ($data['references'] as $item) {
-    if ($item['type'] == 'user') {
-        $user = $imp->prepUser($item);
-        var_dump($user);
-    } else if ($item['type'] == 'group') {
-        $group = $imp->prepGroup($item);
-        var_dump($group);
-    } else if ($item['type'] == 'tag') {
-        // could need these if we work from the parsed message text
-        // otherwise, the #blarf in orig text is fine.
-    } else if ($item['type'] == 'thread') {
-        // Shouldn't need thread info; we'll reconstruct conversations
-        // from the reply-to chains.
-    } else if ($item['type'] == 'message') {
-        // If we're processing everything, then we don't need the refs here.
-    } else {
-        echo "(skipping unknown ref: " . $item['type'] . ")\n";
-    }
+// @fixme follow paging
+foreach ($data as $item) {
+    $user = $imp->prepUser($item);
+    var_dump($user);
 }
 
-// Process in reverse chron order...
+/*
+$data = $yam->messages();
+var_dump($data);
 // @fixme follow paging
 $messages = $data['messages'];
-array_reverse($messages);
+$messages = array_reverse($messages);
 foreach ($messages as $message) {
     $notice = $imp->prepNotice($message);
     var_dump($notice);
 }
+*/
diff --git a/plugins/YammerImport/yammer-import.php b/plugins/YammerImport/yammer-import.php
index da99c48e90..7241809ba0 100644
--- a/plugins/YammerImport/yammer-import.php
+++ b/plugins/YammerImport/yammer-import.php
@@ -13,44 +13,26 @@ require 'yam-config.php';
 $yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
 $imp = new YammerImporter($yam);
 
+// First, import all the users!
+// @fixme follow paging -- we only get 50 at a time
+$data = $yam->users();
+foreach ($data as $item) {
+    $user = $imp->importUser($item);
+    echo "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)\n";
+}
+
 $data = $yam->messages();
-/*
-  ["messages"]=>
-  ["meta"]=> // followed_user_ids, current_user_id, etc
-  ["threaded_extended"]=> // empty!
-  ["references"]=> // lists the users, threads, replied messages, tags
-*/
-
-// 1) we're getting messages in descending order, but we'll want to process ascending
-// 2) we'll need to pull out all those referenced items too?
-// 3) do we need to page over or anything?
-
-// 20 qualifying messages per hit...
-// use older_than to grab more
-// (better if we can go in reverse though!)
-// meta: The older-available element indicates whether messages older than those shown are available to be fetched. See the older_than parameter mentioned above.
+// @fixme pull the full group list; this'll be a partial list with less data
+// and only for groups referenced in the message set.
 foreach ($data['references'] as $item) {
-    if ($item['type'] == 'user') {
-        $user = $imp->importUser($item);
-        echo "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)\n";
-    } else if ($item['type'] == 'group') {
+    if ($item['type'] == 'group') {
         $group = $imp->importGroup($item);
         echo "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)\n";
-    } else if ($item['type'] == 'tag') {
-        // could need these if we work from the parsed message text
-        // otherwise, the #blarf in orig text is fine.
-    } else if ($item['type'] == 'thread') {
-        // Shouldn't need thread info; we'll reconstruct conversations
-        // from the reply-to chains.
-    } else if ($item['type'] == 'message') {
-        // If we're processing everything, then we don't need the refs here.
-    } else {
-        echo "(skipping unknown ref: " . $item['type'] . ")\n";
     }
 }
 
 // Process in reverse chron order...
-// @fixme follow paging
+// @fixme follow paging -- we only get 20 at a time, and start at the most recent!
 $messages = $data['messages'];
 $messages = array_reverse($messages);
 foreach ($messages as $item) {
diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
index bb6db73528..583ed3a8c1 100644
--- a/plugins/YammerImport/yammerimporter.php
+++ b/plugins/YammerImport/yammerimporter.php
@@ -158,16 +158,60 @@ class YammerImporter
         $options['nickname'] = $item['name'];
         $options['fullname'] = trim($item['full_name']);
 
-        // We don't appear to have full bio avail here!
-        $options['bio'] = $item['job_title'];
-
-        // What about other data like emails?
-
         // Avatar... this will be the "_small" variant.
         // Remove that (pre-extension) suffix to get the orig-size image.
         $avatar = $item['mugshot_url'];
 
-        // Warning: we don't have following state for other users?
+        // The following info is only available in full data, not in the reference version.
+
+        // There can be extensive contact info, but for now we'll only pull the primary email.
+        if (isset($item['contact'])) {
+            foreach ($item['contact']['email_addresses'] as $addr) {
+                if ($addr['type'] == 'primary') {
+                    $options['email'] = $addr['address'];
+                    $options['email_confirmed'] = true;
+                    break;
+                }
+            }
+        }
+
+        // There can be multiple external URLs; for now pull the first one as home page.
+        if (isset($item['external_urls'])) {
+            foreach ($item['external_urls'] as $url) {
+                if (common_valid_http_url($url)) {
+                    $options['homepage'] = $url;
+                    break;
+                }
+            }
+        }
+
+        // Combine a few bits into the bio...
+        $bio = array();
+        if (!empty($item['job_title'])) {
+            $bio[] = $item['job_title'];
+        }
+        if (!empty($item['summary'])) {
+            $bio[] = $item['summary'];
+        }
+        if (!empty($item['expertise'])) {
+            $bio[] = _m('Expertise:') . ' ' . $item['expertise'];
+        }
+        $options['bio'] = implode("\n\n", $bio);
+
+        // Pull raw location string, may be lookupable
+        if (!empty($item['location'])) {
+            $options['location'] = $item['location'];
+        }
+
+        // Timezone is in format like 'Pacific Time (US & Canada)'
+        // We need to convert that to a zone id. :P
+        // @fixme timezone not yet supported at registration time :)
+        if (!empty($item['timezone'])) {
+            $tz = $this->timezone($item['timezone']);
+            if ($tz) {
+                $options['timezone'] = $tz;
+            }
+        }
 
         return array('orig_id' => $origId,
                      'orig_url' => $origUrl,
@@ -325,6 +369,18 @@ class YammerImporter
         return common_sql_date(strtotime($ts));
     }
 
+    private function timezone($tz)
+    {
+        // Blaaaaaarf!
+        $known = array('Pacific Time (US & Canada)' => 'America/Los_Angeles',
+                       'Eastern Time (US & Canada)' => 'America/New_York');
+        if (array_key_exists($known, $tz)) {
+            return $known[$tz];
+        } else {
+            return false;
+        }
+    }
+
     /**
      * Download and update given avatar image
      *

From 0ed506ee9384f95bd01cdcd72a253d08ae89b92e Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 18:21:36 -0700
Subject: [PATCH 16/35] Add group link on Yammer import (won't work until
 memberships are fixed)

---
 plugins/YammerImport/yammerimporter.php | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
index 583ed3a8c1..6dc72f4766 100644
--- a/plugins/YammerImport/yammerimporter.php
+++ b/plugins/YammerImport/yammerimporter.php
@@ -289,6 +289,12 @@ class YammerImporter
             $groupId = $this->findImportedGroup($item['group_id']);
             if ($groupId) {
                 $options['groups'] = array($groupId);
+
+                // @fixme if we see a group link inline, don't add this?
+                $group = User_group::staticGet('id', $groupId);
+                if ($group) {
+                    $content .= ' !' . $group->nickname;
+                }
             }
         }
 

From 7a381f2533c5f5da9702c6968655d52a76cb47dc Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 22:00:25 -0700
Subject: [PATCH 17/35] Support non-image file uploads in Yammer import

---
 plugins/YammerImport/yammerimporter.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
index 6dc72f4766..bfe486770c 100644
--- a/plugins/YammerImport/yammerimporter.php
+++ b/plugins/YammerImport/yammerimporter.php
@@ -306,8 +306,8 @@ class YammerImporter
 
         $attachments = array();
         foreach ($item['attachments'] as $attach) {
-            if ($attach['type'] == 'image') {
-                $attachments[] = $attach['image']['url'];
+            if ($attach['type'] == 'image' || $attach['type'] == 'file') {
+                $attachments[] = $attach[$attach['type']]['url'];
             } else {
                 common_log(LOG_WARNING, "Unrecognized Yammer attachment type: " . $attach['type']);
             }

From acd76139333aa29a415de6103fc87e3955b232b1 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Tue, 21 Sep 2010 23:19:36 -0700
Subject: [PATCH 18/35] Fixes for Yammer groups import: pulling explicit list,
 fixed avatar fetch

---
 plugins/YammerImport/sn_yammerclient.php | 18 ++++++++++++++++-
 plugins/YammerImport/yammer-import.php   | 16 +++++++--------
 plugins/YammerImport/yammerimporter.php  | 25 ++++++++++++++----------
 3 files changed, 40 insertions(+), 19 deletions(-)

diff --git a/plugins/YammerImport/sn_yammerclient.php b/plugins/YammerImport/sn_yammerclient.php
index 0f244ced6a..8f9f1d4131 100644
--- a/plugins/YammerImport/sn_yammerclient.php
+++ b/plugins/YammerImport/sn_yammerclient.php
@@ -100,7 +100,7 @@ class SN_YammerClient
      *
      * @throws Exception for HTTP error or bad JSON return
      */
-    protected function api($method, $params=array())
+    public function api($method, $params=array())
     {
         $body = $this->fetchApi("api/v1/$method.json", $params);
         $data = json_decode($body, true);
@@ -217,4 +217,20 @@ class SN_YammerClient
     {
         return $this->api('users', $params);
     }
+
+    /**
+     * High-level API hit: fetch all groups in the network (up to 20 at a time).
+     * Return data is the full JSON array returned, listing user items.
+     *
+     * The matching messages themselves will be in the 'users' item within.
+     *
+     * @param array $options optional set of additional params for the request.
+     * @return array of JSON-sourced user data arrays
+     *
+     * @throws Exception on low-level or HTTP error
+     */
+    public function groups($params=array())
+    {
+        return $this->api('groups', $params);
+    }
 }
diff --git a/plugins/YammerImport/yammer-import.php b/plugins/YammerImport/yammer-import.php
index 7241809ba0..4931d1bc52 100644
--- a/plugins/YammerImport/yammer-import.php
+++ b/plugins/YammerImport/yammer-import.php
@@ -21,18 +21,18 @@ foreach ($data as $item) {
     echo "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)\n";
 }
 
-$data = $yam->messages();
-// @fixme pull the full group list; this'll be a partial list with less data
-// and only for groups referenced in the message set.
-foreach ($data['references'] as $item) {
-    if ($item['type'] == 'group') {
-        $group = $imp->importGroup($item);
-        echo "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)\n";
-    }
+// Groups!
+// @fixme follow paging -- we only get 20 at a time
+$data = $yam->groups();
+foreach ($data as $item) {
+    $group = $imp->importGroup($item);
+    echo "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)\n";
 }
 
+// Messages!
 // Process in reverse chron order...
 // @fixme follow paging -- we only get 20 at a time, and start at the most recent!
+$data = $yam->messages();
 $messages = $data['messages'];
 $messages = array_reverse($messages);
 foreach ($messages as $item) {
diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/yammerimporter.php
index bfe486770c..9ce0d1e588 100644
--- a/plugins/YammerImport/yammerimporter.php
+++ b/plugins/YammerImport/yammerimporter.php
@@ -51,10 +51,12 @@ class YammerImporter
         } else {
             $user = User::register($data['options']);
             $profile = $user->getProfile();
-            try {
-                $this->saveAvatar($data['avatar'], $profile);
-            } catch (Exception $e) {
-                common_log(LOG_ERROR, "Error importing Yammer avatar: " . $e->getMessage());
+            if ($data['avatar']) {
+                try {
+                    $this->saveAvatar($data['avatar'], $profile);
+                } catch (Exception $e) {
+                    common_log(LOG_ERR, "Error importing Yammer avatar: " . $e->getMessage());
+                }
             }
             $this->recordImportedUser($data['orig_id'], $profile->id);
             return $profile;
@@ -76,10 +78,12 @@ class YammerImporter
             return User_group::staticGet('id', $groupId);
         } else {
             $group = User_group::register($data['options']);
-            try {
-                $this->saveAvatar($data['avatar'], $group);
-            } catch (Exception $e) {
-                common_log(LOG_ERROR, "Error importing Yammer avatar: " . $e->getMessage());
+            if ($data['avatar']) {
+                try {
+                    $this->saveAvatar($data['avatar'], $group);
+                } catch (Exception $e) {
+                    common_log(LOG_ERR, "Error importing Yammer avatar: " . $e->getMessage());
+                }
             }
             $this->recordImportedGroup($data['orig_id'], $group->id);
             return $group;
@@ -111,7 +115,7 @@ class YammerImporter
                     $content .= ' ' . $upload->shortUrl();
                     $uploads[] = $upload;
                 } catch (Exception $e) {
-                    common_log(LOG_ERROR, "Error importing Yammer attachment: " . $e->getMessage());
+                    common_log(LOG_ERR, "Error importing Yammer attachment: " . $e->getMessage());
                 }
             }
 
@@ -254,7 +258,8 @@ class YammerImporter
         $options['local'] = true;
         return array('orig_id' => $origId,
                      'orig_url' => $origUrl,
-                     'options' => $options);
+                     'options' => $options,
+                     'avatar' => $avatar);
     }
 
     /**

From 12ec7efe901d5667e26c86e11543010105656d84 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Wed, 22 Sep 2010 12:52:34 -0700
Subject: [PATCH 19/35] Split Yammer importer files into subdirs before I get
 too lost adding UI

---
 plugins/YammerImport/YammerImportPlugin.php          | 4 +---
 plugins/YammerImport/{ => lib}/sn_yammerclient.php   | 0
 plugins/YammerImport/{ => lib}/yammerimporter.php    | 0
 plugins/YammerImport/{ => scripts}/yamdump.php       | 2 +-
 plugins/YammerImport/{ => scripts}/yammer-import.php | 2 +-
 5 files changed, 3 insertions(+), 5 deletions(-)
 rename plugins/YammerImport/{ => lib}/sn_yammerclient.php (100%)
 rename plugins/YammerImport/{ => lib}/yammerimporter.php (100%)
 rename plugins/YammerImport/{ => scripts}/yamdump.php (90%)
 rename plugins/YammerImport/{ => scripts}/yammer-import.php (94%)

diff --git a/plugins/YammerImport/YammerImportPlugin.php b/plugins/YammerImport/YammerImportPlugin.php
index a3520d8a86..79b8260b69 100644
--- a/plugins/YammerImport/YammerImportPlugin.php
+++ b/plugins/YammerImport/YammerImportPlugin.php
@@ -68,9 +68,7 @@ class YammerImportPlugin extends Plugin
         switch ($lower) {
         case 'sn_yammerclient':
         case 'yammerimporter':
-        case 'yammerimqueuehandler':
-        case 'importyammeraction':
-            require_once "$base/$lower.php";
+            require_once "$base/lib/$lower.php";
             return false;
         default:
             return true;
diff --git a/plugins/YammerImport/sn_yammerclient.php b/plugins/YammerImport/lib/sn_yammerclient.php
similarity index 100%
rename from plugins/YammerImport/sn_yammerclient.php
rename to plugins/YammerImport/lib/sn_yammerclient.php
diff --git a/plugins/YammerImport/yammerimporter.php b/plugins/YammerImport/lib/yammerimporter.php
similarity index 100%
rename from plugins/YammerImport/yammerimporter.php
rename to plugins/YammerImport/lib/yammerimporter.php
diff --git a/plugins/YammerImport/yamdump.php b/plugins/YammerImport/scripts/yamdump.php
similarity index 90%
rename from plugins/YammerImport/yamdump.php
rename to plugins/YammerImport/scripts/yamdump.php
index 809baa1223..a358777ad1 100644
--- a/plugins/YammerImport/yamdump.php
+++ b/plugins/YammerImport/scripts/yamdump.php
@@ -4,7 +4,7 @@ if (php_sapi_name() != 'cli') {
     die('no');
 }
 
-define('INSTALLDIR', dirname(dirname(dirname(__FILE__))));
+define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__)))));
 
 require INSTALLDIR . "/scripts/commandline.inc";
 
diff --git a/plugins/YammerImport/yammer-import.php b/plugins/YammerImport/scripts/yammer-import.php
similarity index 94%
rename from plugins/YammerImport/yammer-import.php
rename to plugins/YammerImport/scripts/yammer-import.php
index 4931d1bc52..ac258e1c7d 100644
--- a/plugins/YammerImport/yammer-import.php
+++ b/plugins/YammerImport/scripts/yammer-import.php
@@ -4,7 +4,7 @@ if (php_sapi_name() != 'cli') {
     die('no');
 }
 
-define('INSTALLDIR', dirname(dirname(dirname(__FILE__))));
+define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__)))));
 
 require INSTALLDIR . "/scripts/commandline.inc";
 

From a0052104388b4b73866aaa4d4cfafd08694247e0 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Wed, 22 Sep 2010 13:12:39 -0700
Subject: [PATCH 20/35] Initial README for yammer importer

---
 plugins/YammerImport/README | 75 +++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 plugins/YammerImport/README

diff --git a/plugins/YammerImport/README b/plugins/YammerImport/README
new file mode 100644
index 0000000000..5ab080647a
--- /dev/null
+++ b/plugins/YammerImport/README
@@ -0,0 +1,75 @@
+Yammer Import Plugin
+====================
+
+This plugin allows a one-time import pulling user accounts, groups, and
+public messages from an existing Yammer instance, using Yammer's public API.
+
+Requirements
+------------
+
+* An account on the Yammer network you wish to import from
+* An administrator account on the target StatusNet instance
+* This YammerImport plugin enabled on your StatusNet instance
+
+Setup
+-----
+
+The import process will be runnable through an administration panel on
+your StatusNet site.
+
+The user interface and OAuth setup has not yet been completed, you will
+have to manually initiate the OAuth authentication to get a token.
+
+Be patient, there will be a UI soon. ;)
+
+
+Limitations
+-----------
+
+Paging has not yet been added, so the importer will only pull up to:
+* first 50 users
+* first 20 groups
+* last 20 public messages
+
+
+Subscriptions and group memberships
+-----------------------------------
+
+Yammer's API does not expose user/tag subscriptions or group memberships
+except for the authenticating user. As a result, users will need to re-join
+groups and re-follow their fellow users after the import.
+
+(This limitation may be lifted in future for sites on the Silver or Gold
+plans where the import is done by a verified admin, as it should be possible
+to fetch the information for each user via the admin account.)
+
+
+Authentication
+--------------
+
+Account passwords cannot be retrieved, but the primary e-mail address is
+retained so users can reset their passwords by mail if you're not using a
+custom authentication system like LDAP.
+
+
+Private messages and groups
+---------------------------
+
+At this time, only public messages are imported; private direct and group
+messages are ignored. (This may change with Silver and Gold plans in future.)
+
+Yammer groups may be either public or private. Groups in StatusNet currently
+have no privacy option, so any private groups will become public groups in the
+imported site.
+
+
+Attachments
+-----------
+
+Attached image and document files will be copied in as if they had been
+uploaded to the StatusNet site. Currently images do not display inline like
+they do on Yammer; they will be linked instead.
+
+File type and size limitations on attachments will be applied, so beware some
+attachments may not make it through.
+

From 084befc32f6ddefc8aa7743dc0ad332a985c98f5 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Wed, 22 Sep 2010 17:51:50 -0700
Subject: [PATCH 21/35] WORK IN PROGRESS: Starting infrastructure to initiate
 Yammer import from web UI and process it in the background queues. Totally
 not complete yet.

---
 plugins/YammerImport/YammerImportPlugin.php   |  61 +++++-
 .../YammerImport/actions/yammeradminpanel.php | 153 +++++++++++++++
 plugins/YammerImport/actions/yammerauth.php   |  17 ++
 .../YammerImport/classes/Yammer_common.php    | 165 +++++++++++++++++
 plugins/YammerImport/classes/Yammer_group.php |  79 ++++++++
 .../YammerImport/classes/Yammer_notice.php    |  79 ++++++++
 .../classes/Yammer_notice_stub.php            | 174 ++++++++++++++++++
 plugins/YammerImport/classes/Yammer_state.php |  37 ++++
 plugins/YammerImport/classes/Yammer_user.php  |  79 ++++++++
 plugins/YammerImport/lib/yammerimporter.php   |  27 +--
 .../YammerImport/lib/yammerqueuehandler.php   |  47 +++++
 11 files changed, 892 insertions(+), 26 deletions(-)
 create mode 100644 plugins/YammerImport/actions/yammeradminpanel.php
 create mode 100644 plugins/YammerImport/actions/yammerauth.php
 create mode 100644 plugins/YammerImport/classes/Yammer_common.php
 create mode 100644 plugins/YammerImport/classes/Yammer_group.php
 create mode 100644 plugins/YammerImport/classes/Yammer_notice.php
 create mode 100644 plugins/YammerImport/classes/Yammer_notice_stub.php
 create mode 100644 plugins/YammerImport/classes/Yammer_state.php
 create mode 100644 plugins/YammerImport/classes/Yammer_user.php
 create mode 100644 plugins/YammerImport/lib/yammerqueuehandler.php

diff --git a/plugins/YammerImport/YammerImportPlugin.php b/plugins/YammerImport/YammerImportPlugin.php
index 79b8260b69..f55169a55b 100644
--- a/plugins/YammerImport/YammerImportPlugin.php
+++ b/plugins/YammerImport/YammerImportPlugin.php
@@ -22,9 +22,7 @@
  * @maintainer Brion Vibber <brion@status.net>
  */
 
-if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); }
-
-set_include_path(get_include_path() . PATH_SEPARATOR . dirname(__FILE__) . '/extlib/');
+if (!defined('STATUSNET')) { exit(1); }
 
 class YammerImportPlugin extends Plugin
 {
@@ -36,8 +34,8 @@ class YammerImportPlugin extends Plugin
      */
     function onRouterInitialized($m)
     {
-        $m->connect('admin/import/yammer',
-                    array('action' => 'importyammer'));
+        $m->connect('admin/yammer',
+                    array('action' => 'yammeradminpanel'));
         return true;
     }
 
@@ -53,6 +51,56 @@ class YammerImportPlugin extends Plugin
         return true;
     }
 
+    /**
+     * Set up all our tables...
+     */
+    function onCheckSchema()
+    {
+        $schema = Schema::get();
+
+        $tables = array('Yammer_state',
+                        'Yammer_user',
+                        'Yammer_group',
+                        'Yammer_notice',
+                        'Yammer_notice_stub');
+        foreach ($tables as $table) {
+            $schema->ensureTable($table, $table::schemaDef());
+        }
+
+        return true;
+    }
+
+    /**
+     * If the plugin's installed, this should be accessible to admins.
+     */
+    function onAdminPanelCheck($name, &$isOK)
+    {
+        if ($name == 'yammer') {
+            $isOK = true;
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Add the Yammer admin panel to the list...
+     */
+    function onEndAdminPanelNav($nav)
+    {
+        if (AdminPanelAction::canAdmin('yammer')) {
+            $action_name = $nav->action->trimmed('action');
+
+            $nav->out->menuItem(common_local_url('yammeradminpanel'),
+                                _m('Yammer'),
+                                _m('Yammer import'),
+                                $action_name == 'yammeradminpanel',
+                                'nav_yammer_admin_panel');
+        }
+
+        return true;
+    }
+
     /**
      * Automatically load the actions and libraries used by the plugin
      *
@@ -70,6 +118,9 @@ class YammerImportPlugin extends Plugin
         case 'yammerimporter':
             require_once "$base/lib/$lower.php";
             return false;
+        case 'yammeradminpanelaction':
+            require_once "$base/actions/yammeradminpanel.php";
+            return false;
         default:
             return true;
         }
diff --git a/plugins/YammerImport/actions/yammeradminpanel.php b/plugins/YammerImport/actions/yammeradminpanel.php
new file mode 100644
index 0000000000..875debac92
--- /dev/null
+++ b/plugins/YammerImport/actions/yammeradminpanel.php
@@ -0,0 +1,153 @@
+<?php
+/**
+ * StatusNet, the distributed open-source microblogging tool
+ *
+ * Yammer import administration panel
+ *
+ * PHP version 5
+ *
+ * LICENCE: This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @category  Settings
+ * @package   StatusNet
+ * @author    Zach Copley <zach@status.net>
+ * @copyright 2010 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
+ * @link      http://status.net/
+ */
+
+if (!defined('STATUSNET')) {
+    exit(1);
+}
+
+class YammeradminpanelAction extends AdminPanelAction
+{
+    /**
+     * Returns the page title
+     *
+     * @return string page title
+     */
+    function title()
+    {
+        return _m('Yammer Import');
+    }
+
+    /**
+     * Instructions for using this form.
+     *
+     * @return string instructions
+     */
+    function getInstructions()
+    {
+        return _m('Yammer import tool');
+    }
+
+    /**
+     * Show the Yammer admin panel form
+     *
+     * @return void
+     */
+    function showForm()
+    {
+        $form = new YammerAdminPanelForm($this);
+        $form->show();
+        return;
+    }
+}
+
+class YammerAdminPanelForm extends AdminForm
+{
+    /**
+     * ID of the form
+     *
+     * @return string ID of the form
+     */
+    function id()
+    {
+        return 'yammeradminpanel';
+    }
+
+    /**
+     * class of the form
+     *
+     * @return string class of the form
+     */
+    function formClass()
+    {
+        return 'form_settings';
+    }
+
+    /**
+     * Action of the form
+     *
+     * @return string URL of the action
+     */
+    function action()
+    {
+        return common_local_url('yammeradminpanel');
+    }
+
+    /**
+     * Data elements of the form
+     *
+     * @return void
+     */
+    function formData()
+    {
+        $this->out->element('p', array(), 'yammer import IN DA HOUSE');
+        
+        /*
+        Possible states of the yammer import process:
+        - null (not doing any sort of import)
+        - requesting-auth
+        - authenticated
+        - import-users
+        - import-groups
+        - fetch-messages
+        - import-messages
+        - done
+        */
+        $yammerState = Yammer_state::staticGet('id', 1);
+        $state = $yammerState ? $yammerState->state || null;
+        
+        switch($state)
+        {
+            case null:
+                $this->out->element('p', array(), 'Time to start auth:');
+                $this->showAuthForm();
+                break;
+            case 'requesting-auth':
+                $this->out->element('p', array(), 'Need to finish auth!');
+                $this->showAuthForm();
+                break;
+            case 'import-users':
+            case 'import-groups':
+            case 'import-messages':
+            case 'save-messages':
+                $this->showImportState();
+                break;
+            
+        }
+    }
+
+    /**
+     * Action elements
+     *
+     * @return void
+     */
+    function formActions()
+    {
+        // No submit buttons needed at bottom
+    }
+}
diff --git a/plugins/YammerImport/actions/yammerauth.php b/plugins/YammerImport/actions/yammerauth.php
new file mode 100644
index 0000000000..7e6e7204ae
--- /dev/null
+++ b/plugins/YammerImport/actions/yammerauth.php
@@ -0,0 +1,17 @@
+<?php
+
+
+function showYammerAuth()
+{
+    $token = $yam->requestToken();
+    $url = $yam->authorizeUrl($token);
+
+    // We're going to try doing this in an iframe; if that's not happy
+    // we can redirect but there doesn't seem to be a way to get Yammer's
+    // oauth to call us back instead of the manual copy. :(
+
+    //common_redirect($url, 303);
+    $this->element('iframe', array('id' => 'yammer-oauth',
+                                   'src' => $url));
+}
+
diff --git a/plugins/YammerImport/classes/Yammer_common.php b/plugins/YammerImport/classes/Yammer_common.php
new file mode 100644
index 0000000000..81e302ab29
--- /dev/null
+++ b/plugins/YammerImport/classes/Yammer_common.php
@@ -0,0 +1,165 @@
+<?php
+/**
+ * Data class for remembering Yammer import mappings
+ *
+ * PHP version 5
+ *
+ * @category Data
+ * @package  StatusNet
+ * @author   Brion Vibber <brion@status.net>
+ * @license  http://www.fsf.org/licensing/licenses/agpl.html AGPLv3
+ * @link     http://status.net/
+ *
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.     See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+if (!defined('STATUSNET')) {
+    exit(1);
+}
+
+/**
+ * Common base class for the Yammer import mappings for users, groups, and notices.
+ *
+ * Child classes must override these static methods, since we need to run
+ * on PHP 5.2.x which has no late static binding:
+ * - staticGet (as our other classes)
+ * - schemaDef (call self::doSchemaDef)
+ * - record (call self::doRecord)
+ */
+
+class Yammer_common extends Memcached_DataObject
+{
+    public $__table = 'yammer_XXXX'; // table name
+    public $__field = 'XXXX_id';     // field name to save into
+    public $id;                      // int  primary_key not_null
+    public $user_id;                 // int(4)
+    public $created;                 // datetime
+
+    /**
+     * @fixme add a 'references' thing for the foreign key when we support that
+     */
+    protected static function doSchemaDef($field)
+    {
+        return array(new ColumnDef('id', 'bigint', null,
+                                   false, 'PRI'),
+                     new ColumnDef($field, 'integer', null,
+                                   false, 'UNI'),
+                     new ColumnDef('created', 'datetime', null,
+                                   false));
+    }
+
+    /**
+     * return table definition for DB_DataObject
+     *
+     * DB_DataObject needs to know something about the table to manipulate
+     * instances. This method provides all the DB_DataObject needs to know.
+     *
+     * @return array array of column definitions
+     */
+
+    function table()
+    {
+        return array('id'           => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
+                     $this->__field => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
+                     'created'      => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL);
+    }
+
+    /**
+     * return key definitions for DB_DataObject
+     *
+     * DB_DataObject needs to know about keys that the table has, since it
+     * won't appear in StatusNet's own keys list. In most cases, this will
+     * simply reference your keyTypes() function.
+     *
+     * @return array list of key field names
+     */
+
+    function keys()
+    {
+        return array_keys($this->keyTypes());
+    }
+
+    /**
+     * return key definitions for Memcached_DataObject
+     *
+     * Our caching system uses the same key definitions, but uses a different
+     * method to get them. This key information is used to store and clear
+     * cached data, so be sure to list any key that will be used for static
+     * lookups.
+     *
+     * @return array associative array of key definitions, field name to type:
+     *         'K' for primary key: for compound keys, add an entry for each component;
+     *         'U' for unique keys: compound keys are not well supported here.
+     */
+
+    function keyTypes()
+    {
+        return array('id' => 'K', $this->__field => 'U');
+    }
+
+    /**
+     * Magic formula for non-autoincrementing integer primary keys
+     *
+     * If a table has a single integer column as its primary key, DB_DataObject
+     * assumes that the column is auto-incrementing and makes a sequence table
+     * to do this incrementation. Since we don't need this for our class, we
+     * overload this method and return the magic formula that DB_DataObject needs.
+     *
+     * @return array magic three-false array that stops auto-incrementing.
+     */
+
+    function sequenceKey()
+    {
+        return array(false, false, false);
+    }
+
+    /**
+     * Save a mapping between a remote Yammer and local imported user.
+     *
+     * @param integer $user_id ID of the status in StatusNet
+     * @param integer $orig_id ID of the notice in Yammer
+     *
+     * @return Yammer_common new object for this value
+     */
+
+    protected static function doRecord($class, $field, $orig_id, $local_id)
+    {
+        $map = self::staticGet('id', $orig_id);
+
+        if (!empty($map)) {
+            return $map;
+        }
+
+        $map = self::staticGet($field, $local_id);
+
+        if (!empty($map)) {
+            return $map;
+        }
+
+        common_debug("Mapping Yammer $field {$orig_id} to local $field {$local_id}");
+
+        $map = new $class();
+
+        $map->id = $orig_id;
+        $map->$field = $local_id;
+        $map->created = common_sql_now();
+
+        $map->insert();
+
+        return $map;
+    }
+}
diff --git a/plugins/YammerImport/classes/Yammer_group.php b/plugins/YammerImport/classes/Yammer_group.php
new file mode 100644
index 0000000000..4e7a6ebd03
--- /dev/null
+++ b/plugins/YammerImport/classes/Yammer_group.php
@@ -0,0 +1,79 @@
+<?php
+/**
+ * Data class for remembering Yammer import mappings
+ *
+ * PHP version 5
+ *
+ * @category Data
+ * @package  StatusNet
+ * @author   Brion Vibber <brion@status.net>
+ * @license  http://www.fsf.org/licensing/licenses/agpl.html AGPLv3
+ * @link     http://status.net/
+ *
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.     See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+if (!defined('STATUSNET')) {
+    exit(1);
+}
+
+class Yammer_group extends Yammer_common
+{
+    public $__table = 'yammer_group'; // table name
+    public $__field = 'group_id';     // field to map to
+    public $group_id;                 // int
+
+    /**
+     * Get an instance by key
+     *
+     * This is a utility method to get a single instance with a given key value.
+     *
+     * @param string $k Key to use to lookup
+     * @param mixed  $v Value to lookup
+     *
+     * @return Yammer_group object found, or null for no hits
+     *
+     */
+
+    function staticGet($k, $v=null)
+    {
+        return Memcached_DataObject::staticGet('Yammer_group', $k, $v);
+    }
+
+    /**
+     * Return schema definition to set this table up in onCheckSchema
+     */
+
+    static function schemaDef()
+    {
+        return self::doSchemaDef('group_id');
+    }
+
+    /**
+     * Save a mapping between a remote Yammer and local imported group.
+     *
+     * @param integer $orig_id ID of the notice in Yammer
+     * @param integer $group_id ID of the status in StatusNet
+     *
+     * @return Yammer_group new object for this value
+     */
+
+    static function record($orig_id, $group_id)
+    {
+        return self::doRecord('Yammer_group', 'group_id', $orig_id, $group_id);
+    }
+}
diff --git a/plugins/YammerImport/classes/Yammer_notice.php b/plugins/YammerImport/classes/Yammer_notice.php
new file mode 100644
index 0000000000..0f63db6303
--- /dev/null
+++ b/plugins/YammerImport/classes/Yammer_notice.php
@@ -0,0 +1,79 @@
+<?php
+/**
+ * Data class for remembering Yammer import mappings
+ *
+ * PHP version 5
+ *
+ * @category Data
+ * @package  StatusNet
+ * @author   Brion Vibber <brion@status.net>
+ * @license  http://www.fsf.org/licensing/licenses/agpl.html AGPLv3
+ * @link     http://status.net/
+ *
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.     See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+if (!defined('STATUSNET')) {
+    exit(1);
+}
+
+class Yammer_notice extends Yammer_common
+{
+    public $__table = 'yammer_notice'; // table name
+    public $__field = 'notice_id';     // field to map to
+    public $notice_id;                 // int
+
+    /**
+     * Get an instance by key
+     *
+     * This is a utility method to get a single instance with a given key value.
+     *
+     * @param string $k Key to use to lookup
+     * @param mixed  $v Value to lookup
+     *
+     * @return Yammer_notice object found, or null for no hits
+     *
+     */
+
+    function staticGet($k, $v=null)
+    {
+        return Memcached_DataObject::staticGet('Yammer_notice', $k, $v);
+    }
+
+    /**
+     * Return schema definition to set this table up in onCheckSchema
+     */
+
+    static function schemaDef()
+    {
+        return self::doSchemaDef('notice_id');
+    }
+
+    /**
+     * Save a mapping between a remote Yammer and local imported notice.
+     *
+     * @param integer $orig_id ID of the notice in Yammer
+     * @param integer $notice_id ID of the status in StatusNet
+     *
+     * @return Yammer_notice new object for this value
+     */
+
+    static function record($orig_id, $notice_id)
+    {
+        return self::doRecord('Yammer_notice', 'notice_id', $orig_id, $notice_id);
+    }
+}
diff --git a/plugins/YammerImport/classes/Yammer_notice_stub.php b/plugins/YammerImport/classes/Yammer_notice_stub.php
new file mode 100644
index 0000000000..98a5e2cf77
--- /dev/null
+++ b/plugins/YammerImport/classes/Yammer_notice_stub.php
@@ -0,0 +1,174 @@
+<?php
+/**
+ * Data class for remembering Yammer import mappings
+ *
+ * PHP version 5
+ *
+ * @category Data
+ * @package  StatusNet
+ * @author   Brion Vibber <brion@status.net>
+ * @license  http://www.fsf.org/licensing/licenses/agpl.html AGPLv3
+ * @link     http://status.net/
+ *
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.     See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+if (!defined('STATUSNET')) {
+    exit(1);
+}
+
+/**
+ * Temporary storage for imported Yammer messages between fetching and saving
+ * as local notices.
+ *
+ * The Yammer API only allows us to page down from the most recent items; in
+ * order to start saving the oldest notices first, we have to pull them all
+ * down in reverse chronological order, then go back over them from oldest to
+ * newest and actually save them into our notice table.
+ */
+
+class Yammer_notice_stub extends Memcached_DataObject
+{
+    public $__table = 'yammer_notice_stub'; // table name
+    public $id;                             // int  primary_key not_null
+    public $json_data;                      // text
+    public $created;                        // datetime
+
+    /**
+     * Return schema definition to set this table up in onCheckSchema
+     */
+    static function schemaDef($field)
+    {
+        return array(new ColumnDef('id', 'bigint', null,
+                                   false, 'PRI'),
+                     new ColumnDef('json_data', 'text', null,
+                                   false),
+                     new ColumnDef('created', 'datetime', null,
+                                   false));
+    }
+
+    /**
+     * return table definition for DB_DataObject
+     *
+     * DB_DataObject needs to know something about the table to manipulate
+     * instances. This method provides all the DB_DataObject needs to know.
+     *
+     * @return array array of column definitions
+     */
+
+    function table()
+    {
+        return array('id'           => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
+                     'json_data'    => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
+                     'created'      => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL);
+    }
+
+    /**
+     * return key definitions for DB_DataObject
+     *
+     * DB_DataObject needs to know about keys that the table has, since it
+     * won't appear in StatusNet's own keys list. In most cases, this will
+     * simply reference your keyTypes() function.
+     *
+     * @return array list of key field names
+     */
+
+    function keys()
+    {
+        return array_keys($this->keyTypes());
+    }
+
+    /**
+     * return key definitions for Memcached_DataObject
+     *
+     * Our caching system uses the same key definitions, but uses a different
+     * method to get them. This key information is used to store and clear
+     * cached data, so be sure to list any key that will be used for static
+     * lookups.
+     *
+     * @return array associative array of key definitions, field name to type:
+     *         'K' for primary key: for compound keys, add an entry for each component;
+     *         'U' for unique keys: compound keys are not well supported here.
+     */
+
+    function keyTypes()
+    {
+        return array('id' => 'K');
+    }
+
+    /**
+     * Magic formula for non-autoincrementing integer primary keys
+     *
+     * If a table has a single integer column as its primary key, DB_DataObject
+     * assumes that the column is auto-incrementing and makes a sequence table
+     * to do this incrementation. Since we don't need this for our class, we
+     * overload this method and return the magic formula that DB_DataObject needs.
+     *
+     * @return array magic three-false array that stops auto-incrementing.
+     */
+
+    function sequenceKey()
+    {
+        return array(false, false, false);
+    }
+
+    /**
+     * Save the native Yammer API representation of a message for the pending
+     * import. Since they come in in reverse chronological order, we need to
+     * record them all as stubs and then go through from the beginning and
+     * save them as native notices, or we'll lose ordering and threading
+     * data.
+     *
+     * @param integer $orig_id ID of the notice on Yammer
+     * @param array $data the message record fetched out of Yammer API returnd data
+     *
+     * @return Yammer_notice_stub new object for this value
+     */
+
+    static function record($orig_id, $data)
+    {
+        common_debug("Recording Yammer message stub {$orig_id} for pending import...");
+
+        $stub = new Yammer_notice_stub();
+
+        $stub->id = $orig_id;
+        $stub->json_data = json_encode($data);
+        $stub->created = common_sql_now();
+
+        $stub->insert();
+
+        return $stub;
+    }
+
+    /**
+     * Save a mapping between a remote Yammer and local imported user.
+     *
+     * @param integer $user_id ID of the status in StatusNet
+     *
+     * @return Yammer_notice_stub new object for this value
+     */
+
+    static function retrieve($orig_id)
+    {
+        $stub = self::staticGet('id', $orig_id);
+        if ($stub) {
+            return json_decode($stub->json_data, true);
+        } else {
+            return false;
+        }
+    }
+}
diff --git a/plugins/YammerImport/classes/Yammer_state.php b/plugins/YammerImport/classes/Yammer_state.php
new file mode 100644
index 0000000000..a476fd3bec
--- /dev/null
+++ b/plugins/YammerImport/classes/Yammer_state.php
@@ -0,0 +1,37 @@
+<?php
+
+/*
+
+yammer_state
+    id
+    state
+    request_token
+    oauth_token
+    users_page
+    groups_page
+    messages_oldest
+    created
+    modified
+
+yammer_user
+    id
+    user_id
+    created
+
+yammer_group
+    id
+    group_id
+    created
+
+yammer_notice
+    id
+    notice_id
+    created
+
+yammer_notice_stub
+    id
+    json_data
+    created
+
+*/
+
diff --git a/plugins/YammerImport/classes/Yammer_user.php b/plugins/YammerImport/classes/Yammer_user.php
new file mode 100644
index 0000000000..f6fcd6c3cb
--- /dev/null
+++ b/plugins/YammerImport/classes/Yammer_user.php
@@ -0,0 +1,79 @@
+<?php
+/**
+ * Data class for remembering Yammer import mappings
+ *
+ * PHP version 5
+ *
+ * @category Data
+ * @package  StatusNet
+ * @author   Brion Vibber <brion@status.net>
+ * @license  http://www.fsf.org/licensing/licenses/agpl.html AGPLv3
+ * @link     http://status.net/
+ *
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.     See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+if (!defined('STATUSNET')) {
+    exit(1);
+}
+
+class Yammer_user extends Yammer_common
+{
+    public $__table = 'yammer_user'; // table name
+    public $__field = 'user_id';     // field to map to
+    public $user_id;                 // int
+
+    /**
+     * Get an instance by key
+     *
+     * This is a utility method to get a single instance with a given key value.
+     *
+     * @param string $k Key to use to lookup
+     * @param mixed  $v Value to lookup
+     *
+     * @return Yammer_user object found, or null for no hits
+     *
+     */
+
+    function staticGet($k, $v=null)
+    {
+        return Memcached_DataObject::staticGet('Yammer_user', $k, $v);
+    }
+
+    /**
+     * Return schema definition to set this table up in onCheckSchema
+     */
+
+    static function schemaDef()
+    {
+        return self::doSchemaDef('user_id');
+    }
+
+    /**
+     * Save a mapping between a remote Yammer and local imported user.
+     *
+     * @param integer $orig_id ID of the notice in Yammer
+     * @param integer $user_id ID of the status in StatusNet
+     *
+     * @return Yammer_user new object for this value
+     */
+
+    static function record($orig_id, $user_id)
+    {
+        return self::doRecord('Yammer_user', 'user_id', $orig_id, $user_id);
+    }
+}
diff --git a/plugins/YammerImport/lib/yammerimporter.php b/plugins/YammerImport/lib/yammerimporter.php
index 9ce0d1e588..b1d2815b9e 100644
--- a/plugins/YammerImport/lib/yammerimporter.php
+++ b/plugins/YammerImport/lib/yammerimporter.php
@@ -26,9 +26,6 @@
 class YammerImporter
 {
     protected $client;
-    protected $users=array();
-    protected $groups=array();
-    protected $notices=array();
 
     function __construct(SN_YammerClient $client)
     {
@@ -330,44 +327,32 @@ class YammerImporter
 
     private function findImportedUser($origId)
     {
-        if (isset($this->users[$origId])) {
-            return $this->users[$origId];
-        } else {
-            return false;
-        }
+        return Yammer_user::staticGet('id', $origId);
     }
 
     private function findImportedGroup($origId)
     {
-        if (isset($this->groups[$origId])) {
-            return $this->groups[$origId];
-        } else {
-            return false;
-        }
+        return Yammer_group::staticGet('id', $origId);
     }
 
     private function findImportedNotice($origId)
     {
-        if (isset($this->notices[$origId])) {
-            return $this->notices[$origId];
-        } else {
-            return false;
-        }
+        return Yammer_notice::staticGet('id', $origId);
     }
 
     private function recordImportedUser($origId, $userId)
     {
-        $this->users[$origId] = $userId;
+        Yammer_user::record($origId, $userId);
     }
 
     private function recordImportedGroup($origId, $groupId)
     {
-        $this->groups[$origId] = $groupId;
+        Yammer_group::record($origId, $groupId);
     }
 
     private function recordImportedNotice($origId, $noticeId)
     {
-        $this->notices[$origId] = $noticeId;
+        Yammer_notice::record($origId, $noticeId);
     }
 
     /**
diff --git a/plugins/YammerImport/lib/yammerqueuehandler.php b/plugins/YammerImport/lib/yammerqueuehandler.php
new file mode 100644
index 0000000000..ca81cbb344
--- /dev/null
+++ b/plugins/YammerImport/lib/yammerqueuehandler.php
@@ -0,0 +1,47 @@
+<?php
+/*
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+if (!defined('STATUSNET')) {
+    exit(1);
+}
+
+/**
+ * Queue handler for bumping the next chunk of Yammer import activity!
+ *
+ * @package YammerImportPlugin
+ * @author Brion Vibber <brion@status.net>
+ */
+class YammerQueueHandler extends QueueHandler
+{
+    function transport()
+    {
+        return 'yammer';
+    }
+
+    function handle($notice)
+    {
+        $importer = new YammerImporter();
+        if ($importer->hasWork()) {
+            return $importer->iterate();
+        } else {
+            // We're done!
+            return true;
+        }
+    }
+}

From 44ff13c947095fb2307d6e9a10362e940ef1b9af Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Wed, 22 Sep 2010 17:53:38 -0700
Subject: [PATCH 22/35] More doc comments on SN_YammerClient

---
 plugins/YammerImport/lib/sn_yammerclient.php | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/plugins/YammerImport/lib/sn_yammerclient.php b/plugins/YammerImport/lib/sn_yammerclient.php
index 8f9f1d4131..830f9dabb8 100644
--- a/plugins/YammerImport/lib/sn_yammerclient.php
+++ b/plugins/YammerImport/lib/sn_yammerclient.php
@@ -139,8 +139,11 @@ class SN_YammerClient
     }
 
     /**
+     * Encode a key-value pair for use in an authentication header.
+     *
      * @param string $key
      * @param string $val
+     * @return string
      */
     protected function authHeaderChunk($key, $val)
     {
@@ -148,6 +151,9 @@ class SN_YammerClient
     }
 
     /**
+     * Ask the Yammer server for a request token, which can be passed on
+     * to authorizeUrl() for the user to start the authentication process.
+     *
      * @return array of oauth return data; should contain nice things
      */
     public function requestToken()
@@ -162,6 +168,9 @@ class SN_YammerClient
     }
 
     /**
+     * Get a final access token from the verifier/PIN code provided to
+     * the user from Yammer's auth pages.
+     *
      * @return array of oauth return data; should contain nice things
      */
     public function accessToken($verifier)
@@ -175,7 +184,7 @@ class SN_YammerClient
     }
 
     /**
-     * Give the URL to send users to to authorize a new app setup
+     * Give the URL to send users to to authorize a new app setup.
      *
      * @param string $token as returned from accessToken()
      * @return string URL

From 5183997e3585d9201e08e5934307b90921ae92a1 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Thu, 23 Sep 2010 12:52:58 -0700
Subject: [PATCH 23/35] A little more cleanup...

---
 plugins/YammerImport/YammerImportPlugin.php   |  10 +-
 .../classes/Yammer_notice_stub.php            |   4 +-
 plugins/YammerImport/classes/Yammer_state.php | 155 ++++++++++++++----
 plugins/YammerImport/scripts/yamdump.php      |   6 +-
 4 files changed, 141 insertions(+), 34 deletions(-)

diff --git a/plugins/YammerImport/YammerImportPlugin.php b/plugins/YammerImport/YammerImportPlugin.php
index f55169a55b..58fc8b772a 100644
--- a/plugins/YammerImport/YammerImportPlugin.php
+++ b/plugins/YammerImport/YammerImportPlugin.php
@@ -64,7 +64,7 @@ class YammerImportPlugin extends Plugin
                         'Yammer_notice',
                         'Yammer_notice_stub');
         foreach ($tables as $table) {
-            $schema->ensureTable($table, $table::schemaDef());
+            $schema->ensureTable(strtolower($table), $table::schemaDef());
         }
 
         return true;
@@ -121,6 +121,14 @@ class YammerImportPlugin extends Plugin
         case 'yammeradminpanelaction':
             require_once "$base/actions/yammeradminpanel.php";
             return false;
+        case 'yammer_state':
+        case 'yammer_notice_stub':
+        case 'yammer_common':
+        case 'yammer_user':
+        case 'yammer_group':
+        case 'yammer_notice':
+            require_once "$base/classes/$cls.php";
+            return false;
         default:
             return true;
         }
diff --git a/plugins/YammerImport/classes/Yammer_notice_stub.php b/plugins/YammerImport/classes/Yammer_notice_stub.php
index 98a5e2cf77..cc52554dea 100644
--- a/plugins/YammerImport/classes/Yammer_notice_stub.php
+++ b/plugins/YammerImport/classes/Yammer_notice_stub.php
@@ -51,7 +51,7 @@ class Yammer_notice_stub extends Memcached_DataObject
     /**
      * Return schema definition to set this table up in onCheckSchema
      */
-    static function schemaDef($field)
+    static function schemaDef()
     {
         return array(new ColumnDef('id', 'bigint', null,
                                    false, 'PRI'),
@@ -73,7 +73,7 @@ class Yammer_notice_stub extends Memcached_DataObject
     function table()
     {
         return array('id'           => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
-                     'json_data'    => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
+                     'json_data'    => DB_DATAOBJECT_STR + DB_DATAOBJECT_NOTNULL,
                      'created'      => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL);
     }
 
diff --git a/plugins/YammerImport/classes/Yammer_state.php b/plugins/YammerImport/classes/Yammer_state.php
index a476fd3bec..98a656bfc5 100644
--- a/plugins/YammerImport/classes/Yammer_state.php
+++ b/plugins/YammerImport/classes/Yammer_state.php
@@ -1,37 +1,136 @@
 <?php
+/**
+ * Data class for remembering Yammer import state
+ *
+ * PHP version 5
+ *
+ * @category Data
+ * @package  StatusNet
+ * @author   Brion Vibber <brion@status.net>
+ * @license  http://www.fsf.org/licensing/licenses/agpl.html AGPLv3
+ * @link     http://status.net/
+ *
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.     See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
 
-/*
+if (!defined('STATUSNET')) {
+    exit(1);
+}
 
-yammer_state
-    id
-    state
-    request_token
-    oauth_token
-    users_page
-    groups_page
-    messages_oldest
-    created
-    modified
+class Yammer_state extends Memcached_DataObject
+{
+    public $__table = 'yammer_state'; // table name
+    public $id;                       // int  primary_key not_null
+    public $state;                    // import state key
+    public $request_token;            // oauth request token; clear when auth is complete.
+    public $oauth_token;              // actual oauth token! clear when import is done?
+    public $users_page;               // last page of users we've fetched
+    public $groups_page;              // last page of groups we've fetched
+    public $messages_oldest;          // oldest message ID we've fetched
+    public $messages_newest;          // newest message ID we've imported
+    public $created;                  // datetime
+    public $modified;                 // datetime
 
-yammer_user
-    id
-    user_id
-    created
+    /**
+     * Return schema definition to set this table up in onCheckSchema
+     */
+    static function schemaDef()
+    {
+        return array(new ColumnDef('id', 'int', null,
+                                   false, 'PRI'),
+                     new ColumnDef('state', 'text'),
+                     new ColumnDef('request_token', 'text'),
+                     new ColumnDef('oauth_token', 'text'),
+                     new ColumnDef('users_page', 'int'),
+                     new ColumnDef('groups_page', 'int'),
+                     new ColumnDef('messages_oldest', 'bigint'),
+                     new ColumnDef('messages_newest', 'bigint'),
+                     new ColumnDef('created', 'datetime'),
+                     new ColumnDef('modified', 'datetime'));
+    }
 
-yammer_group
-    id
-    group_id
-    created
+    /**
+     * return table definition for DB_DataObject
+     *
+     * DB_DataObject needs to know something about the table to manipulate
+     * instances. This method provides all the DB_DataObject needs to know.
+     *
+     * @return array array of column definitions
+     */
 
-yammer_notice
-    id
-    notice_id
-    created
+    function table()
+    {
+        return array('id'              => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
+                     'state'           => DB_DATAOBJECT_STR,
+                     'request_token'   => DB_DATAOBJECT_STR,
+                     'oauth_token'     => DB_DATAOBJECT_STR,
+                     'users_page'      => DB_DATAOBJECT_INT,
+                     'groups_page'     => DB_DATAOBJECT_INT,
+                     'messages_oldest' => DB_DATAOBJECT_INT,
+                     'messages_newest' => DB_DATAOBJECT_INT,
+                     'created'         => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL);
+    }
 
-yammer_notice_stub
-    id
-    json_data
-    created
+    /**
+     * return key definitions for DB_DataObject
+     *
+     * DB_DataObject needs to know about keys that the table has, since it
+     * won't appear in StatusNet's own keys list. In most cases, this will
+     * simply reference your keyTypes() function.
+     *
+     * @return array list of key field names
+     */
 
-*/
+    function keys()
+    {
+        return array_keys($this->keyTypes());
+    }
 
+    /**
+     * return key definitions for Memcached_DataObject
+     *
+     * Our caching system uses the same key definitions, but uses a different
+     * method to get them. This key information is used to store and clear
+     * cached data, so be sure to list any key that will be used for static
+     * lookups.
+     *
+     * @return array associative array of key definitions, field name to type:
+     *         'K' for primary key: for compound keys, add an entry for each component;
+     *         'U' for unique keys: compound keys are not well supported here.
+     */
+
+    function keyTypes()
+    {
+        return array('id' => 'K');
+    }
+
+    /**
+     * Magic formula for non-autoincrementing integer primary keys
+     *
+     * If a table has a single integer column as its primary key, DB_DataObject
+     * assumes that the column is auto-incrementing and makes a sequence table
+     * to do this incrementation. Since we don't need this for our class, we
+     * overload this method and return the magic formula that DB_DataObject needs.
+     *
+     * @return array magic three-false array that stops auto-incrementing.
+     */
+
+    function sequenceKey()
+    {
+        return array(false, false, false);
+    }
+}
diff --git a/plugins/YammerImport/scripts/yamdump.php b/plugins/YammerImport/scripts/yamdump.php
index a358777ad1..944ee2e499 100644
--- a/plugins/YammerImport/scripts/yamdump.php
+++ b/plugins/YammerImport/scripts/yamdump.php
@@ -13,6 +13,7 @@ require 'yam-config.php';
 $yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
 $imp = new YammerImporter($yam);
 
+/*
 $data = $yam->users();
 var_dump($data);
 // @fixme follow paging
@@ -20,9 +21,9 @@ foreach ($data as $item) {
     $user = $imp->prepUser($item);
     var_dump($user);
 }
+*/
 
-/*
-$data = $yam->messages();
+$data = $yam->messages(array('newer_than' => 1));
 var_dump($data);
 // @fixme follow paging
 $messages = $data['messages'];
@@ -31,4 +32,3 @@ foreach ($messages as $message) {
     $notice = $imp->prepNotice($message);
     var_dump($notice);
 }
-*/

From eb8be9988eed03b285fa2095a1d99010f928b117 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Thu, 23 Sep 2010 15:23:56 -0700
Subject: [PATCH 24/35] Work in progress: YammerRunner state machine wrapper
 for running the Yammer import in chunks.

---
 plugins/YammerImport/YammerImportPlugin.php   |   1 +
 .../YammerImport/actions/yammeradminpanel.php |   2 +-
 plugins/YammerImport/classes/Yammer_state.php |   3 +
 plugins/YammerImport/lib/yammerrunner.php     | 198 ++++++++++++++++++
 4 files changed, 203 insertions(+), 1 deletion(-)
 create mode 100644 plugins/YammerImport/lib/yammerrunner.php

diff --git a/plugins/YammerImport/YammerImportPlugin.php b/plugins/YammerImport/YammerImportPlugin.php
index 58fc8b772a..85eab74c04 100644
--- a/plugins/YammerImport/YammerImportPlugin.php
+++ b/plugins/YammerImport/YammerImportPlugin.php
@@ -116,6 +116,7 @@ class YammerImportPlugin extends Plugin
         switch ($lower) {
         case 'sn_yammerclient':
         case 'yammerimporter':
+        case 'yammerrunner':
             require_once "$base/lib/$lower.php";
             return false;
         case 'yammeradminpanelaction':
diff --git a/plugins/YammerImport/actions/yammeradminpanel.php b/plugins/YammerImport/actions/yammeradminpanel.php
index 875debac92..9f935bbefb 100644
--- a/plugins/YammerImport/actions/yammeradminpanel.php
+++ b/plugins/YammerImport/actions/yammeradminpanel.php
@@ -133,7 +133,7 @@ class YammerAdminPanelForm extends AdminForm
                 break;
             case 'import-users':
             case 'import-groups':
-            case 'import-messages':
+            case 'fetch-messages':
             case 'save-messages':
                 $this->showImportState();
                 break;
diff --git a/plugins/YammerImport/classes/Yammer_state.php b/plugins/YammerImport/classes/Yammer_state.php
index 98a656bfc5..0174ead15d 100644
--- a/plugins/YammerImport/classes/Yammer_state.php
+++ b/plugins/YammerImport/classes/Yammer_state.php
@@ -38,6 +38,7 @@ class Yammer_state extends Memcached_DataObject
     public $state;                    // import state key
     public $request_token;            // oauth request token; clear when auth is complete.
     public $oauth_token;              // actual oauth token! clear when import is done?
+    public $oauth_secret;             // actual oauth secret! clear when import is done?
     public $users_page;               // last page of users we've fetched
     public $groups_page;              // last page of groups we've fetched
     public $messages_oldest;          // oldest message ID we've fetched
@@ -55,6 +56,7 @@ class Yammer_state extends Memcached_DataObject
                      new ColumnDef('state', 'text'),
                      new ColumnDef('request_token', 'text'),
                      new ColumnDef('oauth_token', 'text'),
+                     new ColumnDef('oauth_secret', 'text'),
                      new ColumnDef('users_page', 'int'),
                      new ColumnDef('groups_page', 'int'),
                      new ColumnDef('messages_oldest', 'bigint'),
@@ -78,6 +80,7 @@ class Yammer_state extends Memcached_DataObject
                      'state'           => DB_DATAOBJECT_STR,
                      'request_token'   => DB_DATAOBJECT_STR,
                      'oauth_token'     => DB_DATAOBJECT_STR,
+                     'oauth_secret'    => DB_DATAOBJECT_STR,
                      'users_page'      => DB_DATAOBJECT_INT,
                      'groups_page'     => DB_DATAOBJECT_INT,
                      'messages_oldest' => DB_DATAOBJECT_INT,
diff --git a/plugins/YammerImport/lib/yammerrunner.php b/plugins/YammerImport/lib/yammerrunner.php
new file mode 100644
index 0000000000..e229b2acbe
--- /dev/null
+++ b/plugins/YammerImport/lib/yammerrunner.php
@@ -0,0 +1,198 @@
+<?php
+/*
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2010, StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+if (!defined('STATUSNET')) {
+    exit(1);
+}
+
+/**
+ * State machine for running through Yammer import.
+ *
+ * @package YammerImportPlugin
+ * @author Brion Vibber <brion@status.net>
+ */
+class YammerRunner
+{
+    private $state;
+    private $client;
+    private $importer;
+
+    function __construct()
+    {
+        $state = Yammer_state::staticGet('id', 1);
+        if (!$state) {
+            common_log(LOG_ERR, "No YammerImport state during import run. Should not happen!");
+            throw new ServerException('No YammerImport state during import run.');
+        }
+
+        $this->state = $state;
+        $this->client = new SN_YammerClient(
+            common_config('yammer', 'consumer_key'),
+            common_config('yammer', 'consumer_secret'),
+            $this->state->oauth_token,
+            $this->state->oauth_secret);
+        $this->importer = new YammerImporter($client);
+    }
+
+    public function iterate()
+    {
+
+        switch($state->state)
+        {
+            case null:
+            case 'requesting-auth':
+                // Neither of these should reach our background state!
+                common_log(LOG_ERR, "Non-background YammerImport state '$state->state' during import run!");
+                return false;
+            case 'import-users':
+                return $this->iterateUsers();
+            case 'import-groups':
+                return $this->iterateGroups();
+            case 'fetch-messages':
+                return $this->iterateFetchMessages();
+            case 'save-messages':
+                return $this->iterateSaveMessages();
+            default:
+                common_log(LOG_ERR, "Invalid YammerImport state '$state->state' during import run!");
+                return false;
+        }
+    }
+
+    /**
+     * Trundle through one 'page' return of up to 50 user accounts retrieved
+     * from the Yammer API, importing them as we go.
+     *
+     * When we run out of users, move on to groups.
+     *
+     * @return boolean success
+     */
+    private function iterateUsers()
+    {
+        $old = clone($this->state);
+
+        $page = intval($this->state->users_page) + 1;
+        $data = $this->client->users(array('page' => $page));
+
+        if (count($data) == 0) {
+            common_log(LOG_INFO, "Finished importing Yammer users; moving on to groups.");
+            $this->state->state = 'import-groups';
+        } else {
+            foreach ($data as $item) {
+                $user = $imp->importUser($item);
+                common_log(LOG_INFO, "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)");
+            }
+            $this->state->users_page = $page;
+        }
+        $this->state->update($old);
+        return true;
+    }
+
+    /**
+     * Trundle through one 'page' return of up to 20 user groups retrieved
+     * from the Yammer API, importing them as we go.
+     *
+     * When we run out of groups, move on to messages.
+     *
+     * @return boolean success
+     */
+    private function iterateGroups()
+    {
+        $old = clone($this->state);
+
+        $page = intval($this->state->groups_page) + 1;
+        $data = $this->client->groups(array('page' => $page));
+
+        if (count($data) == 0) {
+            common_log(LOG_INFO, "Finished importing Yammer groups; moving on to messages.");
+            $this->state->state = 'import-messages';
+        } else {
+            foreach ($data as $item) {
+                $group = $imp->importGroup($item);
+                common_log(LOG_INFO, "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)");
+            }
+            $this->state->groups_page = $page;
+        }
+        $this->state->update($old);
+        return true;
+    }
+
+    /**
+     * Trundle through one 'page' return of up to 20 public messages retrieved
+     * from the Yammer API, saving them to our stub table for future import in
+     * correct chronological order.
+     *
+     * When we run out of messages to fetch, move on to saving the messages.
+     *
+     * @return boolean success
+     */
+    private function iterateFetchMessages()
+    {
+        $old = clone($this->state);
+
+        $oldest = intval($this->state->messages_oldest);
+        if ($oldest) {
+            $params = array('older_than' => $oldest);
+        } else {
+            $params = array();
+        }
+        $data = $this->client->messages($params);
+        $messages = $data['messages'];
+
+        if (count($data) == 0) {
+            common_log(LOG_INFO, "Finished fetching Yammer messages; moving on to save messages.");
+            $this->state->state = 'save-messages';
+        } else {
+            foreach ($data as $item) {
+                Yammer_notice_stub::record($item['id'], $item);
+                $oldest = $item['id'];
+            }
+            $this->state->messages_oldest = $oldest;
+        }
+        $this->state->update($old);
+        return true;
+    }
+
+    private function iterateSaveMessages()
+    {
+        $old = clone($this->state);
+
+        $newest = intval($this->state->messages_newest);
+        if ($newest) {
+            $stub->addWhere('id > ' . $newest);
+        }
+        $stub->limit(20);
+        $stub->find();
+        
+        if ($stub->N == 0) {
+            common_log(LOG_INFO, "Finished saving Yammer messages; import complete!");
+            $this->state->state = 'done';
+        } else {
+            while ($stub->fetch()) {
+                $item = json_decode($stub->json_data);
+                $notice = $this->importer->importNotice($item);
+                common_log(LOG_INFO, "Imported Yammer notice " . $item['id'] . " as $notice->id");
+                $newest = $item['id'];
+            }
+            $this->state->messages_newest = $newest;
+        }
+        $this->state->update($old);
+        return true;
+    }
+
+}

From dd414db9ea71ca7b2d0fca901d51931557bde6bd Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Thu, 23 Sep 2010 16:40:22 -0700
Subject: [PATCH 25/35] Work in progress: most of the infrastructure for
 running import via BG queues or CLI script is now in place (untested, no UI,
 needs tweaks & fixes)

---
 plugins/YammerImport/README                   |  42 ++++++++
 .../YammerImport/lib/yammerqueuehandler.php   |  18 +++-
 plugins/YammerImport/lib/yammerrunner.php     | 102 +++++++++++++++++-
 .../YammerImport/scripts/yammer-import.php    |  59 +++++-----
 4 files changed, 186 insertions(+), 35 deletions(-)

diff --git a/plugins/YammerImport/README b/plugins/YammerImport/README
index 5ab080647a..1bac69a243 100644
--- a/plugins/YammerImport/README
+++ b/plugins/YammerImport/README
@@ -73,3 +73,45 @@ they do on Yammer; they will be linked instead.
 File type and size limitations on attachments will be applied, so beware some
 attachments may not make it through.
 
+
+
+
+Code structure
+==============
+
+Standalone classes
+------------------
+
+YammerRunner: encapsulates the iterative process of retrieving the various users,
+              groups, and messages via SN_YammerClient and saving them locally
+              via YammerImporter.
+
+SN_YammerClient: encapsulates HTTP+OAuth interface to Yammer API, returns data
+                 as straight decoded JSON object trees.
+
+YammerImporter: encapsulates logic to pull information from the returned API data
+                and convert them to native StatusNet users, groups, and messages.
+
+Web UI actions
+-------------
+
+YammeradminpanelAction: web panel for site administrator to initiate and monitor
+                        the import process.
+
+Command-line scripts
+--------------------
+
+yammer-import.php: CLI script to start a Yammer import run in one go.
+
+Database objects
+----------------
+
+Yammer_state: data object storing YammerRunner's state between iterations.
+
+Yammer_notice_stub: data object for temporary storage of fetched Yammer messages
+                    between fetching them (reverse chron order) and saving them
+                    to local messages (forward chron order).
+Yammer_user,
+Yammer_group,
+Yammer_notice: data objects mapping original Yammer item IDs to their local copies.
+
diff --git a/plugins/YammerImport/lib/yammerqueuehandler.php b/plugins/YammerImport/lib/yammerqueuehandler.php
index ca81cbb344..5fc3777835 100644
--- a/plugins/YammerImport/lib/yammerqueuehandler.php
+++ b/plugins/YammerImport/lib/yammerqueuehandler.php
@@ -36,11 +36,23 @@ class YammerQueueHandler extends QueueHandler
 
     function handle($notice)
     {
-        $importer = new YammerImporter();
-        if ($importer->hasWork()) {
-            return $importer->iterate();
+        $runner = YammerRunner::init();
+        if ($runner->hasWork()) {
+            if ($runner->iterate()) {
+                if ($runner->hasWork()) {
+                    // More to do? Shove us back on the queue...
+                    $qm = QueueManager::get();
+                    $qm->enqueue('YammerImport', 'yammer');
+                }
+                return true;
+            } else {
+                // Something failed?
+                // @fixme should we be trying again here, or should we give warning?
+                return false;
+            }
         } else {
             // We're done!
+            common_log(LOG_INFO, "Yammer import has no work to do at this time; discarding.");
             return true;
         }
     }
diff --git a/plugins/YammerImport/lib/yammerrunner.php b/plugins/YammerImport/lib/yammerrunner.php
index e229b2acbe..95ff783714 100644
--- a/plugins/YammerImport/lib/yammerrunner.php
+++ b/plugins/YammerImport/lib/yammerrunner.php
@@ -33,29 +33,123 @@ class YammerRunner
     private $client;
     private $importer;
 
-    function __construct()
+    public static function init()
     {
         $state = Yammer_state::staticGet('id', 1);
         if (!$state) {
-            common_log(LOG_ERR, "No YammerImport state during import run. Should not happen!");
-            throw new ServerException('No YammerImport state during import run.');
+            $state = new Yammer_state();
+            $state->id = 1;
+            $state->state = 'init';
+            $state->insert();
         }
+        return new YammerRunner($state);
+    }
 
+    private function __construct($state)
+    {
         $this->state = $state;
+
         $this->client = new SN_YammerClient(
             common_config('yammer', 'consumer_key'),
             common_config('yammer', 'consumer_secret'),
             $this->state->oauth_token,
             $this->state->oauth_secret);
+
         $this->importer = new YammerImporter($client);
     }
 
+    /**
+     * Check which state we're in
+     *
+     * @return string
+     */
+    public function state()
+    {
+        return $this->state->state;
+    }
+
+    /**
+     * Is the import done, finished, complete, finito?
+     *
+     * @return boolean
+     */
+    public function isDone()
+    {
+        $workStates = array('import-users', 'import-groups', 'fetch-messages', 'save-messages');
+        return ($this->state() == 'done');
+    }
+
+    /**
+     * Check if we have work to do in iterate().
+     */
+    public function hasWork()
+    {
+        $workStates = array('import-users', 'import-groups', 'fetch-messages', 'save-messages');
+        return in_array($this->state(), $workStates);
+    }
+
+    /**
+     * Start the authentication process! If all goes well, we'll get back a URL.
+     * Have the user visit that URL, log in on Yammer and verify the importer's
+     * permissions. They'll get back a verification code, which needs to be passed
+     * on to saveAuthToken().
+     *
+     * @return string URL
+     */
+    public function requestAuth()
+    {
+        if ($this->state->state != 'init') {
+            throw ServerError("Cannot request Yammer auth; already there!");
+        }
+
+        $old = clone($this->state);
+        $this->state->state = 'requesting-auth';
+        $this->state->request_token = $client->requestToken();
+        $this->state->update($old);
+
+        return $this->client->authorizeUrl($this->state->request_token);
+    }
+
+    /**
+     * Now that the user's given us this verification code from Yammer, we can
+     * request a final OAuth token/secret pair which we can use to access the
+     * API.
+     *
+     * After success here, we'll be ready to move on and run through iterate()
+     * until the import is complete.
+     *
+     * @param string $verifier
+     * @return boolean success
+     */
+    public function saveAuthToken($verifier)
+    {
+        if ($this->state->state != 'requesting-auth') {
+            throw ServerError("Cannot save auth token in Yammer import state {$this->state->state}");
+        }
+
+        $old = clone($this->state);
+        list($token, $secret) = $this->client->getAuthToken($verifier);
+        $this->state->verifier = '';
+        $this->state->oauth_token = $token;
+        $this->state->oauth_secret = $secret;
+
+        $this->state->update($old);
+
+        return true;
+    }
+
+    /**
+     * Once authentication is complete, we need to call iterate() a bunch of times
+     * until state() returns 'done'.
+     *
+     * @return boolean success
+     */
     public function iterate()
     {
 
         switch($state->state)
         {
-            case null:
+            case 'init':
             case 'requesting-auth':
                 // Neither of these should reach our background state!
                 common_log(LOG_ERR, "Non-background YammerImport state '$state->state' during import run!");
diff --git a/plugins/YammerImport/scripts/yammer-import.php b/plugins/YammerImport/scripts/yammer-import.php
index ac258e1c7d..24307d6cd4 100644
--- a/plugins/YammerImport/scripts/yammer-import.php
+++ b/plugins/YammerImport/scripts/yammer-import.php
@@ -8,34 +8,37 @@ define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__)))));
 
 require INSTALLDIR . "/scripts/commandline.inc";
 
-// temp stuff
-require 'yam-config.php';
-$yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
-$imp = new YammerImporter($yam);
+$runner = YammerRunner::init();
 
-// First, import all the users!
-// @fixme follow paging -- we only get 50 at a time
-$data = $yam->users();
-foreach ($data as $item) {
-    $user = $imp->importUser($item);
-    echo "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)\n";
-}
+switch ($runner->state())
+{
+    case 'init':
+        $url = $runner->requestAuth();
+        echo "Log in to Yammer at the following URL and confirm permissions:\n";
+        echo "\n";
+        echo "    $url\n";
+        echo "\n";
+        echo "Pass the resulting code back by running:\n"
+        echo "\n"
+        echo "    php yammer-import.php --auth=####\n";
+        echo "\n";
+        break;
 
-// Groups!
-// @fixme follow paging -- we only get 20 at a time
-$data = $yam->groups();
-foreach ($data as $item) {
-    $group = $imp->importGroup($item);
-    echo "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)\n";
-}
+    case 'requesting-auth':
+        if (empty($options['auth'])) {
+            echo "Please finish authenticating!\n";
+            break;
+        }
+        $runner->saveAuthToken($options['auth']);
+        // Fall through...
 
-// Messages!
-// Process in reverse chron order...
-// @fixme follow paging -- we only get 20 at a time, and start at the most recent!
-$data = $yam->messages();
-$messages = $data['messages'];
-$messages = array_reverse($messages);
-foreach ($messages as $item) {
-    $notice = $imp->importNotice($item);
-    echo "Imported Yammer notice " . $item['id'] . " as $notice->id\n";
-}
+    default:
+        while (true) {
+            echo "... {$runner->state->state}\n";
+            if (!$runner->iterate()) {
+                echo "... done.\n";
+                break;
+            }
+        }
+        break;
+}
\ No newline at end of file

From bdd8a587e7b9f64fb0d3a39b851dde3f9cb562ab Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Thu, 23 Sep 2010 17:55:13 -0700
Subject: [PATCH 26/35] Ok, command-line workflow for YammerImportPlugin seems
 to mostly work, at least on tiny test site :D

---
 .../YammerImport/classes/Yammer_common.php    |  4 +-
 .../classes/Yammer_notice_stub.php            | 45 ++++++-----
 plugins/YammerImport/classes/Yammer_state.php | 20 ++++-
 plugins/YammerImport/lib/sn_yammerclient.php  |  7 +-
 plugins/YammerImport/lib/yammerimporter.php   | 11 ++-
 plugins/YammerImport/lib/yammerrunner.php     | 77 ++++++++++++++-----
 .../YammerImport/scripts/yammer-import.php    | 43 ++++++++---
 7 files changed, 144 insertions(+), 63 deletions(-)

diff --git a/plugins/YammerImport/classes/Yammer_common.php b/plugins/YammerImport/classes/Yammer_common.php
index 81e302ab29..6ec6fc9041 100644
--- a/plugins/YammerImport/classes/Yammer_common.php
+++ b/plugins/YammerImport/classes/Yammer_common.php
@@ -138,13 +138,13 @@ class Yammer_common extends Memcached_DataObject
 
     protected static function doRecord($class, $field, $orig_id, $local_id)
     {
-        $map = self::staticGet('id', $orig_id);
+        $map = parent::staticGet($class, 'id', $orig_id);
 
         if (!empty($map)) {
             return $map;
         }
 
-        $map = self::staticGet($field, $local_id);
+        $map = parent::staticGet($class, $field, $local_id);
 
         if (!empty($map)) {
             return $map;
diff --git a/plugins/YammerImport/classes/Yammer_notice_stub.php b/plugins/YammerImport/classes/Yammer_notice_stub.php
index cc52554dea..e10300c4c7 100644
--- a/plugins/YammerImport/classes/Yammer_notice_stub.php
+++ b/plugins/YammerImport/classes/Yammer_notice_stub.php
@@ -48,6 +48,23 @@ class Yammer_notice_stub extends Memcached_DataObject
     public $json_data;                      // text
     public $created;                        // datetime
 
+    /**
+     * Get an instance by key
+     *
+     * This is a utility method to get a single instance with a given key value.
+     *
+     * @param string $k Key to use to lookup
+     * @param mixed  $v Value to lookup
+     *
+     * @return Yammer_notice_stub object found, or null for no hits
+     *
+     */
+
+    function staticGet($k, $v=null)
+    {
+        return Memcached_DataObject::staticGet('Yammer_notice_stub', $k, $v);
+    }
+
     /**
      * Return schema definition to set this table up in onCheckSchema
      */
@@ -126,6 +143,16 @@ class Yammer_notice_stub extends Memcached_DataObject
         return array(false, false, false);
     }
 
+    /**
+     * Decode the stored data structure.
+     *
+     * @return mixed
+     */
+    public function getData()
+    {
+        return json_decode($this->json_data, true);
+    }
+
     /**
      * Save the native Yammer API representation of a message for the pending
      * import. Since they come in in reverse chronological order, we need to
@@ -153,22 +180,4 @@ class Yammer_notice_stub extends Memcached_DataObject
 
         return $stub;
     }
-
-    /**
-     * Save a mapping between a remote Yammer and local imported user.
-     *
-     * @param integer $user_id ID of the status in StatusNet
-     *
-     * @return Yammer_notice_stub new object for this value
-     */
-
-    static function retrieve($orig_id)
-    {
-        $stub = self::staticGet('id', $orig_id);
-        if ($stub) {
-            return json_decode($stub->json_data, true);
-        } else {
-            return false;
-        }
-    }
 }
diff --git a/plugins/YammerImport/classes/Yammer_state.php b/plugins/YammerImport/classes/Yammer_state.php
index 0174ead15d..2f1fd7780b 100644
--- a/plugins/YammerImport/classes/Yammer_state.php
+++ b/plugins/YammerImport/classes/Yammer_state.php
@@ -36,7 +36,6 @@ class Yammer_state extends Memcached_DataObject
     public $__table = 'yammer_state'; // table name
     public $id;                       // int  primary_key not_null
     public $state;                    // import state key
-    public $request_token;            // oauth request token; clear when auth is complete.
     public $oauth_token;              // actual oauth token! clear when import is done?
     public $oauth_secret;             // actual oauth secret! clear when import is done?
     public $users_page;               // last page of users we've fetched
@@ -46,6 +45,23 @@ class Yammer_state extends Memcached_DataObject
     public $created;                  // datetime
     public $modified;                 // datetime
 
+    /**
+     * Get an instance by key
+     *
+     * This is a utility method to get a single instance with a given key value.
+     *
+     * @param string $k Key to use to lookup
+     * @param mixed  $v Value to lookup
+     *
+     * @return Yammer_state object found, or null for no hits
+     *
+     */
+
+    function staticGet($k, $v=null)
+    {
+        return Memcached_DataObject::staticGet('Yammer_state', $k, $v);
+    }
+
     /**
      * Return schema definition to set this table up in onCheckSchema
      */
@@ -54,7 +70,6 @@ class Yammer_state extends Memcached_DataObject
         return array(new ColumnDef('id', 'int', null,
                                    false, 'PRI'),
                      new ColumnDef('state', 'text'),
-                     new ColumnDef('request_token', 'text'),
                      new ColumnDef('oauth_token', 'text'),
                      new ColumnDef('oauth_secret', 'text'),
                      new ColumnDef('users_page', 'int'),
@@ -78,7 +93,6 @@ class Yammer_state extends Memcached_DataObject
     {
         return array('id'              => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
                      'state'           => DB_DATAOBJECT_STR,
-                     'request_token'   => DB_DATAOBJECT_STR,
                      'oauth_token'     => DB_DATAOBJECT_STR,
                      'oauth_secret'    => DB_DATAOBJECT_STR,
                      'users_page'      => DB_DATAOBJECT_INT,
diff --git a/plugins/YammerImport/lib/sn_yammerclient.php b/plugins/YammerImport/lib/sn_yammerclient.php
index 830f9dabb8..5da1cc5e7e 100644
--- a/plugins/YammerImport/lib/sn_yammerclient.php
+++ b/plugins/YammerImport/lib/sn_yammerclient.php
@@ -104,7 +104,8 @@ class SN_YammerClient
     {
         $body = $this->fetchApi("api/v1/$method.json", $params);
         $data = json_decode($body, true);
-        if (!$data) {
+        if ($data === null) {
+            common_log(LOG_ERR, "Invalid JSON response from Yammer API: " . $body);
             throw new Exception("Invalid JSON response from Yammer API");
         }
         return $data;
@@ -161,7 +162,7 @@ class SN_YammerClient
         if ($this->token || $this->tokenSecret) {
             throw new Exception("Requesting a token, but already set up with a token");
         }
-        $data = $this->fetch('oauth/request_token');
+        $data = $this->fetchApi('oauth/request_token');
         $arr = array();
         parse_str($data, $arr);
         return $arr;
@@ -176,7 +177,7 @@ class SN_YammerClient
     public function accessToken($verifier)
     {
         $this->verifier = $verifier;
-        $data = $this->fetch('oauth/access_token');
+        $data = $this->fetchApi('oauth/access_token');
         $this->verifier = null;
         $arr = array();
         parse_str($data, $arr);
diff --git a/plugins/YammerImport/lib/yammerimporter.php b/plugins/YammerImport/lib/yammerimporter.php
index b1d2815b9e..0425b8b04e 100644
--- a/plugins/YammerImport/lib/yammerimporter.php
+++ b/plugins/YammerImport/lib/yammerimporter.php
@@ -327,17 +327,20 @@ class YammerImporter
 
     private function findImportedUser($origId)
     {
-        return Yammer_user::staticGet('id', $origId);
+        $map = Yammer_user::staticGet('id', $origId);
+        return $map ? $map->user_id : null;
     }
 
     private function findImportedGroup($origId)
     {
-        return Yammer_group::staticGet('id', $origId);
+        $map = Yammer_group::staticGet('id', $origId);
+        return $map ? $map->group_id : null;
     }
 
     private function findImportedNotice($origId)
     {
-        return Yammer_notice::staticGet('id', $origId);
+        $map = Yammer_notice::staticGet('id', $origId);
+        return $map ? $map->notice_id : null;
     }
 
     private function recordImportedUser($origId, $userId)
@@ -370,7 +373,7 @@ class YammerImporter
         // Blaaaaaarf!
         $known = array('Pacific Time (US & Canada)' => 'America/Los_Angeles',
                        'Eastern Time (US & Canada)' => 'America/New_York');
-        if (array_key_exists($known, $tz)) {
+        if (array_key_exists($tz, $known)) {
             return $known[$tz];
         } else {
             return false;
diff --git a/plugins/YammerImport/lib/yammerrunner.php b/plugins/YammerImport/lib/yammerrunner.php
index 95ff783714..c4db48399c 100644
--- a/plugins/YammerImport/lib/yammerrunner.php
+++ b/plugins/YammerImport/lib/yammerrunner.php
@@ -33,18 +33,31 @@ class YammerRunner
     private $client;
     private $importer;
 
+    /**
+     * Normalize our singleton state and give us a YammerRunner object to play with!
+     *
+     * @return YammerRunner
+     */
     public static function init()
     {
         $state = Yammer_state::staticGet('id', 1);
         if (!$state) {
-            $state = new Yammer_state();
-            $state->id = 1;
-            $state->state = 'init';
-            $state->insert();
+            $state = self::initState();
         }
         return new YammerRunner($state);
     }
 
+    private static function initState()
+    {
+        $state = new Yammer_state();
+        $state->id = 1;
+        $state->state = 'init';
+        $state->created = common_sql_now();
+        $state->modified = common_sql_now();
+        $state->insert();
+        return $state;
+    }
+
     private function __construct($state)
     {
         $this->state = $state;
@@ -55,7 +68,7 @@ class YammerRunner
             $this->state->oauth_token,
             $this->state->oauth_secret);
 
-        $this->importer = new YammerImporter($client);
+        $this->importer = new YammerImporter($this->client);
     }
 
     /**
@@ -81,6 +94,8 @@ class YammerRunner
 
     /**
      * Check if we have work to do in iterate().
+     *
+     * @return boolean
      */
     public function hasWork()
     {
@@ -88,6 +103,15 @@ class YammerRunner
         return in_array($this->state(), $workStates);
     }
 
+    /**
+     * Blow away any current state!
+     */
+    public function reset()
+    {
+        $this->state->delete();
+        $this->state = self::initState();
+    }
+
     /**
      * Start the authentication process! If all goes well, we'll get back a URL.
      * Have the user visit that URL, log in on Yammer and verify the importer's
@@ -102,12 +126,16 @@ class YammerRunner
             throw ServerError("Cannot request Yammer auth; already there!");
         }
 
+        $data = $this->client->requestToken();
+
         $old = clone($this->state);
         $this->state->state = 'requesting-auth';
-        $this->state->request_token = $client->requestToken();
+        $this->state->oauth_token = $data['oauth_token'];
+        $this->state->oauth_secret = $data['oauth_token_secret'];
+        $this->state->modified = common_sql_now();
         $this->state->update($old);
 
-        return $this->client->authorizeUrl($this->state->request_token);
+        return $this->client->authorizeUrl($this->state->oauth_token);
     }
 
     /**
@@ -127,12 +155,13 @@ class YammerRunner
             throw ServerError("Cannot save auth token in Yammer import state {$this->state->state}");
         }
 
-        $old = clone($this->state);
-        list($token, $secret) = $this->client->getAuthToken($verifier);
-        $this->state->verifier = '';
-        $this->state->oauth_token = $token;
-        $this->state->oauth_secret = $secret;
+        $data = $this->client->accessToken($verifier);
 
+        $old = clone($this->state);
+        $this->state->state = 'import-users';
+        $this->state->oauth_token = $data['oauth_token'];
+        $this->state->oauth_secret = $data['oauth_token_secret'];
+        $this->state->modified = common_sql_now();
         $this->state->update($old);
 
         return true;
@@ -146,8 +175,7 @@ class YammerRunner
      */
     public function iterate()
     {
-
-        switch($state->state)
+        switch($this->state())
         {
             case 'init':
             case 'requesting-auth':
@@ -188,11 +216,12 @@ class YammerRunner
             $this->state->state = 'import-groups';
         } else {
             foreach ($data as $item) {
-                $user = $imp->importUser($item);
+                $user = $this->importer->importUser($item);
                 common_log(LOG_INFO, "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)");
             }
             $this->state->users_page = $page;
         }
+        $this->state->modified = common_sql_now();
         $this->state->update($old);
         return true;
     }
@@ -214,14 +243,15 @@ class YammerRunner
 
         if (count($data) == 0) {
             common_log(LOG_INFO, "Finished importing Yammer groups; moving on to messages.");
-            $this->state->state = 'import-messages';
+            $this->state->state = 'fetch-messages';
         } else {
             foreach ($data as $item) {
-                $group = $imp->importGroup($item);
+                $group = $this->importer->importGroup($item);
                 common_log(LOG_INFO, "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)");
             }
             $this->state->groups_page = $page;
         }
+        $this->state->modified = common_sql_now();
         $this->state->update($old);
         return true;
     }
@@ -248,16 +278,17 @@ class YammerRunner
         $data = $this->client->messages($params);
         $messages = $data['messages'];
 
-        if (count($data) == 0) {
+        if (count($messages) == 0) {
             common_log(LOG_INFO, "Finished fetching Yammer messages; moving on to save messages.");
             $this->state->state = 'save-messages';
         } else {
-            foreach ($data as $item) {
+            foreach ($messages as $item) {
                 Yammer_notice_stub::record($item['id'], $item);
                 $oldest = $item['id'];
             }
             $this->state->messages_oldest = $oldest;
         }
+        $this->state->modified = common_sql_now();
         $this->state->update($old);
         return true;
     }
@@ -267,10 +298,13 @@ class YammerRunner
         $old = clone($this->state);
 
         $newest = intval($this->state->messages_newest);
+
+        $stub = new Yammer_notice_stub();
         if ($newest) {
-            $stub->addWhere('id > ' . $newest);
+            $stub->whereAdd('id > ' . $newest);
         }
         $stub->limit(20);
+        $stub->orderBy('id');
         $stub->find();
         
         if ($stub->N == 0) {
@@ -278,13 +312,14 @@ class YammerRunner
             $this->state->state = 'done';
         } else {
             while ($stub->fetch()) {
-                $item = json_decode($stub->json_data);
+                $item = $stub->getData();
                 $notice = $this->importer->importNotice($item);
                 common_log(LOG_INFO, "Imported Yammer notice " . $item['id'] . " as $notice->id");
                 $newest = $item['id'];
             }
             $this->state->messages_newest = $newest;
         }
+        $this->state->modified = common_sql_now();
         $this->state->update($old);
         return true;
     }
diff --git a/plugins/YammerImport/scripts/yammer-import.php b/plugins/YammerImport/scripts/yammer-import.php
index 24307d6cd4..1491cfd308 100644
--- a/plugins/YammerImport/scripts/yammer-import.php
+++ b/plugins/YammerImport/scripts/yammer-import.php
@@ -4,41 +4,60 @@ if (php_sapi_name() != 'cli') {
     die('no');
 }
 
+
 define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__)))));
 
+$longoptions = array('verify=', 'reset');
 require INSTALLDIR . "/scripts/commandline.inc";
 
+echo "Checking current state...\n";
 $runner = YammerRunner::init();
 
+if (have_option('reset')) {
+    echo "Resetting Yammer import state...\n";
+    $runner->reset();
+}
+
 switch ($runner->state())
 {
     case 'init':
+        echo "Requesting authentication to Yammer API...\n";
         $url = $runner->requestAuth();
         echo "Log in to Yammer at the following URL and confirm permissions:\n";
         echo "\n";
         echo "    $url\n";
         echo "\n";
-        echo "Pass the resulting code back by running:\n"
-        echo "\n"
-        echo "    php yammer-import.php --auth=####\n";
+        echo "Pass the resulting code back by running:\n";
+        echo "\n";
+        echo "    php yammer-import.php --verify=####\n";
         echo "\n";
         break;
 
     case 'requesting-auth':
-        if (empty($options['auth'])) {
-            echo "Please finish authenticating!\n";
-            break;
+        if (!have_option('verify')) {
+            echo "Awaiting authentication...\n";
+            echo "\n";
+            echo "If you need to start over, reset the state:\n";
+            echo "\n";
+            echo "    php yammer-import.php --reset\n";
+            echo "\n";
+            exit(1);
         }
-        $runner->saveAuthToken($options['auth']);
+        echo "Saving final authentication token for Yammer API...\n";
+        $runner->saveAuthToken(get_option_value('verify'));
         // Fall through...
 
     default:
-        while (true) {
-            echo "... {$runner->state->state}\n";
+        while ($runner->hasWork()) {
+            echo "... {$runner->state()}\n";
             if (!$runner->iterate()) {
-                echo "... done.\n";
-                break;
+                echo "FAIL??!?!?!\n";
             }
         }
+        if ($runner->isDone()) {
+            echo "... done.\n";
+        } else {
+            echo "... no more import work scheduled.\n";
+        }
         break;
-}
\ No newline at end of file
+}

From d962f7092f42566602c4f3a9f37f39b13b2437a3 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Fri, 24 Sep 2010 14:52:51 -0700
Subject: [PATCH 27/35] Initial progress display of Yammer import state in
 admin panel

---
 .../YammerImport/actions/yammeradminpanel.php | 123 +++++++++++++-----
 plugins/YammerImport/css/admin.css            |  11 ++
 plugins/YammerImport/lib/yammerrunner.php     |  47 +++++++
 3 files changed, 151 insertions(+), 30 deletions(-)
 create mode 100644 plugins/YammerImport/css/admin.css

diff --git a/plugins/YammerImport/actions/yammeradminpanel.php b/plugins/YammerImport/actions/yammeradminpanel.php
index 9f935bbefb..2c9f412a24 100644
--- a/plugins/YammerImport/actions/yammeradminpanel.php
+++ b/plugins/YammerImport/actions/yammeradminpanel.php
@@ -64,6 +64,12 @@ class YammeradminpanelAction extends AdminPanelAction
         $form->show();
         return;
     }
+
+    function showStylesheets()
+    {
+        parent::showStylesheets();
+        $this->cssLink('plugins/YammerImport/css/admin.css', null, 'screen, projection, tv');
+    }
 }
 
 class YammerAdminPanelForm extends AdminForm
@@ -105,40 +111,97 @@ class YammerAdminPanelForm extends AdminForm
      */
     function formData()
     {
-        $this->out->element('p', array(), 'yammer import IN DA HOUSE');
-        
-        /*
-        Possible states of the yammer import process:
-        - null (not doing any sort of import)
-        - requesting-auth
-        - authenticated
-        - import-users
-        - import-groups
-        - fetch-messages
-        - import-messages
-        - done
-        */
-        $yammerState = Yammer_state::staticGet('id', 1);
-        $state = $yammerState ? $yammerState->state || null;
-        
-        switch($state)
+        $runner = YammerRunner::init();
+
+        switch($runner->state())
         {
-            case null:
-                $this->out->element('p', array(), 'Time to start auth:');
-                $this->showAuthForm();
-                break;
+            case 'init':
             case 'requesting-auth':
-                $this->out->element('p', array(), 'Need to finish auth!');
                 $this->showAuthForm();
-                break;
-            case 'import-users':
-            case 'import-groups':
-            case 'fetch-messages':
-            case 'save-messages':
-                $this->showImportState();
-                break;
-            
+            default:
         }
+        $this->showImportState($runner);
+    }
+
+    private function showAuthForm()
+    {
+        $this->out->element('p', array(), 'show an auth form');
+    }
+
+    private function showImportState(YammerRunner $runner)
+    {
+        $userCount = $runner->countUsers();
+        $groupCount = $runner->countGroups();
+        $fetchedCount = $runner->countFetchedNotices();
+        $savedCount = $runner->countSavedNotices();
+
+        $labels = array(
+            'init' => array(
+                'label' => _m("Initialize"),
+                'progress' => _m('No import running'),
+                'complete' => _m('Initiated Yammer server connection...'),
+            ),
+            'requesting-auth' => array(
+                'label' => _m('Connect to Yammer'),
+                'progress' => _m('Awaiting authorization...'),
+                'complete' => _m('Connected.'),
+            ),
+            'import-users' => array(
+                'label' => _m('Import user accounts'),
+                'progress' => sprintf(_m("Importing %d user...", "Importing %d users...", $userCount), $userCount),
+                'complete' => sprintf(_m("Imported %d user.", "Imported %d users.", $userCount), $userCount),
+            ),
+            'import-groups' => array(
+                'label' => _m('Import user groups'),
+                'progress' => sprintf(_m("Importing %d group...", "Importing %d groups...", $groupCount), $groupCount),
+                'complete' => sprintf(_m("Imported %d group.", "Imported %d groups.", $groupCount), $groupCount),
+            ),
+            'fetch-messages' => array(
+                'label' => _m('Prepare public notices for import'),
+                'progress' => sprintf(_m("Preparing %d notice...", "Preparing %d notices...", $fetchedCount), $fetchedCount),
+                'complete' => sprintf(_m("Prepared %d notice.", "Prepared %d notices.", $fetchedCount), $fetchedCount),
+            ),
+            'save-messages' => array(
+                'label' => _m('Import public notices'),
+                'progress' => sprintf(_m("Importing %d notice...", "Importing %d notices...", $savedCount), $savedCount),
+                'complete' => sprintf(_m("Imported %d notice.", "Imported %d notices.", $savedCount), $savedCount),
+            ),
+            'done' => array(
+                'label' => _m('Done'),
+                'progress' => sprintf(_m("Import is complete!")),
+                'complete' => sprintf(_m("Import is complete!")),
+            )
+        );
+        $steps = array_keys($labels);
+        $currentStep = array_search($runner->state(), $steps);
+
+        foreach ($steps as $step => $state) {
+            if ($step < $currentStep) {
+                // This step is done
+                $this->progressBar($labels[$state]['label'],
+                                   $labels[$state]['complete'],
+                                   'complete');
+            } else if ($step == $currentStep) {
+                // This step is in progress
+                $this->progressBar($labels[$state]['label'],
+                                   $labels[$state]['progress'],
+                                   'progress');
+            } else {
+                // This step has not yet been done.
+                $this->progressBar($labels[$state]['label'],
+                                   _m("Waiting..."),
+                                   'waiting');
+            }
+        }
+    }
+
+    private function progressBar($label, $status, $class)
+    {
+        // @fixme prettify ;)
+        $this->out->elementStart('div', array('class' => $class));
+        $this->out->element('p', array(), $label);
+        $this->out->element('p', array(), $status);
+        $this->out->elementEnd('div');
     }
 
     /**
diff --git a/plugins/YammerImport/css/admin.css b/plugins/YammerImport/css/admin.css
new file mode 100644
index 0000000000..c1462237a5
--- /dev/null
+++ b/plugins/YammerImport/css/admin.css
@@ -0,0 +1,11 @@
+.waiting {
+    color: #888;
+}
+
+.progress {
+    color: blue;
+}
+
+.done {
+    color: black;
+}
diff --git a/plugins/YammerImport/lib/yammerrunner.php b/plugins/YammerImport/lib/yammerrunner.php
index c4db48399c..e0aadff2c3 100644
--- a/plugins/YammerImport/lib/yammerrunner.php
+++ b/plugins/YammerImport/lib/yammerrunner.php
@@ -324,4 +324,51 @@ class YammerRunner
         return true;
     }
 
+    /**
+     * Count the number of Yammer users we've mapped into our system!
+     *
+     * @return int
+     */
+    public function countUsers()
+    {
+        $map = new Yammer_user();
+        return $map->count();
+    }
+
+
+    /**
+     * Count the number of Yammer groups we've mapped into our system!
+     *
+     * @return int
+     */
+    public function countGroups()
+    {
+        $map = new Yammer_group();
+        return $map->count();
+    }
+
+
+    /**
+     * Count the number of Yammer notices we've pulled down for pending import...
+     *
+     * @return int
+     */
+    public function countFetchedNotices()
+    {
+        $map = new Yammer_notice_stub();
+        return $map->count();
+    }
+
+
+    /**
+     * Count the number of Yammer notices we've mapped into our system!
+     *
+     * @return int
+     */
+    public function countSavedNotices()
+    {
+        $map = new Yammer_notice();
+        return $map->count();
+    }
+
 }

From 35119f4072f93911e1cc2a8f99e36c757ae25e3d Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Fri, 24 Sep 2010 16:15:45 -0700
Subject: [PATCH 28/35] Pretty up the Yammer import status display a bit

---
 .../YammerImport/actions/yammeradminpanel.php |  31 ++++++-----
 plugins/YammerImport/css/admin.css            |  49 +++++++++++++++++-
 plugins/YammerImport/css/done.png             | Bin 0 -> 991 bytes
 plugins/YammerImport/css/icon_processing.gif  | Bin 0 -> 673 bytes
 4 files changed, 66 insertions(+), 14 deletions(-)
 create mode 100644 plugins/YammerImport/css/done.png
 create mode 100644 plugins/YammerImport/css/icon_processing.gif

diff --git a/plugins/YammerImport/actions/yammeradminpanel.php b/plugins/YammerImport/actions/yammeradminpanel.php
index 2c9f412a24..12df3c2022 100644
--- a/plugins/YammerImport/actions/yammeradminpanel.php
+++ b/plugins/YammerImport/actions/yammeradminpanel.php
@@ -175,32 +175,37 @@ class YammerAdminPanelForm extends AdminForm
         $steps = array_keys($labels);
         $currentStep = array_search($runner->state(), $steps);
 
+        $this->out->elementStart('div', array('class' => 'yammer-import'));
         foreach ($steps as $step => $state) {
             if ($step < $currentStep) {
                 // This step is done
-                $this->progressBar($labels[$state]['label'],
-                                   $labels[$state]['complete'],
-                                   'complete');
+                $this->progressBar($state,
+                                   'complete',
+                                   $labels[$state]['label'],
+                                   $labels[$state]['complete']);
             } else if ($step == $currentStep) {
                 // This step is in progress
-                $this->progressBar($labels[$state]['label'],
-                                   $labels[$state]['progress'],
-                                   'progress');
+                $this->progressBar($state,
+                                   'progress',
+                                   $labels[$state]['label'],
+                                   $labels[$state]['progress']);
             } else {
                 // This step has not yet been done.
-                $this->progressBar($labels[$state]['label'],
-                                   _m("Waiting..."),
-                                   'waiting');
+                $this->progressBar($state,
+                                   'waiting',
+                                   $labels[$state]['label'],
+                                   _m("Waiting..."));
             }
         }
+        $this->out->elementEnd('div');
     }
 
-    private function progressBar($label, $status, $class)
+    private function progressBar($state, $class, $label, $status)
     {
         // @fixme prettify ;)
-        $this->out->elementStart('div', array('class' => $class));
-        $this->out->element('p', array(), $label);
-        $this->out->element('p', array(), $status);
+        $this->out->elementStart('div', array('class' => "import-step import-step-$state $class"));
+        $this->out->element('div', array('class' => 'import-label'), $label);
+        $this->out->element('div', array('class' => 'import-status'), $status);
         $this->out->elementEnd('div');
     }
 
diff --git a/plugins/YammerImport/css/admin.css b/plugins/YammerImport/css/admin.css
index c1462237a5..28d52d07c6 100644
--- a/plugins/YammerImport/css/admin.css
+++ b/plugins/YammerImport/css/admin.css
@@ -1,11 +1,58 @@
+.yammer-import {
+    background-color: #eee;
+
+    border: solid 1px;
+    border-radius: 8px;
+    -moz-border-radius: 8px;
+    -webkit-border-radius: 8px;
+    -opera-border-radius: 8px;
+
+    padding: 16px;
+}
+
+.import-step {
+    padding: 8px;
+}
+.import-label {
+    font-weight: bold;
+}
+.import-status {
+    margin-left: 20px;
+    padding-left: 20px;
+}
+
+
 .waiting {
     color: #888;
 }
 
 .progress {
+    background-color: #fff;
+    border-radius: 8px;
+    -moz-border-radius: 8px;
+    -webkit-border-radius: 8px;
+    -opera-border-radius: 8px;
+}
+
+.progress .import-label {
     color: blue;
 }
 
-.done {
+.progress .import-status {
+    background-image: url(icon_processing.gif);
+    background-repeat: no-repeat;
+}
+
+.complete {
     color: black;
 }
+
+.complete .import-status {
+    background-image: url(done.png);
+    background-repeat: no-repeat;
+}
+
+.import-step-done .import-status {
+    /* override */
+    background: none !important;
+}
diff --git a/plugins/YammerImport/css/done.png b/plugins/YammerImport/css/done.png
new file mode 100644
index 0000000000000000000000000000000000000000..1f3f4115014653df14a79c51bfaa86340bbe2e03
GIT binary patch
literal 991
zcmV<510ei~P)<h;3K|Lk000e1NJLTq000sI000sQ1^@s6R?d!B000A~Nkl<ZIE{6b
zPi$0G6vlt=zW3(Mv@>ma9mi6K=}4q%MH*rRO%S0%qN!987bXUXN<vsjOl8B)1a*UH
zbiv9U5loZ>m5K>%x*)bti$qbxQsm#HQYf{poj<Sdz5DKU(Wz;ooa8L-$^Gv6&bi+?
z2KO<;#7em2g-H_-gcYv)b4=a$J|LC`Vkz9VjhXHjIydZUNY`d#j*W^G^IjmY{XRb2
zcjD~F_eSN{TK)e+3LxETZh3v*i^pQFpBcM1f))}X3^W=YLCQ|^h`XMVGrxW}aAb66
zSYdV%`!Az(tJ$&pK-a0U*`ZiK3Cl4sZ3AHf8lp%O1)9*;<Qv+#bH)4d)ZvSp1CI)T
z3Fugg#@F86IB=tMzAmJU<rvtGz>XVOj=;2xg`66+g6X-tY=34?vtNI^>FN)E^#V!&
zv~F;_1A8Xp%QA)lVHyYvOxwVYiz>T?xNY0)?dYN7u^sGq`RzB-tIX8^LYP>q*XMT?
zrmp<wCxjs|mYi#ydfwZ1f-P-30m!GG678FEy8$8ME_vaotWp}SG+Kd*G%M;?^ZwQ|
ztf*~A`kKagj)R@2S(DvN7=}nGQITR*OWQg?2|GbHQW70$R0K+D*5|wUptFybWlysI
zxi5LTc{>MR>Ziqh0x2cF?-K+8XRm$3WTnsmU@1u;r9?@Hq{h*4UR}GNxShl>3>p#{
zx;Gs`YfThIl*?sGr6R}A_t5vtQC#t`1cV4Zt}7L(NNS=`<5fHke7=oKg>xvS&|0IE
zA`C-{#UfMFcRAeu4*i#o;+P4h?-a)XWy*OfxwXlz$n=&=xgb|RoM&+KEAs9d>YNAg
zJdaYT#GjKlIrQyYT)K6RDAZ)!CN2+7d{UfH7fitL{Bqd#Z1$C6aSkOH8Xkl`L!&3j
z)U2e|bQv4J$e~kjQkc9-B*978Bt-4h&%M)oR6u9~x;&+RYg}n>P3IC>-}jM$Mo9?6
zkdZ%5Q}N0i8~Tu$;uMAf@uWp_Q<mZ5g?+QP)Ciy!M<nLd(fa(}<-PSQ$j!{m5>ymX
zsL*Ob)WTd)c|2*+Soa_oPELF}db-k$W*$&xb>5uFCo23@8i*y$M_bw&bG6HC>>6-V
zCN=dIDK}0!vkbgMW$?4{k8XVL_W(W+0>&aE0mOkCAd_5fbmZ67bUyTWvMo_>x++4>
zP6XpOFZ?}x`(pTo^tcSn0rP-dER7{QNdOiQtGZQnRRsh~!Bva`_y_%#tFU>{Dfj>Y
N002ovPDHLkV1mlL*&P4?

literal 0
HcmV?d00001

diff --git a/plugins/YammerImport/css/icon_processing.gif b/plugins/YammerImport/css/icon_processing.gif
new file mode 100644
index 0000000000000000000000000000000000000000..d0bce1542342e912da81a2c260562df172f30d73
GIT binary patch
literal 673
zcmZ?wbhEHb6krfw_{6~Q|Nnmm28Kh24mmkF0U1e2Nli^nlO|14{Lk&@8WQa67~pE8
zXTZz|lvDgC<y@4SSdyBeP@Y+mp^%uBSdo*Tn4*`NmzK|<_>+Z`3#dv5h=E26FfcG1
zbL_hF&)}42ws10s6^G;;cE1^EoUR)U5A70}d2pLv!jVIT7j&Z~EblI3x0K*v_sV|m
z0kj3v921Z^em#l`(k(o@H$3ZdDRc@9NidXDNbqrumReCGv$gd8+e8WW28HVqkJ_9i
zH>s*<31KtHjANIPvi2#*6BEu%3Dak5O_t&NBI)<h(<yPr>H?V$TxT}#l{vOTn<?_G
z_#ejR!~8}oQ>5naXTfF^&~Hhq+NX@#Ccc>y7T?;vjI&jdhsDsPJyAw*m0Qz>i}K7#
zL9w50<qa%!r20=RDYEpz>Ng{fT}A5JUe8lRK1h7_Y2;BWJDd=c6f&i?Wv5(5q?6|P
zQw{>maxZP<537OA37Uk}7@%_$4o$EWe_Zl>&#id|lE-BpDC#+Fn|msJ%_2h{Hg1vP
z#N8WAzfWasG}yq|xqE)DrWaOofX=z|?*pgc%{ig5vl!pqDlC|q&~Z0$&Rvsft&VO-
z4MZj+%-+Vx%W}v;V76hyp=;+R;x+~t^Q%*xuFTQAF2})fSfTHDAs>sO!OBw`)&)o$
c0!CNZt))x~rAZP^^P&YOFfdqy5)K#u0POD40{{R3

literal 0
HcmV?d00001


From 19adb7c8d3aadcc71a24b9d30ad0742b43ecb328 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Fri, 24 Sep 2010 16:27:33 -0700
Subject: [PATCH 29/35] Pretty it up a bit more

---
 plugins/YammerImport/actions/yammeradminpanel.php |  5 +++--
 plugins/YammerImport/css/admin.css                | 11 ++---------
 2 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/plugins/YammerImport/actions/yammeradminpanel.php b/plugins/YammerImport/actions/yammeradminpanel.php
index 12df3c2022..13c95e37f7 100644
--- a/plugins/YammerImport/actions/yammeradminpanel.php
+++ b/plugins/YammerImport/actions/yammeradminpanel.php
@@ -175,7 +175,8 @@ class YammerAdminPanelForm extends AdminForm
         $steps = array_keys($labels);
         $currentStep = array_search($runner->state(), $steps);
 
-        $this->out->elementStart('div', array('class' => 'yammer-import'));
+        $this->out->elementStart('fieldset', array('class' => 'yammer-import'));
+        $this->out->element('legend', array(), _m('Import status'));
         foreach ($steps as $step => $state) {
             if ($step < $currentStep) {
                 // This step is done
@@ -197,7 +198,7 @@ class YammerAdminPanelForm extends AdminForm
                                    _m("Waiting..."));
             }
         }
-        $this->out->elementEnd('div');
+        $this->out->elementEnd('fieldset');
     }
 
     private function progressBar($state, $class, $label, $status)
diff --git a/plugins/YammerImport/css/admin.css b/plugins/YammerImport/css/admin.css
index 28d52d07c6..4c1aaacd64 100644
--- a/plugins/YammerImport/css/admin.css
+++ b/plugins/YammerImport/css/admin.css
@@ -1,12 +1,4 @@
 .yammer-import {
-    background-color: #eee;
-
-    border: solid 1px;
-    border-radius: 8px;
-    -moz-border-radius: 8px;
-    -webkit-border-radius: 8px;
-    -opera-border-radius: 8px;
-
     padding: 16px;
 }
 
@@ -27,7 +19,8 @@
 }
 
 .progress {
-    background-color: #fff;
+    background-color: white;
+    border: solid 1px blue;
     border-radius: 8px;
     -moz-border-radius: 8px;
     -webkit-border-radius: 8px;

From ebbbaba378bbe174ceb5e76477db89ef647304e5 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Fri, 24 Sep 2010 17:22:44 -0700
Subject: [PATCH 30/35] Work in progress on getting the frontend Yammer import
 form going....

---
 plugins/YammerImport/YammerImportPlugin.php   |   8 +-
 .../YammerImport/actions/yammeradminpanel.php | 205 +++++-------------
 plugins/YammerImport/actions/yammerauth.php   |  88 ++++++--
 .../YammerImport/lib/yammerauthinitform.php   |  71 ++++++
 .../YammerImport/lib/yammerauthverifyform.php |  82 +++++++
 .../YammerImport/lib/yammerprogressform.php   | 128 +++++++++++
 6 files changed, 420 insertions(+), 162 deletions(-)
 create mode 100644 plugins/YammerImport/lib/yammerauthinitform.php
 create mode 100644 plugins/YammerImport/lib/yammerauthverifyform.php
 create mode 100644 plugins/YammerImport/lib/yammerprogressform.php

diff --git a/plugins/YammerImport/YammerImportPlugin.php b/plugins/YammerImport/YammerImportPlugin.php
index 85eab74c04..bb1e976186 100644
--- a/plugins/YammerImport/YammerImportPlugin.php
+++ b/plugins/YammerImport/YammerImportPlugin.php
@@ -36,6 +36,8 @@ class YammerImportPlugin extends Plugin
     {
         $m->connect('admin/yammer',
                     array('action' => 'yammeradminpanel'));
+        $m->connect('admin/yammer/auth',
+                    array('action' => 'yammerauth'));
         return true;
     }
 
@@ -117,10 +119,14 @@ class YammerImportPlugin extends Plugin
         case 'sn_yammerclient':
         case 'yammerimporter':
         case 'yammerrunner':
+        case 'yammerauthinitform':
+        case 'yammerauthverifyform':
+        case 'yammerprogressform':
             require_once "$base/lib/$lower.php";
             return false;
         case 'yammeradminpanelaction':
-            require_once "$base/actions/yammeradminpanel.php";
+            $crop = substr($lower, 0, strlen($lower) - strlen('action'));
+            require_once "$base/actions/$crop.php";
             return false;
         case 'yammer_state':
         case 'yammer_notice_stub':
diff --git a/plugins/YammerImport/actions/yammeradminpanel.php b/plugins/YammerImport/actions/yammeradminpanel.php
index 13c95e37f7..56e721d03c 100644
--- a/plugins/YammerImport/actions/yammeradminpanel.php
+++ b/plugins/YammerImport/actions/yammeradminpanel.php
@@ -53,6 +53,43 @@ class YammeradminpanelAction extends AdminPanelAction
         return _m('Yammer import tool');
     }
 
+    function prepare($args)
+    {
+        $ok = parent::prepare($args);
+
+        $this->init_auth = $this->trimmed('init_auth');
+        $this->verify_token = $this->trimmed('verify_token');
+
+        return $ok;
+    }
+
+    function handle($args)
+    {
+        if ($this->init_auth) {
+            $url = $runner->requestAuth();
+            $form = new YammerAuthVerifyForm($this, $url);
+            return $this->showAjaxForm($form);
+        } else if ($this->verify_token) {
+            $runner->saveAuthToken($this->verify_token);
+            $form = new YammerAuthProgressForm();
+            return $this->showAjaxForm($form);
+        }
+
+        return parent::handle($args);
+    }
+
+    function showAjaxForm($form)
+    {
+        $this->startHTML('text/xml;charset=utf-8');
+        $this->elementStart('head');
+        $this->element('title', null, _m('Yammer import'));
+        $this->elementEnd('head');
+        $this->elementStart('body');
+        $form->show();
+        $this->elementEnd('body');
+        $this->elementEnd('html');
+    }
+
     /**
      * Show the Yammer admin panel form
      *
@@ -60,9 +97,24 @@ class YammeradminpanelAction extends AdminPanelAction
      */
     function showForm()
     {
-        $form = new YammerAdminPanelForm($this);
+        $this->elementStart('fieldset');
+
+        $runner = YammerRunner::init();
+
+        switch($runner->state())
+        {
+            case 'init':
+                $form = new YammerAuthInitForm($this);
+                break;
+            case 'requesting-auth':
+                $form = new YammerAuthVerifyForm($this, $runner);
+                break;
+            default:
+                $form = new YammerProgressForm($this, $runner);
+        }
         $form->show();
-        return;
+
+        $this->elementEnd('fieldset');
     }
 
     function showStylesheets()
@@ -70,153 +122,10 @@ class YammeradminpanelAction extends AdminPanelAction
         parent::showStylesheets();
         $this->cssLink('plugins/YammerImport/css/admin.css', null, 'screen, projection, tv');
     }
-}
 
-class YammerAdminPanelForm extends AdminForm
-{
-    /**
-     * ID of the form
-     *
-     * @return string ID of the form
-     */
-    function id()
+    function showScripts()
     {
-        return 'yammeradminpanel';
-    }
-
-    /**
-     * class of the form
-     *
-     * @return string class of the form
-     */
-    function formClass()
-    {
-        return 'form_settings';
-    }
-
-    /**
-     * Action of the form
-     *
-     * @return string URL of the action
-     */
-    function action()
-    {
-        return common_local_url('yammeradminpanel');
-    }
-
-    /**
-     * Data elements of the form
-     *
-     * @return void
-     */
-    function formData()
-    {
-        $runner = YammerRunner::init();
-
-        switch($runner->state())
-        {
-            case 'init':
-            case 'requesting-auth':
-                $this->showAuthForm();
-            default:
-        }
-        $this->showImportState($runner);
-    }
-
-    private function showAuthForm()
-    {
-        $this->out->element('p', array(), 'show an auth form');
-    }
-
-    private function showImportState(YammerRunner $runner)
-    {
-        $userCount = $runner->countUsers();
-        $groupCount = $runner->countGroups();
-        $fetchedCount = $runner->countFetchedNotices();
-        $savedCount = $runner->countSavedNotices();
-
-        $labels = array(
-            'init' => array(
-                'label' => _m("Initialize"),
-                'progress' => _m('No import running'),
-                'complete' => _m('Initiated Yammer server connection...'),
-            ),
-            'requesting-auth' => array(
-                'label' => _m('Connect to Yammer'),
-                'progress' => _m('Awaiting authorization...'),
-                'complete' => _m('Connected.'),
-            ),
-            'import-users' => array(
-                'label' => _m('Import user accounts'),
-                'progress' => sprintf(_m("Importing %d user...", "Importing %d users...", $userCount), $userCount),
-                'complete' => sprintf(_m("Imported %d user.", "Imported %d users.", $userCount), $userCount),
-            ),
-            'import-groups' => array(
-                'label' => _m('Import user groups'),
-                'progress' => sprintf(_m("Importing %d group...", "Importing %d groups...", $groupCount), $groupCount),
-                'complete' => sprintf(_m("Imported %d group.", "Imported %d groups.", $groupCount), $groupCount),
-            ),
-            'fetch-messages' => array(
-                'label' => _m('Prepare public notices for import'),
-                'progress' => sprintf(_m("Preparing %d notice...", "Preparing %d notices...", $fetchedCount), $fetchedCount),
-                'complete' => sprintf(_m("Prepared %d notice.", "Prepared %d notices.", $fetchedCount), $fetchedCount),
-            ),
-            'save-messages' => array(
-                'label' => _m('Import public notices'),
-                'progress' => sprintf(_m("Importing %d notice...", "Importing %d notices...", $savedCount), $savedCount),
-                'complete' => sprintf(_m("Imported %d notice.", "Imported %d notices.", $savedCount), $savedCount),
-            ),
-            'done' => array(
-                'label' => _m('Done'),
-                'progress' => sprintf(_m("Import is complete!")),
-                'complete' => sprintf(_m("Import is complete!")),
-            )
-        );
-        $steps = array_keys($labels);
-        $currentStep = array_search($runner->state(), $steps);
-
-        $this->out->elementStart('fieldset', array('class' => 'yammer-import'));
-        $this->out->element('legend', array(), _m('Import status'));
-        foreach ($steps as $step => $state) {
-            if ($step < $currentStep) {
-                // This step is done
-                $this->progressBar($state,
-                                   'complete',
-                                   $labels[$state]['label'],
-                                   $labels[$state]['complete']);
-            } else if ($step == $currentStep) {
-                // This step is in progress
-                $this->progressBar($state,
-                                   'progress',
-                                   $labels[$state]['label'],
-                                   $labels[$state]['progress']);
-            } else {
-                // This step has not yet been done.
-                $this->progressBar($state,
-                                   'waiting',
-                                   $labels[$state]['label'],
-                                   _m("Waiting..."));
-            }
-        }
-        $this->out->elementEnd('fieldset');
-    }
-
-    private function progressBar($state, $class, $label, $status)
-    {
-        // @fixme prettify ;)
-        $this->out->elementStart('div', array('class' => "import-step import-step-$state $class"));
-        $this->out->element('div', array('class' => 'import-label'), $label);
-        $this->out->element('div', array('class' => 'import-status'), $status);
-        $this->out->elementEnd('div');
-    }
-
-    /**
-     * Action elements
-     *
-     * @return void
-     */
-    function formActions()
-    {
-        // No submit buttons needed at bottom
+        parent::showScripts();
+        $this->script('plugins/YammerImport/js/yammer-admin.js');
     }
 }
diff --git a/plugins/YammerImport/actions/yammerauth.php b/plugins/YammerImport/actions/yammerauth.php
index 7e6e7204ae..d0d4b40c71 100644
--- a/plugins/YammerImport/actions/yammerauth.php
+++ b/plugins/YammerImport/actions/yammerauth.php
@@ -1,17 +1,79 @@
 <?php
+/**
+ * StatusNet, the distributed open-source microblogging tool
+ *
+ * Yammer import administration panel
+ *
+ * PHP version 5
+ *
+ * LICENCE: This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @category  Settings
+ * @package   StatusNet
+ * @author    Zach Copley <zach@status.net>
+ * @copyright 2010 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
+ * @link      http://status.net/
+ */
 
-
-function showYammerAuth()
-{
-    $token = $yam->requestToken();
-    $url = $yam->authorizeUrl($token);
-
-    // We're going to try doing this in an iframe; if that's not happy
-    // we can redirect but there doesn't seem to be a way to get Yammer's
-    // oauth to call us back instead of the manual copy. :(
-
-    //common_redirect($url, 303);
-    $this->element('iframe', array('id' => 'yammer-oauth',
-                                   'src' => $url));
+if (!defined('STATUSNET')) {
+    exit(1);
+}
+
+class YammerauthAction extends AdminPanelAction
+{
+
+    /**
+     * Show the Yammer admin panel form
+     *
+     * @return void
+     */
+    function prepare($args)
+    {
+        parent::prepare($args);
+        
+        $this->verify_token = $this->trim('verify_token');
+    }
+
+    /**
+     * Handle request
+     *
+     * Does the subscription and returns results.
+     *
+     * @param Array $args unused.
+     *
+     * @return void
+     */
+
+    function handle($args)
+    {
+        if ($this->verify_token) {
+            $runner->saveAuthToken($this->verify_token);
+            $form = new YammerAuthProgressForm();
+        } else {
+            $url = $runner->requestAuth();
+            $form = new YammerAuthVerifyForm($this, $url);
+        }
+
+        $this->startHTML('text/xml;charset=utf-8');
+        $this->elementStart('head');
+        $this->element('title', null, _m('Connect to Yammer'));
+        $this->elementEnd('head');
+        $this->elementStart('body');
+        $form->show();
+        $this->elementEnd('body');
+        $this->elementEnd('html');
+    }
 }
 
diff --git a/plugins/YammerImport/lib/yammerauthinitform.php b/plugins/YammerImport/lib/yammerauthinitform.php
new file mode 100644
index 0000000000..559ec4e7cc
--- /dev/null
+++ b/plugins/YammerImport/lib/yammerauthinitform.php
@@ -0,0 +1,71 @@
+<?php
+
+class YammerAuthInitForm extends Form
+{
+    /**
+     * ID of the form
+     *
+     * @return int ID of the form
+     */
+
+    function id()
+    {
+        return 'yammer-auth-init-form';
+    }
+
+
+    /**
+     * class of the form
+     *
+     * @return string of the form class
+     */
+
+    function formClass()
+    {
+        return 'form_yammer_auth_init';
+    }
+
+
+    /**
+     * Action of the form
+     *
+     * @return string URL of the action
+     */
+
+    function action()
+    {
+        return common_local_url('yammeradminpanel');
+    }
+
+
+    /**
+     * Legend of the Form
+     *
+     * @return void
+     */
+    function formLegend()
+    {
+        $this->out->element('legend', null, _m('Connect to Yammer'));
+    }
+
+    /**
+     * Data elements of the form
+     *
+     * @return void
+     */
+
+    function formData()
+    {
+    }
+
+    /**
+     * Action elements
+     *
+     * @return void
+     */
+
+    function formActions()
+    {
+        $this->out->submit('submit', _m('Connect to Yammer'), 'submit', null, _m('Request authorization to connect to Yammer account'));
+    }
+}
diff --git a/plugins/YammerImport/lib/yammerauthverifyform.php b/plugins/YammerImport/lib/yammerauthverifyform.php
new file mode 100644
index 0000000000..488b5b8d1f
--- /dev/null
+++ b/plugins/YammerImport/lib/yammerauthverifyform.php
@@ -0,0 +1,82 @@
+<?php
+
+class YammerAuthVerifyForm extends Form
+{
+    private $verify_url;
+
+    function __construct($out, $auth_url)
+    {
+        parent::__construct($out);
+        $this->verify_url = $auth_url;
+    }
+
+    /**
+     * ID of the form
+     *
+     * @return int ID of the form
+     */
+
+    function id()
+    {
+        return 'yammer-auth-verify-form';
+    }
+
+
+    /**
+     * class of the form
+     *
+     * @return string of the form class
+     */
+
+    function formClass()
+    {
+        return 'form_yammer_auth_verify';
+    }
+
+
+    /**
+     * Action of the form
+     *
+     * @return string URL of the action
+     */
+
+    function action()
+    {
+        return common_local_url('yammeradminpanel');
+    }
+
+
+    /**
+     * Legend of the Form
+     *
+     * @return void
+     */
+    function formLegend()
+    {
+        $this->out->element('legend', null, _m('Connect to Yammer'));
+    }
+
+    /**
+     * Data elements of the form
+     *
+     * @return void
+     */
+
+    function formData()
+    {
+    }
+
+    /**
+     * Action elements
+     *
+     * @return void
+     */
+
+    function formActions()
+    {
+        $this->out->input('verify-code', _m('Verification code:'), '', _m("Click through and paste the code it gives you below..."));
+        $this->out->submit('submit', _m('Verify code'), 'submit', null, _m('Verification code'));
+        $this->element('iframe', array('id' => 'yammer-oauth',
+                                       'src' => $this->auth_url));
+    }
+}
diff --git a/plugins/YammerImport/lib/yammerprogressform.php b/plugins/YammerImport/lib/yammerprogressform.php
new file mode 100644
index 0000000000..776efa100f
--- /dev/null
+++ b/plugins/YammerImport/lib/yammerprogressform.php
@@ -0,0 +1,128 @@
+<?php
+
+class YammerProgressForm extends Form
+{
+    /**
+     * ID of the form
+     *
+     * @return string ID of the form
+     */
+    function id()
+    {
+        return 'yammer-progress';
+    }
+
+    /**
+     * class of the form
+     *
+     * @return string class of the form
+     */
+    function formClass()
+    {
+        return 'form_settings';
+    }
+
+    /**
+     * Action of the form
+     *
+     * @return string URL of the action
+     */
+    function action()
+    {
+        return common_local_url('yammeradminpanel');
+    }
+
+    /**
+     * Data elements of the form
+     *
+     * @return void
+     */
+    function formData()
+    {
+        $runner = YammerRunner::init();
+
+        $userCount = $runner->countUsers();
+        $groupCount = $runner->countGroups();
+        $fetchedCount = $runner->countFetchedNotices();
+        $savedCount = $runner->countSavedNotices();
+
+        $labels = array(
+            'init' => array(
+                'label' => _m("Initialize"),
+                'progress' => _m('No import running'),
+                'complete' => _m('Initiated Yammer server connection...'),
+            ),
+            'requesting-auth' => array(
+                'label' => _m('Connect to Yammer'),
+                'progress' => _m('Awaiting authorization...'),
+                'complete' => _m('Connected.'),
+            ),
+            'import-users' => array(
+                'label' => _m('Import user accounts'),
+                'progress' => sprintf(_m("Importing %d user...", "Importing %d users...", $userCount), $userCount),
+                'complete' => sprintf(_m("Imported %d user.", "Imported %d users.", $userCount), $userCount),
+            ),
+            'import-groups' => array(
+                'label' => _m('Import user groups'),
+                'progress' => sprintf(_m("Importing %d group...", "Importing %d groups...", $groupCount), $groupCount),
+                'complete' => sprintf(_m("Imported %d group.", "Imported %d groups.", $groupCount), $groupCount),
+            ),
+            'fetch-messages' => array(
+                'label' => _m('Prepare public notices for import'),
+                'progress' => sprintf(_m("Preparing %d notice...", "Preparing %d notices...", $fetchedCount), $fetchedCount),
+                'complete' => sprintf(_m("Prepared %d notice.", "Prepared %d notices.", $fetchedCount), $fetchedCount),
+            ),
+            'save-messages' => array(
+                'label' => _m('Import public notices'),
+                'progress' => sprintf(_m("Importing %d notice...", "Importing %d notices...", $savedCount), $savedCount),
+                'complete' => sprintf(_m("Imported %d notice.", "Imported %d notices.", $savedCount), $savedCount),
+            ),
+            'done' => array(
+                'label' => _m('Done'),
+                'progress' => sprintf(_m("Import is complete!")),
+                'complete' => sprintf(_m("Import is complete!")),
+            )
+        );
+        $steps = array_keys($labels);
+        $currentStep = array_search($runner->state(), $steps);
+
+        $this->out->elementStart('fieldset', array('class' => 'yammer-import'));
+        $this->out->element('legend', array(), _m('Import status'));
+        foreach ($steps as $step => $state) {
+            if ($state == 'init') {
+                // Don't show 'init', it's boring.
+                continue;
+            }
+            if ($step < $currentStep) {
+                // This step is done
+                $this->progressBar($state,
+                                   'complete',
+                                   $labels[$state]['label'],
+                                   $labels[$state]['complete']);
+            } else if ($step == $currentStep) {
+                // This step is in progress
+                $this->progressBar($state,
+                                   'progress',
+                                   $labels[$state]['label'],
+                                   $labels[$state]['progress']);
+            } else {
+                // This step has not yet been done.
+                $this->progressBar($state,
+                                   'waiting',
+                                   $labels[$state]['label'],
+                                   _m("Waiting..."));
+            }
+        }
+        $this->out->elementEnd('fieldset');
+    }
+
+    private function progressBar($state, $class, $label, $status)
+    {
+        // @fixme prettify ;)
+        $this->out->elementStart('div', array('class' => "import-step import-step-$state $class"));
+        $this->out->element('div', array('class' => 'import-label'), $label);
+        $this->out->element('div', array('class' => 'import-status'), $status);
+        $this->out->elementEnd('div');
+    }
+
+}

From eeaab2bc0072a7f836000c1d817d59c663423998 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Mon, 27 Sep 2010 12:24:10 -0700
Subject: [PATCH 31/35] Work in progress on fixing auth... looks like the
 iframe doesn't work though. Sigh.

---
 .../YammerImport/actions/yammeradminpanel.php | 38 +++++++++++--------
 .../YammerImport/lib/yammerauthinitform.php   |  1 +
 .../YammerImport/lib/yammerauthverifyform.php | 12 +++---
 plugins/YammerImport/lib/yammerrunner.php     | 21 ++++++++--
 .../YammerImport/scripts/yammer-import.php    |  2 +
 5 files changed, 49 insertions(+), 25 deletions(-)

diff --git a/plugins/YammerImport/actions/yammeradminpanel.php b/plugins/YammerImport/actions/yammeradminpanel.php
index 56e721d03c..71651cdf56 100644
--- a/plugins/YammerImport/actions/yammeradminpanel.php
+++ b/plugins/YammerImport/actions/yammeradminpanel.php
@@ -33,6 +33,8 @@ if (!defined('STATUSNET')) {
 
 class YammeradminpanelAction extends AdminPanelAction
 {
+    private $runner;
+
     /**
      * Returns the page title
      *
@@ -59,23 +61,29 @@ class YammeradminpanelAction extends AdminPanelAction
 
         $this->init_auth = $this->trimmed('init_auth');
         $this->verify_token = $this->trimmed('verify_token');
+        $this->runner = YammerRunner::init();
 
         return $ok;
     }
 
     function handle($args)
     {
-        if ($this->init_auth) {
-            $url = $runner->requestAuth();
-            $form = new YammerAuthVerifyForm($this, $url);
-            return $this->showAjaxForm($form);
-        } else if ($this->verify_token) {
-            $runner->saveAuthToken($this->verify_token);
-            $form = new YammerAuthProgressForm();
-            return $this->showAjaxForm($form);
+        if ($_SERVER['REQUEST_METHOD'] == 'POST') {
+            $this->checkSessionToken();
+            if ($this->init_auth) {
+                $url = $this->runner->requestAuth();
+                $form = new YammerAuthVerifyForm($this, $this->runner);
+                return $this->showAjaxForm($form);
+            } else if ($this->verify_token) {
+                $this->runner->saveAuthToken($this->verify_token);
+                $form = new YammerAuthProgressForm();
+                return $this->showAjaxForm($form);
+            } else {
+                throw new ClientException('Invalid POST');
+            }
+        } else {
+            return parent::handle($args);
         }
-
-        return parent::handle($args);
     }
 
     function showAjaxForm($form)
@@ -99,18 +107,16 @@ class YammeradminpanelAction extends AdminPanelAction
     {
         $this->elementStart('fieldset');
 
-        $runner = YammerRunner::init();
-
-        switch($runner->state())
+        switch($this->runner->state())
         {
             case 'init':
-                $form = new YammerAuthInitForm($this);
+                $form = new YammerAuthInitForm($this, $this->runner);
                 break;
             case 'requesting-auth':
-                $form = new YammerAuthVerifyForm($this, $runner);
+                $form = new YammerAuthVerifyForm($this, $this->runner);
                 break;
             default:
-                $form = new YammerProgressForm($this, $runner);
+                $form = new YammerProgressForm($this, $this->runner);
         }
         $form->show();
 
diff --git a/plugins/YammerImport/lib/yammerauthinitform.php b/plugins/YammerImport/lib/yammerauthinitform.php
index 559ec4e7cc..5a83a06c21 100644
--- a/plugins/YammerImport/lib/yammerauthinitform.php
+++ b/plugins/YammerImport/lib/yammerauthinitform.php
@@ -56,6 +56,7 @@ class YammerAuthInitForm extends Form
 
     function formData()
     {
+        $this->out->hidden('init_auth', '1');
     }
 
     /**
diff --git a/plugins/YammerImport/lib/yammerauthverifyform.php b/plugins/YammerImport/lib/yammerauthverifyform.php
index 488b5b8d1f..dc9d2ce1b2 100644
--- a/plugins/YammerImport/lib/yammerauthverifyform.php
+++ b/plugins/YammerImport/lib/yammerauthverifyform.php
@@ -2,12 +2,12 @@
 
 class YammerAuthVerifyForm extends Form
 {
-    private $verify_url;
+    private $runner;
 
-    function __construct($out, $auth_url)
+    function __construct($out, YammerRunner $runner)
     {
         parent::__construct($out);
-        $this->verify_url = $auth_url;
+        $this->runner = $runner;
     }
 
     /**
@@ -64,6 +64,9 @@ class YammerAuthVerifyForm extends Form
 
     function formData()
     {
+        $this->out->input('verify_token', _m('Verification code:'), '', _m("Click through and paste the code it gives you below..."));
+        $this->out->element('iframe', array('id' => 'yammer-oauth',
+                                            'src' => $this->runner->getAuthUrl()));
     }
 
     /**
@@ -74,9 +77,6 @@ class YammerAuthVerifyForm extends Form
 
     function formActions()
     {
-        $this->out->input('verify-code', _m('Verification code:'), '', _m("Click through and paste the code it gives you below..."));
         $this->out->submit('submit', _m('Verify code'), 'submit', null, _m('Verification code'));
-        $this->element('iframe', array('id' => 'yammer-oauth',
-                                       'src' => $this->auth_url));
     }
 }
diff --git a/plugins/YammerImport/lib/yammerrunner.php b/plugins/YammerImport/lib/yammerrunner.php
index e0aadff2c3..aee6b17e15 100644
--- a/plugins/YammerImport/lib/yammerrunner.php
+++ b/plugins/YammerImport/lib/yammerrunner.php
@@ -123,7 +123,7 @@ class YammerRunner
     public function requestAuth()
     {
         if ($this->state->state != 'init') {
-            throw ServerError("Cannot request Yammer auth; already there!");
+            throw new ServerException("Cannot request Yammer auth; already there!");
         }
 
         $data = $this->client->requestToken();
@@ -135,7 +135,22 @@ class YammerRunner
         $this->state->modified = common_sql_now();
         $this->state->update($old);
 
-        return $this->client->authorizeUrl($this->state->oauth_token);
+        return $this->getAuthUrl();
+    }
+
+    /**
+     * When already in requesting-auth state, grab the URL to send the user to
+     * to complete OAuth setup.
+     *
+     * @return string URL
+     */
+    function getAuthUrl()
+    {
+        if ($this->state() == 'requesting-auth') {
+            return $this->client->authorizeUrl($this->state->oauth_token);
+        } else {
+            throw new ServerException('Cannot get Yammer auth URL when not in requesting-auth state!');
+        }
     }
 
     /**
@@ -152,7 +167,7 @@ class YammerRunner
     public function saveAuthToken($verifier)
     {
         if ($this->state->state != 'requesting-auth') {
-            throw ServerError("Cannot save auth token in Yammer import state {$this->state->state}");
+            throw new ServerException("Cannot save auth token in Yammer import state {$this->state->state}");
         }
 
         $data = $this->client->accessToken($verifier);
diff --git a/plugins/YammerImport/scripts/yammer-import.php b/plugins/YammerImport/scripts/yammer-import.php
index 1491cfd308..b4aa921e50 100644
--- a/plugins/YammerImport/scripts/yammer-import.php
+++ b/plugins/YammerImport/scripts/yammer-import.php
@@ -16,6 +16,8 @@ $runner = YammerRunner::init();
 if (have_option('reset')) {
     echo "Resetting Yammer import state...\n";
     $runner->reset();
+    echo "done.\n";
+    exit(0);
 }
 
 switch ($runner->state())

From 05c12c58bb99332c8117160e85a724319333d1f1 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Mon, 27 Sep 2010 12:34:01 -0700
Subject: [PATCH 32/35] Ok, got the AJAX clicky-throughs working for yammer
 auth (if app is already registered), but needs prettification. Yammer ignores
 callback URLs unless they're pre-registered with the app, and this apparently
 requires manual intervention to become a 'trusted' app, you don't get it on
 those you register yourself. Sigh. Also can't use an iframe since it breaks
 out of the frame (fair 'nuff)

---
 plugins/YammerImport/actions/yammeradminpanel.php | 2 +-
 plugins/YammerImport/lib/yammerauthverifyform.php | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/plugins/YammerImport/actions/yammeradminpanel.php b/plugins/YammerImport/actions/yammeradminpanel.php
index 71651cdf56..fdf7a084f4 100644
--- a/plugins/YammerImport/actions/yammeradminpanel.php
+++ b/plugins/YammerImport/actions/yammeradminpanel.php
@@ -76,7 +76,7 @@ class YammeradminpanelAction extends AdminPanelAction
                 return $this->showAjaxForm($form);
             } else if ($this->verify_token) {
                 $this->runner->saveAuthToken($this->verify_token);
-                $form = new YammerAuthProgressForm();
+                $form = new YammerProgressForm($this, $this->runner);
                 return $this->showAjaxForm($form);
             } else {
                 throw new ClientException('Invalid POST');
diff --git a/plugins/YammerImport/lib/yammerauthverifyform.php b/plugins/YammerImport/lib/yammerauthverifyform.php
index dc9d2ce1b2..96decea102 100644
--- a/plugins/YammerImport/lib/yammerauthverifyform.php
+++ b/plugins/YammerImport/lib/yammerauthverifyform.php
@@ -65,8 +65,17 @@ class YammerAuthVerifyForm extends Form
     function formData()
     {
         $this->out->input('verify_token', _m('Verification code:'), '', _m("Click through and paste the code it gives you below..."));
+        
+        // iframe would be nice to avoid leaving -- since they don't seem to have callback url O_O
+        /*
         $this->out->element('iframe', array('id' => 'yammer-oauth',
                                             'src' => $this->runner->getAuthUrl()));
+        */
+        // yeah, it ignores the callback_url
+        $this->out->element('a',
+            array('href' => $this->runner->getAuthUrl(),
+                  'target' => '_blank'),
+            'clicky click');
     }
 
     /**

From 585c7f35ca5141d85a8d2f20e9b62a99cbf03f4e Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Mon, 27 Sep 2010 13:34:35 -0700
Subject: [PATCH 33/35] Yammer import (work run via background queues) now can
 be started from the admin panel! :DDDD Still requires that the app be
 registered on your network manually first.

---
 plugins/YammerImport/YammerImportPlugin.php   |  3 ++-
 .../YammerImport/actions/yammeradminpanel.php |  4 ++++
 .../YammerImport/lib/yammerauthverifyform.php | 19 ++++++++++++++-----
 .../YammerImport/lib/yammerqueuehandler.php   |  3 +--
 plugins/YammerImport/lib/yammerrunner.php     |  9 +++++++++
 5 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/plugins/YammerImport/YammerImportPlugin.php b/plugins/YammerImport/YammerImportPlugin.php
index bb1e976186..547870936b 100644
--- a/plugins/YammerImport/YammerImportPlugin.php
+++ b/plugins/YammerImport/YammerImportPlugin.php
@@ -48,7 +48,7 @@ class YammerImportPlugin extends Plugin
      */
     function onEndInitializeQueueManager(QueueManager $qm)
     {
-        $qm->connect('importym', 'ImportYmQueueHandler');
+        $qm->connect('yammer', 'YammerQueueHandler');
 
         return true;
     }
@@ -122,6 +122,7 @@ class YammerImportPlugin extends Plugin
         case 'yammerauthinitform':
         case 'yammerauthverifyform':
         case 'yammerprogressform':
+        case 'yammerqueuehandler':
             require_once "$base/lib/$lower.php";
             return false;
         case 'yammeradminpanelaction':
diff --git a/plugins/YammerImport/actions/yammeradminpanel.php b/plugins/YammerImport/actions/yammeradminpanel.php
index fdf7a084f4..04ef26d512 100644
--- a/plugins/YammerImport/actions/yammeradminpanel.php
+++ b/plugins/YammerImport/actions/yammeradminpanel.php
@@ -76,6 +76,10 @@ class YammeradminpanelAction extends AdminPanelAction
                 return $this->showAjaxForm($form);
             } else if ($this->verify_token) {
                 $this->runner->saveAuthToken($this->verify_token);
+                
+                // Haho! Now we can make THE FUN HAPPEN
+                $this->runner->startBackgroundImport();
+                
                 $form = new YammerProgressForm($this, $this->runner);
                 return $this->showAjaxForm($form);
             } else {
diff --git a/plugins/YammerImport/lib/yammerauthverifyform.php b/plugins/YammerImport/lib/yammerauthverifyform.php
index 96decea102..2b3efbcb1a 100644
--- a/plugins/YammerImport/lib/yammerauthverifyform.php
+++ b/plugins/YammerImport/lib/yammerauthverifyform.php
@@ -64,7 +64,20 @@ class YammerAuthVerifyForm extends Form
 
     function formData()
     {
-        $this->out->input('verify_token', _m('Verification code:'), '', _m("Click through and paste the code it gives you below..."));
+        $this->out->elementStart('p');
+        $this->out->text(_m('Follow this link to confirm authorization at Yammer; you will be prompted to log in if necessary:'));
+        $this->out->elementEnd('p');
+
+        $this->out->elementStart('blockquote');
+        $this->out->element('a',
+            array('href' => $this->runner->getAuthUrl(),
+                  'target' => '_blank'),
+            _m('Open Yammer authentication window'));
+        $this->out->elementEnd('blockquote');
+        
+        $this->out->element('p', array(), _m('Copy the verification code you are given into the form below:'));
+
+        $this->out->input('verify_token', _m('Verification code:'));
         
         // iframe would be nice to avoid leaving -- since they don't seem to have callback url O_O
         /*
@@ -72,10 +85,6 @@ class YammerAuthVerifyForm extends Form
                                             'src' => $this->runner->getAuthUrl()));
         */
         // yeah, it ignores the callback_url
-        $this->out->element('a',
-            array('href' => $this->runner->getAuthUrl(),
-                  'target' => '_blank'),
-            'clicky click');
     }
 
     /**
diff --git a/plugins/YammerImport/lib/yammerqueuehandler.php b/plugins/YammerImport/lib/yammerqueuehandler.php
index 5fc3777835..acc8073115 100644
--- a/plugins/YammerImport/lib/yammerqueuehandler.php
+++ b/plugins/YammerImport/lib/yammerqueuehandler.php
@@ -41,8 +41,7 @@ class YammerQueueHandler extends QueueHandler
             if ($runner->iterate()) {
                 if ($runner->hasWork()) {
                     // More to do? Shove us back on the queue...
-                    $qm = QueueManager::get();
-                    $qm->enqueue('YammerImport', 'yammer');
+                    $runner->startBackgroundImport();
                 }
                 return true;
             } else {
diff --git a/plugins/YammerImport/lib/yammerrunner.php b/plugins/YammerImport/lib/yammerrunner.php
index aee6b17e15..e0aec0d166 100644
--- a/plugins/YammerImport/lib/yammerrunner.php
+++ b/plugins/YammerImport/lib/yammerrunner.php
@@ -386,4 +386,13 @@ class YammerRunner
         return $map->count();
     }
 
+    /**
+     * Start running import work in the background queues...
+     */
+    public function startBackgroundImport()
+    {
+        $qm = QueueManager::get();
+        $qm->enqueue('YammerImport', 'yammer');
+    }
+
 }

From f528cc55488046c1ef58a8d95dd7f89d81c80cae Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Mon, 27 Sep 2010 16:56:48 -0700
Subject: [PATCH 34/35] Yammer import API keys can now be overridden by the
 admin.

---
 plugins/YammerImport/YammerImportPlugin.php   |   1 +
 .../YammerImport/actions/yammeradminpanel.php |  89 +++++++++-----
 plugins/YammerImport/css/admin.css            |   4 +
 plugins/YammerImport/lib/yammerapikeyform.php | 112 ++++++++++++++++++
 .../YammerImport/lib/yammerauthinitform.php   |  10 +-
 .../YammerImport/lib/yammerauthverifyform.php |  37 ++++--
 plugins/YammerImport/scripts/yamdump.php      |  34 ------
 7 files changed, 209 insertions(+), 78 deletions(-)
 create mode 100644 plugins/YammerImport/lib/yammerapikeyform.php
 delete mode 100644 plugins/YammerImport/scripts/yamdump.php

diff --git a/plugins/YammerImport/YammerImportPlugin.php b/plugins/YammerImport/YammerImportPlugin.php
index 547870936b..98c6ecd0ab 100644
--- a/plugins/YammerImport/YammerImportPlugin.php
+++ b/plugins/YammerImport/YammerImportPlugin.php
@@ -119,6 +119,7 @@ class YammerImportPlugin extends Plugin
         case 'sn_yammerclient':
         case 'yammerimporter':
         case 'yammerrunner':
+        case 'yammerapikeyform':
         case 'yammerauthinitform':
         case 'yammerauthverifyform':
         case 'yammerprogressform':
diff --git a/plugins/YammerImport/actions/yammeradminpanel.php b/plugins/YammerImport/actions/yammeradminpanel.php
index 04ef26d512..13960d9051 100644
--- a/plugins/YammerImport/actions/yammeradminpanel.php
+++ b/plugins/YammerImport/actions/yammeradminpanel.php
@@ -52,15 +52,18 @@ class YammeradminpanelAction extends AdminPanelAction
      */
     function getInstructions()
     {
-        return _m('Yammer import tool');
+        return _m('This Yammer import tool is still undergoing testing, ' .
+                  'and is incomplete in some areas. ' .
+                'Currently user subscriptions and group memberships are not ' .
+                'transferred; in the future this may be supported for ' .
+                'imports done by verified administrators on the Yammer side.');
     }
 
     function prepare($args)
     {
         $ok = parent::prepare($args);
 
-        $this->init_auth = $this->trimmed('init_auth');
-        $this->verify_token = $this->trimmed('verify_token');
+        $this->subaction = $this->trimmed('subaction');
         $this->runner = YammerRunner::init();
 
         return $ok;
@@ -68,26 +71,48 @@ class YammeradminpanelAction extends AdminPanelAction
 
     function handle($args)
     {
+        // @fixme move this to saveSettings and friends?
         if ($_SERVER['REQUEST_METHOD'] == 'POST') {
             $this->checkSessionToken();
-            if ($this->init_auth) {
-                $url = $this->runner->requestAuth();
-                $form = new YammerAuthVerifyForm($this, $this->runner);
-                return $this->showAjaxForm($form);
-            } else if ($this->verify_token) {
-                $this->runner->saveAuthToken($this->verify_token);
-                
+            if ($this->subaction == 'change-apikey') {
+                $form = new YammerApiKeyForm($this);
+            } else if ($this->subaction == 'apikey') {
+                if ($this->saveKeys()) {
+                    $form = new YammerAuthInitForm($this, $this->runner);
+                } else {
+                    $form = new YammerApiKeyForm($this);
+                }
+            } else if ($this->subaction == 'authinit') {
+                // hack
+                if ($this->arg('change-apikey')) {
+                    $form = new YammerApiKeyForm($this);
+                } else {
+                    $url = $this->runner->requestAuth();
+                    $form = new YammerAuthVerifyForm($this, $this->runner);
+                }
+            } else if ($this->subaction == 'authverify') {
+                $this->runner->saveAuthToken($this->trimmed('verify_token'));
+
                 // Haho! Now we can make THE FUN HAPPEN
                 $this->runner->startBackgroundImport();
-                
+
                 $form = new YammerProgressForm($this, $this->runner);
-                return $this->showAjaxForm($form);
             } else {
                 throw new ClientException('Invalid POST');
             }
-        } else {
-            return parent::handle($args);
+            return $this->showAjaxForm($form);
         }
+        return parent::handle($args);
+    }
+
+    function saveKeys()
+    {
+        $key = $this->trimmed('consumer_key');
+        $secret = $this->trimmed('consumer_secret');
+        Config::save('yammer', 'consumer_key', $key);
+        Config::save('yammer', 'consumer_secret', $secret);
+
+        return !empty($key) && !empty($secret);
     }
 
     function showAjaxForm($form)
@@ -102,6 +127,27 @@ class YammeradminpanelAction extends AdminPanelAction
         $this->elementEnd('html');
     }
 
+    /**
+     * Fetch the appropriate form for our current state.
+     * @return Form
+     */
+    function statusForm()
+    {
+        if (!(common_config('yammer', 'consumer_key'))
+            || !(common_config('yammer', 'consumer_secret'))) {
+            return new YammerApiKeyForm($this);
+        }
+        switch($this->runner->state())
+        {
+            case 'init':
+                return new YammerAuthInitForm($this, $this->runner);
+            case 'requesting-auth':
+                return new YammerAuthVerifyForm($this, $this->runner);
+            default:
+                return new YammerProgressForm($this, $this->runner);
+        }
+    }
+
     /**
      * Show the Yammer admin panel form
      *
@@ -110,20 +156,7 @@ class YammeradminpanelAction extends AdminPanelAction
     function showForm()
     {
         $this->elementStart('fieldset');
-
-        switch($this->runner->state())
-        {
-            case 'init':
-                $form = new YammerAuthInitForm($this, $this->runner);
-                break;
-            case 'requesting-auth':
-                $form = new YammerAuthVerifyForm($this, $this->runner);
-                break;
-            default:
-                $form = new YammerProgressForm($this, $this->runner);
-        }
-        $form->show();
-
+        $this->statusForm()->show();
         $this->elementEnd('fieldset');
     }
 
diff --git a/plugins/YammerImport/css/admin.css b/plugins/YammerImport/css/admin.css
index 4c1aaacd64..9c99a0b880 100644
--- a/plugins/YammerImport/css/admin.css
+++ b/plugins/YammerImport/css/admin.css
@@ -49,3 +49,7 @@
     /* override */
     background: none !important;
 }
+
+.magiclink {
+    margin-left: 40px;
+}
\ No newline at end of file
diff --git a/plugins/YammerImport/lib/yammerapikeyform.php b/plugins/YammerImport/lib/yammerapikeyform.php
new file mode 100644
index 0000000000..b2acec4ede
--- /dev/null
+++ b/plugins/YammerImport/lib/yammerapikeyform.php
@@ -0,0 +1,112 @@
+<?php
+
+class YammerApikeyForm extends Form
+{
+    private $runner;
+
+    function __construct($out)
+    {
+        parent::__construct($out);
+        $this->runner = $runner;
+    }
+
+    /**
+     * ID of the form
+     *
+     * @return int ID of the form
+     */
+
+    function id()
+    {
+        return 'yammer-apikey-form';
+    }
+
+
+    /**
+     * class of the form
+     *
+     * @return string of the form class
+     */
+
+    function formClass()
+    {
+        return 'form_yammer_apikey form_settings';
+    }
+
+
+    /**
+     * Action of the form
+     *
+     * @return string URL of the action
+     */
+
+    function action()
+    {
+        return common_local_url('yammeradminpanel');
+    }
+
+
+    /**
+     * Legend of the Form
+     *
+     * @return void
+     */
+    function formLegend()
+    {
+        $this->out->element('legend', null, _m('Yammer API registration'));
+    }
+
+    /**
+     * Data elements of the form
+     *
+     * @return void
+     */
+
+    function formData()
+    {
+        $this->out->hidden('subaction', 'apikey');
+
+        $this->out->elementStart('fieldset');
+
+        $this->out->elementStart('p');
+        $this->out->text(_m('Before we can connect to your Yammer network, ' .
+                            'you will need to register the importer as an ' .
+                            'application authorized to pull data on your behalf. ' .
+                            'This registration will work only for your own network. ' .
+                            'Follow this link to register the app at Yammer; ' .
+                            'you will be prompted to log in if necessary:'));
+        $this->out->elementEnd('p');
+
+        $this->out->elementStart('p', array('class' => 'magiclink'));
+        $this->out->element('a',
+            array('href' => 'https://www.yammer.com/client_applications/new',
+                  'target' => '_blank'),
+            _m('Open Yammer application registration form'));
+        $this->out->elementEnd('p');
+
+        $this->out->element('p', array(), _m('Copy the consumer key and secret you are given into the form below:'));
+
+        $this->out->elementStart('ul', array('class' => 'form_data'));
+        $this->out->elementStart('li');
+        $this->out->input('consumer_key', _m('Consumer key:'), common_config('yammer', 'consumer_key'));
+        $this->out->elementEnd('li');
+        $this->out->elementStart('li');
+        $this->out->input('consumer_secret', _m('Consumer secret:'), common_config('yammer', 'consumer_secret'));
+        $this->out->elementEnd('li');
+        $this->out->elementEnd('ul');
+
+        $this->out->submit('submit', _m('Save'), 'submit', null, _m('Save these consumer keys'));
+
+        $this->out->elementEnd('fieldset');
+    }
+
+    /**
+     * Action elements
+     *
+     * @return void
+     */
+
+    function formActions()
+    {
+    }
+}
diff --git a/plugins/YammerImport/lib/yammerauthinitform.php b/plugins/YammerImport/lib/yammerauthinitform.php
index 5a83a06c21..9f48fd82a5 100644
--- a/plugins/YammerImport/lib/yammerauthinitform.php
+++ b/plugins/YammerImport/lib/yammerauthinitform.php
@@ -22,7 +22,7 @@ class YammerAuthInitForm extends Form
 
     function formClass()
     {
-        return 'form_yammer_auth_init';
+        return 'form_yammer_auth_init form_settings';
     }
 
 
@@ -56,7 +56,12 @@ class YammerAuthInitForm extends Form
 
     function formData()
     {
-        $this->out->hidden('init_auth', '1');
+        $this->out->hidden('subaction', 'authinit');
+
+        $this->out->elementStart('fieldset');
+        $this->out->submit('submit', _m('Start authentication'), 'submit', null, _m('Request authorization to connect to Yammer account'));
+        $this->out->submit('change-apikey', _m('Change API key'));
+        $this->out->elementEnd('fieldset');
     }
 
     /**
@@ -67,6 +72,5 @@ class YammerAuthInitForm extends Form
 
     function formActions()
     {
-        $this->out->submit('submit', _m('Connect to Yammer'), 'submit', null, _m('Request authorization to connect to Yammer account'));
     }
 }
diff --git a/plugins/YammerImport/lib/yammerauthverifyform.php b/plugins/YammerImport/lib/yammerauthverifyform.php
index 2b3efbcb1a..e119be96f7 100644
--- a/plugins/YammerImport/lib/yammerauthverifyform.php
+++ b/plugins/YammerImport/lib/yammerauthverifyform.php
@@ -30,7 +30,7 @@ class YammerAuthVerifyForm extends Form
 
     function formClass()
     {
-        return 'form_yammer_auth_verify';
+        return 'form_yammer_auth_verify form_settings';
     }
 
 
@@ -64,27 +64,39 @@ class YammerAuthVerifyForm extends Form
 
     function formData()
     {
+        $this->out->hidden('subaction', 'authverify');
+
+        $this->out->elementStart('fieldset');
+
         $this->out->elementStart('p');
         $this->out->text(_m('Follow this link to confirm authorization at Yammer; you will be prompted to log in if necessary:'));
         $this->out->elementEnd('p');
 
-        $this->out->elementStart('blockquote');
-        $this->out->element('a',
-            array('href' => $this->runner->getAuthUrl(),
-                  'target' => '_blank'),
-            _m('Open Yammer authentication window'));
-        $this->out->elementEnd('blockquote');
-        
-        $this->out->element('p', array(), _m('Copy the verification code you are given into the form below:'));
-
-        $this->out->input('verify_token', _m('Verification code:'));
-        
         // iframe would be nice to avoid leaving -- since they don't seem to have callback url O_O
         /*
         $this->out->element('iframe', array('id' => 'yammer-oauth',
                                             'src' => $this->runner->getAuthUrl()));
         */
         // yeah, it ignores the callback_url
+        // soo... crappy link. :(
+
+        $this->out->elementStart('p', array('class' => 'magiclink'));
+        $this->out->element('a',
+            array('href' => $this->runner->getAuthUrl(),
+                  'target' => '_blank'),
+            _m('Open Yammer authentication window'));
+        $this->out->elementEnd('p');
+        
+        $this->out->element('p', array(), _m('Copy the verification code you are given below:'));
+
+        $this->out->elementStart('ul', array('class' => 'form_data'));
+        $this->out->elementStart('li');
+        $this->out->input('verify_token', _m('Verification code:'));
+        $this->out->elementEnd('li');
+        $this->out->elementEnd('ul');
+        
+        $this->out->submit('submit', _m('Continue'), 'submit', null, _m('Save code and begin import'));
+        $this->out->elementEnd('fieldset');
     }
 
     /**
@@ -95,6 +107,5 @@ class YammerAuthVerifyForm extends Form
 
     function formActions()
     {
-        $this->out->submit('submit', _m('Verify code'), 'submit', null, _m('Verification code'));
     }
 }
diff --git a/plugins/YammerImport/scripts/yamdump.php b/plugins/YammerImport/scripts/yamdump.php
deleted file mode 100644
index 944ee2e499..0000000000
--- a/plugins/YammerImport/scripts/yamdump.php
+++ /dev/null
@@ -1,34 +0,0 @@
-<?php
-
-if (php_sapi_name() != 'cli') {
-    die('no');
-}
-
-define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__)))));
-
-require INSTALLDIR . "/scripts/commandline.inc";
-
-// temp stuff
-require 'yam-config.php';
-$yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
-$imp = new YammerImporter($yam);
-
-/*
-$data = $yam->users();
-var_dump($data);
-// @fixme follow paging
-foreach ($data as $item) {
-    $user = $imp->prepUser($item);
-    var_dump($user);
-}
-*/
-
-$data = $yam->messages(array('newer_than' => 1));
-var_dump($data);
-// @fixme follow paging
-$messages = $data['messages'];
-$messages = array_reverse($messages);
-foreach ($messages as $message) {
-    $notice = $imp->prepNotice($message);
-    var_dump($notice);
-}

From 0477101af7bda9d8e58f3315837a81fca17d06b4 Mon Sep 17 00:00:00 2001
From: Brion Vibber <brion@pobox.com>
Date: Mon, 27 Sep 2010 17:12:06 -0700
Subject: [PATCH 35/35] update README for YammerImport

---
 plugins/YammerImport/README | 97 +++++++++++++++++++++++++++++++------
 1 file changed, 81 insertions(+), 16 deletions(-)

diff --git a/plugins/YammerImport/README b/plugins/YammerImport/README
index 1bac69a243..975faa2132 100644
--- a/plugins/YammerImport/README
+++ b/plugins/YammerImport/README
@@ -8,28 +8,93 @@ Requirements
 ------------
 
 * An account on the Yammer network you wish to import from
-* An administrator account on the target StatusNet instance
+* An administrator account on the target StatusNet instance, or
+  command-line administrative access
 * This YammerImport plugin enabled on your StatusNet instance
 
-Setup
------
-
-The import process will be runnable through an administration panel on
-your StatusNet site.
-
-The user interface and OAuth setup has not yet been completed, you will
-have to manually initiate the OAuth authentication to get a token.
-
-Be patient, there will be a UI soon. ;)
-
 
 Limitations
 -----------
 
-Paging has not yet been added, so the importer will only pull up to:
-* first 50 users
-* first 20 groups
-* last 20 public messages
+Yammer API key registrations only work for your own network unless you make
+arrangements for a 'trusted app' key, so for now users will need to register
+the app themselves. There is a helper in the admin panel for this.
+
+In theory any number of users, groups, and messages should be supported, but
+it hasn't been fully tested on non-trivial-sized sites.
+
+No provision has yet been made for dealing with conflicting usernames or
+group names, or names which are not considered valid by StatusNet. Errors
+are possible.
+
+Running via the web admin interface requires having queueing enabled, and is
+fairly likely to have problems with the application key registration step in
+a small installation at this time.
+
+
+Web setup
+---------
+
+The import process is runnable through an administration panel on your
+StatusNet site. The user interface is still a bit flaky, however, and if
+errors occur during import the process may stop with no way to restart it
+visible.
+
+The admin interface will probably kinda blow up if JS/AJAX isn't working.
+
+You'll be prompted to register the application and authenticate into Yammer,
+after which a progress screen will display.
+
+Two big warnings:
+* The progress display does not currently auto-refresh.
+* If anything fails once actual import has begun, it'll just keep showing
+  the current state. You won't see an error message, and there's no way
+  to reset or restart from the web UI yet.
+
+You can continue or reset the import state using the command-line script.
+
+
+CLI setup
+---------
+
+You'll need to register an application consumer key to allow the importer
+to connect to your Yammer network; this requires logging into Yammer:
+
+  https://www.yammer.com/client_applications/new
+
+Check all the 'read' options; no 'write' options are required, but Yammer
+seems to end up setting them anyway.
+
+You can set the resulting keys directly in config.php:
+
+    $config['yammer']['consumer_key'] = '#####';
+    $config['yammer']['consumer_secret'] = '##########';
+
+Initiate authentication by starting up the importer script:
+
+    php plugins/YammerImport/scripts/yammer-import.php
+
+Since you haven't yet authenticated, this will request an auth token and
+give you a URL to open in your web browser. Once logged in and authorized
+there, you'll be given a confirmation code. Pass this back:
+
+    php plugins/YammerImport/scripts/yammer-import.php --verify=####
+
+If all is well, the import process will begin and run through the end.
+
+In case of error or manual abort, you should be able to continue the
+import from where you left off by running the script again:
+
+    php plugins/YammerImport/scripts/yammer-import.php
+
+To reset the Yammer import state -- without removing any of the items
+that have already been imported -- you can pass the --reset option:
+
+    php plugins/YammerImport/scripts/yammer-import.php --reset
+
+This'll let you start over from the requesting-authentication stage.
+Any users, groups, or notices that have already been imported will be
+retained.
 
 
 Subscriptions and group memberships