Ok, command-line workflow for YammerImportPlugin seems to mostly work, at least on tiny test site :D

This commit is contained in:
Brion Vibber 2010-09-23 17:55:13 -07:00
parent dd414db9ea
commit bdd8a587e7
7 changed files with 144 additions and 63 deletions

View File

@ -138,13 +138,13 @@ class Yammer_common extends Memcached_DataObject
protected static function doRecord($class, $field, $orig_id, $local_id) protected static function doRecord($class, $field, $orig_id, $local_id)
{ {
$map = self::staticGet('id', $orig_id); $map = parent::staticGet($class, 'id', $orig_id);
if (!empty($map)) { if (!empty($map)) {
return $map; return $map;
} }
$map = self::staticGet($field, $local_id); $map = parent::staticGet($class, $field, $local_id);
if (!empty($map)) { if (!empty($map)) {
return $map; return $map;

View File

@ -48,6 +48,23 @@ class Yammer_notice_stub extends Memcached_DataObject
public $json_data; // text public $json_data; // text
public $created; // datetime public $created; // datetime
/**
* Get an instance by key
*
* This is a utility method to get a single instance with a given key value.
*
* @param string $k Key to use to lookup
* @param mixed $v Value to lookup
*
* @return Yammer_notice_stub object found, or null for no hits
*
*/
function staticGet($k, $v=null)
{
return Memcached_DataObject::staticGet('Yammer_notice_stub', $k, $v);
}
/** /**
* Return schema definition to set this table up in onCheckSchema * Return schema definition to set this table up in onCheckSchema
*/ */
@ -126,6 +143,16 @@ class Yammer_notice_stub extends Memcached_DataObject
return array(false, false, false); return array(false, false, false);
} }
/**
* Decode the stored data structure.
*
* @return mixed
*/
public function getData()
{
return json_decode($this->json_data, true);
}
/** /**
* Save the native Yammer API representation of a message for the pending * Save the native Yammer API representation of a message for the pending
* import. Since they come in in reverse chronological order, we need to * import. Since they come in in reverse chronological order, we need to
@ -153,22 +180,4 @@ class Yammer_notice_stub extends Memcached_DataObject
return $stub; return $stub;
} }
/**
* Save a mapping between a remote Yammer and local imported user.
*
* @param integer $user_id ID of the status in StatusNet
*
* @return Yammer_notice_stub new object for this value
*/
static function retrieve($orig_id)
{
$stub = self::staticGet('id', $orig_id);
if ($stub) {
return json_decode($stub->json_data, true);
} else {
return false;
}
}
} }

View File

@ -36,7 +36,6 @@ class Yammer_state extends Memcached_DataObject
public $__table = 'yammer_state'; // table name public $__table = 'yammer_state'; // table name
public $id; // int primary_key not_null public $id; // int primary_key not_null
public $state; // import state key public $state; // import state key
public $request_token; // oauth request token; clear when auth is complete.
public $oauth_token; // actual oauth token! clear when import is done? public $oauth_token; // actual oauth token! clear when import is done?
public $oauth_secret; // actual oauth secret! clear when import is done? public $oauth_secret; // actual oauth secret! clear when import is done?
public $users_page; // last page of users we've fetched public $users_page; // last page of users we've fetched
@ -46,6 +45,23 @@ class Yammer_state extends Memcached_DataObject
public $created; // datetime public $created; // datetime
public $modified; // datetime public $modified; // datetime
/**
* Get an instance by key
*
* This is a utility method to get a single instance with a given key value.
*
* @param string $k Key to use to lookup
* @param mixed $v Value to lookup
*
* @return Yammer_state object found, or null for no hits
*
*/
function staticGet($k, $v=null)
{
return Memcached_DataObject::staticGet('Yammer_state', $k, $v);
}
/** /**
* Return schema definition to set this table up in onCheckSchema * Return schema definition to set this table up in onCheckSchema
*/ */
@ -54,7 +70,6 @@ class Yammer_state extends Memcached_DataObject
return array(new ColumnDef('id', 'int', null, return array(new ColumnDef('id', 'int', null,
false, 'PRI'), false, 'PRI'),
new ColumnDef('state', 'text'), new ColumnDef('state', 'text'),
new ColumnDef('request_token', 'text'),
new ColumnDef('oauth_token', 'text'), new ColumnDef('oauth_token', 'text'),
new ColumnDef('oauth_secret', 'text'), new ColumnDef('oauth_secret', 'text'),
new ColumnDef('users_page', 'int'), new ColumnDef('users_page', 'int'),
@ -78,7 +93,6 @@ class Yammer_state extends Memcached_DataObject
{ {
return array('id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL, return array('id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
'state' => DB_DATAOBJECT_STR, 'state' => DB_DATAOBJECT_STR,
'request_token' => DB_DATAOBJECT_STR,
'oauth_token' => DB_DATAOBJECT_STR, 'oauth_token' => DB_DATAOBJECT_STR,
'oauth_secret' => DB_DATAOBJECT_STR, 'oauth_secret' => DB_DATAOBJECT_STR,
'users_page' => DB_DATAOBJECT_INT, 'users_page' => DB_DATAOBJECT_INT,

View File

@ -104,7 +104,8 @@ class SN_YammerClient
{ {
$body = $this->fetchApi("api/v1/$method.json", $params); $body = $this->fetchApi("api/v1/$method.json", $params);
$data = json_decode($body, true); $data = json_decode($body, true);
if (!$data) { if ($data === null) {
common_log(LOG_ERR, "Invalid JSON response from Yammer API: " . $body);
throw new Exception("Invalid JSON response from Yammer API"); throw new Exception("Invalid JSON response from Yammer API");
} }
return $data; return $data;
@ -161,7 +162,7 @@ class SN_YammerClient
if ($this->token || $this->tokenSecret) { if ($this->token || $this->tokenSecret) {
throw new Exception("Requesting a token, but already set up with a token"); throw new Exception("Requesting a token, but already set up with a token");
} }
$data = $this->fetch('oauth/request_token'); $data = $this->fetchApi('oauth/request_token');
$arr = array(); $arr = array();
parse_str($data, $arr); parse_str($data, $arr);
return $arr; return $arr;
@ -176,7 +177,7 @@ class SN_YammerClient
public function accessToken($verifier) public function accessToken($verifier)
{ {
$this->verifier = $verifier; $this->verifier = $verifier;
$data = $this->fetch('oauth/access_token'); $data = $this->fetchApi('oauth/access_token');
$this->verifier = null; $this->verifier = null;
$arr = array(); $arr = array();
parse_str($data, $arr); parse_str($data, $arr);

View File

@ -327,17 +327,20 @@ class YammerImporter
private function findImportedUser($origId) private function findImportedUser($origId)
{ {
return Yammer_user::staticGet('id', $origId); $map = Yammer_user::staticGet('id', $origId);
return $map ? $map->user_id : null;
} }
private function findImportedGroup($origId) private function findImportedGroup($origId)
{ {
return Yammer_group::staticGet('id', $origId); $map = Yammer_group::staticGet('id', $origId);
return $map ? $map->group_id : null;
} }
private function findImportedNotice($origId) private function findImportedNotice($origId)
{ {
return Yammer_notice::staticGet('id', $origId); $map = Yammer_notice::staticGet('id', $origId);
return $map ? $map->notice_id : null;
} }
private function recordImportedUser($origId, $userId) private function recordImportedUser($origId, $userId)
@ -370,7 +373,7 @@ class YammerImporter
// Blaaaaaarf! // Blaaaaaarf!
$known = array('Pacific Time (US & Canada)' => 'America/Los_Angeles', $known = array('Pacific Time (US & Canada)' => 'America/Los_Angeles',
'Eastern Time (US & Canada)' => 'America/New_York'); 'Eastern Time (US & Canada)' => 'America/New_York');
if (array_key_exists($known, $tz)) { if (array_key_exists($tz, $known)) {
return $known[$tz]; return $known[$tz];
} else { } else {
return false; return false;

View File

@ -33,16 +33,29 @@ class YammerRunner
private $client; private $client;
private $importer; private $importer;
/**
* Normalize our singleton state and give us a YammerRunner object to play with!
*
* @return YammerRunner
*/
public static function init() public static function init()
{ {
$state = Yammer_state::staticGet('id', 1); $state = Yammer_state::staticGet('id', 1);
if (!$state) { if (!$state) {
$state = self::initState();
}
return new YammerRunner($state);
}
private static function initState()
{
$state = new Yammer_state(); $state = new Yammer_state();
$state->id = 1; $state->id = 1;
$state->state = 'init'; $state->state = 'init';
$state->created = common_sql_now();
$state->modified = common_sql_now();
$state->insert(); $state->insert();
} return $state;
return new YammerRunner($state);
} }
private function __construct($state) private function __construct($state)
@ -55,7 +68,7 @@ class YammerRunner
$this->state->oauth_token, $this->state->oauth_token,
$this->state->oauth_secret); $this->state->oauth_secret);
$this->importer = new YammerImporter($client); $this->importer = new YammerImporter($this->client);
} }
/** /**
@ -81,6 +94,8 @@ class YammerRunner
/** /**
* Check if we have work to do in iterate(). * Check if we have work to do in iterate().
*
* @return boolean
*/ */
public function hasWork() public function hasWork()
{ {
@ -88,6 +103,15 @@ class YammerRunner
return in_array($this->state(), $workStates); return in_array($this->state(), $workStates);
} }
/**
* Blow away any current state!
*/
public function reset()
{
$this->state->delete();
$this->state = self::initState();
}
/** /**
* Start the authentication process! If all goes well, we'll get back a URL. * Start the authentication process! If all goes well, we'll get back a URL.
* Have the user visit that URL, log in on Yammer and verify the importer's * Have the user visit that URL, log in on Yammer and verify the importer's
@ -102,12 +126,16 @@ class YammerRunner
throw ServerError("Cannot request Yammer auth; already there!"); throw ServerError("Cannot request Yammer auth; already there!");
} }
$data = $this->client->requestToken();
$old = clone($this->state); $old = clone($this->state);
$this->state->state = 'requesting-auth'; $this->state->state = 'requesting-auth';
$this->state->request_token = $client->requestToken(); $this->state->oauth_token = $data['oauth_token'];
$this->state->oauth_secret = $data['oauth_token_secret'];
$this->state->modified = common_sql_now();
$this->state->update($old); $this->state->update($old);
return $this->client->authorizeUrl($this->state->request_token); return $this->client->authorizeUrl($this->state->oauth_token);
} }
/** /**
@ -127,12 +155,13 @@ class YammerRunner
throw ServerError("Cannot save auth token in Yammer import state {$this->state->state}"); throw ServerError("Cannot save auth token in Yammer import state {$this->state->state}");
} }
$old = clone($this->state); $data = $this->client->accessToken($verifier);
list($token, $secret) = $this->client->getAuthToken($verifier);
$this->state->verifier = '';
$this->state->oauth_token = $token;
$this->state->oauth_secret = $secret;
$old = clone($this->state);
$this->state->state = 'import-users';
$this->state->oauth_token = $data['oauth_token'];
$this->state->oauth_secret = $data['oauth_token_secret'];
$this->state->modified = common_sql_now();
$this->state->update($old); $this->state->update($old);
return true; return true;
@ -146,8 +175,7 @@ class YammerRunner
*/ */
public function iterate() public function iterate()
{ {
switch($this->state())
switch($state->state)
{ {
case 'init': case 'init':
case 'requesting-auth': case 'requesting-auth':
@ -188,11 +216,12 @@ class YammerRunner
$this->state->state = 'import-groups'; $this->state->state = 'import-groups';
} else { } else {
foreach ($data as $item) { foreach ($data as $item) {
$user = $imp->importUser($item); $user = $this->importer->importUser($item);
common_log(LOG_INFO, "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)"); common_log(LOG_INFO, "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)");
} }
$this->state->users_page = $page; $this->state->users_page = $page;
} }
$this->state->modified = common_sql_now();
$this->state->update($old); $this->state->update($old);
return true; return true;
} }
@ -214,14 +243,15 @@ class YammerRunner
if (count($data) == 0) { if (count($data) == 0) {
common_log(LOG_INFO, "Finished importing Yammer groups; moving on to messages."); common_log(LOG_INFO, "Finished importing Yammer groups; moving on to messages.");
$this->state->state = 'import-messages'; $this->state->state = 'fetch-messages';
} else { } else {
foreach ($data as $item) { foreach ($data as $item) {
$group = $imp->importGroup($item); $group = $this->importer->importGroup($item);
common_log(LOG_INFO, "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)"); common_log(LOG_INFO, "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)");
} }
$this->state->groups_page = $page; $this->state->groups_page = $page;
} }
$this->state->modified = common_sql_now();
$this->state->update($old); $this->state->update($old);
return true; return true;
} }
@ -248,16 +278,17 @@ class YammerRunner
$data = $this->client->messages($params); $data = $this->client->messages($params);
$messages = $data['messages']; $messages = $data['messages'];
if (count($data) == 0) { if (count($messages) == 0) {
common_log(LOG_INFO, "Finished fetching Yammer messages; moving on to save messages."); common_log(LOG_INFO, "Finished fetching Yammer messages; moving on to save messages.");
$this->state->state = 'save-messages'; $this->state->state = 'save-messages';
} else { } else {
foreach ($data as $item) { foreach ($messages as $item) {
Yammer_notice_stub::record($item['id'], $item); Yammer_notice_stub::record($item['id'], $item);
$oldest = $item['id']; $oldest = $item['id'];
} }
$this->state->messages_oldest = $oldest; $this->state->messages_oldest = $oldest;
} }
$this->state->modified = common_sql_now();
$this->state->update($old); $this->state->update($old);
return true; return true;
} }
@ -267,10 +298,13 @@ class YammerRunner
$old = clone($this->state); $old = clone($this->state);
$newest = intval($this->state->messages_newest); $newest = intval($this->state->messages_newest);
$stub = new Yammer_notice_stub();
if ($newest) { if ($newest) {
$stub->addWhere('id > ' . $newest); $stub->whereAdd('id > ' . $newest);
} }
$stub->limit(20); $stub->limit(20);
$stub->orderBy('id');
$stub->find(); $stub->find();
if ($stub->N == 0) { if ($stub->N == 0) {
@ -278,13 +312,14 @@ class YammerRunner
$this->state->state = 'done'; $this->state->state = 'done';
} else { } else {
while ($stub->fetch()) { while ($stub->fetch()) {
$item = json_decode($stub->json_data); $item = $stub->getData();
$notice = $this->importer->importNotice($item); $notice = $this->importer->importNotice($item);
common_log(LOG_INFO, "Imported Yammer notice " . $item['id'] . " as $notice->id"); common_log(LOG_INFO, "Imported Yammer notice " . $item['id'] . " as $notice->id");
$newest = $item['id']; $newest = $item['id'];
} }
$this->state->messages_newest = $newest; $this->state->messages_newest = $newest;
} }
$this->state->modified = common_sql_now();
$this->state->update($old); $this->state->update($old);
return true; return true;
} }

View File

@ -4,41 +4,60 @@ if (php_sapi_name() != 'cli') {
die('no'); die('no');
} }
define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__))))); define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__)))));
$longoptions = array('verify=', 'reset');
require INSTALLDIR . "/scripts/commandline.inc"; require INSTALLDIR . "/scripts/commandline.inc";
echo "Checking current state...\n";
$runner = YammerRunner::init(); $runner = YammerRunner::init();
if (have_option('reset')) {
echo "Resetting Yammer import state...\n";
$runner->reset();
}
switch ($runner->state()) switch ($runner->state())
{ {
case 'init': case 'init':
echo "Requesting authentication to Yammer API...\n";
$url = $runner->requestAuth(); $url = $runner->requestAuth();
echo "Log in to Yammer at the following URL and confirm permissions:\n"; echo "Log in to Yammer at the following URL and confirm permissions:\n";
echo "\n"; echo "\n";
echo " $url\n"; echo " $url\n";
echo "\n"; echo "\n";
echo "Pass the resulting code back by running:\n" echo "Pass the resulting code back by running:\n";
echo "\n" echo "\n";
echo " php yammer-import.php --auth=####\n"; echo " php yammer-import.php --verify=####\n";
echo "\n"; echo "\n";
break; break;
case 'requesting-auth': case 'requesting-auth':
if (empty($options['auth'])) { if (!have_option('verify')) {
echo "Please finish authenticating!\n"; echo "Awaiting authentication...\n";
break; echo "\n";
echo "If you need to start over, reset the state:\n";
echo "\n";
echo " php yammer-import.php --reset\n";
echo "\n";
exit(1);
} }
$runner->saveAuthToken($options['auth']); echo "Saving final authentication token for Yammer API...\n";
$runner->saveAuthToken(get_option_value('verify'));
// Fall through... // Fall through...
default: default:
while (true) { while ($runner->hasWork()) {
echo "... {$runner->state->state}\n"; echo "... {$runner->state()}\n";
if (!$runner->iterate()) { if (!$runner->iterate()) {
echo "... done.\n"; echo "FAIL??!?!?!\n";
break;
} }
} }
if ($runner->isDone()) {
echo "... done.\n";
} else {
echo "... no more import work scheduled.\n";
}
break; break;
} }