GNUsocial/gnu-social
4
0
Fork 1
Code Issues Releases Wiki Activity

Work in progress: most of the infrastructure for running import via BG queues or CLI script is now in place (untested, no UI, needs tweaks & fixes)

Browse Source
This commit is contained in:
Brion Vibber 2010-09-23 16:40:22 -07:00
parent e8ad436a99
commit ae507b0485
4 changed files with 186 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
plugins/YammerImport/README
View File

@ -73,3 +73,45 @@ they do on Yammer; they will be linked instead.
File type and size limitations on attachments will be applied, so beware some
attachments may not make it through.
Code structure
==============
Standalone classes
------------------
YammerRunner: encapsulates the iterative process of retrieving the various users,
groups, and messages via SN_YammerClient and saving them locally
via YammerImporter.
SN_YammerClient: encapsulates HTTP+OAuth interface to Yammer API, returns data
as straight decoded JSON object trees.
YammerImporter: encapsulates logic to pull information from the returned API data
and convert them to native StatusNet users, groups, and messages.
Web UI actions
-------------
YammeradminpanelAction: web panel for site administrator to initiate and monitor
the import process.
Command-line scripts
--------------------
yammer-import.php: CLI script to start a Yammer import run in one go.
Database objects
----------------
Yammer_state: data object storing YammerRunner's state between iterations.
Yammer_notice_stub: data object for temporary storage of fetched Yammer messages
between fetching them (reverse chron order) and saving them
to local messages (forward chron order).
Yammer_user,
Yammer_group,
Yammer_notice: data objects mapping original Yammer item IDs to their local copies.

18
plugins/YammerImport/lib/yammerqueuehandler.php
View File

@ -36,11 +36,23 @@ class YammerQueueHandler extends QueueHandler
function handle($notice)
{
$importer = new YammerImporter();
if ($importer->hasWork()) {
return $importer->iterate();
$runner = YammerRunner::init();
if ($runner->hasWork()) {
if ($runner->iterate()) {
if ($runner->hasWork()) {
// More to do? Shove us back on the queue...
$qm = QueueManager::get();
$qm->enqueue('YammerImport', 'yammer');
}
return true;
} else {
// Something failed?
// @fixme should we be trying again here, or should we give warning?
return false;
}
} else {
// We're done!
common_log(LOG_INFO, "Yammer import has no work to do at this time; discarding.");
return true;
}
}

102
plugins/YammerImport/lib/yammerrunner.php
View File

@ -33,29 +33,123 @@ class YammerRunner
private $client;
private $importer;
function __construct()
public static function init()
{
$state = Yammer_state::staticGet('id', 1);
if (!$state) {
common_log(LOG_ERR, "No YammerImport state during import run. Should not happen!");
throw new ServerException('No YammerImport state during import run.');
$state = new Yammer_state();
$state->id = 1;
$state->state = 'init';
$state->insert();
}
return new YammerRunner($state);
}
private function __construct($state)
{
$this->state = $state;
$this->client = new SN_YammerClient(
common_config('yammer', 'consumer_key'),
common_config('yammer', 'consumer_secret'),
$this->state->oauth_token,
$this->state->oauth_secret);
$this->importer = new YammerImporter($client);
}
/**
* Check which state we're in
*
* @return string
*/
public function state()
{
return $this->state->state;
}
/**
* Is the import done, finished, complete, finito?
*
* @return boolean
*/
public function isDone()
{
$workStates = array('import-users', 'import-groups', 'fetch-messages', 'save-messages');
return ($this->state() == 'done');
}
/**
* Check if we have work to do in iterate().
*/
public function hasWork()
{
$workStates = array('import-users', 'import-groups', 'fetch-messages', 'save-messages');
return in_array($this->state(), $workStates);
}
/**
* Start the authentication process! If all goes well, we'll get back a URL.
* Have the user visit that URL, log in on Yammer and verify the importer's
* permissions. They'll get back a verification code, which needs to be passed
* on to saveAuthToken().
*
* @return string URL
*/
public function requestAuth()
{
if ($this->state->state != 'init') {
throw ServerError("Cannot request Yammer auth; already there!");
}
$old = clone($this->state);
$this->state->state = 'requesting-auth';
$this->state->request_token = $client->requestToken();
$this->state->update($old);
return $this->client->authorizeUrl($this->state->request_token);
}
/**
* Now that the user's given us this verification code from Yammer, we can
* request a final OAuth token/secret pair which we can use to access the
* API.
*
* After success here, we'll be ready to move on and run through iterate()
* until the import is complete.
*
* @param string $verifier
* @return boolean success
*/
public function saveAuthToken($verifier)
{
if ($this->state->state != 'requesting-auth') {
throw ServerError("Cannot save auth token in Yammer import state {$this->state->state}");
}
$old = clone($this->state);
list($token, $secret) = $this->client->getAuthToken($verifier);
$this->state->verifier = '';
$this->state->oauth_token = $token;
$this->state->oauth_secret = $secret;
$this->state->update($old);
return true;
}
/**
* Once authentication is complete, we need to call iterate() a bunch of times
* until state() returns 'done'.
*
* @return boolean success
*/
public function iterate()
{
switch($state->state)
{
case null:
case 'init':
case 'requesting-auth':
// Neither of these should reach our background state!
common_log(LOG_ERR, "Non-background YammerImport state '$state->state' during import run!");

59
plugins/YammerImport/scripts/yammer-import.php
View File

@ -8,34 +8,37 @@ define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__)))));
require INSTALLDIR . "/scripts/commandline.inc";
// temp stuff
require 'yam-config.php';
$yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
$imp = new YammerImporter($yam);
$runner = YammerRunner::init();
// First, import all the users!
// @fixme follow paging -- we only get 50 at a time
$data = $yam->users();
foreach ($data as $item) {
$user = $imp->importUser($item);
echo "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)\n";
}
switch ($runner->state())
{
case 'init':
$url = $runner->requestAuth();
echo "Log in to Yammer at the following URL and confirm permissions:\n";
echo "\n";
echo " $url\n";
echo "\n";
echo "Pass the resulting code back by running:\n"
echo "\n"
echo " php yammer-import.php --auth=####\n";
echo "\n";
break;
// Groups!
// @fixme follow paging -- we only get 20 at a time
$data = $yam->groups();
foreach ($data as $item) {
$group = $imp->importGroup($item);
echo "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)\n";
}
case 'requesting-auth':
if (empty($options['auth'])) {
echo "Please finish authenticating!\n";
break;
}
$runner->saveAuthToken($options['auth']);
// Fall through...
// Messages!
// Process in reverse chron order...
// @fixme follow paging -- we only get 20 at a time, and start at the most recent!
$data = $yam->messages();
$messages = $data['messages'];
$messages = array_reverse($messages);
foreach ($messages as $item) {
$notice = $imp->importNotice($item);
echo "Imported Yammer notice " . $item['id'] . " as $notice->id\n";
}
default:
while (true) {
echo "... {$runner->state->state}\n";
if (!$runner->iterate()) {
echo "... done.\n";
break;
}
}
break;
}