Work in progress: most of the infrastructure for running import via BG queues or CLI script is now in place (untested, no UI, needs tweaks & fixes)
This commit is contained in:
Brion Vibber
parent
e8ad436a99
commit
ae507b0485
|
|
@ -73,3 +73,45 @@ they do on Yammer; they will be linked instead.
|
|||
File type and size limitations on attachments will be applied, so beware some
|
||||
attachments may not make it through.
|
||||
|
||||
|
||||
|
||||
|
||||
Code structure
|
||||
==============
|
||||
|
||||
Standalone classes
|
||||
------------------
|
||||
|
||||
YammerRunner: encapsulates the iterative process of retrieving the various users,
|
||||
groups, and messages via SN_YammerClient and saving them locally
|
||||
via YammerImporter.
|
||||
|
||||
SN_YammerClient: encapsulates HTTP+OAuth interface to Yammer API, returns data
|
||||
as straight decoded JSON object trees.
|
||||
|
||||
YammerImporter: encapsulates logic to pull information from the returned API data
|
||||
and convert them to native StatusNet users, groups, and messages.
|
||||
|
||||
Web UI actions
|
||||
-------------
|
||||
|
||||
YammeradminpanelAction: web panel for site administrator to initiate and monitor
|
||||
the import process.
|
||||
|
||||
Command-line scripts
|
||||
--------------------
|
||||
|
||||
yammer-import.php: CLI script to start a Yammer import run in one go.
|
||||
|
||||
Database objects
|
||||
----------------
|
||||
|
||||
Yammer_state: data object storing YammerRunner's state between iterations.
|
||||
|
||||
Yammer_notice_stub: data object for temporary storage of fetched Yammer messages
|
||||
between fetching them (reverse chron order) and saving them
|
||||
to local messages (forward chron order).
|
||||
Yammer_user,
|
||||
Yammer_group,
|
||||
Yammer_notice: data objects mapping original Yammer item IDs to their local copies.
|
||||
|
||||
|
|
|
|||
|
|
@ -36,11 +36,23 @@ class YammerQueueHandler extends QueueHandler
|
|||
|
||||
function handle($notice)
|
||||
{
|
||||
$importer = new YammerImporter();
|
||||
if ($importer->hasWork()) {
|
||||
return $importer->iterate();
|
||||
$runner = YammerRunner::init();
|
||||
if ($runner->hasWork()) {
|
||||
if ($runner->iterate()) {
|
||||
if ($runner->hasWork()) {
|
||||
// More to do? Shove us back on the queue...
|
||||
$qm = QueueManager::get();
|
||||
$qm->enqueue('YammerImport', 'yammer');
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
// Something failed?
|
||||
// @fixme should we be trying again here, or should we give warning?
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
// We're done!
|
||||
common_log(LOG_INFO, "Yammer import has no work to do at this time; discarding.");
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,29 +33,123 @@ class YammerRunner
|
|||
private $client;
|
||||
private $importer;
|
||||
|
||||
function __construct()
|
||||
public static function init()
|
||||
{
|
||||
$state = Yammer_state::staticGet('id', 1);
|
||||
if (!$state) {
|
||||
common_log(LOG_ERR, "No YammerImport state during import run. Should not happen!");
|
||||
throw new ServerException('No YammerImport state during import run.');
|
||||
$state = new Yammer_state();
|
||||
$state->id = 1;
|
||||
$state->state = 'init';
|
||||
$state->insert();
|
||||
}
|
||||
return new YammerRunner($state);
|
||||
}
|
||||
|
||||
private function __construct($state)
|
||||
{
|
||||
$this->state = $state;
|
||||
|
||||
$this->client = new SN_YammerClient(
|
||||
common_config('yammer', 'consumer_key'),
|
||||
common_config('yammer', 'consumer_secret'),
|
||||
$this->state->oauth_token,
|
||||
$this->state->oauth_secret);
|
||||
|
||||
$this->importer = new YammerImporter($client);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check which state we're in
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function state()
|
||||
{
|
||||
return $this->state->state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Is the import done, finished, complete, finito?
|
||||
*
|
||||
* @return boolean
|
||||
*/
|
||||
public function isDone()
|
||||
{
|
||||
$workStates = array('import-users', 'import-groups', 'fetch-messages', 'save-messages');
|
||||
return ($this->state() == 'done');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we have work to do in iterate().
|
||||
*/
|
||||
public function hasWork()
|
||||
{
|
||||
$workStates = array('import-users', 'import-groups', 'fetch-messages', 'save-messages');
|
||||
return in_array($this->state(), $workStates);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the authentication process! If all goes well, we'll get back a URL.
|
||||
* Have the user visit that URL, log in on Yammer and verify the importer's
|
||||
* permissions. They'll get back a verification code, which needs to be passed
|
||||
* on to saveAuthToken().
|
||||
*
|
||||
* @return string URL
|
||||
*/
|
||||
public function requestAuth()
|
||||
{
|
||||
if ($this->state->state != 'init') {
|
||||
throw ServerError("Cannot request Yammer auth; already there!");
|
||||
}
|
||||
|
||||
$old = clone($this->state);
|
||||
$this->state->state = 'requesting-auth';
|
||||
$this->state->request_token = $client->requestToken();
|
||||
$this->state->update($old);
|
||||
|
||||
return $this->client->authorizeUrl($this->state->request_token);
|
||||
}
|
||||
|
||||
/**
|
||||
* Now that the user's given us this verification code from Yammer, we can
|
||||
* request a final OAuth token/secret pair which we can use to access the
|
||||
* API.
|
||||
*
|
||||
* After success here, we'll be ready to move on and run through iterate()
|
||||
* until the import is complete.
|
||||
*
|
||||
* @param string $verifier
|
||||
* @return boolean success
|
||||
*/
|
||||
public function saveAuthToken($verifier)
|
||||
{
|
||||
if ($this->state->state != 'requesting-auth') {
|
||||
throw ServerError("Cannot save auth token in Yammer import state {$this->state->state}");
|
||||
}
|
||||
|
||||
$old = clone($this->state);
|
||||
list($token, $secret) = $this->client->getAuthToken($verifier);
|
||||
$this->state->verifier = '';
|
||||
$this->state->oauth_token = $token;
|
||||
$this->state->oauth_secret = $secret;
|
||||
|
||||
$this->state->update($old);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Once authentication is complete, we need to call iterate() a bunch of times
|
||||
* until state() returns 'done'.
|
||||
*
|
||||
* @return boolean success
|
||||
*/
|
||||
public function iterate()
|
||||
{
|
||||
|
||||
switch($state->state)
|
||||
{
|
||||
case null:
|
||||
case 'init':
|
||||
case 'requesting-auth':
|
||||
// Neither of these should reach our background state!
|
||||
common_log(LOG_ERR, "Non-background YammerImport state '$state->state' during import run!");
|
||||
|
|
|
|||
|
|
@ -8,34 +8,37 @@ define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__)))));
|
|||
|
||||
require INSTALLDIR . "/scripts/commandline.inc";
|
||||
|
||||
// temp stuff
|
||||
require 'yam-config.php';
|
||||
$yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
|
||||
$imp = new YammerImporter($yam);
|
||||
$runner = YammerRunner::init();
|
||||
|
||||
// First, import all the users!
|
||||
// @fixme follow paging -- we only get 50 at a time
|
||||
$data = $yam->users();
|
||||
foreach ($data as $item) {
|
||||
$user = $imp->importUser($item);
|
||||
echo "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)\n";
|
||||
}
|
||||
switch ($runner->state())
|
||||
{
|
||||
case 'init':
|
||||
$url = $runner->requestAuth();
|
||||
echo "Log in to Yammer at the following URL and confirm permissions:\n";
|
||||
echo "\n";
|
||||
echo " $url\n";
|
||||
echo "\n";
|
||||
echo "Pass the resulting code back by running:\n"
|
||||
echo "\n"
|
||||
echo " php yammer-import.php --auth=####\n";
|
||||
echo "\n";
|
||||
break;
|
||||
|
||||
// Groups!
|
||||
// @fixme follow paging -- we only get 20 at a time
|
||||
$data = $yam->groups();
|
||||
foreach ($data as $item) {
|
||||
$group = $imp->importGroup($item);
|
||||
echo "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)\n";
|
||||
}
|
||||
case 'requesting-auth':
|
||||
if (empty($options['auth'])) {
|
||||
echo "Please finish authenticating!\n";
|
||||
break;
|
||||
}
|
||||
$runner->saveAuthToken($options['auth']);
|
||||
// Fall through...
|
||||
|
||||
// Messages!
|
||||
// Process in reverse chron order...
|
||||
// @fixme follow paging -- we only get 20 at a time, and start at the most recent!
|
||||
$data = $yam->messages();
|
||||
$messages = $data['messages'];
|
||||
$messages = array_reverse($messages);
|
||||
foreach ($messages as $item) {
|
||||
$notice = $imp->importNotice($item);
|
||||
echo "Imported Yammer notice " . $item['id'] . " as $notice->id\n";
|
||||
}
|
||||
default:
|
||||
while (true) {
|
||||
echo "... {$runner->state->state}\n";
|
||||
if (!$runner->iterate()) {
|
||||
echo "... done.\n";
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
Loading…
Reference in New Issue