WORK IN PROGRESS: Starting infrastructure to initiate Yammer import from web UI and process it in the background queues. Totally not complete yet.

This commit is contained in:
Brion Vibber 2010-09-22 17:51:50 -07:00
parent a005210438
commit 084befc32f
11 changed files with 892 additions and 26 deletions

View File

@ -22,9 +22,7 @@
* @maintainer Brion Vibber <brion@status.net>
*/
if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); }
set_include_path(get_include_path() . PATH_SEPARATOR . dirname(__FILE__) . '/extlib/');
if (!defined('STATUSNET')) { exit(1); }
class YammerImportPlugin extends Plugin
{
@ -36,8 +34,8 @@ class YammerImportPlugin extends Plugin
*/
function onRouterInitialized($m)
{
$m->connect('admin/import/yammer',
array('action' => 'importyammer'));
$m->connect('admin/yammer',
array('action' => 'yammeradminpanel'));
return true;
}
@ -53,6 +51,56 @@ class YammerImportPlugin extends Plugin
return true;
}
/**
* Set up all our tables...
*/
function onCheckSchema()
{
$schema = Schema::get();
$tables = array('Yammer_state',
'Yammer_user',
'Yammer_group',
'Yammer_notice',
'Yammer_notice_stub');
foreach ($tables as $table) {
$schema->ensureTable($table, $table::schemaDef());
}
return true;
}
/**
* If the plugin's installed, this should be accessible to admins.
*/
function onAdminPanelCheck($name, &$isOK)
{
if ($name == 'yammer') {
$isOK = true;
return false;
}
return true;
}
/**
* Add the Yammer admin panel to the list...
*/
function onEndAdminPanelNav($nav)
{
if (AdminPanelAction::canAdmin('yammer')) {
$action_name = $nav->action->trimmed('action');
$nav->out->menuItem(common_local_url('yammeradminpanel'),
_m('Yammer'),
_m('Yammer import'),
$action_name == 'yammeradminpanel',
'nav_yammer_admin_panel');
}
return true;
}
/**
* Automatically load the actions and libraries used by the plugin
*
@ -70,6 +118,9 @@ class YammerImportPlugin extends Plugin
case 'yammerimporter':
require_once "$base/lib/$lower.php";
return false;
case 'yammeradminpanelaction':
require_once "$base/actions/yammeradminpanel.php";
return false;
default:
return true;
}

View File

@ -0,0 +1,153 @@
<?php
/**
* StatusNet, the distributed open-source microblogging tool
*
* Yammer import administration panel
*
* PHP version 5
*
* LICENCE: This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @category Settings
* @package StatusNet
* @author Zach Copley <zach@status.net>
* @copyright 2010 StatusNet, Inc.
* @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
* @link http://status.net/
*/
if (!defined('STATUSNET')) {
exit(1);
}
class YammeradminpanelAction extends AdminPanelAction
{
/**
* Returns the page title
*
* @return string page title
*/
function title()
{
return _m('Yammer Import');
}
/**
* Instructions for using this form.
*
* @return string instructions
*/
function getInstructions()
{
return _m('Yammer import tool');
}
/**
* Show the Yammer admin panel form
*
* @return void
*/
function showForm()
{
$form = new YammerAdminPanelForm($this);
$form->show();
return;
}
}
class YammerAdminPanelForm extends AdminForm
{
/**
* ID of the form
*
* @return string ID of the form
*/
function id()
{
return 'yammeradminpanel';
}
/**
* class of the form
*
* @return string class of the form
*/
function formClass()
{
return 'form_settings';
}
/**
* Action of the form
*
* @return string URL of the action
*/
function action()
{
return common_local_url('yammeradminpanel');
}
/**
* Data elements of the form
*
* @return void
*/
function formData()
{
$this->out->element('p', array(), 'yammer import IN DA HOUSE');
/*
Possible states of the yammer import process:
- null (not doing any sort of import)
- requesting-auth
- authenticated
- import-users
- import-groups
- fetch-messages
- import-messages
- done
*/
$yammerState = Yammer_state::staticGet('id', 1);
$state = $yammerState ? $yammerState->state || null;
switch($state)
{
case null:
$this->out->element('p', array(), 'Time to start auth:');
$this->showAuthForm();
break;
case 'requesting-auth':
$this->out->element('p', array(), 'Need to finish auth!');
$this->showAuthForm();
break;
case 'import-users':
case 'import-groups':
case 'import-messages':
case 'save-messages':
$this->showImportState();
break;
}
}
/**
* Action elements
*
* @return void
*/
function formActions()
{
// No submit buttons needed at bottom
}
}

View File

@ -0,0 +1,17 @@
<?php
function showYammerAuth()
{
$token = $yam->requestToken();
$url = $yam->authorizeUrl($token);
// We're going to try doing this in an iframe; if that's not happy
// we can redirect but there doesn't seem to be a way to get Yammer's
// oauth to call us back instead of the manual copy. :(
//common_redirect($url, 303);
$this->element('iframe', array('id' => 'yammer-oauth',
'src' => $url));
}

View File

@ -0,0 +1,165 @@
<?php
/**
* Data class for remembering Yammer import mappings
*
* PHP version 5
*
* @category Data
* @package StatusNet
* @author Brion Vibber <brion@status.net>
* @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3
* @link http://status.net/
*
* StatusNet - the distributed open-source microblogging tool
* Copyright (C) 2010, StatusNet, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
if (!defined('STATUSNET')) {
exit(1);
}
/**
* Common base class for the Yammer import mappings for users, groups, and notices.
*
* Child classes must override these static methods, since we need to run
* on PHP 5.2.x which has no late static binding:
* - staticGet (as our other classes)
* - schemaDef (call self::doSchemaDef)
* - record (call self::doRecord)
*/
class Yammer_common extends Memcached_DataObject
{
public $__table = 'yammer_XXXX'; // table name
public $__field = 'XXXX_id'; // field name to save into
public $id; // int primary_key not_null
public $user_id; // int(4)
public $created; // datetime
/**
* @fixme add a 'references' thing for the foreign key when we support that
*/
protected static function doSchemaDef($field)
{
return array(new ColumnDef('id', 'bigint', null,
false, 'PRI'),
new ColumnDef($field, 'integer', null,
false, 'UNI'),
new ColumnDef('created', 'datetime', null,
false));
}
/**
* return table definition for DB_DataObject
*
* DB_DataObject needs to know something about the table to manipulate
* instances. This method provides all the DB_DataObject needs to know.
*
* @return array array of column definitions
*/
function table()
{
return array('id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
$this->__field => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
'created' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL);
}
/**
* return key definitions for DB_DataObject
*
* DB_DataObject needs to know about keys that the table has, since it
* won't appear in StatusNet's own keys list. In most cases, this will
* simply reference your keyTypes() function.
*
* @return array list of key field names
*/
function keys()
{
return array_keys($this->keyTypes());
}
/**
* return key definitions for Memcached_DataObject
*
* Our caching system uses the same key definitions, but uses a different
* method to get them. This key information is used to store and clear
* cached data, so be sure to list any key that will be used for static
* lookups.
*
* @return array associative array of key definitions, field name to type:
* 'K' for primary key: for compound keys, add an entry for each component;
* 'U' for unique keys: compound keys are not well supported here.
*/
function keyTypes()
{
return array('id' => 'K', $this->__field => 'U');
}
/**
* Magic formula for non-autoincrementing integer primary keys
*
* If a table has a single integer column as its primary key, DB_DataObject
* assumes that the column is auto-incrementing and makes a sequence table
* to do this incrementation. Since we don't need this for our class, we
* overload this method and return the magic formula that DB_DataObject needs.
*
* @return array magic three-false array that stops auto-incrementing.
*/
function sequenceKey()
{
return array(false, false, false);
}
/**
* Save a mapping between a remote Yammer and local imported user.
*
* @param integer $user_id ID of the status in StatusNet
* @param integer $orig_id ID of the notice in Yammer
*
* @return Yammer_common new object for this value
*/
protected static function doRecord($class, $field, $orig_id, $local_id)
{
$map = self::staticGet('id', $orig_id);
if (!empty($map)) {
return $map;
}
$map = self::staticGet($field, $local_id);
if (!empty($map)) {
return $map;
}
common_debug("Mapping Yammer $field {$orig_id} to local $field {$local_id}");
$map = new $class();
$map->id = $orig_id;
$map->$field = $local_id;
$map->created = common_sql_now();
$map->insert();
return $map;
}
}

View File

@ -0,0 +1,79 @@
<?php
/**
* Data class for remembering Yammer import mappings
*
* PHP version 5
*
* @category Data
* @package StatusNet
* @author Brion Vibber <brion@status.net>
* @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3
* @link http://status.net/
*
* StatusNet - the distributed open-source microblogging tool
* Copyright (C) 2010, StatusNet, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
if (!defined('STATUSNET')) {
exit(1);
}
class Yammer_group extends Yammer_common
{
public $__table = 'yammer_group'; // table name
public $__field = 'group_id'; // field to map to
public $group_id; // int
/**
* Get an instance by key
*
* This is a utility method to get a single instance with a given key value.
*
* @param string $k Key to use to lookup
* @param mixed $v Value to lookup
*
* @return Yammer_group object found, or null for no hits
*
*/
function staticGet($k, $v=null)
{
return Memcached_DataObject::staticGet('Yammer_group', $k, $v);
}
/**
* Return schema definition to set this table up in onCheckSchema
*/
static function schemaDef()
{
return self::doSchemaDef('group_id');
}
/**
* Save a mapping between a remote Yammer and local imported group.
*
* @param integer $orig_id ID of the notice in Yammer
* @param integer $group_id ID of the status in StatusNet
*
* @return Yammer_group new object for this value
*/
static function record($orig_id, $group_id)
{
return self::doRecord('Yammer_group', 'group_id', $orig_id, $group_id);
}
}

View File

@ -0,0 +1,79 @@
<?php
/**
* Data class for remembering Yammer import mappings
*
* PHP version 5
*
* @category Data
* @package StatusNet
* @author Brion Vibber <brion@status.net>
* @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3
* @link http://status.net/
*
* StatusNet - the distributed open-source microblogging tool
* Copyright (C) 2010, StatusNet, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
if (!defined('STATUSNET')) {
exit(1);
}
class Yammer_notice extends Yammer_common
{
public $__table = 'yammer_notice'; // table name
public $__field = 'notice_id'; // field to map to
public $notice_id; // int
/**
* Get an instance by key
*
* This is a utility method to get a single instance with a given key value.
*
* @param string $k Key to use to lookup
* @param mixed $v Value to lookup
*
* @return Yammer_notice object found, or null for no hits
*
*/
function staticGet($k, $v=null)
{
return Memcached_DataObject::staticGet('Yammer_notice', $k, $v);
}
/**
* Return schema definition to set this table up in onCheckSchema
*/
static function schemaDef()
{
return self::doSchemaDef('notice_id');
}
/**
* Save a mapping between a remote Yammer and local imported notice.
*
* @param integer $orig_id ID of the notice in Yammer
* @param integer $notice_id ID of the status in StatusNet
*
* @return Yammer_notice new object for this value
*/
static function record($orig_id, $notice_id)
{
return self::doRecord('Yammer_notice', 'notice_id', $orig_id, $notice_id);
}
}

View File

@ -0,0 +1,174 @@
<?php
/**
* Data class for remembering Yammer import mappings
*
* PHP version 5
*
* @category Data
* @package StatusNet
* @author Brion Vibber <brion@status.net>
* @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3
* @link http://status.net/
*
* StatusNet - the distributed open-source microblogging tool
* Copyright (C) 2010, StatusNet, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
if (!defined('STATUSNET')) {
exit(1);
}
/**
* Temporary storage for imported Yammer messages between fetching and saving
* as local notices.
*
* The Yammer API only allows us to page down from the most recent items; in
* order to start saving the oldest notices first, we have to pull them all
* down in reverse chronological order, then go back over them from oldest to
* newest and actually save them into our notice table.
*/
class Yammer_notice_stub extends Memcached_DataObject
{
public $__table = 'yammer_notice_stub'; // table name
public $id; // int primary_key not_null
public $json_data; // text
public $created; // datetime
/**
* Return schema definition to set this table up in onCheckSchema
*/
static function schemaDef($field)
{
return array(new ColumnDef('id', 'bigint', null,
false, 'PRI'),
new ColumnDef('json_data', 'text', null,
false),
new ColumnDef('created', 'datetime', null,
false));
}
/**
* return table definition for DB_DataObject
*
* DB_DataObject needs to know something about the table to manipulate
* instances. This method provides all the DB_DataObject needs to know.
*
* @return array array of column definitions
*/
function table()
{
return array('id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
'json_data' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
'created' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL);
}
/**
* return key definitions for DB_DataObject
*
* DB_DataObject needs to know about keys that the table has, since it
* won't appear in StatusNet's own keys list. In most cases, this will
* simply reference your keyTypes() function.
*
* @return array list of key field names
*/
function keys()
{
return array_keys($this->keyTypes());
}
/**
* return key definitions for Memcached_DataObject
*
* Our caching system uses the same key definitions, but uses a different
* method to get them. This key information is used to store and clear
* cached data, so be sure to list any key that will be used for static
* lookups.
*
* @return array associative array of key definitions, field name to type:
* 'K' for primary key: for compound keys, add an entry for each component;
* 'U' for unique keys: compound keys are not well supported here.
*/
function keyTypes()
{
return array('id' => 'K');
}
/**
* Magic formula for non-autoincrementing integer primary keys
*
* If a table has a single integer column as its primary key, DB_DataObject
* assumes that the column is auto-incrementing and makes a sequence table
* to do this incrementation. Since we don't need this for our class, we
* overload this method and return the magic formula that DB_DataObject needs.
*
* @return array magic three-false array that stops auto-incrementing.
*/
function sequenceKey()
{
return array(false, false, false);
}
/**
* Save the native Yammer API representation of a message for the pending
* import. Since they come in in reverse chronological order, we need to
* record them all as stubs and then go through from the beginning and
* save them as native notices, or we'll lose ordering and threading
* data.
*
* @param integer $orig_id ID of the notice on Yammer
* @param array $data the message record fetched out of Yammer API returnd data
*
* @return Yammer_notice_stub new object for this value
*/
static function record($orig_id, $data)
{
common_debug("Recording Yammer message stub {$orig_id} for pending import...");
$stub = new Yammer_notice_stub();
$stub->id = $orig_id;
$stub->json_data = json_encode($data);
$stub->created = common_sql_now();
$stub->insert();
return $stub;
}
/**
* Save a mapping between a remote Yammer and local imported user.
*
* @param integer $user_id ID of the status in StatusNet
*
* @return Yammer_notice_stub new object for this value
*/
static function retrieve($orig_id)
{
$stub = self::staticGet('id', $orig_id);
if ($stub) {
return json_decode($stub->json_data, true);
} else {
return false;
}
}
}

View File

@ -0,0 +1,37 @@
<?php
/*
yammer_state
id
state
request_token
oauth_token
users_page
groups_page
messages_oldest
created
modified
yammer_user
id
user_id
created
yammer_group
id
group_id
created
yammer_notice
id
notice_id
created
yammer_notice_stub
id
json_data
created
*/

View File

@ -0,0 +1,79 @@
<?php
/**
* Data class for remembering Yammer import mappings
*
* PHP version 5
*
* @category Data
* @package StatusNet
* @author Brion Vibber <brion@status.net>
* @license http://www.fsf.org/licensing/licenses/agpl.html AGPLv3
* @link http://status.net/
*
* StatusNet - the distributed open-source microblogging tool
* Copyright (C) 2010, StatusNet, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
if (!defined('STATUSNET')) {
exit(1);
}
class Yammer_user extends Yammer_common
{
public $__table = 'yammer_user'; // table name
public $__field = 'user_id'; // field to map to
public $user_id; // int
/**
* Get an instance by key
*
* This is a utility method to get a single instance with a given key value.
*
* @param string $k Key to use to lookup
* @param mixed $v Value to lookup
*
* @return Yammer_user object found, or null for no hits
*
*/
function staticGet($k, $v=null)
{
return Memcached_DataObject::staticGet('Yammer_user', $k, $v);
}
/**
* Return schema definition to set this table up in onCheckSchema
*/
static function schemaDef()
{
return self::doSchemaDef('user_id');
}
/**
* Save a mapping between a remote Yammer and local imported user.
*
* @param integer $orig_id ID of the notice in Yammer
* @param integer $user_id ID of the status in StatusNet
*
* @return Yammer_user new object for this value
*/
static function record($orig_id, $user_id)
{
return self::doRecord('Yammer_user', 'user_id', $orig_id, $user_id);
}
}

View File

@ -26,9 +26,6 @@
class YammerImporter
{
protected $client;
protected $users=array();
protected $groups=array();
protected $notices=array();
function __construct(SN_YammerClient $client)
{
@ -330,44 +327,32 @@ class YammerImporter
private function findImportedUser($origId)
{
if (isset($this->users[$origId])) {
return $this->users[$origId];
} else {
return false;
}
return Yammer_user::staticGet('id', $origId);
}
private function findImportedGroup($origId)
{
if (isset($this->groups[$origId])) {
return $this->groups[$origId];
} else {
return false;
}
return Yammer_group::staticGet('id', $origId);
}
private function findImportedNotice($origId)
{
if (isset($this->notices[$origId])) {
return $this->notices[$origId];
} else {
return false;
}
return Yammer_notice::staticGet('id', $origId);
}
private function recordImportedUser($origId, $userId)
{
$this->users[$origId] = $userId;
Yammer_user::record($origId, $userId);
}
private function recordImportedGroup($origId, $groupId)
{
$this->groups[$origId] = $groupId;
Yammer_group::record($origId, $groupId);
}
private function recordImportedNotice($origId, $noticeId)
{
$this->notices[$origId] = $noticeId;
Yammer_notice::record($origId, $noticeId);
}
/**

View File

@ -0,0 +1,47 @@
<?php
/*
* StatusNet - the distributed open-source microblogging tool
* Copyright (C) 2010, StatusNet, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
if (!defined('STATUSNET')) {
exit(1);
}
/**
* Queue handler for bumping the next chunk of Yammer import activity!
*
* @package YammerImportPlugin
* @author Brion Vibber <brion@status.net>
*/
class YammerQueueHandler extends QueueHandler
{
function transport()
{
return 'yammer';
}
function handle($notice)
{
$importer = new YammerImporter();
if ($importer->hasWork()) {
return $importer->iterate();
} else {
// We're done!
return true;
}
}
}