GNUsocial/gnu-social
4
0
Fork 1
Code Issues Releases Wiki Activity

- Rewrote SyncTwitterFriends as a daemon

Browse Source 
- Made it use OAuth
- Code clean up
This commit is contained in:
Zach Copley 2009-08-06 07:03:05 +00:00
parent 6d29592ec7
commit 421e33f145
3 changed files with 309 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

329
lib/twitter.php
View File

@ -17,83 +17,20 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
if (!defined('LACONICA')) { exit(1); }
if (!defined('LACONICA')) {
exit(1);
}
define('TWITTER_SERVICE', 1); // Twitter is foreign_service ID 1
function get_twitter_data($uri, $screen_name, $password)
{
$options = array(
CURLOPT_USERPWD => sprintf("%s:%s", $screen_name, $password),
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FAILONERROR => true,
CURLOPT_HEADER => false,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_USERAGENT => "Laconica",
CURLOPT_CONNECTTIMEOUT => 120,
CURLOPT_TIMEOUT => 120,
# Twitter is strict about accepting invalid "Expect" headers
CURLOPT_HTTPHEADER => array('Expect:')
);
$ch = curl_init($uri);
curl_setopt_array($ch, $options);
$data = curl_exec($ch);
$errmsg = curl_error($ch);
if ($errmsg) {
common_debug("Twitter bridge - cURL error: $errmsg - trying to load: $uri with user $screen_name.",
__FILE__);
if (defined('SCRIPT_DEBUG')) {
print "cURL error: $errmsg - trying to load: $uri with user $screen_name.\n";
}
}
curl_close($ch);
return $data;
}
function twitter_json_data($uri, $screen_name, $password)
{
$json_data = get_twitter_data($uri, $screen_name, $password);
if (!$json_data) {
return false;
}
$data = json_decode($json_data);
if (!$data) {
return false;
}
return $data;
}
function twitter_user_info($screen_name, $password)
{
$uri = "http://twitter.com/users/show/$screen_name.json";
return twitter_json_data($uri, $screen_name, $password);
}
function twitter_friends_ids($screen_name, $password)
{
$uri = "http://twitter.com/friends/ids/$screen_name.json";
return twitter_json_data($uri, $screen_name, $password);
}
function update_twitter_user($twitter_id, $screen_name)
{
$uri = 'http://twitter.com/' . $screen_name;
$fuser = new Foreign_user();
$fuser->query('BEGIN');
// Dropping down to SQL because regular db_object udpate stuff doesn't seem
// Dropping down to SQL because regular DB_DataObject udpate stuff doesn't seem
// to work so good with tables that have multiple column primary keys
// Any time we update the uri for a forein user we have to make sure there
@ -102,35 +39,14 @@ function update_twitter_user($twitter_id, $screen_name)
$qry = 'UPDATE foreign_user set uri = \'\' WHERE uri = ';
$qry .= '\'' . $uri . '\'' . ' AND service = ' . TWITTER_SERVICE;
$result = $fuser->query($qry);
if ($result) {
common_debug("Removed uri ($uri) from another foreign_user who was squatting on it.");
if (defined('SCRIPT_DEBUG')) {
print("Removed uri ($uri) from another Twitter user who was squatting on it.\n");
}
}
$fuser->query($qry);
// Update the user
$qry = 'UPDATE foreign_user SET nickname = ';
$qry .= '\'' . $screen_name . '\'' . ', uri = \'' . $uri . '\' ';
$qry .= 'WHERE id = ' . $twitter_id . ' AND service = ' . TWITTER_SERVICE;
$result = $fuser->query($qry);
if (!$result) {
common_log(LOG_WARNING,
"Couldn't update foreign_user data for Twitter user: $screen_name");
common_log_db_error($fuser, 'UPDATE', __FILE__);
if (defined('SCRIPT_DEBUG')) {
print "UPDATE failed: for Twitter user: $twitter_id - $screen_name. - ";
print common_log_objstring($fuser) . "\n";
$error = &PEAR::getStaticProperty('DB_DataObject','lastError');
print "DB_DataObject Error: " . $error->getMessage() . "\n";
}
return false;
}
$fuser->query('COMMIT');
$fuser->free();
@ -147,23 +63,22 @@ function add_twitter_user($twitter_id, $screen_name)
// Clear out any bad old foreign_users with the new user's legit URL
// This can happen when users move around or fakester accounts get
// repoed, and things like that.
$luser = new Foreign_user();
$luser->uri = $new_uri;
$luser->service = TWITTER_SERVICE;
$result = $luser->delete();
if ($result) {
if (empty($result)) {
common_log(LOG_WARNING,
"Twitter bridge - removed invalid Twitter user squatting on uri: $new_uri");
if (defined('SCRIPT_DEBUG')) {
print "Removed invalid Twitter user squatting on uri: $new_uri\n";
}
}
$luser->free();
unset($luser);
// Otherwise, create a new Twitter user
$fuser = new Foreign_user();
$fuser->nickname = $screen_name;
@ -173,21 +88,12 @@ function add_twitter_user($twitter_id, $screen_name)
$fuser->created = common_sql_now();
$result = $fuser->insert();
if (!$result) {
if (empty($result)) {
common_log(LOG_WARNING,
"Twitter bridge - failed to add new Twitter user: $twitter_id - $screen_name.");
common_log_db_error($fuser, 'INSERT', __FILE__);
if (defined('SCRIPT_DEBUG')) {
print "INSERT failed: could not add new Twitter user: $twitter_id - $screen_name. - ";
print common_log_objstring($fuser) . "\n";
$error = &PEAR::getStaticProperty('DB_DataObject','lastError');
print "DB_DataObject Error: " . $error->getMessage() . "\n";
}
} else {
common_debug("Twitter bridge - Added new Twitter user: $screen_name ($twitter_id).");
if (defined('SCRIPT_DEBUG')) {
print "Added new Twitter user: $screen_name ($twitter_id).\n";
}
}
return $result;
@ -199,23 +105,20 @@ function save_twitter_user($twitter_id, $screen_name)
// Check to see whether the Twitter user is already in the system,
// and update its screen name and uri if so.
$fuser = Foreign_user::getForeignUser($twitter_id, TWITTER_SERVICE);
if ($fuser) {
if (!empty($fuser)) {
$result = true;
// Only update if Twitter screen name has changed
if ($fuser->nickname != $screen_name) {
$result = update_twitter_user($twitter_id, $screen_name);
common_debug('Twitter bridge - Updated nickname (and URI) for Twitter user ' .
"$fuser->id to $screen_name, was $fuser->nickname");
if (defined('SCRIPT_DEBUG')) {
print 'Updated nickname (and URI) for Twitter user ' .
"$fuser->id to $screen_name, was $fuser->nickname\n";
}
}
return $result;
@ -230,119 +133,6 @@ function save_twitter_user($twitter_id, $screen_name)
return true;
}
function retreive_twitter_friends($twitter_id, $screen_name, $password)
{
$friends = array();
$uri = "http://twitter.com/statuses/friends/$twitter_id.json?page=";
$friends_ids = twitter_friends_ids($screen_name, $password);
if (!$friends_ids) {
return $friends;
}
if (defined('SCRIPT_DEBUG')) {
print "Twitter 'social graph' ids method says $screen_name has " .
count($friends_ids) . " friends.\n";
}
// Calculate how many pages to get...
$pages = ceil(count($friends_ids) / 100);
if ($pages == 0) {
common_log(LOG_WARNING,
"Twitter bridge - $screen_name seems to have no friends.");
if (defined('SCRIPT_DEBUG')) {
print "$screen_name seems to have no friends.\n";
}
}
for ($i = 1; $i <= $pages; $i++) {
$data = get_twitter_data($uri . $i, $screen_name, $password);
if (!$data) {
common_log(LOG_WARNING,
"Twitter bridge - Couldn't retrieve page $i of $screen_name's friends.");
if (defined('SCRIPT_DEBUG')) {
print "Couldn't retrieve page $i of $screen_name's friends.\n";
}
continue;
}
$more_friends = json_decode($data);
if (!$more_friends) {
common_log(LOG_WARNING,
"Twitter bridge - No data for page $i of $screen_name's friends.");
if (defined('SCRIPT_DEBUG')) {
print "No data for page $i of $screen_name's friends.\n";
}
continue;
}
$friends = array_merge($friends, $more_friends);
}
return $friends;
}
function save_twitter_friends($user, $twitter_id, $screen_name, $password)
{
$friends = retreive_twitter_friends($twitter_id, $screen_name, $password);
if (empty($friends)) {
common_debug("Twitter bridge - Couldn't get friends data from Twitter for $screen_name.");
if (defined('SCRIPT_DEBUG')) {
print "Couldn't get friends data from Twitter for $screen_name.\n";
}
return false;
}
foreach ($friends as $friend) {
$friend_name = $friend->screen_name;
$friend_id = (int) $friend->id;
// Update or create the Foreign_user record
if (!save_twitter_user($friend_id, $friend_name)) {
common_log(LOG_WARNING,
"Twitter bridge - couldn't save $screen_name's friend, $friend_name.");
if (defined('SCRIPT_DEBUG')) {
print "Couldn't save $screen_name's friend, $friend_name.\n";
}
continue;
}
// Check to see if there's a related local user
$flink = Foreign_link::getByForeignID($friend_id, 1);
if ($flink) {
// Get associated user and subscribe her
$friend_user = User::staticGet('id', $flink->user_id);
if (!empty($friend_user)) {
$result = subs_subscribe_to($user, $friend_user);
if ($result === true) {
common_debug("Twitter bridge - subscribed $friend_user->nickname to $user->nickname.");
if (defined('SCRIPT_DEBUG')) {
print("Subscribed $friend_user->nickname to $user->nickname.\n");
}
} else {
if (defined('SCRIPT_DEBUG')) {
print "$result ($friend_user->nickname to $user->nickname)\n";
}
}
}
}
}
return true;
}
function is_twitter_bound($notice, $flink) {
// Check to see if notice should go to Twitter
@ -351,7 +141,7 @@ function is_twitter_bound($notice, $flink) {
// If it's not a Twitter-style reply, or if the user WANTS to send replies.
if (!preg_match('/^@[a-zA-Z0-9_]{1,15}\b/u', $notice->content) ||
($flink->noticesync & FOREIGN_NOTICE_SEND_REPLY)) {
return true;
return true;
}
}
@ -361,7 +151,7 @@ function is_twitter_bound($notice, $flink) {
function broadcast_twitter($notice)
{
$flink = Foreign_link::getByUserID($notice->profile_id,
TWITTER_SERVICE);
TWITTER_SERVICE);
if (is_twitter_bound($notice, $flink)) {
@ -378,54 +168,53 @@ function broadcast_twitter($notice)
$status = $client->statuses_update($statustxt);
} catch (OAuthClientCurlException $e) {
if ($e->getMessage() == 'The requested URL returned error: 401') {
if ($e->getMessage() == 'The requested URL returned error: 401') {
$errmsg = sprintf('User %1$s (user id: %2$s) has an invalid ' .
'Twitter OAuth access token.',
$user->nickname, $user->id);
common_log(LOG_WARNING, $errmsg);
$errmsg = sprintf('User %1$s (user id: %2$s) has an invalid ' .
'Twitter OAuth access token.',
$user->nickname, $user->id);
common_log(LOG_WARNING, $errmsg);
// Bad auth token! We need to delete the foreign_link
// to Twitter and inform the user.
// Bad auth token! We need to delete the foreign_link
// to Twitter and inform the user.
remove_twitter_link($flink);
return true;
remove_twitter_link($flink);
return true;
} else {
} else {
// Some other error happened, so we should probably
// try to send again later.
// Some other error happened, so we should probably
// try to send again later.
$errmsg = sprintf('cURL error trying to send notice to Twitter ' .
'for user %1$s (user id: %2$s) - ' .
'code: %3$s message: $4$s.',
$user->nickname, $user->id,
$e->getCode(), $e->getMessage());
$errmsg = sprintf('cURL error trying to send notice to Twitter ' .
'for user %1$s (user id: %2$s) - ' .
'code: %3$s message: $4$s.',
$user->nickname, $user->id,
$e->getCode(), $e->getMessage());
common_log(LOG_WARNING, $errmsg);
return false;
}
}
if (empty($status)) {
// This could represent a failure posting,
// or the Twitter API might just be behaving flakey.
$errmsg = sprint('No data returned by Twitter API when ' .
'trying to send update for %1$s (user id %2$s).',
$user->nickname, $user->id);
common_log(LOG_WARNING, $errmsg);
return false;
}
}
if (empty($status)) {
// This could represent a failure posting,
// or the Twitter API might just be behaving flakey.
$errmsg = sprint('No data returned by Twitter API when ' .
'trying to send update for %1$s (user id %2$s).',
$user->nickname, $user->id);
common_log(LOG_WARNING, $errmsg);
return false;
}
// Notice crossed the great divide
$msg = sprintf('Twitter bridge posted notice %s to Twitter.',
$notice->id);
common_log(LOG_INFO, $msg);
// Notice crossed the great divide
$msg = sprintf('Twitter bridge posted notice %s to Twitter.',
$notice->id);
common_log(LOG_INFO, $msg);
}
return true;
@ -442,7 +231,7 @@ function remove_twitter_link($flink)
if (empty($result)) {
common_log(LOG_ERR, 'Could not remove Twitter bridge ' .
"Foreign_link for $user->nickname (user id: $user->id)!");
"Foreign_link for $user->nickname (user id: $user->id)!");
common_log_db_error($flink, 'DELETE', __FILE__);
}
@ -450,17 +239,17 @@ function remove_twitter_link($flink)
if (isset($user->email)) {
$result = mail_twitter_bridge_removed($user);
$result = mail_twitter_bridge_removed($user);
if (!$result) {
if (!$result) {
$msg = 'Unable to send email to notify ' .
"$user->nickname (user id: $user->id) " .
'that their Twitter bridge link was ' .
'removed!';
$msg = 'Unable to send email to notify ' .
"$user->nickname (user id: $user->id) " .
'that their Twitter bridge link was ' .
'removed!';
common_log(LOG_WARNING, $msg);
}
common_log(LOG_WARNING, $msg);
}
}
}

40
lib/twitteroauthclient.php
View File

@ -58,4 +58,44 @@ class TwitterOAuthClient extends OAuthClient
return $statuses;
}
function statuses_friends($id = null, $user_id = null, $screen_name = null,
$page = null)
{
$url = "https://twitter.com/statuses/friends.json";
$params = array('id' => $id,
'user_id' => $user_id,
'screen_name' => $screen_name,
'page' => $page);
$qry = http_build_query($params);
if (!empty($qry)) {
$url .= "?$qry";
}
$response = $this->oAuthGet($url);
$ids = json_decode($response);
return $ids;
}
function friends_ids($id = null, $user_id = null, $screen_name = null,
$page = null)
{
$url = "https://twitter.com/friends/ids.json";
$params = array('id' => $id,
'user_id' => $user_id,
'screen_name' => $screen_name,
'page' => $page);
$qry = http_build_query($params);
if (!empty($qry)) {
$url .= "?$qry";
}
$response = $this->oAuthGet($url);
$ids = json_decode($response);
return $ids;
}
}

263
scripts/synctwitterfriends.php
View File

@ -20,85 +20,242 @@
define('INSTALLDIR', realpath(dirname(__FILE__) . '/..'));
// Uncomment this to get useful console output
$shortoptions = 'di::';
$longoptions = array('id::', 'debug');
$helptext = <<<END_OF_TRIM_HELP
Batch script for synching local friends with Twitter friends.
-i --id Identity (default 'generic')
-d --debug Debug (lots of log output)
END_OF_TRIM_HELP;
require_once INSTALLDIR . '/scripts/commandline.inc';
require_once INSTALLDIR . '/lib/parallelizingdaemon.php';
/**
* Daemon to sync local friends with Twitter friends
*
* @category Twitter
* @package Laconica
* @author Zach Copley <zach@controlyourself.ca>
* @author Evan Prodromou <evan@controlyourself.ca>
* @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
* @link http://laconi.ca/
*/
$helptext = <<<END_OF_TWITTER_HELP
Batch script for synching local friends with Twitter friends.
END_OF_TWITTER_HELP;
require_once INSTALLDIR.'/scripts/commandline.inc';
require_once INSTALLDIR . '/scripts/commandline.inc';
require_once INSTALLDIR . '/lib/parallelizingdaemon.php';
// Make a lockfile
$lockfilename = lockFilename();
if (!($lockfile = @fopen($lockfilename, "w"))) {
print "Already running... exiting.\n";
exit(1);
}
class SyncTwitterFriendsDaemon extends ParallelizingDaemon
{
function __construct($id = null, $interval = 60,
$max_children = 2, $debug = null)
{
parent::__construct($id, $interval, $max_children, $debug);
}
// Obtain an exlcusive lock on file (will fail if script is already going)
if (!@flock( $lockfile, LOCK_EX | LOCK_NB, &$wouldblock) || $wouldblock) {
// Script already running - abort
@fclose($lockfile);
print "Already running... exiting.\n";
exit(1);
}
/**
* Name of this daemon
*
* @return string Name of the daemon.
*/
$flink = new Foreign_link();
$flink->service = 1; // Twitter
$flink->orderBy('last_friendsync');
$flink->limit(25); // sync this many users during this run
$cnt = $flink->find();
function name()
{
return ('synctwitterfriendsdaemon.' . $this->_id);
}
print "Updating Twitter friends subscriptions for $cnt users.\n";
function getObjects()
{
$flinks = array();
$flink = new Foreign_link();
while ($flink->fetch()) {
$conn = &$flink->getDatabaseConnection();
if (($flink->friendsync & FOREIGN_FRIEND_RECV) == FOREIGN_FRIEND_RECV) {
$flink->service = TWITTER_SERVICE;
$flink->orderBy('last_friendsync');
$flink->limit(25); // sync this many users during this run
$flink->find();
$user = User::staticGet($flink->user_id);
if (empty($user)) {
common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id);
print "Unmatched user for ID $flink->user_id\n";
continue;
while ($flink->fetch()) {
if (($flink->friendsync & FOREIGN_FRIEND_RECV) == FOREIGN_FRIEND_RECV) {
$flinks[] = clone($flink);
}
}
print "Updating Twitter friends for $user->nickname (Laconica ID: $user->id)... ";
$conn->disconnect();
$fuser = $flink->getForeignUser();
global $_DB_DATAOBJECT;
unset($_DB_DATAOBJECT['CONNECTIONS']);
if (empty($fuser)) {
common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id);
print "Unmatched user for ID $flink->user_id\n";
continue;
}
return $flinks;
}
save_twitter_friends($user, $fuser->id, $fuser->nickname, $flink->credentials);
function childTask($flink) {
// Each child ps needs its own DB connection
// Note: DataObject::getDatabaseConnection() creates
// a new connection if there isn't one already
$conn = &$flink->getDatabaseConnection();
$this->subscribeTwitterFriends($flink);
$flink->last_friendsync = common_sql_now();
$flink->update();
if (defined('SCRIPT_DEBUG')) {
print "\nDONE\n";
} else {
print "DONE\n";
$conn->disconnect();
// XXX: Couldn't find a less brutal way to blow
// away a cached connection
global $_DB_DATAOBJECT;
unset($_DB_DATAOBJECT['CONNECTIONS']);
}
function fetchTwitterFriends($flink)
{
$friends = array();
$client = new TwitterOAuthClient($flink->token, $flink->credentials);
try {
$friends_ids = $client->friends_ids();
} catch (OAuthCurlException $e) {
common_log(LOG_WARNING, $this->name() .
' - cURL error getting friend ids ' .
$e->getCode() . ' - ' . $e->getMessage());
return $friends;
}
}
}
function lockFilename()
{
$piddir = common_config('daemon', 'piddir');
if (!$piddir) {
$piddir = '/var/run';
if (empty($friends_ids)) {
common_debug($this->name() .
" - Twitter user $flink->foreign_id " .
'doesn\'t have any friends!');
return $friends;
}
common_debug($this->name() . ' - Twitter\'s API says Twitter user id ' .
"$flink->foreign_id has " .
count($friends_ids) . ' friends.');
// Calculate how many pages to get...
$pages = ceil(count($friends_ids) / 100);
if ($pages == 0) {
common_debug($this->name() . " - $user seems to have no friends.");
}
for ($i = 1; $i <= $pages; $i++) {
try {
$more_friends = $client->statuses_friends(null, null, null, $i);
} catch (OAuthCurlException $e) {
common_log(LOG_WARNING, $this->name() .
' - cURL error getting Twitter statuses/friends ' .
"page $i - " . $e->getCode() . ' - ' .
$e->getMessage());
}
if (empty($more_friends)) {
common_log(LOG_WARNING, $this->name() .
" - Couldn't retrieve page $i " .
"of Twitter user $flink->foreign_id friends.");
continue;
} else {
$friends = array_merge($friends, $more_friends);
}
}
return $friends;
}
function subscribeTwitterFriends($flink)
{
$friends = $this->fetchTwitterFriends($flink);
if (empty($friends)) {
common_debug($this->name() .
' - Couldn\'t get friends from Twitter for ' .
"Twitter user $flink->foreign_id.");
return false;
}
$user = $flink->getUser();
foreach ($friends as $friend) {
$friend_name = $friend->screen_name;
$friend_id = (int) $friend->id;
// Update or create the Foreign_user record for each
// Twitter friend
if (!save_twitter_user($friend_id, $friend_name)) {
common_log(LOG_WARNING, $this-name() .
" - Couldn't save $screen_name's friend, $friend_name.");
continue;
}
// Check to see if there's a related local user
$friend_flink = Foreign_link::getByForeignID($friend_id,
TWITTER_SERVICE);
if (!empty($friend_flink)) {
// Get associated user and subscribe her
$friend_user = User::staticGet('id', $friend_flink->user_id);
if (!empty($friend_user)) {
$result = subs_subscribe_to($user, $friend_user);
if ($result === true) {
common_log(LOG_INFO,
$this->name() . ' - Subscribed ' .
"$friend_user->nickname to $user->nickname.");
} else {
common_debug($this->name() .
' - Tried subscribing ' .
"$friend_user->nickname to $user->nickname - " .
$result);
}
}
}
}
return true;
}
return $piddir . '/synctwitterfriends.lock';
}
// Cleanup
fclose($lockfile);
unlink($lockfilename);
declare(ticks = 1);
$id = null;
$debug = null;
if (have_option('i')) {
$id = get_option_value('i');
} else if (have_option('--id')) {
$id = get_option_value('--id');
} else if (count($args) > 0) {
$id = $args[0];
} else {
$id = null;
}
if (have_option('d') || have_option('debug')) {
$debug = true;
}
$syncer = new SyncTwitterFriendsDaemon($id, 60, 2, $debug);
$syncer->runOnce();
exit(0);