Merge ActivitySpam plugin

plugins/ActivitySpam/ActivitySpamPlugin.php

@@ -0,0 +1,318 @@
+<?php
+/**
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2011,2012, StatusNet, Inc.
+ *
+ * ActivitySpam Plugin
+ *
+ * PHP version 5
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2011,2012 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+if (!defined('STATUSNET')) {
+    // This check helps protect against security problems;
+    // your code file can't be executed directly from the web.
+    exit(1);
+}
+
+/**
+ * Check new notices with activity spam service.
+ * 
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2011,2012 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+class ActivitySpamPlugin extends Plugin
+{
+    public $server = null;
+    public $hideSpam = false;
+
+    const REVIEWSPAM = 'ActivitySpamPlugin::REVIEWSPAM';
+    const TRAINSPAM = 'ActivitySpamPlugin::TRAINSPAM';
+
+    /**
+     * Initializer
+     *
+     * @return boolean hook value; true means continue processing, false means stop.
+     */
+    function initialize()
+    {
+        $this->filter = new SpamFilter(common_config('activityspam', 'server'),
+                                       common_config('activityspam', 'consumerkey'),
+                                       common_config('activityspam', 'secret'));
+
+        $this->hideSpam = common_config('activityspam', 'hidespam');
+
+        return true;
+    }
+
+    /**
+     * Database schema setup
+     *
+     * @see Schema
+     * @see ColumnDef
+     *
+     * @return boolean hook value; true means continue processing, false means stop.
+     */
+
+    function onCheckSchema()
+    {
+        $schema = Schema::get();
+        $schema->ensureTable('spam_score', Spam_score::schemaDef());
+
+        Spam_score::upgrade();
+
+        return true;
+    }
+
+    /**
+     * Load related modules when needed
+     *
+     * @param string $cls Name of the class to be loaded
+     *
+     * @return boolean hook value; true means continue processing, false means stop.
+     */
+
+    function onAutoload($cls)
+    {
+        $dir = dirname(__FILE__);
+
+        switch ($cls)
+        {
+        case 'TrainAction':
+        case 'SpamAction':
+            include_once $dir . '/' . strtolower(mb_substr($cls, 0, -6)) . '.php';
+            return false;
+        case 'Spam_score':
+            include_once $dir . '/'.$cls.'.php';
+            return false;
+        case 'SpamFilter':
+        case 'SpamNoticeStream':
+        case 'TrainSpamForm':
+        case 'TrainHamForm':
+            include_once $dir . '/'.strtolower($cls).'.php';
+            return false;
+        default:
+            return true;
+        }
+    }
+
+    /**
+     * When a notice is saved, check its spam score
+     * 
+     * @param Notice $notice Notice that was just saved
+     *
+     * @return boolean hook value; true means continue processing, false means stop.
+     */
+
+    function onEndNoticeSave($notice)
+    {
+        try {
+
+            $result = $this->filter->test($notice);
+
+            $score = Spam_score::saveNew($notice, $result);
+
+            $this->log(LOG_INFO, "Notice " . $notice->id . " has spam score " . $score->score);
+
+        } catch (Exception $e) {
+            // Log but continue 
+            $this->log(LOG_ERR, $e->getMessage());
+        }
+
+        return true;
+    }
+
+    function onNoticeDeleteRelated($notice) {
+        $score = Spam_score::staticGet('notice_id', $notice->id);
+        if (!empty($score)) {
+            $score->delete();
+        }
+        return true;
+    }
+
+    function onUserRightsCheck($profile, $right, &$result) {
+        switch ($right) {
+        case self::REVIEWSPAM:
+        case self::TRAINSPAM:
+            $result = ($profile->hasRole(Profile_role::MODERATOR) || $profile->hasRole('modhelper'));
+            return false;
+        default:
+            return true;
+        }
+    }
+
+    function onGetSpamFilter(&$filter) {
+        $filter = $this->filter;
+        return false;
+    }
+
+    function onEndShowNoticeOptionItems($nli)
+    {
+        $profile = Profile::current();
+
+        if (!empty($profile) && $profile->hasRight(self::TRAINSPAM)) {
+
+            $notice = $nli->getNotice();
+            $out = $nli->getOut();
+
+            if (!empty($notice)) {
+
+                $score = $this->getScore($notice);
+
+                if (empty($score)) {
+                    $this->debug("No score for notice " . $notice->id);
+                    // XXX: show a question-mark or something
+                } else if ($score->is_spam) {
+                    $form = new TrainHamForm($out, $notice);
+                    $form->show();
+                } else if (!$score->is_spam) {
+                    $form = new TrainSpamForm($out, $notice);
+                    $form->show();
+                }
+            }
+        }
+
+        return true;
+    }
+    
+    /**
+     * Map URLs to actions
+     *
+     * @param Net_URL_Mapper $m path-to-action mapper
+     *
+     * @return boolean hook value; true means continue processing, false means stop.
+     */
+
+    function onRouterInitialized($m)
+    {
+        $m->connect('main/train/spam',
+                    array('action' => 'train', 'category' => 'spam'));
+        $m->connect('main/train/ham',
+                    array('action' => 'train', 'category' => 'ham'));
+        $m->connect('main/spam',
+                    array('action' => 'spam'));
+        return true;
+    }
+
+    function onEndShowStyles($action)
+    {
+        $action->element('style', null,
+                         '.form-train-spam input.submit { background: url('.$this->path('icons/bullet_black.png').') no-repeat 0px 0px } ' . "\n" .
+                         '.form-train-ham input.submit { background: url('.$this->path('icons/exclamation.png').') no-repeat 0px 0px } ');
+        return true;
+    }
+
+    function onEndPublicGroupNav($nav)
+    {
+        $user = common_current_user();
+
+        if (!empty($user) && $user->hasRight(self::REVIEWSPAM)) {
+            $nav->out->menuItem(common_local_url('spam'),
+                                _m('MENU','Spam'),
+                                // TRANS: Menu item title in search group navigation panel.
+                                _('Notices marked as spam'),
+                                $nav->actionName == 'spam',
+                                'nav_timeline_spam');
+        }
+
+        return true;
+    }
+
+    function onPluginVersion(&$versions)
+    {
+        $versions[] = array('name' => 'ActivitySpam',
+                            'version' => STATUSNET_VERSION,
+                            'author' => 'Evan Prodromou',
+                            'homepage' => 'http://status.net/wiki/Plugin:ActivitySpam',
+                            'description' =>
+                            _m('Test notices against the Activity Spam service.'));
+        return true;
+    }
+
+    function getScore($notice)
+    {
+        $score = Spam_score::staticGet('notice_id', $notice->id);
+        
+        if (!empty($score)) {
+            return $score;
+        }
+
+        try {
+
+            $result = $this->filter->test($notice);
+
+            $score = Spam_score::saveNew($notice, $result);
+
+            $this->log(LOG_INFO, "Notice " . $notice->id . " has spam score " . $score->score);
+
+        } catch (Exception $e) {
+            // Log but continue 
+            $this->log(LOG_ERR, $e->getMessage());
+            $score = null;
+        }
+
+        return $score;
+    }
+
+    function onStartReadWriteTables(&$alwaysRW, &$rwdb) {
+        $alwaysRW[] = 'spam_score';
+        return true;
+    }
+
+
+    function onEndNoticeInScope($notice, $profile, &$bResult)
+    {
+        if ($this->hideSpam) {
+            if ($bResult) {
+
+                $score = Spam_score::staticGet('notice_id', $notice->id);
+
+                if (!empty($score) && $score->is_spam) {
+                    if (empty($profile) ||
+                        ($profile->id !== $notice->profile_id &&
+                         !$profile->hasRight(self::REVIEWSPAM))) {
+                        $bResult = false;
+                    }
+                }
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Pre-cache our spam scores if needed.
+     */
+    function onEndNoticeListPrefill(&$notices, &$profiles, $avatarSize) {
+        if ($this->hideSpam) {
+            foreach ($notices as $notice) {
+                $ids[] = $notice->id;
+            }
+            Memcached_DataObject::multiGet('Spam_score', 'notice_id', $ids);
+        }
+        return true;
+    }
+}

plugins/ActivitySpam/Spam_score.php

@@ -0,0 +1,216 @@
+<?php
+  /**
+   * StatusNet - the distributed open-source microblogging tool
+   * Copyright (C) 2011, StatusNet, Inc.
+   *
+   * Score of a notice by activity spam service
+   * 
+   * PHP version 5
+   *
+   * This program is free software: you can redistribute it and/or modify
+   * it under the terms of the GNU Affero General Public License as published by
+   * the Free Software Foundation, either version 3 of the License, or
+   * (at your option) any later version.
+   *
+   * This program is distributed in the hope that it will be useful,
+   * but WITHOUT ANY WARRANTY; without even the implied warranty of
+   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   * GNU Affero General Public License for more details.
+   *
+   * You should have received a copy of the GNU Affero General Public License
+   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+   *
+   * @category  Spam
+   * @package   StatusNet
+   * @author    Evan Prodromou <evan@status.net>
+   * @copyright 2011 StatusNet, Inc.
+   * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+   * @link      http://status.net/
+   */
+
+if (!defined('STATUSNET')) {
+    exit(1);
+}
+
+/**
+ * Score of a notice per the activity spam service
+ *
+ * @category Spam
+ * @package  StatusNet
+ * @author   Evan Prodromou <evan@status.net>
+ * @license  http://www.fsf.org/licensing/licenses/agpl.html AGPLv3
+ * @link     http://status.net/
+ *
+ * @see      DB_DataObject
+ */
+
+class Spam_score extends Managed_DataObject
+{
+    const MAX_SCALE = 10000;
+    public $__table = 'spam_score'; // table name
+
+    public $notice_id;   // int
+    public $score;       // float
+    public $created;     // datetime
+
+    /**
+     * Get an instance by key
+     *
+     * @param string $k Key to use to lookup (usually 'notice_id' for this class)
+     * @param mixed  $v Value to lookup
+     *
+     * @return Spam_score object found, or null for no hits
+     *
+     */
+    function staticGet($k, $v=null)
+    {
+        return Managed_DataObject::staticGet('Spam_score', $k, $v);
+    }
+
+    function saveNew($notice, $result) {
+
+        $score = new Spam_score();
+
+        $score->notice_id      = $notice->id;
+        $score->score          = $result->probability;
+        $score->is_spam        = $result->isSpam;
+        $score->scaled         = Spam_score::scale($score->score);
+        $score->created        = common_sql_now();
+        $score->notice_created = $notice->created;
+
+        $score->insert();
+
+        self::blow('spam_score:notice_ids');
+
+        return $score;
+    }
+
+    function save($notice, $result) {
+
+        $orig  = null;
+        $score = Spam_score::staticGet('notice_id', $notice->id);
+
+        if (empty($score)) {
+            $score = new Spam_score();
+        } else {
+            $orig = clone($score);
+        }
+
+        $score->notice_id      = $notice->id;
+        $score->score          = $result->probability;
+        $score->is_spam        = $result->isSpam;
+        $score->scaled         = Spam_score::scale($score->score);
+        $score->created        = common_sql_now();
+        $score->notice_created = $notice->created;
+
+        if (empty($orig)) {
+            $score->insert();
+        } else {
+            $score->update($orig);
+        }
+        
+        self::blow('spam_score:notice_ids');
+
+        return $score;
+    }
+
+    function delete()
+    {
+        self::blow('spam_score:notice_ids');
+        self::blow('spam_score:notice_ids;last');
+        parent::delete();
+    }
+
+    /**
+     * The One True Thingy that must be defined and declared.
+     */
+    public static function schemaDef()
+    {
+        return array(
+            'description' => 'score of the notice per activityspam',
+            'fields' => array(
+                'notice_id' => array('type' => 'int',
+                                     'not null' => true,
+                                     'description' => 'notice getting scored'),
+                'score' => array('type' => 'double',
+                                 'not null' => true,
+                                 'description' => 'score for the notice (0.0, 1.0)'),
+                'scaled' => array('type' => 'int',
+                                  'description' => 'scaled score for the notice (0, 10000)'),
+                'is_spam' => array('type' => 'tinyint',
+                                   'description' => 'flag for spamosity'),
+                'created' => array('type' => 'datetime',
+                                   'not null' => true,
+                                   'description' => 'date this record was created'),
+                'notice_created' => array('type' => 'datetime',
+                                          'description' => 'date the notice was created'),
+            ),
+            'primary key' => array('notice_id'),
+            'foreign keys' => array(
+                'spam_score_notice_id_fkey' => array('notice', array('notice_id' => 'id')),
+            ),
+            'indexes' => array(
+                'spam_score_created_idx' => array('created'),
+                'spam_score_scaled_idx' => array('scaled'),
+            ),
+        );
+    }
+
+    public static function upgrade()
+    {
+        Spam_score::upgradeScaled();
+        Spam_score::upgradeIsSpam();
+        Spam_score::upgradeNoticeCreated();
+    }
+
+    protected static function upgradeScaled()
+    {
+        $score = new Spam_score();
+        $score->whereAdd('scaled IS NULL');
+        
+        if ($score->find()) {
+            while ($score->fetch()) {
+                $orig = clone($score);
+                $score->scaled = Spam_score::scale($score->score);
+                $score->update($orig);
+            }
+        }
+    }
+
+    protected static function upgradeIsSpam()
+    {
+        $score = new Spam_score();
+        $score->whereAdd('is_spam IS NULL');
+        
+        if ($score->find()) {
+            while ($score->fetch()) {
+                $orig = clone($score);
+                $score->is_spam = ($score->score >= 0.90) ? 1 : 0;
+                $score->update($orig);
+            }
+        }
+    }
+
+    protected static function upgradeNoticeCreated()
+    {
+        $score = new Spam_score();
+        $score->whereAdd('notice_created IS NULL');
+        
+        if ($score->find()) {
+            while ($score->fetch()) {
+                $notice = Notice::staticGet('id', $score->notice_id);
+                if (!empty($notice)) {
+                    $orig = clone($score);
+                    $score->notice_created = $notice->created;
+                    $score->update($orig);
+                }
+            }
+        }
+    }
+
+    public static function scale($score)
+    {
+        $raw = round($score * Spam_score::MAX_SCALE);
+        return max(0, min(Spam_score::MAX_SCALE, $raw));
+    }
+}

plugins/ActivitySpam/scripts/testuser.php

@@ -0,0 +1,105 @@
+<?php
+/*
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2012 StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+define('INSTALLDIR', realpath(dirname(__FILE__) . '/../../../..'));
+
+$shortoptions = 'i:n:a';
+$longoptions = array('id=', 'nickname=', 'all');
+
+$helptext = <<<END_OF_TESTUSER_HELP
+testuser.php [options]
+Test user activities against the spam filter
+
+  -i --id       ID of user to export
+  -n --nickname nickname of the user to export
+  -a --all      All users
+END_OF_TESTUSER_HELP;
+
+require_once INSTALLDIR.'/scripts/commandline.inc';
+
+function testAllUsers($filter) {
+    $found = false;
+    $offset = 0;
+    $limit  = 1000;
+
+    do {
+
+        $user = new User();
+        $user->orderBy('created');
+        $user->limit($offset, $limit);
+
+        $found = $user->find();
+
+        if ($found) {
+            while ($user->fetch()) {
+                try {
+                    testUser($filter, $user);
+                } catch (Exception $e) {
+                    printfnq("ERROR testing user %s\n: %s", $user->nickname, $e->getMessage());
+                }
+            }
+            $offset += $found;
+        }
+
+    } while ($found > 0);
+}
+
+function testUser($filter, $user) {
+
+    printfnq("Testing user %s\n", $user->nickname);
+
+    $profile = Profile::staticGet('id', $user->id);
+
+    $str = new ProfileNoticeStream($profile, $profile);
+
+    $offset = 0;
+    $limit  = 100;
+
+    do {
+        $notice = $str->getNotices($offset, $limit);
+        while ($notice->fetch()) {
+            try {
+                printfv("Testing notice %d...", $notice->id);
+                $result = $filter->test($notice);
+                Spam_score::save($notice, $result);
+                printfv("%s\n", ($result->isSpam) ? "SPAM" : "HAM");
+            } catch (Exception $e) {
+                printfnq("ERROR testing notice %d: %s\n", $notice->id, $e->getMessage());
+            }
+        }
+        $offset += $notice->N;
+    } while ($notice->N > 0);
+}
+
+try {
+    $filter = null;
+    Event::handle('GetSpamFilter', array(&$filter));
+    if (empty($filter)) {
+        throw new Exception(_("No spam filter."));
+    }
+    if (have_option('a', 'all')) {
+        testAllUsers($filter);
+    } else {
+        $user = getUser();
+        testUser($filter, $user);
+    }
+} catch (Exception $e) {
+    print $e->getMessage()."\n";
+    exit(1);
+}

plugins/ActivitySpam/scripts/trainuser.php

@@ -0,0 +1,81 @@
+<?php
+/*
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2012 StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+define('INSTALLDIR', realpath(dirname(__FILE__) . '/../../../..'));
+
+$shortoptions = 'i:n:t:';
+$longoptions = array('id=', 'nickname=', 'category=');
+
+$helptext = <<<END_OF_TRAINUSER_HELP
+trainuser.php [options]
+Train user activities against the spam filter
+
+  -i --id       ID of user to export
+  -n --nickname nickname of the user to export
+  -t --category Category; one of "spam" or "ham"
+
+END_OF_TRAINUSER_HELP;
+
+require_once INSTALLDIR.'/scripts/commandline.inc';
+
+function trainUser($filter, $user, $category) {
+
+    printfnq("Training user %s\n", $user->nickname);
+
+    $profile = Profile::staticGet('id', $user->id);
+
+    $str = new ProfileNoticeStream($profile, $profile);
+
+    $offset = 0;
+    $limit  = 100;
+
+    do {
+        $notice = $str->getNotices($offset, $limit);
+        while ($notice->fetch()) {
+            try {
+                printfv("Training notice %d...", $notice->id);
+                $filter->trainOnError($notice, $category);
+                $result = $filter->test($notice);
+                $score = Spam_score::save($notice, $result);
+                printfv("%s\n", ($result->isSpam) ? "SPAM" : "HAM");
+            } catch (Exception $e) {
+                printfnq("ERROR training notice %d\n: %s", $notice->id, $e->getMessage());
+            }
+        }
+        $offset += $notice->N;
+    } while ($notice->N > 0);
+}
+
+try {
+    $filter = null;
+    Event::handle('GetSpamFilter', array(&$filter));
+    if (empty($filter)) {
+        throw new Exception(_("No spam filter."));
+    }
+    $user = getUser();
+    $category = get_option_value('t', 'category');
+    if ($category !== SpamFilter::HAM &&
+        $category !== SpamFilter::SPAM) {
+        throw new Exception(_("No such category."));
+    }
+    trainUser($filter, $user, $category);
+} catch (Exception $e) {
+    print $e->getMessage()."\n";
+    exit(1);
+}

plugins/ActivitySpam/spam.php

@@ -0,0 +1,165 @@
+<?php
+/**
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2012, StatusNet, Inc.
+ *
+ * Stream of latest spam messages
+ * 
+ * PHP version 5
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2012 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+if (!defined('STATUSNET')) {
+    // This check helps protect against security problems;
+    // your code file can't be executed directly from the web.
+    exit(1);
+}
+
+require_once INSTALLDIR.'/lib/noticelist.php';
+
+/**
+ * SpamAction
+ * 
+ * Shows the latest spam on the service
+ * 
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2012 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+class SpamAction extends Action
+{
+    var $page = null;
+    var $notices = null;
+
+    function title() {
+        return _("Latest Spam");
+    }
+
+    /**
+     * For initializing members of the class.
+     *
+     * @param array $argarray misc. arguments
+     *
+     * @return boolean true
+     */
+
+    function prepare($argarray)
+    {
+        parent::prepare($argarray);
+
+        $this->page = ($this->arg('page')) ? ($this->arg('page')+0) : 1;
+
+        // User must be logged in.
+
+        $user = common_current_user();
+
+        if (empty($user)) {
+            throw new ClientException(_("You must be logged in to review."), 403);
+        }
+
+        // User must have the right to review spam
+
+        if (!$user->hasRight(ActivitySpamPlugin::REVIEWSPAM)) {
+            throw new ClientException(_('You cannot review spam on this site.'), 403);
+        }
+
+        $stream = new SpamNoticeStream($user->getProfile());
+
+        $this->notices = $stream->getNotices(($this->page-1)*NOTICES_PER_PAGE,
+                                             NOTICES_PER_PAGE + 1);
+
+        if($this->page > 1 && $this->notices->N == 0) {
+            throw new ClientException(_('No such page.'), 404);
+        }
+
+        return true;
+    }
+
+    /**
+     * Handler method
+     *
+     * @param array $argarray is ignored since it's now passed in in prepare()
+     *
+     * @return void
+     */
+
+    function handle($argarray=null)
+    {
+        parent::handle($args);
+
+        $this->showPage();
+    }
+
+    /**
+     * Fill the content area
+     *
+     * Shows a list of the notices in the public stream, with some pagination
+     * controls.
+     *
+     * @return void
+     */
+
+    function showContent()
+    {
+        $nl = new NoticeList($this->notices, $this);
+
+        $cnt = $nl->show();
+
+        if ($cnt == 0) {
+            $this->showEmptyList();
+        }
+
+        $this->pagination($this->page > 1, 
+                          $cnt > NOTICES_PER_PAGE,
+                          $this->page,
+                          'spam');
+    }
+
+    function showEmptyList()
+    {
+        // TRANS: Text displayed for public feed when there are no public notices.
+        $message = _('This is the timeline of spam messages for %%site.name%% but none have been detected yet.');
+
+        $this->elementStart('div', 'guide');
+        $this->raw(common_markup_to_html($message));
+        $this->elementEnd('div');
+    }
+
+    /**
+     * Return true if read only.
+     *
+     * MAY override
+     *
+     * @param array $args other arguments
+     *
+     * @return boolean is read only action?
+     */
+
+    function isReadOnly($args)
+    {
+        return true;
+    }
+}

plugins/ActivitySpam/spamfilter.php

@@ -0,0 +1,171 @@
+<?php
+  /**
+   * StatusNet - the distributed open-source microblogging tool
+   * Copyright (C) 2012, StatusNet, Inc.
+   *
+   * Spam filter class
+   * 
+   * PHP version 5
+   *
+   * This program is free software: you can redistribute it and/or modify
+   * it under the terms of the GNU Affero General Public License as published by
+   * the Free Software Foundation, either version 3 of the License, or
+   * (at your option) any later version.
+   *
+   * This program is distributed in the hope that it will be useful,
+   * but WITHOUT ANY WARRANTY; without even the implied warranty of
+   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   * GNU Affero General Public License for more details.
+   *
+   * You should have received a copy of the GNU Affero General Public License
+   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+   *
+   * @category  Spam
+   * @package   StatusNet
+   * @author    Evan Prodromou <evan@status.net>
+   * @copyright 2012 StatusNet, Inc.
+   * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+   * @link      http://status.net/
+   */
+
+if (!defined('STATUSNET')) {
+    // This check helps protect against security problems;
+    // your code file can't be executed directly from the web.
+    exit(1);
+}
+
+/**
+ * Spam filter class
+ *
+ * Local proxy for remote filter
+ *
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2012 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+class SpamFilter extends OAuthClient {
+
+    const HAM  = 'ham';
+    const SPAM = 'spam';
+
+    public $server;
+
+    function __construct($server, $consumerKey, $secret) {
+        parent::__construct($consumerKey, $secret);
+        $this->server = $server;
+    }
+
+    protected function toActivity($notice) {
+        // FIXME: need this to autoload ActivityStreamsMediaLink
+        $doc = new ActivityStreamJSONDocument();
+
+        $activity = $notice->asActivity(null);
+
+        return $activity;
+    }
+
+    public function test($notice) {
+
+        $activity = $this->toActivity($notice);
+        return $this->testActivity($activity);
+    }
+    
+    public function testActivity($activity) {
+
+        $response = $this->postJSON($this->server . "/is-this-spam", $activity->asArray());
+
+        $result = json_decode($response->getBody());
+
+        return $result;
+    }
+
+    public function train($notice, $category) {
+
+        $activity = $this->toActivity($notice);
+        return $this->trainActivity($activity, $category);
+
+    }
+
+    public function trainActivity($activity, $category) {
+
+        switch ($category) {
+        case self::HAM:
+            $endpoint = '/this-is-ham';
+            break;
+        case self::SPAM:
+            $endpoint = '/this-is-spam';
+            break;
+        default:
+            throw new Exception("Unknown category: " + $category);
+        }
+
+        $response = $this->postJSON($this->server . $endpoint, $activity->asArray());
+
+        // We don't do much with the results
+        return true;
+    }
+
+    public function trainOnError($notice, $category) {
+
+        $activity = $this->toActivity($notice);
+
+        return $this->trainActivityOnError($activity, $category);
+    }
+    
+    public function trainActivityOnError($activity, $category) {
+
+        $result = $this->testActivity($activity);
+
+        if (($category === self::SPAM && $result->isSpam) ||
+            ($category === self::HAM && !$result->isSpam)) {
+            return true;
+        } else {
+            return $this->trainActivity($activity, $category);
+        }
+    }
+
+    function postJSON($url, $body)
+    {
+        $request = OAuthRequest::from_consumer_and_token($this->consumer,
+                                                         $this->token,
+                                                         'POST',
+                                                         $url);
+
+        $request->sign_request($this->sha1_method,
+                               $this->consumer,
+                               $this->token);
+
+        $hclient = new HTTPClient($url);
+
+        $hclient->setConfig(array('connect_timeout' => 120,
+                                  'timeout' => 120,
+                                  'follow_redirects' => true,
+                                  'ssl_verify_peer' => false,
+                                  'ssl_verify_host' => false));
+
+        $hclient->setMethod(HTTP_Request2::METHOD_POST);
+        $hclient->setBody(json_encode($body));
+        $hclient->setHeader('Content-Type', 'application/json');
+        $hclient->setHeader($request->to_header());
+
+        // Twitter is strict about accepting invalid "Expect" headers
+        // No reason not to clear it still here -ESP
+
+        $hclient->setHeader('Expect', '');
+
+        try {
+            $response = $hclient->send();
+            $code = $response->getStatus();
+            if (!$response->isOK()) {
+                throw new OAuthClientException($response->getBody(), $code);
+            }
+            return $response;
+        } catch (Exception $e) {
+            throw new OAuthClientException($e->getMessage(), $e->getCode());
+        }
+    }
+}

plugins/ActivitySpam/spamnoticestream.php

@@ -0,0 +1,101 @@
+<?php
+/**
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2012, StatusNet, Inc.
+ *
+ * Spam notice stream
+ * 
+ * PHP version 5
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2012 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+if (!defined('STATUSNET')) {
+    // This check helps protect against security problems;
+    // your code file can't be executed directly from the web.
+    exit(1);
+}
+
+/**
+ * Spam notice stream
+ *
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2012 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+class SpamNoticeStream extends ScopingNoticeStream
+{
+    function __construct($tag, $profile = -1)
+    {
+        if (is_int($profile) && $profile == -1) {
+            $profile = Profile::current();
+        }
+        parent::__construct(new CachingNoticeStream(new RawSpamNoticeStream(),
+                                                    'spam_score:notice_ids'));
+    }
+}
+
+/**
+ * Raw stream of spammy notices
+ *
+ * @category  Stream
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2011 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+class RawSpamNoticeStream extends NoticeStream
+{
+    function getNoticeIds($offset, $limit, $since_id, $max_id)
+    {
+        $ss = new Spam_score();
+
+        $ss->is_spam = 1;
+
+        $ss->selectAdd();
+        $ss->selectAdd('notice_id');
+
+        Notice::addWhereSinceId($ss, $since_id, 'notice_id');
+        Notice::addWhereMaxId($ss, $max_id, 'notice_id');
+
+        $ss->orderBy('notice_created DESC, notice_id DESC');
+
+        if (!is_null($offset)) {
+            $ss->limit($offset, $limit);
+        }
+
+        $ids = array();
+
+        if ($ss->find()) {
+            while ($ss->fetch()) {
+                $ids[] = $ss->notice_id;
+            }
+        }
+
+        return $ids;
+    }
+}

plugins/ActivitySpam/train.php

@@ -0,0 +1,155 @@
+<?php
+/**
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2012, StatusNet, Inc.
+ *
+ * Train a notice as spam
+ * 
+ * PHP version 5
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2012 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+if (!defined('STATUSNET')) {
+    // This check helps protect against security problems;
+    // your code file can't be executed directly from the web.
+    exit(1);
+}
+
+/**
+ * Train a notice as spam
+ *
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2012 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+class TrainAction extends Action
+{
+    protected $notice = null;
+    protected $filter = null;
+    protected $category = null;
+
+    /**
+     * For initializing members of the class.
+     *
+     * @param array $argarray misc. arguments
+     *
+     * @return boolean true
+     */
+
+    function prepare($argarray)
+    {
+        parent::prepare($argarray);
+
+        // User must be logged in.
+
+        $user = common_current_user();
+
+        if (empty($user)) {
+            throw new ClientException(_("You must be logged in to train spam."), 403);
+        }
+
+        // User must have the right to review spam
+
+        if (!$user->hasRight(ActivitySpamPlugin::TRAINSPAM)) {
+            throw new ClientException(_('You cannot review spam on this site.'), 403);
+        }
+
+        $id = $this->trimmed('notice');
+
+        $this->notice = Notice::staticGet('id', $id);
+
+        if (empty($this->notice)) {
+            throw new ClientException(_("No such notice."));
+        }
+
+        $this->checkSessionToken();
+
+        $filter = null;
+
+        Event::handle('GetSpamFilter', array(&$filter));
+
+        if (empty($filter)) {
+            throw new ServerException(_("No spam filter configured."));
+        }
+
+        $this->filter = $filter;
+
+        $this->category = $this->trimmed('category');
+
+        if ($this->category !== SpamFilter::SPAM &&
+            $this->category !== SpamFilter::HAM)
+        {
+            throw new ClientException(_("No such category."));
+        }
+
+        return true;
+    }
+
+    /**
+     * Handler method
+     *
+     * @param array $argarray is ignored since it's now passed in in prepare()
+     *
+     * @return void
+     */
+
+    function handle($argarray=null)
+    {
+        // Train
+
+        $this->filter->trainOnError($this->notice, $this->category);
+
+        // Re-test
+
+        $result = $this->filter->test($this->notice);
+
+        // Update or insert
+
+        $score = Spam_score::save($this->notice, $result);
+
+        // Show new toggle form
+
+        if ($this->category === SpamFilter::SPAM) {
+            $form = new TrainHamForm($this, $this->notice);
+        } else {
+            $form = new TrainSpamForm($this, $this->notice);
+        }
+
+        if ($this->boolean('ajax')) {
+            $this->startHTML('text/xml;charset=utf-8');
+            $this->elementStart('head');
+            // TRANS: Page title for page on which favorite notices can be unfavourited.
+            $this->element('title', null, _('Disfavor favorite.'));
+            $this->elementEnd('head');
+            $this->elementStart('body');
+            $form->show();
+            $this->elementEnd('body');
+            $this->elementEnd('html');
+        } else {
+            common_redirect(common_local_url('spam'), 303);
+        }
+    }
+}

plugins/ActivitySpam/trainhamform.php

@@ -0,0 +1,146 @@
+<?php
+/**
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2011, StatusNet, Inc.
+ *
+ * Toggle indicating spam, click to train as ham
+ * 
+ * PHP version 5
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2011 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+if (!defined('STATUSNET')) {
+    // This check helps protect against security problems;
+    // your code file can't be executed directly from the web.
+    exit(1);
+}
+
+/**
+ * Form 
+ *
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2011 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+class TrainHamForm extends Form {
+
+    var $notice  = null;
+
+    function __construct($out, $notice) {
+        parent::__construct($out);
+        $this->notice = $notice;
+    }
+
+    /**
+     * Name of the form
+     *
+     * Sub-classes should overload this with the name of their form.
+     *
+     * @return void
+     */
+
+    function formLegend()
+    {
+        return _("Train ham");
+    }
+
+    /**
+     * Visible or invisible data elements
+     *
+     * Display the form fields that make up the data of the form.
+     * Sub-classes should overload this to show their data.
+     *
+     * @return void
+     */
+
+    function formData()
+    {
+        $this->hidden('notice', $this->notice->id);
+    }
+
+    /**
+     * Buttons for form actions
+     *
+     * Submit and cancel buttons (or whatever)
+     * Sub-classes should overload this to show their own buttons.
+     *
+     * @return void
+     */
+
+    function formActions()
+    {
+        $this->submit('train-ham-submit-' . $this->notice->id,
+                      _('Clear spam'),
+                      'submit',
+                      null,
+                      _("Clear spam"));
+    }
+
+    /**
+     * ID of the form
+     *
+     * Should be unique on the page. Sub-classes should overload this
+     * to show their own IDs.
+     *
+     * @return int ID of the form
+     */
+
+    function id()
+    {
+        return 'train-ham-' . $this->notice->id;
+    }
+
+    /**
+     * Action of the form.
+     *
+     * URL to post to. Should be overloaded by subclasses to give
+     * somewhere to post to.
+     *
+     * @return string URL to post to
+     */
+
+    function action()
+    {
+        return common_local_url('train', array('category' => 'ham'));
+    }
+
+    /**
+     * Class of the form. May include space-separated list of multiple classes.
+     *
+     * If 'ajax' is included, the form will automatically be submitted with
+     * an 'ajax=1' parameter added, and the resulting form or error message
+     * will replace the form after submission.
+     *
+     * It's up to you to make sure that the target action supports this!
+     *
+     * @return string the form's class
+     */
+
+    function formClass()
+    {
+        return 'form-train-ham ajax';
+    }
+}

plugins/ActivitySpam/trainspamform.php

@@ -0,0 +1,146 @@
+<?php
+/**
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2011, StatusNet, Inc.
+ *
+ * Toggle indicating ham, click to train as spam
+ * 
+ * PHP version 5
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2011 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+if (!defined('STATUSNET')) {
+    // This check helps protect against security problems;
+    // your code file can't be executed directly from the web.
+    exit(1);
+}
+
+/**
+ * Form 
+ *
+ * @category  Spam
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2011 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+class TrainSpamForm extends Form {
+
+    var $notice  = null;
+
+    function __construct($out, $notice) {
+        parent::__construct($out);
+        $this->notice = $notice;
+    }
+
+    /**
+     * Name of the form
+     *
+     * Sub-classes should overload this with the name of their form.
+     *
+     * @return void
+     */
+
+    function formLegend()
+    {
+        return _("Train spam");
+    }
+
+    /**
+     * Visible or invisible data elements
+     *
+     * Display the form fields that make up the data of the form.
+     * Sub-classes should overload this to show their data.
+     *
+     * @return void
+     */
+
+    function formData()
+    {
+        $this->hidden('notice', $this->notice->id);
+    }
+
+    /**
+     * Buttons for form actions
+     *
+     * Submit and cancel buttons (or whatever)
+     * Sub-classes should overload this to show their own buttons.
+     *
+     * @return void
+     */
+
+    function formActions()
+    {
+        $this->submit('train-spam-submit-' . $this->notice->id,
+                      _('Train spam'),
+                      'submit',
+                      null,
+                      _("Mark as spam"));
+    }
+
+    /**
+     * ID of the form
+     *
+     * Should be unique on the page. Sub-classes should overload this
+     * to show their own IDs.
+     *
+     * @return int ID of the form
+     */
+
+    function id()
+    {
+        return 'train-spam-' . $this->notice->id;
+    }
+
+    /**
+     * Action of the form.
+     *
+     * URL to post to. Should be overloaded by subclasses to give
+     * somewhere to post to.
+     *
+     * @return string URL to post to
+     */
+
+    function action()
+    {
+        return common_local_url('train', array('category' => 'spam'));
+    }
+
+    /**
+     * Class of the form. May include space-separated list of multiple classes.
+     *
+     * If 'ajax' is included, the form will automatically be submitted with
+     * an 'ajax=1' parameter added, and the resulting form or error message
+     * will replace the form after submission.
+     *
+     * It's up to you to make sure that the target action supports this!
+     *
+     * @return string the form's class
+     */
+
+    function formClass()
+    {
+        return 'form-train-spam ajax';
+    }
+}