From ddfb56d9207c2ac371b0ea2fcfd786301b1c72a9 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Fri, 23 Oct 2015 18:03:44 +0000 Subject: [PATCH 01/17] TwitterBridge also check for dupe by uri In case a twitter item came in from some other source (such as linkback). --- plugins/TwitterBridge/lib/twitterimport.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/plugins/TwitterBridge/lib/twitterimport.php b/plugins/TwitterBridge/lib/twitterimport.php index d929fecf83..5b0c34d9d7 100644 --- a/plugins/TwitterBridge/lib/twitterimport.php +++ b/plugins/TwitterBridge/lib/twitterimport.php @@ -102,6 +102,17 @@ class TwitterImport return Notice::getKV('id', $n2s->notice_id); } + $dupe = Notice::getKV('uri', $statusUri); + if($dupe instanceof Notice) { + // Add it to our record + Notice_to_status::saveNew($dupe->id, $statusId); + common_log( + LOG_INFO, + __METHOD__ . " - Ignoring duplicate import: {$statusId}" + ); + return $dupe; + } + // If it's a retweet, save it as a repeat! if (!empty($status->retweeted_status)) { common_log(LOG_INFO, "Status {$statusId} is a retweet of " . twitter_id($status->retweeted_status) . "."); From 033ed7e4aa3b07656cb39934f487395b9e2bfdfa Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Fri, 23 Oct 2015 19:00:08 +0000 Subject: [PATCH 02/17] Mark up link to original as a repost for repeats http://indiewebcamp.com/repost --- lib/noticelistitem.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/noticelistitem.php b/lib/noticelistitem.php index dc171409f4..5c613df859 100644 --- a/lib/noticelistitem.php +++ b/lib/noticelistitem.php @@ -516,6 +516,19 @@ class NoticeListItem extends Widget if (!$this->notice->isLocal()) { $class .= ' external'; } + + try { + if($this->repeat) { + $this->out->element('a', + array('href' => $this->repeat->getUrl(), + 'class' => 'u-url'), + ''); + $class = str_replace('u-url', 'u-repost-of', $class); + } + } catch (InvalidUrlException $e) { + // no permalink available + } + try { $this->out->element('a', array('href' => $this->notice->getUrl(), From 96fb18da3fd2966bee188ef18a97fd3baf63fe58 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Thu, 22 Oct 2015 17:17:14 +0000 Subject: [PATCH 03/17] Initial helpers for verification and microformats This will form the backend of all incoming linkbacks. We verify the linkback is real and then try to form a rich notice out of it. --- plugins/Linkback/lib/util.php | 363 ++++++++++++++++++++++++++++++++++ 1 file changed, 363 insertions(+) create mode 100644 plugins/Linkback/lib/util.php diff --git a/plugins/Linkback/lib/util.php b/plugins/Linkback/lib/util.php new file mode 100644 index 0000000000..e4893bc644 --- /dev/null +++ b/plugins/Linkback/lib/util.php @@ -0,0 +1,363 @@ +get($source); + } catch(Exception $ex) { + return NULL; + } + + $body = htmlspecialchars_decode($response->getBody()); + // We're slightly more lenient in our link detection than the spec requires + if(!linkback_lenient_target_match($body, $target)) { + return NULL; + } + + return $response; +} + +function linkback_get_target($target) { + // TODO: linkback to a user should work for attention + // TODO: ignore remote notices and users + // Resolve target (https://github.com/converspace/webmention/issues/43) + $request = HTTPClient::start(); + + try { + $response = $request->head($target); + } catch(Exception $ex) { + return NULL; + } + + try { + return Notice::fromUri($response->getEffectiveUrl()); + } catch(UnknownUriException $ex) { + preg_match('/\/notice\/(\d+)(?:#.*)?$/', $response->getEffectiveUrl(), $match); + return Notice::getKV('id', $match[1]); + } + + return NULL; +} + +// Based on https://github.com/acegiak/Semantic-Linkbacks/blob/master/semantic-linkbacks-microformats-handler.php, GPL-2.0+ +function linkback_find_entry($mf2, $target) { + if(isset($mf2['items'][0]['type']) && in_array("h-feed", $mf2['items'][0]["type"]) && isset($mf2['items'][0]['children'])) { + $mf2['items'] = $mf2['items'][0]['children']; + } + + $entries = array_filter($mf2['items'], function($x) { return isset($x['type']) && in_array('h-entry', $x['type']); }); + + foreach ($entries as $entry) { + foreach ((array)$entry['properties'] as $key => $values) { + if(count(array_filter($values, function($x) use ($target) { return linkback_lenient_target_match($x, $target); })) > 0) { + return $entry['properties']; + } + + // check included h-* formats and their links + foreach ($values as $obj) { + if(isset($obj['type']) && array_intersect(array('h-cite', 'h-entry'), $obj['type']) && + isset($obj['properties']) && isset($obj['properties']['url']) && + count(array_filter($obj['properties']['url'], + function($x) use ($target) { return linkback_lenient_target_match($x, $target); })) > 0 + ) { + return $entry['properties']; + } + } + + // check content for the link + if ($key == "content" && preg_match_all("/]+?".preg_quote($target, "/")."[^>]*>([^>]+?)<\/a>/i", htmlspecialchars_decode($values[0]['html']), $context)) { + return $entry['properties']; + // check summary for the link + } elseif ($key == "summary" && preg_match_all("/]+?".preg_quote($target, "/")."[^>]*>([^>]+?)<\/a>/i", htmlspecialchars_decode($values[0]), $context)) { + return $entry['properties']; + } + } + } + + // Default to first one + if(count($entries) > 0) { + return $entries[0]['properties']; + } + + return NULL; +} + +function linkback_entry_type($entry, $mf2, $target) { + if(!$entry) { return 'mention'; } + + if($mf2['rels'] && $mf2['rels']['in-reply-to']) { + foreach($mf2['rels']['in-reply-to'] as $url) { + if(linkback_lenient_target_match($url, $target)) { + return 'reply'; + } + } + } + + $classes = array( + 'in-reply-to' => 'reply', + 'repost-of' => 'repost', + 'like-of' => 'like', + 'tag-of' => 'tag' + ); + + foreach((array)$entry as $key => $values) { + if(count(array_filter($values, function($x) use ($target) { return linkback_lenient_target_match($x, $target); })) > 0) { + if($classes[$key]) { return $classes[$key]; } + } + + foreach ($values as $obj) { + if(isset($obj['type']) && array_intersect(array('h-cite', 'h-entry'), $obj['type']) && + isset($obj['properties']) && isset($obj['properties']['url']) && + count(array_filter($obj['properties']['url'], + function($x) use ($target) { return linkback_lenient_target_match($x, $target); })) > 0 + ) { + if($classes[$key]) { return $classes[$key]; } + } + } + } + + return 'mention'; +} + +function linkback_is_dupe($key, $url) { + $dupe = Notice::getKV('uri', $url); + if ($dupe instanceof Notice) { + common_log(LOG_INFO, "Linkback: ignoring duplicate post: $url"); + return $dupe; + } + + return false; +} + + +function linkback_hcard($mf2, $url) { + if(empty($mf2['items'])) { + return null; + } + + $hcards = array(); + foreach($mf2['items'] as $item) { + if(!in_array('h-card', $item['type'])) { + continue; + } + + // We found a match, return it immediately + if(isset($item['properties']['url']) && in_array($url, $item['properties']['url'])) { + return $item['properties']; + + // Let's keep all the hcards for later, to return one of them at least + $hcards[] = $item['properties']; + } + } + + // No match immediately for the url we expected, but there were h-cards found + if (count($hcards) > 0) { + return $hcards[0]; + } + + return null; +} + +function linkback_notice($source, $notice, $entry, $author, $mf2) { + $content = $entry['content'] ? $entry['content'][0]['html'] : + ($entry['summary'] ? $entry['sumary'][0] : $entry['name'][0]); + + $rendered = common_purify($content); + + if($entry['type'] == 'mention') { + $name = $entry['name'] ? $entry['name'][0] : substr(common_strip_html($content), 0, 20).'…'; + $rendered = _m('linked to this from '.htmlspecialchars($name).''); + } + + $content = common_strip_html($rendered); + $shortened = common_shorten_links($content); + if(Notice::contentTooLong($shortened)) { + $content = substr($content, + 0, + Notice::maxContent() - (mb_strlen($source) + 2)); + $rendered = $content . '…'; + $content .= ' ' . $source; + } + + $options = array('is_local' => Notice::REMOTE, + 'url' => $entry['url'][0], + 'uri' => $source, + 'rendered' => $rendered, + 'replies' => array(), + 'groups' => array(), + 'peopletags' => array(), + 'tags' => array(), + 'urls' => array()); + + // TODO: when mentioning a user and not a post, neither of these but set replies above + if($entry['type'] == 'repost') { + $options['repeat_of'] = $notice->id; + } else { + $options['reply_to'] = $notice->id; + } + + if($entry['published'] || $entry['updated']) { + $options['created'] = $entry['published'] ? common_sql_date($entry['published'][0]) : common_sql_date($entry['updated'][0]); + } + + if($entry['photo']) { + $options['urls'][] = $entry['photo'][0]; + } + + foreach((array)$entry['category'] as $tag) { + $tag = common_canonical_tag($tag); + if($tag) { $options['tags'][] = $tag; } + } + + + if($mf2['rels'] && $mf2['rels']['enclosure']) { + foreach($mf2['rels']['enclosure'] as $url) { + $options['urls'][] = $url; + } + } + + if($mf2['rels'] && $mf2['rels']['tag']) { + foreach($mf2['rels']['tag'] as $url) { + preg_match('/\/([^\/]+)\/*$/', $url, $match); + $tag = common_canonical_tag($match[1]); + if($tag) { $options['tags'][] = $tag; } + } + } + + if($entry['type'] != 'reply' && $entry['type'] != 'repost') { + $options['urls'] = array(); + } + + return array($content, $options); +} + +function linkback_profile($entry, $mf2, $response, $target) { + if(isset($entry['properties']['author']) && isset($entry['properties']['author'][0]['properties'])) { + $author = $entry['properties']['author'][0]['properties']; + } else { + $author = linkback_hcard($mf2, $response->getEffectiveUrl()); + } + + if(!$author) { + $author = array('name' => array($entry['name'])); + } + + if(!$author['url']) { + $author['url'] = array($response->getEffectiveUrl()); + } + + $user = User::getKV('uri', $author['url'][0]); + if ($user instanceof User) { + common_log(LOG_INFO, "Linkback: ignoring linkback from local user: $url"); + return true; + } + + $profile = Profile::fromUri($author['url'][0]); + if(!($profile instanceof Profile)) { + $profile = Profile::getKV('profileurl', $author['url'][0]); + } + + if(!($profile instanceof Profile)) { + $profile = new Profile(); + $profile->profileurl = $author['url'][0]; + $profile->fullname = $author['name'][0]; + $profile->nickname = $author['nickname'] ? $author['nickname'][0] : str_replace(' ', '', $author['name'][0]); + $profile->created = common_sql_now(); + $profile->insert(); + } + + return array($profile, $author); +} + +function linkback_save($source, $target, $response, $notice) { + if($dupe = linkback_is_dupe('uri', $response->getEffectiveUrl())) { return $dupe->getLocalUrl(); } + if($dupe = linkback_is_dupe('url', $response->getEffectiveUrl())) { return $dupe->getLocalUrl(); } + if($dupe = linkback_is_dupe('uri', $source)) { return $dupe->getLocalUrl(); } + if($dupe = linkback_is_dupe('url', $source)) { return $dupe->getLocalUrl(); } + + $mf2 = new Mf2\Parser($response->getBody(), $response->getEffectiveUrl()); + $mf2 = $mf2->parse(); + + $entry = linkback_find_entry($mf2, $target); + if(!$entry) { + preg_match('/([^<]+)', $response->getBody(), $match); + $entry = array( + 'content' => array('html' => $response->getBody()), + 'name' => $match[1] ? htmlspecialchars_decode($match[1]) : $source + ); + } + + if(!$entry['url']) { + $entry['url'] = array($response->getEffectiveUrl()); + } + + if($dupe = linkback_is_dupe('uri', $entry['url'][0])) { return $dupe->getLocalUrl(); } + if($dupe = linkback_is_dupe('url', $entry['url'][0])) { return $dupe->getLocalUrl(); } + + $entry['type'] = linkback_entry_type($entry, $mf2, $target); + list($profile, $author) = linkback_profile($entry, $mf2, $response, $target); + list($content, $options) = linkback_notice($source, $notice, $entry, $author, $mf2); + + if($entry['type'] == 'like' || ($entry['type'] == 'reply' && $entry['rsvp'])) { + $act = new Activity(); + $act->type = ActivityObject::ACTIVITY; + $act->time = $options['created'] ? strtotime($options['created']) : time(); + $act->title = $entry["name"] ? $entry["name"][0] : _m("Favor"); + $act->actor = $profile->asActivityObject(); + $act->target = $notice->asActivityObject(); + $act->objects = array(clone($act->target)); + + // TRANS: Message that is the "content" of a favorite (%1$s is the actor's nickname, %2$ is the favorited + // notice's nickname and %3$s is the content of the favorited notice.) + $act->content = sprintf(_('%1$s favorited something by %2$s: %3$s'), + $profile->getNickname(), $notice->getProfile()->getNickname(), + $notice->rendered ?: $notice->content); + if($entry['rsvp']) { + $act->content = $options['rendered']; + } + + $act->verb = ActivityVerb::FAVORITE; + if(strtolower($entry['rsvp'][0]) == 'yes') { + $act->verb = 'http://activitystrea.ms/schema/1.0/rsvp-yes'; + } else if(strtolower($entry['rsvp'][0]) == 'no') { + $act->verb = 'http://activitystrea.ms/schema/1.0/rsvp-no'; + } else if(strtolower($entry['rsvp'][0]) == 'maybe') { + $act->verb = 'http://activitystrea.ms/schema/1.0/rsvp-maybe'; + } + + $act->id = $source; + $act->link = $entry['url'][0]; + + $options['source'] = 'linkback'; + $options['mentions'] = $options['replies']; + unset($options['reply_to']); + unset($options['repeat_of']); + + try { + $saved = Notice::saveActivity($act, $profile, $options); + } catch (Exception $e) { + common_log(LOG_ERR, "Linkback save of remote message $source failed: " . $e->getMessage()); + return false; + } + common_log(LOG_INFO, "Linkback saved remote message $source as notice id $saved->id"); + } else { + // Fallback is to make a notice manually + try { + $saved = Notice::saveNew($profile->id, + $content, + 'linkback', + $options); + } catch (Exception $e) { + common_log(LOG_ERR, "Linkback save of remote message $source failed: " . $e->getMessage()); + return false; + } + common_log(LOG_INFO, "Linkback saved remote message $source as notice id $saved->id"); + } + + return $saved->getLocalUrl(); +} From dbfb69885900b25c3385f484e17c1338a94d491c Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Thu, 22 Oct 2015 17:18:20 +0000 Subject: [PATCH 04/17] Webmention frontend --- plugins/Linkback/LinkbackPlugin.php | 7 +++ plugins/Linkback/actions/webmention.php | 73 +++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 plugins/Linkback/actions/webmention.php diff --git a/plugins/Linkback/LinkbackPlugin.php b/plugins/Linkback/LinkbackPlugin.php index e3519dac9e..2aeb8ea016 100644 --- a/plugins/Linkback/LinkbackPlugin.php +++ b/plugins/Linkback/LinkbackPlugin.php @@ -32,6 +32,7 @@ if (!defined('STATUSNET')) { } require_once('Auth/Yadis/Yadis.php'); +require_once(__DIR__ . '/lib/util.php'); define('LINKBACKPLUGIN_VERSION', '0.1'); @@ -306,6 +307,12 @@ class LinkbackPlugin extends Plugin } } + + public function onRouterInitialized(URLMapper $m) + { + $m->connect('main/linkback/webmention', array('action' => 'webmention')); + } + public function version() { return LINKBACKPLUGIN_VERSION; diff --git a/plugins/Linkback/actions/webmention.php b/plugins/Linkback/actions/webmention.php new file mode 100644 index 0000000000..30bc42cea9 --- /dev/null +++ b/plugins/Linkback/actions/webmention.php @@ -0,0 +1,73 @@ +<?php +/* + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +class WebmentionAction extends Action +{ + protected function handle() + { + GNUsocial::setApi(true); // Minimize error messages to aid in debugging + parent::handle(); + if ($this->isPost()) { + return $this->handlePost(); + } + + return false; + } + + function handlePost() + { + $source = $this->arg('source'); + $target = $this->arg('target'); + + header('Content-Type: text/plain; charset=utf-8'); + + if(!$source) { + echo _m('"source" is missing')."\n"; + throw new ServerException(_m('"source" is missing'), 400); + } + + if(!$target) { + echo _m('"target" is missing')."\n"; + throw new ServerException(_m('"target" is missing'), 400); + } + + $response = linkback_get_source($source, $target); + if(!$response) { + echo _m('Source does not link to target.')."\n"; + throw new ServerException(_m('Source does not link to target.'), 400); + } + + $notice = linkback_get_target($target); + if(!$notice) { + echo _m('Target not found')."\n"; + throw new ServerException(_m('Target not found'), 404); + } + + $url = linkback_save($source, $target, $response, $notice); + if(!$url) { + echo _m('An error occured while saving.')."\n"; + throw new ServerException(_m('An error occured while saving.'), 500); + } + + echo $url."\n"; + + return true; + } +} From 1e3cf08d82eedfbdd0ca2bc10ced971fd1adc3dc Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Thu, 22 Oct 2015 17:20:03 +0000 Subject: [PATCH 05/17] Ignore non-local notices as targets --- plugins/Linkback/lib/util.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/Linkback/lib/util.php b/plugins/Linkback/lib/util.php index e4893bc644..803c47763d 100644 --- a/plugins/Linkback/lib/util.php +++ b/plugins/Linkback/lib/util.php @@ -24,7 +24,6 @@ function linkback_get_source($source, $target) { function linkback_get_target($target) { // TODO: linkback to a user should work for attention - // TODO: ignore remote notices and users // Resolve target (https://github.com/converspace/webmention/issues/43) $request = HTTPClient::start(); @@ -35,10 +34,14 @@ function linkback_get_target($target) { } try { - return Notice::fromUri($response->getEffectiveUrl()); + $notice = Notice::fromUri($response->getEffectiveUrl()); } catch(UnknownUriException $ex) { preg_match('/\/notice\/(\d+)(?:#.*)?$/', $response->getEffectiveUrl(), $match); - return Notice::getKV('id', $match[1]); + $notice = Notice::getKV('id', $match[1]); + } + + if($notice instanceof Notice && $notice->isLocal()) { + return $notice; } return NULL; From b9971e8a80c4655eba238b3332fca72e6259e03b Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Thu, 22 Oct 2015 17:39:15 +0000 Subject: [PATCH 06/17] Linkback to user should work --- plugins/Linkback/lib/util.php | 41 +++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/plugins/Linkback/lib/util.php b/plugins/Linkback/lib/util.php index 803c47763d..95444571dc 100644 --- a/plugins/Linkback/lib/util.php +++ b/plugins/Linkback/lib/util.php @@ -23,7 +23,6 @@ function linkback_get_source($source, $target) { } function linkback_get_target($target) { - // TODO: linkback to a user should work for attention // Resolve target (https://github.com/converspace/webmention/issues/43) $request = HTTPClient::start(); @@ -42,6 +41,21 @@ function linkback_get_target($target) { if($notice instanceof Notice && $notice->isLocal()) { return $notice; + } else { + $user = User::getKV('uri', $response->getEffectiveUrl()); + if(!$user) { + preg_match('/\/user\/(\d+)(?:#.*)?$/', $response->getEffectiveUrl(), $match); + $user = User::getKV('id', $match[1]); + } + if(!$user) { + preg_match('/\/([^\/\?#]+)(?:#.*)?$/', $response->getEffectiveUrl(), $match); + if(linkback_lenient_target_match(common_profile_url($match[1]), $response->getEffectiveUrl())) { + $user = User::getKV('nickname', $match[1]); + } + } + if($user instanceof User) { + return $user; + } } return NULL; @@ -166,13 +180,13 @@ function linkback_hcard($mf2, $url) { return null; } -function linkback_notice($source, $notice, $entry, $author, $mf2) { +function linkback_notice($source, $notice_or_user, $entry, $author, $mf2) { $content = $entry['content'] ? $entry['content'][0]['html'] : ($entry['summary'] ? $entry['sumary'][0] : $entry['name'][0]); $rendered = common_purify($content); - if($entry['type'] == 'mention') { + if($notice_or_user instanceof Notice && $entry['type'] == 'mention') { $name = $entry['name'] ? $entry['name'][0] : substr(common_strip_html($content), 0, 20).'…'; $rendered = _m('linked to this from <a href="'.htmlspecialchars($source).'">'.htmlspecialchars($name).'</a>'); } @@ -197,11 +211,14 @@ function linkback_notice($source, $notice, $entry, $author, $mf2) { 'tags' => array(), 'urls' => array()); - // TODO: when mentioning a user and not a post, neither of these but set replies above - if($entry['type'] == 'repost') { - $options['repeat_of'] = $notice->id; + if($notice_or_user instanceof User) { + $options['replies'][] = $notice_or_user->getUri(); } else { - $options['reply_to'] = $notice->id; + if($entry['type'] == 'repost') { + $options['repeat_of'] = $notice_or_user->id; + } else { + $options['reply_to'] = $notice_or_user->id; + } } if($entry['published'] || $entry['updated']) { @@ -277,7 +294,7 @@ function linkback_profile($entry, $mf2, $response, $target) { return array($profile, $author); } -function linkback_save($source, $target, $response, $notice) { +function linkback_save($source, $target, $response, $notice_or_user) { if($dupe = linkback_is_dupe('uri', $response->getEffectiveUrl())) { return $dupe->getLocalUrl(); } if($dupe = linkback_is_dupe('url', $response->getEffectiveUrl())) { return $dupe->getLocalUrl(); } if($dupe = linkback_is_dupe('uri', $source)) { return $dupe->getLocalUrl(); } @@ -304,7 +321,7 @@ function linkback_save($source, $target, $response, $notice) { $entry['type'] = linkback_entry_type($entry, $mf2, $target); list($profile, $author) = linkback_profile($entry, $mf2, $response, $target); - list($content, $options) = linkback_notice($source, $notice, $entry, $author, $mf2); + list($content, $options) = linkback_notice($source, $notice_or_user, $entry, $author, $mf2); if($entry['type'] == 'like' || ($entry['type'] == 'reply' && $entry['rsvp'])) { $act = new Activity(); @@ -312,14 +329,14 @@ function linkback_save($source, $target, $response, $notice) { $act->time = $options['created'] ? strtotime($options['created']) : time(); $act->title = $entry["name"] ? $entry["name"][0] : _m("Favor"); $act->actor = $profile->asActivityObject(); - $act->target = $notice->asActivityObject(); + $act->target = $notice_or_user->asActivityObject(); $act->objects = array(clone($act->target)); // TRANS: Message that is the "content" of a favorite (%1$s is the actor's nickname, %2$ is the favorited // notice's nickname and %3$s is the content of the favorited notice.) $act->content = sprintf(_('%1$s favorited something by %2$s: %3$s'), - $profile->getNickname(), $notice->getProfile()->getNickname(), - $notice->rendered ?: $notice->content); + $profile->getNickname(), $notice_or_user->getProfile()->getNickname(), + $notice_or_user->rendered ?: $notice_or_user->content); if($entry['rsvp']) { $act->content = $options['rendered']; } From 64ac344efab356ffc9debe2ace001ebeb91b7c6f Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Thu, 22 Oct 2015 18:21:46 +0000 Subject: [PATCH 07/17] Update on duplicate linkback This especially allows mentioning mulitple users, etc. --- plugins/Linkback/lib/util.php | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/plugins/Linkback/lib/util.php b/plugins/Linkback/lib/util.php index 95444571dc..36801c12d2 100644 --- a/plugins/Linkback/lib/util.php +++ b/plugins/Linkback/lib/util.php @@ -144,7 +144,6 @@ function linkback_entry_type($entry, $mf2, $target) { function linkback_is_dupe($key, $url) { $dupe = Notice::getKV('uri', $url); if ($dupe instanceof Notice) { - common_log(LOG_INFO, "Linkback: ignoring duplicate post: $url"); return $dupe; } @@ -295,10 +294,10 @@ function linkback_profile($entry, $mf2, $response, $target) { } function linkback_save($source, $target, $response, $notice_or_user) { - if($dupe = linkback_is_dupe('uri', $response->getEffectiveUrl())) { return $dupe->getLocalUrl(); } - if($dupe = linkback_is_dupe('url', $response->getEffectiveUrl())) { return $dupe->getLocalUrl(); } - if($dupe = linkback_is_dupe('uri', $source)) { return $dupe->getLocalUrl(); } - if($dupe = linkback_is_dupe('url', $source)) { return $dupe->getLocalUrl(); } + $dupe = linkback_is_dupe('uri', $response->getEffectiveUrl()); + if(!$dupe) { $dupe = linkback_is_dupe('url', $response->getEffectiveUrl()); } + if(!$dupe) { $dupe = linkback_is_dupe('uri', $source); } + if(!$dupe) { $dupe = linkback_is_dupe('url', $source); } $mf2 = new Mf2\Parser($response->getBody(), $response->getEffectiveUrl()); $mf2 = $mf2->parse(); @@ -316,14 +315,31 @@ function linkback_save($source, $target, $response, $notice_or_user) { $entry['url'] = array($response->getEffectiveUrl()); } - if($dupe = linkback_is_dupe('uri', $entry['url'][0])) { return $dupe->getLocalUrl(); } - if($dupe = linkback_is_dupe('url', $entry['url'][0])) { return $dupe->getLocalUrl(); } + if(!$dupe) { $dupe = linkback_is_dupe('uri', $entry['url'][0]); } + if(!$dupe) { $dupe = linkback_is_dupe('url', $entry['url'][0]); } $entry['type'] = linkback_entry_type($entry, $mf2, $target); list($profile, $author) = linkback_profile($entry, $mf2, $response, $target); list($content, $options) = linkback_notice($source, $notice_or_user, $entry, $author, $mf2); - if($entry['type'] == 'like' || ($entry['type'] == 'reply' && $entry['rsvp'])) { + if($dupe) { + $orig = clone($dupe); + + try { + // Ignore duplicate save error + try { $dupe->saveKnownReplies($options['replies']); } catch (ServerException $ex) {} + try { $dupe->saveKnownTags($options['tags']); } catch (ServerException $ex) {} + try { $dupe->saveKnownUrls($options['urls']); } catch (ServerException $ex) {} + + if($options['reply_to']) { $dupe->reply_to = $options['reply_to']; } + if($options['repost_of']) { $dupe->repost_of = $options['repost_of']; } + if($dupe->update($orig)) { $saved = $dupe; } + } catch (Exception $e) { + common_log(LOG_ERR, "Linkback update of remote message $source failed: " . $e->getMessage()); + return false; + } + common_log(LOG_INFO, "Linkback updated remote message $source as notice id $saved->id"); + } else if($entry['type'] == 'like' || ($entry['type'] == 'reply' && $entry['rsvp'])) { $act = new Activity(); $act->type = ActivityObject::ACTIVITY; $act->time = $options['created'] ? strtotime($options['created']) : time(); From c5bdf6924d6ad302952e91c557a81a4943720ce8 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Thu, 22 Oct 2015 19:13:16 +0000 Subject: [PATCH 08/17] Fix duplicate detection --- plugins/Linkback/lib/util.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/Linkback/lib/util.php b/plugins/Linkback/lib/util.php index 36801c12d2..16454b5873 100644 --- a/plugins/Linkback/lib/util.php +++ b/plugins/Linkback/lib/util.php @@ -142,7 +142,7 @@ function linkback_entry_type($entry, $mf2, $target) { } function linkback_is_dupe($key, $url) { - $dupe = Notice::getKV('uri', $url); + $dupe = Notice::getKV($key, $url); if ($dupe instanceof Notice) { return $dupe; } From 96e0819f00ac0c2f9ea3fa35d58e50b4c3a9b49b Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Thu, 22 Oct 2015 19:13:56 +0000 Subject: [PATCH 09/17] Catch exception that this generates when there is no result --- plugins/Linkback/lib/util.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/Linkback/lib/util.php b/plugins/Linkback/lib/util.php index 16454b5873..cf3f55c5ff 100644 --- a/plugins/Linkback/lib/util.php +++ b/plugins/Linkback/lib/util.php @@ -276,7 +276,10 @@ function linkback_profile($entry, $mf2, $response, $target) { return true; } - $profile = Profile::fromUri($author['url'][0]); + try { + $profile = Profile::fromUri($author['url'][0]); + } catch(UnknownUriException $ex) {} + if(!($profile instanceof Profile)) { $profile = Profile::getKV('profileurl', $author['url'][0]); } From aa4d880148ea9bd62a973e24c3c2632e0b82f42c Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Thu, 22 Oct 2015 19:29:04 +0000 Subject: [PATCH 10/17] Search in sub-items for links to the target as well --- plugins/Linkback/lib/util.php | 60 +++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/plugins/Linkback/lib/util.php b/plugins/Linkback/lib/util.php index cf3f55c5ff..694dd84b0d 100644 --- a/plugins/Linkback/lib/util.php +++ b/plugins/Linkback/lib/util.php @@ -61,6 +61,41 @@ function linkback_get_target($target) { return NULL; } +function linkback_is_contained_in($entry, $target) { + foreach ((array)$entry['properties'] as $key => $values) { + if(count(array_filter($values, function($x) use ($target) { return linkback_lenient_target_match($x, $target); })) > 0) { + return $entry['properties']; + } + + // check included h-* formats and their links + foreach ($values as $obj) { + if(isset($obj['type']) && array_intersect(array('h-cite', 'h-entry'), $obj['type']) && + isset($obj['properties']) && isset($obj['properties']['url']) && + count(array_filter($obj['properties']['url'], + function($x) use ($target) { return linkback_lenient_target_match($x, $target); })) > 0 + ) { + return $entry['properties']; + } + } + + // check content for the link + if ($key == "content" && preg_match_all("/<a[^>]+?".preg_quote($target, "/")."[^>]*>([^>]+?)<\/a>/i", htmlspecialchars_decode($values[0]['html']), $context)) { + return $entry['properties']; + // check summary for the link + } elseif ($key == "summary" && preg_match_all("/<a[^>]+?".preg_quote($target, "/")."[^>]*>([^>]+?)<\/a>/i", htmlspecialchars_decode($values[0]), $context)) { + return $entry['properties']; + } + } + + foreach((array)$entry['children'] as $mf2) { + if(linkback_is_contained_in($mf2, $target)) { + return $entry['properties']; + } + } + + return null; +} + // Based on https://github.com/acegiak/Semantic-Linkbacks/blob/master/semantic-linkbacks-microformats-handler.php, GPL-2.0+ function linkback_find_entry($mf2, $target) { if(isset($mf2['items'][0]['type']) && in_array("h-feed", $mf2['items'][0]["type"]) && isset($mf2['items'][0]['children'])) { @@ -70,29 +105,8 @@ function linkback_find_entry($mf2, $target) { $entries = array_filter($mf2['items'], function($x) { return isset($x['type']) && in_array('h-entry', $x['type']); }); foreach ($entries as $entry) { - foreach ((array)$entry['properties'] as $key => $values) { - if(count(array_filter($values, function($x) use ($target) { return linkback_lenient_target_match($x, $target); })) > 0) { - return $entry['properties']; - } - - // check included h-* formats and their links - foreach ($values as $obj) { - if(isset($obj['type']) && array_intersect(array('h-cite', 'h-entry'), $obj['type']) && - isset($obj['properties']) && isset($obj['properties']['url']) && - count(array_filter($obj['properties']['url'], - function($x) use ($target) { return linkback_lenient_target_match($x, $target); })) > 0 - ) { - return $entry['properties']; - } - } - - // check content for the link - if ($key == "content" && preg_match_all("/<a[^>]+?".preg_quote($target, "/")."[^>]*>([^>]+?)<\/a>/i", htmlspecialchars_decode($values[0]['html']), $context)) { - return $entry['properties']; - // check summary for the link - } elseif ($key == "summary" && preg_match_all("/<a[^>]+?".preg_quote($target, "/")."[^>]*>([^>]+?)<\/a>/i", htmlspecialchars_decode($values[0]), $context)) { - return $entry['properties']; - } + if($prop = linkback_is_contained_in($entry, $target)) { + return $prop; } } From fa1e4620cf970db14572d113f07d04285e214fb5 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Thu, 22 Oct 2015 20:15:44 +0000 Subject: [PATCH 11/17] Add pingback frontend --- plugins/Linkback/LinkbackPlugin.php | 6 ++ plugins/Linkback/actions/pingback.php | 87 +++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 plugins/Linkback/actions/pingback.php diff --git a/plugins/Linkback/LinkbackPlugin.php b/plugins/Linkback/LinkbackPlugin.php index 2aeb8ea016..ac95b4ab25 100644 --- a/plugins/Linkback/LinkbackPlugin.php +++ b/plugins/Linkback/LinkbackPlugin.php @@ -311,6 +311,12 @@ class LinkbackPlugin extends Plugin public function onRouterInitialized(URLMapper $m) { $m->connect('main/linkback/webmention', array('action' => 'webmention')); + $m->connect('main/linkback/pingback', array('action' => 'pingback')); + } + + public function onStartShowHTML($action) + { + header('X-Pingback: ' . common_local_url('pingback')); } public function version() diff --git a/plugins/Linkback/actions/pingback.php b/plugins/Linkback/actions/pingback.php new file mode 100644 index 0000000000..8d2c87fb02 --- /dev/null +++ b/plugins/Linkback/actions/pingback.php @@ -0,0 +1,87 @@ +<?php +/* + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +if (!defined('STATUSNET')) { + exit(1); +} + +class PingbackAction extends Action +{ + protected function handle() + { + GNUsocial::setApi(true); // Minimize error messages to aid in debugging + parent::handle(); + if ($this->isPost()) { + return $this->handlePost(); + } + + return false; + } + + function handlePost() + { + + $server = xmlrpc_server_create(); + xmlrpc_server_register_method($server, 'pingback.ping', array($this, 'ping')); + echo xmlrpc_server_call_method($server, file_get_contents('php://input'), null, array('encoding' => 'utf-8')); + xmlrpc_server_destroy($server); + return true; + } + + function ping($method, $parameters) { + list($source, $target) = $parameters; + + if(!$source) { + return array( + 'faultCode' => 0x0010, + 'faultString' => '"source" is missing' + ); + } + + if(!$target) { + return array( + 'faultCode' => 0x0020, + 'faultString' => '"target" is missing' + ); + } + + $response = linkback_get_source($source, $target); + if(!$response) { + return array( + 'faultCode' => 0x0011, + 'faultString' => 'Source does not link to target' + ); + } + + $notice = linkback_get_target($target); + if(!$notice) { + return array( + 'faultCode' => 0x0021, + 'faultString' => 'Target not found' + ); + } + + $url = linkback_save($source, $target, $response, $notice); + if(!$url) { + return array( + 'faultCode' => 0, + 'faultString' => 'An error occured while saving.' + ); + } + + return array('Success'); + } +} From 6aba21fcd84f7470c4a4c74e14bb014e8e3bb4ec Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Fri, 23 Oct 2015 18:41:00 +0000 Subject: [PATCH 12/17] Use author-declared URL as URI Will make brid.gy work better, for example. --- plugins/Linkback/lib/util.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/Linkback/lib/util.php b/plugins/Linkback/lib/util.php index 694dd84b0d..2f2c52bae1 100644 --- a/plugins/Linkback/lib/util.php +++ b/plugins/Linkback/lib/util.php @@ -216,7 +216,7 @@ function linkback_notice($source, $notice_or_user, $entry, $author, $mf2) { $options = array('is_local' => Notice::REMOTE, 'url' => $entry['url'][0], - 'uri' => $source, + 'uri' => $entry['url'][0], 'rendered' => $rendered, 'replies' => array(), 'groups' => array(), From b95b3180520aad4e5c97b7142ab8db47aa8dd1eb Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Sun, 25 Oct 2015 16:50:59 +0000 Subject: [PATCH 13/17] That never was meant to be in that if --- plugins/Linkback/lib/util.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/Linkback/lib/util.php b/plugins/Linkback/lib/util.php index 2f2c52bae1..8f8388a3d2 100644 --- a/plugins/Linkback/lib/util.php +++ b/plugins/Linkback/lib/util.php @@ -179,10 +179,10 @@ function linkback_hcard($mf2, $url) { // We found a match, return it immediately if(isset($item['properties']['url']) && in_array($url, $item['properties']['url'])) { return $item['properties']; - - // Let's keep all the hcards for later, to return one of them at least - $hcards[] = $item['properties']; } + + // Let's keep all the hcards for later, to return one of them at least + $hcards[] = $item['properties']; } // No match immediately for the url we expected, but there were h-cards found From f09a82268f146f0c22a3a38ad0a88e999234fa5d Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Sun, 25 Oct 2015 17:30:51 +0000 Subject: [PATCH 14/17] Ignore self pings --- plugins/Linkback/lib/util.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/Linkback/lib/util.php b/plugins/Linkback/lib/util.php index 8f8388a3d2..94f0d992f4 100644 --- a/plugins/Linkback/lib/util.php +++ b/plugins/Linkback/lib/util.php @@ -5,6 +5,13 @@ function linkback_lenient_target_match($body, $target) { } function linkback_get_source($source, $target) { + // Check if we are pinging ourselves and ignore + $localprefix = common_config('site', 'server') . '/' . common_config('site', 'path'); + if(linkback_lenient_target_match($source, $localprefix)) { + common_debug('Ignoring self ping from ' . $source . ' to ' . $target); + return NULL; + } + $request = HTTPClient::start(); try { From 47db1ab063fbac28a00edbfe4c526676409bb46d Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Sun, 25 Oct 2015 18:42:14 +0000 Subject: [PATCH 15/17] Autodiscovery header for webmention --- plugins/Linkback/LinkbackPlugin.php | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/Linkback/LinkbackPlugin.php b/plugins/Linkback/LinkbackPlugin.php index ac95b4ab25..642909112d 100644 --- a/plugins/Linkback/LinkbackPlugin.php +++ b/plugins/Linkback/LinkbackPlugin.php @@ -316,6 +316,7 @@ class LinkbackPlugin extends Plugin public function onStartShowHTML($action) { + header('Link: <' . common_local_url('webmention') . '>; rel="webmention"', false); header('X-Pingback: ' . common_local_url('pingback')); } From 11810bbf54b74a049173f99c9c1137052f7b2527 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Tue, 27 Oct 2015 03:15:38 +0000 Subject: [PATCH 16/17] Update conversation when we update reply_to Or repeat_of --- plugins/Linkback/lib/util.php | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/plugins/Linkback/lib/util.php b/plugins/Linkback/lib/util.php index 94f0d992f4..ee498a4771 100644 --- a/plugins/Linkback/lib/util.php +++ b/plugins/Linkback/lib/util.php @@ -356,8 +356,20 @@ function linkback_save($source, $target, $response, $notice_or_user) { try { $dupe->saveKnownUrls($options['urls']); } catch (ServerException $ex) {} if($options['reply_to']) { $dupe->reply_to = $options['reply_to']; } - if($options['repost_of']) { $dupe->repost_of = $options['repost_of']; } + if($options['repeat_of']) { $dupe->repeat_of = $options['repeat_of']; } + if($dupe->reply_to != $orig->reply_to || $dupe->repeat_of != $orig->repeat_of) { + $parent = Notice::getKV('id', $dupe->repost_of ? $dupe->repost_of : $dupe->reply_to); + if($parent instanceof Notice) { + // If we changed the reply_to or repeat_of we might live in a new conversation now + $dupe->conversation = $parent->conversation; + } + } if($dupe->update($orig)) { $saved = $dupe; } + if($dupe->conversation != $orig->conversation && Conversation::noticeCount($orig->conversation) < 1) { + // Delete empty conversation + $emptyConversation = Conversation::getKV('id', $orig->conversation); + $emptyConversation->delete(); + } } catch (Exception $e) { common_log(LOG_ERR, "Linkback update of remote message $source failed: " . $e->getMessage()); return false; From 4f0fae9e90381ff1c107fd746a9b28275f5dd4fc Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber <singpolyma@singpolyma.net> Date: Tue, 27 Oct 2015 17:13:11 +0000 Subject: [PATCH 17/17] Show better source for linback items --- plugins/Linkback/LinkbackPlugin.php | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/plugins/Linkback/LinkbackPlugin.php b/plugins/Linkback/LinkbackPlugin.php index 642909112d..4e6214c714 100644 --- a/plugins/Linkback/LinkbackPlugin.php +++ b/plugins/Linkback/LinkbackPlugin.php @@ -358,4 +358,34 @@ class LinkbackPlugin extends Plugin $action_name === 'linkbacksettings'); return true; } + + function onStartNoticeSourceLink($notice, &$name, &$url, &$title) + { + // If we don't handle this, keep the event handler going + if (!in_array($notice->source, array('linkback'))) { + return true; + } + + try { + $url = $notice->getUrl(); + // If getUrl() throws exception, $url is never set + + $bits = parse_url($url); + $domain = $bits['host']; + if (substr($domain, 0, 4) == 'www.') { + $name = substr($domain, 4); + } else { + $name = $domain; + } + + // TRANS: Title. %s is a domain name. + $title = sprintf(_m('Sent from %s via Linkback'), $domain); + + // Abort event handler, we have a name and URL! + return false; + } catch (InvalidUrlException $e) { + // This just means we don't have the notice source data + return true; + } + } }