From 6dcb293ba0e4fd4f38e2e3f4eb5b75df5f8d771f Mon Sep 17 00:00:00 2001 From: Mikael Nordfeldth Date: Thu, 23 Jun 2016 23:03:58 +0200 Subject: [PATCH 01/10] Unnecessarily verbose code --- plugins/LRDD/lib/discovery.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/LRDD/lib/discovery.php b/plugins/LRDD/lib/discovery.php index 4049113408..6818e12557 100644 --- a/plugins/LRDD/lib/discovery.php +++ b/plugins/LRDD/lib/discovery.php @@ -99,7 +99,7 @@ class Discovery common_debug("LRDD discovery method for '$uri': {$class}"); $lrdd = new $class; - $links = call_user_func(array($lrdd, 'discover'), $uri); + $links = $lrdd->discover($uri); $link = Discovery::getService($links, Discovery::LRDD_REL); // Load the LRDD XRD From a4051945fd71a812132febe0e6041775b75cd2ea Mon Sep 17 00:00:00 2001 From: Mikael Nordfeldth Date: Thu, 23 Jun 2016 23:27:18 +0200 Subject: [PATCH 02/10] Handle exception from Magic Envelope toXML function --- plugins/Diaspora/DiasporaPlugin.php | 7 ++++++- plugins/OStatus/OStatusPlugin.php | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/Diaspora/DiasporaPlugin.php b/plugins/Diaspora/DiasporaPlugin.php index 451d6b904e..c054d3b767 100644 --- a/plugins/Diaspora/DiasporaPlugin.php +++ b/plugins/Diaspora/DiasporaPlugin.php @@ -225,7 +225,12 @@ class DiasporaPlugin extends Plugin public function onSalmonSlap($endpoint_uri, MagicEnvelope $magic_env, Profile $target=null) { - $envxml = $magic_env->toXML($target, 'diaspora'); + try { + $envxml = $magic_env->toXML($target, 'diaspora'); + } catch (Exception $e) { + common_log(LOG_ERR, sprintf('Could not generate Magic Envelope XML (diaspora flavour) for profile id=='.$target->getID().': '.$e->getMessage())); + return false; + } // Diaspora wants another POST format (base64url-encoded POST variable 'xml') $headers = array('Content-Type: application/x-www-form-urlencoded'); diff --git a/plugins/OStatus/OStatusPlugin.php b/plugins/OStatus/OStatusPlugin.php index 9751772fdc..d455f1f478 100644 --- a/plugins/OStatus/OStatusPlugin.php +++ b/plugins/OStatus/OStatusPlugin.php @@ -1407,7 +1407,12 @@ class OStatusPlugin extends Plugin public function onSalmonSlap($endpoint_uri, MagicEnvelope $magic_env, Profile $target=null) { - $envxml = $magic_env->toXML($target); + try { + $envxml = $magic_env->toXML($target); + } catch (Exception $e) { + common_log(LOG_ERR, sprintf('Could not generate Magic Envelope XML for profile id=='.$target->getID().': '.$e->getMessage())); + return false; + } $headers = array('Content-Type: application/magic-envelope+xml'); From 39e8c13afbc460f28a309a71dcb5f548f477c775 Mon Sep 17 00:00:00 2001 From: Mikael Nordfeldth Date: Fri, 24 Jun 2016 13:51:40 +0200 Subject: [PATCH 03/10] Properly parse incoming bookmarks --- lib/activityobject.php | 20 +++++++++++++++++--- plugins/Bookmark/classes/Bookmark.php | 10 +++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/lib/activityobject.php b/lib/activityobject.php index ca6390b725..d211136d3c 100644 --- a/lib/activityobject.php +++ b/lib/activityobject.php @@ -28,9 +28,7 @@ * @link http://status.net/ */ -if (!defined('STATUSNET')) { - exit(1); -} +if (!defined('GNUSOCIAL')) { exit(1); } require_once(INSTALLDIR.'/lib/activitystreamjsondocument.php'); @@ -269,6 +267,22 @@ class ActivityObject if (empty($this->id) && !empty($this->link)) { // fallback if there's no ID $this->id = $this->link; } + + $els = $element->childNodes; + $out = array(); + + for ($i = 0; $i < $els->length; $i++) { + $link = $els->item($i); + if ($link->localName == ActivityUtils::LINK && $link->namespaceURI == ActivityUtils::ATOM) { + $attrs = array(); + foreach ($link->attributes as $attrName=>$attrNode) { + $attrs[$attrName] = $attrNode->nodeValue; + } + $this->extra[] = [$link->localName, + $attrs, + $link->nodeValue]; + } + } } // @todo FIXME: rationalize with Activity::_fromRssItem() diff --git a/plugins/Bookmark/classes/Bookmark.php b/plugins/Bookmark/classes/Bookmark.php index b593bc1909..02b4d3d2e3 100644 --- a/plugins/Bookmark/classes/Bookmark.php +++ b/plugins/Bookmark/classes/Bookmark.php @@ -162,19 +162,19 @@ class Bookmark extends Managed_DataObject $url = null; // each extra element is array('tagname', array('attr'=>'val', ...), 'content') foreach ($actobj->extra as $extra) { - if ($extra[1]['rel'] !== 'related') { + if ($extra[0] !== ActivityUtils::LINK || $extra[1][ActivityUtils::REL] !== 'related') { continue; } - if ($url===null && strlen($extra[1]['href'])>0) { - $url = $extra[1]['href']; + if ($url===null && strlen($extra[1][ActivityUtils::HREF])>0) { + $url = $extra[1][ActivityUtils::HREF]; } elseif ($url !== null) { // TRANS: Client exception thrown when a bookmark is formatted incorrectly. - throw new ClientException(sprintf(_m('Expected exactly 1 link rel=related in a Bookmark, got %1$d.'), count($relLinkEls))); + throw new ClientException(sprintf(_m('Expected exactly 1 link rel=related in a Bookmark, got more than that.'))); } } if (is_null($url)) { // TRANS: Client exception thrown when a bookmark is formatted incorrectly. - throw new ClientException(sprintf(_m('Expected exactly 1 link rel=related in a Bookmark, got %1$d.'), count($relLinkEls))); + throw new ClientException(sprintf(_m('Expected exactly 1 link rel=related in a Bookmark, got 0.'))); } if (!strlen($actobj->title)) { From da365be5a2011bcb7939130fa60320f7d7198ed0 Mon Sep 17 00:00:00 2001 From: Mikael Nordfeldth Date: Fri, 24 Jun 2016 14:49:52 +0200 Subject: [PATCH 04/10] ParagonIE\ConstantTime required PHP7, going to v1.x branch --- extlib/ParagonIE/ConstantTime/Base32.php | 57 +++++++------- extlib/ParagonIE/ConstantTime/Base32Hex.php | 9 +-- extlib/ParagonIE/ConstantTime/Base64.php | 19 ++--- .../ParagonIE/ConstantTime/Base64DotSlash.php | 5 +- .../ConstantTime/Base64DotSlashOrdered.php | 5 +- .../ParagonIE/ConstantTime/Base64UrlSafe.php | 5 +- extlib/ParagonIE/ConstantTime/Binary.php | 9 +-- .../ConstantTime/EncoderInterface.php | 10 +-- extlib/ParagonIE/ConstantTime/Encoding.php | 45 ++++++----- extlib/ParagonIE/ConstantTime/Hex.php | 23 ++---- extlib/ParagonIE/ConstantTime/RFC4648.php | 31 ++++---- extlib/ParagonIE/README.md | 74 +++++++++++++++++++ 12 files changed, 167 insertions(+), 125 deletions(-) create mode 100644 extlib/ParagonIE/README.md diff --git a/extlib/ParagonIE/ConstantTime/Base32.php b/extlib/ParagonIE/ConstantTime/Base32.php index a65c1c7343..28b8ca200e 100644 --- a/extlib/ParagonIE/ConstantTime/Base32.php +++ b/extlib/ParagonIE/ConstantTime/Base32.php @@ -1,5 +1,4 @@ > 2)) & 0xff) ); $err |= ($c0 | $c1 | $c2) >> 8; - } elseif ($i + 1 < $srcLen) { + } elseif($i + 1 < $srcLen) { $c1 = static::decode6Bits($chunk[2]); $dest .= \pack( 'C', @@ -179,7 +170,7 @@ abstract class Base64 implements EncoderInterface * @param int $src * @return int */ - protected static function decode6Bits(int $src): int + protected static function decode6Bits($src) { $ret = -1; @@ -208,7 +199,7 @@ abstract class Base64 implements EncoderInterface * @param int $src * @return string */ - protected static function encode6Bits(int $src): string + protected static function encode6Bits($src) { $diff = 0x41; diff --git a/extlib/ParagonIE/ConstantTime/Base64DotSlash.php b/extlib/ParagonIE/ConstantTime/Base64DotSlash.php index 7f975139b1..160459782e 100644 --- a/extlib/ParagonIE/ConstantTime/Base64DotSlash.php +++ b/extlib/ParagonIE/ConstantTime/Base64DotSlash.php @@ -1,5 +1,4 @@ Date: Fri, 24 Jun 2016 15:19:24 +0200 Subject: [PATCH 05/10] fixes issue #189 with a script lacking exception handling --- plugins/OStatus/scripts/renew-feeds.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/OStatus/scripts/renew-feeds.php b/plugins/OStatus/scripts/renew-feeds.php index 8f8ac3ee1d..d63f448827 100755 --- a/plugins/OStatus/scripts/renew-feeds.php +++ b/plugins/OStatus/scripts/renew-feeds.php @@ -37,7 +37,11 @@ try { while ($sub->fetch()) { echo "Renewing feed subscription\n\tExp.: {$sub->sub_end}\n\tFeed: {$sub->uri}\n\tHub: {$sub->huburi}\n"; - $sub->renew(); + try { + $sub->renew(); + } catch (Exception $e) { + echo "FAILED: {$e->getMessage()}\n"; + } } echo "Done!"; From 0adb7af9a03a16fd45d900c5e15e8d386c454778 Mon Sep 17 00:00:00 2001 From: Mikael Nordfeldth Date: Fri, 24 Jun 2016 15:43:20 +0200 Subject: [PATCH 06/10] Allow a quickHead request, will only return headers --- lib/httpclient.php | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/httpclient.php b/lib/httpclient.php index 1e399bd83e..04a365274d 100644 --- a/lib/httpclient.php +++ b/lib/httpclient.php @@ -206,6 +206,29 @@ class HTTPClient extends HTTP_Request2 return $data; } + /** + * If you want an Accept header, put it in $headers + */ + public static function quickHead($url, array $params=array(), array $headers=array()) + { + if (!empty($params)) { + $params = http_build_query($params, null, '&'); + if (strpos($url, '?') === false) { + $url .= '?' . $params; + } else { + $url .= '&' . $params; + } + } + + $client = new HTTPClient(); + $response = $client->head($url, $headers); + if (!$response->isOk()) { + // TRANS: Exception. %s is the URL we tried to GET. + throw new Exception(sprintf(_m('Could not GET URL %s.'), $url), $response->getStatus()); + } + return $response->getHeader(); + } + /** * Convenience function to run a GET request. * From f1e3314bb790de9f372b47f3bc5af0f2e85b3710 Mon Sep 17 00:00:00 2001 From: Mikael Nordfeldth Date: Fri, 24 Jun 2016 15:47:02 +0200 Subject: [PATCH 07/10] StoreRemoteMedia avoids too large files --- .../StoreRemoteMediaPlugin.php | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php b/plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php index f38ca4713e..c9964869a4 100644 --- a/plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php +++ b/plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php @@ -77,22 +77,44 @@ class StoreRemoteMediaPlugin extends Plugin return true; } - if (!$this->checkWhiteList($file->getUrl()) || - !$this->checkBlackList($file->getUrl())) { + $remoteUrl = $file->getUrl(); + + if (!$this->checkWhiteList($remoteUrl) || + !$this->checkBlackList($remoteUrl)) { return true; } - // First we download the file to memory and test whether it's actually an image file - common_debug(sprintf('Downloading remote file id==%u with URL: %s', $file->getID(), _ve($file->getUrl()))); try { - $imgData = HTTPClient::quickGet($file->getUrl()); + $http = new HTTPClient(); + common_debug(sprintf('Performing HEAD request for remote file id==%u to avoid unnecessarily downloading too large files. URL: %s', $file->getID(), $remoteUrl)); + $head = $http->head($remoteUrl); + $headers = $head->getHeader(); + if (!isset($headers['content-length'])) { + // file size not specified on remote server + common_debug(sprintf('%s: Ignoring remote media because we did not get a content length for file id==%u', __CLASS__, $file->getID())); + return true; + } elseif (intval($headers['content-length']) > common_config('attachments', 'file_quota')) { + // file too big + common_debug(sprintf('%s: Skipping remote media because content length (%u) is larger than file_quota (%u) for file id==%u', __CLASS__, intval($headers['content-length']), common_config('attachments', 'file_quota'), $file->getID())); + return true; + } + + $remoteUrl = $head->effectiveUrl; // to avoid going through redirects again + if (!$this->checkBlackList($remoteUrl)) { + common_log(LOG_WARN, sprintf('%s: Non-blacklisted URL %s redirected to blacklisted URL %s', __CLASS__, $file->getUrl(), $remoteUrl)); + return true; + } + + // Then we download the file to memory and test whether it's actually an image file + common_debug(sprintf('Downloading remote file id==%u with effective URL: %s', $file->getID(), _ve($remoteUrl))); + $imgData = $http->get($remoteUrl); } catch (HTTP_Request2_ConnectionException $e) { common_log(LOG_ERR, __CLASS__.': quickGet on URL: '._ve($file->getUrl()).' threw exception: '.$e->getMessage()); return true; } $info = @getimagesizefromstring($imgData); if ($info === false) { - throw new UnsupportedMediaException(_('Remote file format was not identified as an image.'), $file->getUrl()); + throw new UnsupportedMediaException(_('Remote file format was not identified as an image.'), $remoteUrl); } elseif (!$info[0] || !$info[1]) { throw new UnsupportedMediaException(_('Image file had impossible geometry (0 width or height)')); } From 1dfac3ad63017071cca551a3e7f557da3f6d24aa Mon Sep 17 00:00:00 2001 From: Mikael Nordfeldth Date: Fri, 24 Jun 2016 15:53:23 +0200 Subject: [PATCH 08/10] Allow getting filesize by function --- classes/File.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/classes/File.php b/classes/File.php index c85d919fd7..1a38aefa72 100644 --- a/classes/File.php +++ b/classes/File.php @@ -263,6 +263,11 @@ class File extends Managed_DataObject return self::tryFilename($this->filename); } + public function getSize() + { + return intval($this->size); + } + // where should the file go? static function filename(Profile $profile, $origname, $mimetype) From af23c9f7cd4530c70fcbcceffba278648ec95ab0 Mon Sep 17 00:00:00 2001 From: Mikael Nordfeldth Date: Fri, 24 Jun 2016 15:56:14 +0200 Subject: [PATCH 09/10] StoreRemoteMedia now checks remote filesize before downloading --- .../StoreRemoteMediaPlugin.php | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php b/plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php index c9964869a4..36e3544efa 100644 --- a/plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php +++ b/plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php @@ -85,28 +85,33 @@ class StoreRemoteMediaPlugin extends Plugin } try { + /* $http = new HTTPClient(); common_debug(sprintf('Performing HEAD request for remote file id==%u to avoid unnecessarily downloading too large files. URL: %s', $file->getID(), $remoteUrl)); $head = $http->head($remoteUrl); - $headers = $head->getHeader(); - if (!isset($headers['content-length'])) { - // file size not specified on remote server - common_debug(sprintf('%s: Ignoring remote media because we did not get a content length for file id==%u', __CLASS__, $file->getID())); - return true; - } elseif (intval($headers['content-length']) > common_config('attachments', 'file_quota')) { - // file too big - common_debug(sprintf('%s: Skipping remote media because content length (%u) is larger than file_quota (%u) for file id==%u', __CLASS__, intval($headers['content-length']), common_config('attachments', 'file_quota'), $file->getID())); - return true; - } - $remoteUrl = $head->effectiveUrl; // to avoid going through redirects again if (!$this->checkBlackList($remoteUrl)) { common_log(LOG_WARN, sprintf('%s: Non-blacklisted URL %s redirected to blacklisted URL %s', __CLASS__, $file->getUrl(), $remoteUrl)); return true; } + $headers = $head->getHeader(); + $filesize = isset($headers['content-length']) ? $headers['content-length'] : null; + */ + $filesize = $file->getSize(); + if (empty($filesize)) { + // file size not specified on remote server + common_debug(sprintf('%s: Ignoring remote media because we did not get a content length for file id==%u', __CLASS__, $file->getID())); + return true; + } elseif ($filesize > common_config('attachments', 'file_quota')) { + // file too big + common_debug(sprintf('%s: Skipping remote media because content length (%u) is larger than file_quota (%u) for file id==%u', __CLASS__, intval($filesize), common_config('attachments', 'file_quota'), $file->getID())); + return true; + } + + $http = new HTTPClient(); // Then we download the file to memory and test whether it's actually an image file - common_debug(sprintf('Downloading remote file id==%u with effective URL: %s', $file->getID(), _ve($remoteUrl))); + common_debug(sprintf('Downloading remote file id==%u (should be size %u) with effective URL: %s', $file->getID(), $filesize, _ve($remoteUrl))); $imgData = $http->get($remoteUrl); } catch (HTTP_Request2_ConnectionException $e) { common_log(LOG_ERR, __CLASS__.': quickGet on URL: '._ve($file->getUrl()).' threw exception: '.$e->getMessage()); From d7a4098b5606a885f8a8ec2f28cd701c3ac7c689 Mon Sep 17 00:00:00 2001 From: Mikael Nordfeldth Date: Fri, 24 Jun 2016 16:07:57 +0200 Subject: [PATCH 10/10] Use a separate max download limit for remote files than file_quota too --- plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php b/plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php index 36e3544efa..64c6355b2b 100644 --- a/plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php +++ b/plugins/StoreRemoteMedia/StoreRemoteMediaPlugin.php @@ -18,6 +18,8 @@ class StoreRemoteMediaPlugin extends Plugin public $domain_blacklist = array(); public $check_blacklist = false; + public $max_image_bytes = 5242880; // 5MiB max image size by default + protected $imgData = array(); // these should be declared protected everywhere @@ -103,16 +105,20 @@ class StoreRemoteMediaPlugin extends Plugin // file size not specified on remote server common_debug(sprintf('%s: Ignoring remote media because we did not get a content length for file id==%u', __CLASS__, $file->getID())); return true; + } elseif ($filesize > $this->max_image_bytes) { + //FIXME: When we perhaps start fetching videos etc. we'll need to differentiate max_image_bytes from that... + // file too big according to plugin configuration + common_debug(sprintf('%s: Skipping remote media because content length (%u) is larger than plugin configured max_image_bytes (%u) for file id==%u', __CLASS__, intval($filesize), $this->max_image_bytes, $file->getID())); + return true; } elseif ($filesize > common_config('attachments', 'file_quota')) { - // file too big + // file too big according to site configuration common_debug(sprintf('%s: Skipping remote media because content length (%u) is larger than file_quota (%u) for file id==%u', __CLASS__, intval($filesize), common_config('attachments', 'file_quota'), $file->getID())); return true; } - $http = new HTTPClient(); // Then we download the file to memory and test whether it's actually an image file common_debug(sprintf('Downloading remote file id==%u (should be size %u) with effective URL: %s', $file->getID(), $filesize, _ve($remoteUrl))); - $imgData = $http->get($remoteUrl); + $imgData = HTTPClient::quickGet($remoteUrl); } catch (HTTP_Request2_ConnectionException $e) { common_log(LOG_ERR, __CLASS__.': quickGet on URL: '._ve($file->getUrl()).' threw exception: '.$e->getMessage()); return true;