From e8f45636333fec389d95482f9a4284f5c17c2bdd Mon Sep 17 00:00:00 2001 From: Hugo Sales Date: Mon, 20 Sep 2021 13:08:17 +0100 Subject: [PATCH] [UTIL][Formatting][COMPONENT][Link][Tag] Refactor code from formatting into Link and Tag, where appropriate. Drop 'perfect url regex' as the one used in v2 is better --- components/Link/Link.php | 221 +++++++++++++++++++++++++++++++++++-- components/Tag/Tag.php | 35 +++++- src/Util/Formatting.php | 229 +-------------------------------------- 3 files changed, 246 insertions(+), 239 deletions(-) diff --git a/components/Link/Link.php b/components/Link/Link.php index 7457296aea..a9e54891e1 100644 --- a/components/Link/Link.php +++ b/components/Link/Link.php @@ -27,17 +27,11 @@ use App\Core\Modules\Component; use App\Entity; use App\Entity\NoteToLink; use App\Util\Common; +use App\Util\HTML; use InvalidArgumentException; class Link extends Component { - /** - * "Perfect URL Regex", courtesy of https://urlregex.com/ - */ - const URL_REGEX = <<getURLRegex(), $content, $matched_urls); + $matched_urls = array_unique($matched_urls); foreach ($matched_urls as $match) { try { - $link_id = Entity\Link::getOrCreate($match[0])->getId(); + $link_id = Entity\Link::getOrCreate($match)->getId(); DB::persist(NoteToLink::create(['link_id' => $link_id, 'note_id' => $note_id])); } catch (InvalidArgumentException) { continue; @@ -57,4 +52,212 @@ END; } return Event::next; } + + public function onRenderContent(string &$text) + { + $text = $this->replaceURLs($text); + } + + public function getURLRegex(): string + { + $geouri_labeltext_regex = '\pN\pL\-'; + $geouri_mark_regex = '\-\_\.\!\~\*\\\'\(\)'; // the \\\' is really pretty + $geouri_unreserved_regex = '\pN\pL' . $geouri_mark_regex; + $geouri_punreserved_regex = '\[\]\:\&\+\$'; + $geouri_pctencoded_regex = '(?:\%[0-9a-fA-F][0-9a-fA-F])'; + $geouri_paramchar_regex = $geouri_unreserved_regex . $geouri_punreserved_regex; //FIXME: add $geouri_pctencoded_regex here so it works + + return '#' . + '(?:^|[\s\<\>\(\)\[\]\{\}\\\'\\\";]+)(?![\@\!\#])' . + '(' . + '(?:' . + '(?:' . //Known protocols + '(?:' . + '(?:(?:' . implode('|', $this->URLSchemes(self::URL_SCHEME_COLON_DOUBLE_SLASH)) . ')://)' . + '|' . + '(?:(?:' . implode('|', $this->URLSchemes(self::URL_SCHEME_SINGLE_COLON)) . '):)' . + ')' . + '(?:[\pN\pL\-\_\+\%\~]+(?::[\pN\pL\-\_\+\%\~]+)?\@)?' . //user:pass@ + '(?:' . + '(?:' . + '\[[\pN\pL\-\_\:\.]+(?URLSchemes(self::URL_SCHEME_COLON_COORDINATES)) . '):' . + // There's an order that must be followed here too, if ;crs= is used, it must precede ;u= + // Also 'crsp' (;crs=$crsp) must match $geouri_labeltext_regex + // Also 'uval' (;u=$uval) must be a pnum: \-?[0-9]+ + '(?:' . + '(?:[0-9]+(?:\.[0-9]+)?(?:\,[0-9]+(?:\.[0-9]+)?){1,2})' . // 1(.23)?(,4(.56)){1,2} + '(?:\;(?:[' . $geouri_labeltext_regex . ']+)(?:\=[' . $geouri_paramchar_regex . ']+)*)*' . + ')' . + ')' . + // URLs without domain name, like magnet:?xt=... + '|(?:(?:' . implode('|', $this->URLSchemes(self::URL_SCHEME_NO_DOMAIN)) . '):(?=\?))' . // zero-length lookahead requires ? after : + (Common::config('linkify', 'ipv4') // Convert IPv4 addresses to hyperlinks + ? '|(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' + : '') . + (Common::config('linkify', 'ipv6') // Convert IPv6 addresses to hyperlinks + ? '|(?:' . //IPv6 + '\[?(?:(?:(?:[0-9A-Fa-f]{1,4}:){7}(?:(?:[0-9A-Fa-f]{1,4})|:))|(?:(?:[0-9A-Fa-f]{1,4}:){6}(?::|(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})|(?::[0-9A-Fa-f]{1,4})))|(?:(?:[0-9A-Fa-f]{1,4}:){5}(?:(?::(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})?)|(?:(?::[0-9A-Fa-f]{1,4}){1,2})))|(?:(?:[0-9A-Fa-f]{1,4}:){4}(?::[0-9A-Fa-f]{1,4}){0,1}(?:(?::(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})?)|(?:(?::[0-9A-Fa-f]{1,4}){1,2})))|(?:(?:[0-9A-Fa-f]{1,4}:){3}(?::[0-9A-Fa-f]{1,4}){0,2}(?:(?::(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})?)|(?:(?::[0-9A-Fa-f]{1,4}){1,2})))|(?:(?:[0-9A-Fa-f]{1,4}:){2}(?::[0-9A-Fa-f]{1,4}){0,3}(?:(?::(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})?)|(?:(?::[0-9A-Fa-f]{1,4}){1,2})))|(?:(?:[0-9A-Fa-f]{1,4}:)(?::[0-9A-Fa-f]{1,4}){0,4}(?:(?::(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})?)|(?:(?::[0-9A-Fa-f]{1,4}){1,2})))|(?::(?::[0-9A-Fa-f]{1,4}){0,5}(?:(?::(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})?)|(?:(?::[0-9A-Fa-f]{1,4}){1,2})))|(?:(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})){3})))\]?(? self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'https' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'ftp' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'ftps' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'mms' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'rtsp' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'gopher' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'news' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'nntp' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'telnet' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'wais' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'file' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'prospero' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'webcal' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'irc' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'ircs' => self::URL_SCHEME_COLON_DOUBLE_SLASH, + 'aim' => self::URL_SCHEME_SINGLE_COLON, + 'bitcoin' => self::URL_SCHEME_SINGLE_COLON, + 'fax' => self::URL_SCHEME_SINGLE_COLON, + 'jabber' => self::URL_SCHEME_SINGLE_COLON, + 'mailto' => self::URL_SCHEME_SINGLE_COLON, + 'tel' => self::URL_SCHEME_SINGLE_COLON, + 'xmpp' => self::URL_SCHEME_SINGLE_COLON, + 'magnet' => self::URL_SCHEME_NO_DOMAIN, + 'geo' => self::URL_SCHEME_COLON_COORDINATES, + ]; + + return array_keys(array_filter($schemes, fn ($scheme) => is_null($filter) || ($scheme & $filter))); + } + + /** + * Find links in the given text and pass them to the given callback function. + * + * @param string $text + */ + public function replaceURLs(string $text): string + { + $regex = $this->getURLRegex(); + return preg_replace_callback($regex, fn ($matches) => $this->callbackHelper($matches, [$this, 'linkify']), $text); + } + + /** + * Intermediate callback for `replaceURLs()`, which helps resolve some + * ambiguous link forms before passing on to the final callback. + * + * @param array $matches + * @param callable(string $text): string $callback: return replacement text + * + * @return string + */ + private function callbackHelper(array $matches, callable $callback): string + { + $url = $matches[1]; + $left = strpos($matches[0], $url); + $right = $left + strlen($url); + + $groupSymbolSets = [ + [ + 'left' => '(', + 'right' => ')', + ], + [ + 'left' => '[', + 'right' => ']', + ], + [ + 'left' => '{', + 'right' => '}', + ], + [ + 'left' => '<', + 'right' => '>', + ], + ]; + + $cannotEndWith = ['.', '?', ',', '#']; + do { + $original_url = $url; + foreach ($groupSymbolSets as $groupSymbolSet) { + if (substr($url, -1) == $groupSymbolSet['right']) { + $group_left_count = substr_count($url, $groupSymbolSet['left']); + $group_right_count = substr_count($url, $groupSymbolSet['right']); + if ($group_left_count < $group_right_count) { + --$right; + $url = substr($url, 0, -1); + } + } + } + if (in_array(substr($url, -1), $cannotEndWith)) { + --$right; + $url = substr($url, 0, -1); + } + } while ($original_url != $url); + + $result = $callback($url); + return substr($matches[0], 0, $left) . $result . substr($matches[0], $right); + } + + /** + * Convert a plain text $url to HTML + */ + public function linkify(string $url): string + { + // It comes in special'd, so we unspecial it before passing to the stringifying + // functions + $url = htmlspecialchars_decode($url); + + if (strpos($url, '@') !== false && strpos($url, ':') === false && ($email = filter_var($url, FILTER_VALIDATE_EMAIL)) !== false) { + //url is an email address without the mailto: protocol + $url = "mailto:{$email}"; + } + + $attrs = ['href' => $url, 'title' => $url]; + + // TODO Check to see whether this is a known "attachment" URL. + + // Whether to nofollow + $nf = Common::config('nofollow', 'external'); + if ($nf == 'never') { + $attrs['rel'] = 'external'; + } else { + $attrs['rel'] = 'noopener nofollow external noreferrer'; + } + + return HTML::html(['a' => ['attrs' => $attrs, $url]], options: ['indent' => false]); + } } diff --git a/components/Tag/Tag.php b/components/Tag/Tag.php index cb0a734011..bdd62fa248 100644 --- a/components/Tag/Tag.php +++ b/components/Tag/Tag.php @@ -24,18 +24,29 @@ namespace Component\Tag; use App\Core\DB\DB; use App\Core\Event; use App\Core\Modules\Component; +use App\Core\Router\Router; use App\Entity\NoteTag; +use App\Util\Formatting; +use App\Util\HTML; /** * Component responsible for extracting tags from posted notes, as well as normalizing them * - * @author Hugo Sales + * @author Hugo Sales * @copyright 2021 Free Software Foundation, Inc http://www.fsf.org * @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later */ class Tag extends Component { - const TAG_REGEX = '/(?:^|\s)#([\pL\pN_\-\.]{1,64})/u'; // Brion Vibber 2011-02-23 v2:classes/Notice.php:367 function saveTags + const MAX_TAG_LENGTH = 64; + const TAG_REGEX = '/(^|\\s)(#[\\pL\\pN_\\-\\.]{1,64})/u'; // Brion Vibber 2011-02-23 v2:classes/Notice.php:367 function saveTags + const TAG_SLUG_REGEX = '[A-Za-z0-9]{1,64}'; + + public function onAddRoute($r): bool + { + $r->connect('tag', '/tag/{tag<' . self::TAG_SLUG_REGEX . '>}' , [Controller\Tag::class, 'tag']); + return Event::next; + } /** * Process note by extracting any tags present @@ -46,7 +57,8 @@ class Tag extends Component $processed_tags = false; preg_match_all(self::TAG_REGEX, $content, $matched_tags, PREG_SET_ORDER); foreach ($matched_tags as $match) { - DB::persist($tag = NoteTag::create(['tag' => $match[0], 'note_id' => $note_id])); + $tag = $match[2]; + DB::persist(NoteTag::create(['tag' => $tag, 'canonical' => $this->canonicalTag($tag), 'note_id' => $note_id])); $processed_tags = true; } if ($processed_tags) { @@ -54,9 +66,20 @@ class Tag extends Component } } - public function onAddRoute($r): bool + public function onRenderContent(string &$text) { - $r->connect('tag', '/tag/{tag' . self::TAG_REGEX . '}' , [Controller\Tag::class, 'tag']); - return Event::next; + $text = preg_replace_callback(self::TAG_REGEX, fn ($m) => $m[1] . $this->tagLink($m[2]), $text); + } + + private function tagLink(string $tag): string + { + $canonical = $this->canonicalTag($tag); + $url = Router::url('tag', ['tag' => $canonical]); + return HTML::html(['a' => ['attrs' => ['href' => $url, 'title' => $tag, 'rel' => 'tag'], $tag]], options: ['indent' => false]); + } + + public function canonicalTag(string $tag): string + { + return substr(Formatting::slugify($tag), 0, self::MAX_TAG_LENGTH); } } diff --git a/src/Util/Formatting.php b/src/Util/Formatting.php index 4bd2efb9d7..c9c26e3a7b 100644 --- a/src/Util/Formatting.php +++ b/src/Util/Formatting.php @@ -32,7 +32,6 @@ namespace App\Util; use App\Core\Event; use App\Core\Log; -use App\Core\Router\Router; use App\Entity\Actor; use App\Entity\Group; use App\Entity\Note; @@ -263,11 +262,9 @@ abstract class Formatting // Split \n\n into paragraphs, process each paragrah and merge $text = implode("\n", F\map(explode("\n\n", $text), function (string $paragraph) { $paragraph = nl2br($paragraph, use_xhtml: false); - $paragraph = self::replaceURLs($paragraph, [self::class, 'linkify']); - $paragraph = preg_replace_callback('/(^|\"\;|\'|\(|\[|\{|\s+)(#[\pL\pN_\-\.]{1,64})/u', - fn ($m) => "{$m[1]}" . self::tagLink($m[2]), $paragraph); + Event::handle('RenderContent', [&$paragraph]); - return HTML::html(['p' => [$paragraph]], options: ['raw' => true]); + return HTML::html(['p' => [$paragraph]], options: ['raw' => true, 'indent' => false]); })); return $text; @@ -286,232 +283,16 @@ abstract class Formatting return preg_replace(['/[\x{0}-\x{8}\x{b}-\x{c}\x{e}-\x{19}\x{200b}-\x{200f}\x{202a}-\x{202e}]/u', '/\R/u'], ['', "\n"], $text); } - const URL_SCHEME_COLON_DOUBLE_SLASH = 1; - const URL_SCHEME_SINGLE_COLON = 2; - const URL_SCHEME_NO_DOMAIN = 4; - const URL_SCHEME_COLON_COORDINATES = 8; - - public static function URLSchemes($filter = null) - { - // TODO: move these to config - $schemes = [ - 'http' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'https' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'ftp' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'ftps' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'mms' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'rtsp' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'gopher' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'news' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'nntp' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'telnet' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'wais' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'file' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'prospero' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'webcal' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'irc' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'ircs' => self::URL_SCHEME_COLON_DOUBLE_SLASH, - 'aim' => self::URL_SCHEME_SINGLE_COLON, - 'bitcoin' => self::URL_SCHEME_SINGLE_COLON, - 'fax' => self::URL_SCHEME_SINGLE_COLON, - 'jabber' => self::URL_SCHEME_SINGLE_COLON, - 'mailto' => self::URL_SCHEME_SINGLE_COLON, - 'tel' => self::URL_SCHEME_SINGLE_COLON, - 'xmpp' => self::URL_SCHEME_SINGLE_COLON, - 'magnet' => self::URL_SCHEME_NO_DOMAIN, - 'geo' => self::URL_SCHEME_COLON_COORDINATES, - ]; - - return array_keys(array_filter($schemes, fn ($scheme) => is_null($filter) || ($scheme & $filter))); - } - - /** - * Find links in the given text and pass them to the given callback function. - * - * @param string $text - * @param callable(string $text, mixed $arg): string $callback: return replacement text - * @param mixed $arg: optional argument will be passed on to the callback - */ - public static function replaceURLs(string $text, callable $callback, mixed $arg = null) - { - $geouri_labeltext_regex = '\pN\pL\-'; - $geouri_mark_regex = '\-\_\.\!\~\*\\\'\(\)'; // the \\\' is really pretty - $geouri_unreserved_regex = '\pN\pL' . $geouri_mark_regex; - $geouri_punreserved_regex = '\[\]\:\&\+\$'; - $geouri_pctencoded_regex = '(?:\%[0-9a-fA-F][0-9a-fA-F])'; - $geouri_paramchar_regex = $geouri_unreserved_regex . $geouri_punreserved_regex; //FIXME: add $geouri_pctencoded_regex here so it works - - // Start off with a regex - $regex = '#' . - '(?:^|[\s\<\>\(\)\[\]\{\}\\\'\\\";]+)(?![\@\!\#])' . - '(' . - '(?:' . - '(?:' . //Known protocols - '(?:' . - '(?:(?:' . implode('|', self::URLSchemes(self::URL_SCHEME_COLON_DOUBLE_SLASH)) . ')://)' . - '|' . - '(?:(?:' . implode('|', self::URLSchemes(self::URL_SCHEME_SINGLE_COLON)) . '):)' . - ')' . - '(?:[\pN\pL\-\_\+\%\~]+(?::[\pN\pL\-\_\+\%\~]+)?\@)?' . //user:pass@ - '(?:' . - '(?:' . - '\[[\pN\pL\-\_\:\.]+(? self::callbackHelper($matches, $callback, $arg), $text); - } - - /** - * Intermediate callback for common_replace_links(), helps resolve some - * ambiguous link forms before passing on to the final callback. - * - * @param array $matches - * @param callable $callback - * @param mixed $arg optional argument to pass on as second param to callback - * - * @return string - * - */ - private static function callbackHelper(array $matches, callable $callback, mixed $arg = null): string - { - $url = $matches[1]; - $left = strpos($matches[0], $url); - $right = $left + strlen($url); - - $groupSymbolSets = [ - [ - 'left' => '(', - 'right' => ')', - ], - [ - 'left' => '[', - 'right' => ']', - ], - [ - 'left' => '{', - 'right' => '}', - ], - [ - 'left' => '<', - 'right' => '>', - ], - ]; - - $cannotEndWith = ['.', '?', ',', '#']; - do { - $original_url = $url; - foreach ($groupSymbolSets as $groupSymbolSet) { - if (substr($url, -1) == $groupSymbolSet['right']) { - $group_left_count = substr_count($url, $groupSymbolSet['left']); - $group_right_count = substr_count($url, $groupSymbolSet['right']); - if ($group_left_count < $group_right_count) { - --$right; - $url = substr($url, 0, -1); - } - } - } - if (in_array(substr($url, -1), $cannotEndWith)) { - --$right; - $url = substr($url, 0, -1); - } - } while ($original_url != $url); - - $result = call_user_func_array($callback, [$url, $arg]); - return substr($matches[0], 0, $left) . $result . substr($matches[0], $right); - } - - /** - * Convert a plain text $url to HTML - */ - public static function linkify(string $url): string - { - // It comes in special'd, so we unspecial it before passing to the stringifying - // functions - $url = htmlspecialchars_decode($url); - - if (strpos($url, '@') !== false && strpos($url, ':') === false && ($email = filter_var($url, FILTER_VALIDATE_EMAIL)) !== false) { - //url is an email address without the mailto: protocol - $url = "mailto:{$email}"; - } - - $attrs = ['href' => $url, 'title' => $url]; - - // TODO Check to see whether this is a known "attachment" URL. - - // Whether to nofollow - $nf = Common::config('nofollow', 'external'); - if ($nf == 'never') { - $attrs['rel'] = 'external'; - } else { - $attrs['rel'] = 'noopener nofollow external noreferrer'; - } - - return HTML::html(['a' => ['attrs' => $attrs, $url]]); - } - - public static function tagLink(string $tag): string - { - $canonical = self::canonicalTag($tag); - $url = Router::url('tag', ['tag' => $canonical]); - return HTML::html(['a' => ['attrs' => ['href' => $url, 'title' => $tag, 'rel' => 'tag'], $tag]]); - } - - public static function canonicalTag(string $tag): string - { - return substr(self::slugify($tag), 0, 64); - } - /** * Convert $str to it's closest ASCII representation */ - public static function slugify(string $str): string + public static function slugify(string $str, int $length = 64): string { // php-intl is highly recommended... if (!function_exists('transliterator_transliterate')) { $str = preg_replace('/[^\pL\pN]/u', '', $str); $str = mb_convert_case($str, MB_CASE_LOWER, 'UTF-8'); - $str = substr($str, 0, 64); + $str = substr($str, 0, $length); return $str; } $str = transliterator_transliterate('Any-Latin;' . // any charset to latin compatible @@ -522,7 +303,7 @@ abstract class Formatting 'Lower();' . // turn into lowercase 'Latin-ASCII;', // get ASCII equivalents (ð to d for example) $str); - return preg_replace('/[^\pL\pN]/u', '', $str); + return substr(preg_replace('/[^\pL\pN]/u', '', $str), 0, $length); } /**