[ENTITY] Refactor RemoteURL entities to Link

RemoteURL was being an awfully confusing term.
This commit is contained in:
Diogo Peralta Cordeiro 2021-08-13 20:09:20 +01:00 committed by Hugo Sales
parent fb28a3656a
commit 9e4cac0123
Signed by: someonewithpc
GPG Key ID: 7D0C7EAFC9D835A0
8 changed files with 129 additions and 119 deletions

View File

@ -31,9 +31,9 @@ use App\Core\Modules\Component;
use App\Entity\Attachment;
use App\Entity\AttachmentToNote;
use App\Entity\GSActorToAttachment;
use App\Entity\Link;
use App\Entity\Note;
use App\Entity\RemoteURL;
use App\Entity\RemoteURLToNote;
use App\Entity\NoteToLink;
use App\Util\Common;
use App\Util\Exception\InvalidFormException;
use App\Util\Exception\RedirectException;
@ -142,8 +142,8 @@ END;
preg_match_all(self::URL_REGEX, $content, $matched_urls, PREG_SET_ORDER);
foreach ($matched_urls as $match) {
try {
$remoteurl_id = RemoteURL::getOrCreate($match[0])->getId();
DB::persist(RemoteURLToNote::create(['remoteurl_id' => $remoteurl_id, 'note_id' => $note->getId()]));
$link_id = Link::getOrCreate($match[0])->getId();
DB::persist(NoteToLink::create(['link_id' => $link_id, 'note_id' => $note->getId()]));
$processed_urls = true;
} catch (InvalidArgumentException) {
continue;

View File

@ -46,8 +46,8 @@ use App\Core\Modules\Plugin;
use App\Core\Router\RouteLoader;
use App\Core\Router\Router;
use App\Entity\Attachment;
use App\Entity\Link;
use App\Entity\Note;
use App\Entity\RemoteURL;
use App\Util\Common;
use App\Util\Exception\DuplicateFoundException;
use App\Util\Exception\NotFoundException;
@ -134,24 +134,24 @@ class Embed extends Plugin
*
* @return bool
*/
public function onViewRemoteUrl(array $vars, array &$res): bool
public function onViewLink(array $vars, array &$res): bool
{
$remote_url = $vars['remote_url'];
$link = $vars['link'];
try {
$embed = Cache::get('attachment-embed-' . $remote_url->getId(),
fn () => DB::findOneBy('attachment_embed', ['remoteurl_id' => $remote_url->getId()]));
$embed = Cache::get('attachment-embed-' . $link->getId(),
fn () => DB::findOneBy('attachment_embed', ['link_id' => $link->getId()]));
} catch (DuplicateFoundException $e) {
Log::warning($e);
return Event::next;
} catch (NotFoundException) {
Log::debug("Embed doesn\\'t have a representation for the remote_url id={$remote_url->getId()}. Must have been stored before the plugin was enabled.");
Log::debug("Embed doesn't have a representation for the link id={$link->getId()}. Must have been stored before the plugin was enabled.");
return Event::next;
}
$attributes = $embed->getImageHTMLAttributes(['class' => 'u-photo embed']);
$res[] = Formatting::twigRenderFile('embed/embedView.html.twig',
['embed' => $embed, 'attributes' => $attributes, 'remote_url' => $remote_url]);
['embed' => $embed, 'attributes' => $attributes, 'link' => $link]);
return Event::stop;
}
@ -159,31 +159,32 @@ class Embed extends Plugin
/**
* Save embedding information for an Attachment, if applicable.
*
* @param RemoteURL $remote_url
* @param Note $note
* @param Link $link
* @param Note $note
*
* @throws DuplicateFoundException
*@throws DuplicateFoundException
*
* @return bool
*
*/
public function onNewRemoteURLFromNote(RemoteURL $remote_url, Note $note): bool
public function onNewLinkFromNote(Link $link, Note $note): bool
{
// Only handle text mime
$mimetype = $remote_url->getMimetype();
$mimetype = $link->getMimetype();
if (!(Formatting::startsWith($mimetype, 'text/html') || Formatting::startsWith($mimetype, 'application/xhtml+xml'))) {
return Event::next;
}
// Ignore if already handled
$attachment_embed = DB::find('attachment_embed', ['remoteurl_id' => $remote_url->getId()]);
$attachment_embed = DB::find('attachment_embed', ['link_id' => $link->getId()]);
if (!is_null($attachment_embed)) {
return Event::next;
}
// If an attachment already exist, do not create an Embed for it. Some other plugin must have done things
$remote_url_to_attachment = DB::find('remoteurl_to_attachment', ['remoteurl_id' => $remote_url->getId()]);
if (!is_null($remote_url_to_attachment)) {
$attachment_id = $remote_url_to_attachment->getAttachmentId();
$link_to_attachment = DB::find('link_to_attachment', ['link_id' => $link->getId()]);
if (!is_null($link_to_attachment)) {
$attachment_id = $link_to_attachment->getAttachmentId();
try {
$attachment = DB::findOneBy('attachment', ['id' => $attachment_id]);
$attachment->livesIncrementAndGet();
@ -194,15 +195,15 @@ class Embed extends Plugin
}
// Create an Embed representation for this URL
$embed_data = $this->getEmbedLibMetadata($remote_url->getRemoteUrl());
$embed_data['remoteurl_id'] = $remote_url->getId();
$img_data = $this->downloadThumbnail($embed_data['thumbnail_url']);
$embed_data = $this->getEmbedLibMetadata($link->getUrl());
$embed_data['link_id'] = $link->getId();
$img_data = $this->downloadThumbnail($embed_data['thumbnail_url']);
switch ($img_data) {
case null: // URL isn't usable
$embed_data['thumbnail_url'] = null;
// no break
case false: // Thumbnail isn't acceptable
DB::persist($attachment = Attachment::create(['mimetype' => $remote_url->getMimetype()]));
DB::persist($attachment = Attachment::create(['mimetype' => $link->getMimetype()]));
Event::handle('AttachmentStoreNew', [&$attachment]);
break;
default: // String is valid image data
@ -286,7 +287,7 @@ class Embed extends Plugin
*
* @return bool true if allowed by the lists, false otherwise
*/
private function allowedRemoteUrl(string $url): bool
private function allowedLink(string $url): bool
{
return true;
if ($this->check_whitelist ?? false) {
@ -309,30 +310,30 @@ class Embed extends Plugin
* - checks if respects file quota and whitelist/blacklist, returns false if not;
* - downloads the thumbnail, returns a string if successful.
*
* @param string $remote_url URL to the remote thumbnail
* @param string $url URL to the remote thumbnail
*
* @return null|bool|string
*/
private function downloadThumbnail(string $remote_url): bool|string|null
private function downloadThumbnail(string $url): bool|string|null
{
// Is this a valid URL?
if (!Common::isValidHttpUrl($remote_url)) {
Log::debug("Invalid URL ({$remote_url}) in Embed->downloadThumbnail.");
if (!Common::isValidHttpUrl($url)) {
Log::debug("Invalid URL ({$url}) in Embed->downloadThumbnail.");
return null;
}
// Is this URL trusted?
if (!$this->allowedRemoteUrl($remote_url)) {
Log::info("Blocked URL ({$remote_url}) in Embed->downloadThumbnail.");
if (!$this->allowedLink($url)) {
Log::info("Blocked URL ({$url}) in Embed->downloadThumbnail.");
return false;
}
// Validate if the URL really does point to a remote image
$head = HTTPClient::head($remote_url);
$head = HTTPClient::head($url);
$headers = $head->getHeaders();
$headers = array_change_key_case($headers, CASE_LOWER);
if (empty($headers['content-type']) || GSFile::mimetypeMajor($headers['content-type'][0]) !== 'image') {
Log::debug("URL ({$remote_url}) doesn't point to an image (content-type: " . (!empty($headers['content-type'][0]) ? $headers['content-type'][0] : 'not available') . ') in Embed->downloadThumbnail.');
Log::debug("URL ({$url}) doesn't point to an image (content-type: " . (!empty($headers['content-type'][0]) ? $headers['content-type'][0] : 'not available') . ') in Embed->downloadThumbnail.');
return null;
}
@ -345,8 +346,8 @@ class Embed extends Plugin
}
// Download and return the file
Log::debug("Downloading remote thumbnail from URL: {$remote_url} in Embed->downloadThumbnail.");
return HTTPClient::get($remote_url)->getContent();
Log::debug("Downloading remote thumbnail from URL: {$url} in Embed->downloadThumbnail.");
return HTTPClient::get($url)->getContent();
}
/**

View File

@ -46,7 +46,7 @@ class AttachmentEmbed extends Entity
{
// {{{ Autocode
// @codeCoverageIgnoreStart
private int $remoteurl_id;
private int $link_id;
private int $attachment_id;
private ?string $title;
private ?string $description;
@ -57,15 +57,15 @@ class AttachmentEmbed extends Entity
private ?string $thumbnail_url;
private \DateTimeInterface $modified;
public function setRemoteUrlId(int $remoteurl_id): self
public function setLinkId(int $link_id): self
{
$this->remoteurl_id = $remoteurl_id;
$this->link_id = $link_id;
return $this;
}
public function getRemoteUrlId(): int
public function getLinkId(): int
{
return $this->remoteurl_id;
return $this->link_id;
}
/**
@ -180,7 +180,7 @@ class AttachmentEmbed extends Entity
return [
'name' => 'attachment_embed',
'fields' => [
'remoteurl_id' => ['type' => 'int', 'not null' => true, 'description' => 'Embed for that URL/file'],
'link_id' => ['type' => 'int', 'not null' => true, 'description' => 'Embed for that URL/file'],
'attachment_id' => ['type' => 'int', 'not null' => true, 'description' => 'Attachment relation, used to show previews'],
'provider_name' => ['type' => 'text', 'description' => 'name of this Embed provider'],
'provider_url' => ['type' => 'text', 'description' => 'URL of this Embed provider'],
@ -190,9 +190,9 @@ class AttachmentEmbed extends Entity
'thumbnail_url' => ['type' => 'text', 'description' => 'URL for this Embed resource when applicable (photo, link)'],
'modified' => ['type' => 'timestamp', 'not null' => true, 'description' => 'date this record was modified'],
],
'primary key' => ['remoteurl_id'],
'primary key' => ['link_id'],
'foreign keys' => [
'attachment_embed_remoteurl_id_fkey' => ['remoteurl', ['remoteurl_id' => 'id']],
'attachment_embed_link_id_fkey' => ['link', ['link_id' => 'id']],
'attachment_embed_attachment_id_fkey' => ['attachment', ['attachment_id' => 'id']],
],
];

View File

@ -4,7 +4,7 @@
<img class="u-photo embed" width="{{attributes['width']}}" height="{{attributes['height']}}" src="{{attributes['src']}}" />
{% endif %}
<h5 class="p-name embed">
<a class="u-url" href="{{attachment.getRemoteUrl()}}">{{embed.getTitle() | escape}}</a>
<a class="u-url" href="{{link.getUrl()}}">{{embed.getTitle() | escape}}</a>
</h5>
<div class="p-author embed">
{% if embed.getAuthorName() is not null %}

View File

@ -24,11 +24,14 @@ use App\Core\GSFile;
use App\Core\HTTPClient;
use App\Core\Modules\Plugin;
use App\Entity\AttachmentThumbnail;
use App\Entity\AttachmentToLink;
use App\Entity\AttachmentToNote;
use App\Entity\Link;
use App\Entity\Note;
use App\Entity\RemoteURL;
use App\Entity\RemoteURLToAttachment;
use App\Util\Common;
use App\Util\Exception\DuplicateFoundException;
use App\Util\Exception\ServerException;
use App\Util\Exception\TemporaryFileException;
use App\Util\TemporaryFile;
/**
@ -82,43 +85,48 @@ class StoreRemoteMedia extends Plugin
}
/**
* @param RemoteURL $remote_url
* @param Link $link
* @param Note $note
*
* @throws DuplicateFoundException
* @throws ServerException
* @throws TemporaryFileException
*
* @return bool
*/
public function onNewRemoteURLFromNote(RemoteURL $remote_url, Note $note): bool
public function onNewLinkFromNote(Link $link, Note $note): bool
{
// Embed is the plugin to handle these
if ($remote_url->getMimetypeMajor() === 'text') {
if ($link->getMimetypeMajor() === 'text') {
return Event::next;
}
// Have we handled it already?
$remoteurl_to_attachment = DB::find('remoteurl_to_attachment',
['remoteurl_id' => $remote_url->getId()]);
$attachment_to_link = DB::find('attachment_to_link',
['link_id' => $link->getId()]);
// If it was handled already
if (!is_null($remoteurl_to_attachment)) {
if (!is_null($attachment_to_link)) {
// Relate the note with the existing attachment
DB::persist(AttachmentToNote::create([
'attachment_id' => $remoteurl_to_attachment->getAttachmentId(),
'attachment_id' => $attachment_to_link->getAttachmentId(),
'note_id' => $note->getId(),
]));
DB::flush();
return Event::stop;
} else {
// Retrieve media
$get_response = HTTPClient::get($remote_url->getRemoteUrl());
$get_response = HTTPClient::get($link->getUrl());
$media = $get_response->getContent();
$mimetype = $get_response->getHeaders()['content-type'][0];
unset($get_response);
// Ensure we still want to handle it
if ($mimetype != $remote_url->getMimetype()) {
$remote_url->setMimetype($mimetype);
DB::persist($remote_url);
if ($mimetype != $link->getMimetype()) {
$link->setMimetype($mimetype);
DB::persist($link);
DB::flush();
if ($remote_url->getMimetypeMajor() === 'text') {
if ($link->getMimetypeMajor() === 'text') {
return Event::next;
}
}
@ -128,9 +136,9 @@ class StoreRemoteMedia extends Plugin
$temp_file->write($media);
$attachment = GSFile::sanitizeAndStoreFileAsAttachment($temp_file);
// Relate the remoteurl with the attachment
DB::persist(RemoteURLToAttachment::create([
'remoteurl_id' => $remote_url->getId(),
// Relate the link with the attachment
DB::persist(AttachmentToLink::create([
'link_id' => $link->getId(),
'attachment_id' => $attachment->getId(),
]));

View File

@ -32,12 +32,12 @@ use DateTimeInterface;
* @copyright 2021 Free Software Foundation, Inc http://www.fsf.org
* @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later
*/
class RemoteURLToAttachment extends Entity
class AttachmentToLink extends Entity
{
// {{{ Autocode
// @codeCoverageIgnoreStart
private int $attachment_id;
private int $remoteurl_id;
private int $link_id;
private \DateTimeInterface $modified;
public function setAttachmentId(int $attachment_id): self
@ -51,15 +51,15 @@ class RemoteURLToAttachment extends Entity
return $this->attachment_id;
}
public function setRemoteURLId(int $remoteurl_id): self
public function setLinkId(int $link_id): self
{
$this->remoteurl_id = $remoteurl_id;
$this->link_id = $link_id;
return $this;
}
public function getRemoteURLId(): int
public function getLinkId(): int
{
return $this->remoteurl_id;
return $this->link_id;
}
public function setModified(DateTimeInterface $modified): self
@ -79,15 +79,15 @@ class RemoteURLToAttachment extends Entity
public static function schemaDef(): array
{
return [
'name' => 'remoteurl_to_attachment',
'name' => 'attachment_to_link',
'fields' => [
'remoteurl_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'RemoteURL.id', 'multiplicity' => 'one to one', 'name' => 'attachment_to_note_note_id_fkey', 'not null' => true, 'description' => 'id of the note it belongs to'],
'link_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'Link.id', 'multiplicity' => 'one to one', 'name' => 'attachment_to_note_note_id_fkey', 'not null' => true, 'description' => 'id of the note it belongs to'],
'attachment_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'Attachment.id', 'multiplicity' => 'one to one', 'name' => 'attachment_to_note_attachment_id_fkey', 'not null' => true, 'description' => 'id of attachment'],
'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'],
],
'primary key' => ['remoteurl_id'],
'primary key' => ['link_id'],
'indexes' => [
'remoteurl_id_idx' => ['remoteurl_id'],
'link_id_idx' => ['link_id'],
'attachment_id_idx' => ['attachment_id'],
],
];

View File

@ -31,7 +31,7 @@ use DateTimeInterface;
use InvalidArgumentException;
/**
* Entity for representing a RemoteURL
* Entity for representing a Link
*
* @category DB
* @package GNUsocial
@ -40,13 +40,13 @@ use InvalidArgumentException;
* @copyright 2021 Free Software Foundation, Inc http://www.fsf.org
* @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later
*/
class RemoteURL extends Entity
class Link extends Entity
{
// {{{ Autocode
// @codeCoverageIgnoreStart
private int $id;
private ?string $remote_url;
private ?string $remote_url_hash;
private ?string $url;
private ?string $url_hash;
private ?string $mimetype;
private DateTimeInterface $modified;
@ -61,26 +61,26 @@ class RemoteURL extends Entity
return $this->id;
}
public function getRemoteUrl(): ?string
public function getUrl(): ?string
{
return $this->remote_url;
return $this->url;
}
public function setRemoteUrl(?string $remote_url): self
public function setUrl(?string $url): self
{
$this->remote_url = $remote_url;
$this->url = $url;
return $this;
}
public function setRemoteUrlHash(?string $remote_url_hash): self
public function setUrlHash(?string $url_hash): self
{
$this->remote_url_hash = $remote_url_hash;
$this->url_hash = $url_hash;
return $this;
}
public function getRemoteUrlHash(): ?string
public function getUrlHash(): ?string
{
return $this->remote_url_hash;
return $this->url_hash;
}
public function setMimetype(?string $mimetype): self
@ -127,10 +127,11 @@ class RemoteURL extends Entity
*
* @param string $url
*
*@throws InvalidArgumentException
* @throws DuplicateFoundException
* @throws InvalidArgumentException
*
* @return RemoteURL
* @return Link
*
*/
public static function getOrCreate(string $url): self
{
@ -141,17 +142,17 @@ class RemoteURL extends Entity
$url = $head->getInfo('url'); // The last effective url (after getHeaders, so it follows redirects)
$url_hash = hash(self::URLHASH_ALGO, $url);
try {
return DB::findOneBy('remoteurl', ['remote_url_hash' => $url_hash]);
return DB::findOneBy('link', ['url_hash' => $url_hash]);
} catch (NotFoundException) {
$headers = array_change_key_case($headers, CASE_LOWER);
$remoteurl = self::create([
'remote_url' => $url,
'remote_url_hash' => $url_hash,
'mimetype' => $headers['content-type'][0],
$headers = array_change_key_case($headers, CASE_LOWER);
$link = self::create([
'url' => $url,
'url_hash' => $url_hash,
'mimetype' => $headers['content-type'][0],
]);
DB::persist($remoteurl);
Event::handle('RemoteURLStoredNew', [&$remoteurl]);
return $remoteurl;
DB::persist($link);
Event::handle('LinkStoredNew', [&$link]);
return $link;
}
} else {
throw new InvalidArgumentException();
@ -161,17 +162,17 @@ class RemoteURL extends Entity
public static function schemaDef(): array
{
return [
'name' => 'remoteurl',
'name' => 'link',
'fields' => [
'id' => ['type' => 'serial', 'not null' => true],
'remote_url' => ['type' => 'text', 'description' => 'URL after following possible redirections'],
'remote_url_hash' => ['type' => 'varchar', 'length' => 64, 'description' => 'sha256 of destination URL (url field)'],
'mimetype' => ['type' => 'varchar', 'length' => 50, 'description' => 'mime type of resource'],
'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'],
'id' => ['type' => 'serial', 'not null' => true],
'url' => ['type' => 'text', 'description' => 'URL after following possible redirections'],
'url_hash' => ['type' => 'varchar', 'length' => 64, 'description' => 'sha256 of destination URL (url field)'],
'mimetype' => ['type' => 'varchar', 'length' => 50, 'description' => 'mime type of resource'],
'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'],
],
'primary key' => ['id'],
'indexes' => [
'gsactor_remote_url_hash_idx' => ['remote_url_hash'],
'gsactor_url_hash_idx' => ['url_hash'],
],
];
}

View File

@ -25,7 +25,7 @@ use App\Core\Event;
use DateTimeInterface;
/**
* Entity for relating a RemoteURL to a post
* Entity for relating a Link to a post
*
* @category DB
* @package GNUsocial
@ -34,23 +34,23 @@ use DateTimeInterface;
* @copyright 2021 Free Software Foundation, Inc http://www.fsf.org
* @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later
*/
class RemoteURLToNote extends Entity
class NoteToLink extends Entity
{
// {{{ Autocode
// @codeCoverageIgnoreStart
private int $remoteurl_id;
private int $link_id;
private int $note_id;
private \DateTimeInterface $modified;
public function setRemoteURLId(int $remoteurl_id): self
public function setLinkId(int $link_id): self
{
$this->remoteurl_id = $remoteurl_id;
$this->link_id = $link_id;
return $this;
}
public function getRemoteURLId(): int
public function getLinkId(): int
{
return $this->remoteurl_id;
return $this->link_id;
}
public function setNoteId(int $note_id): self
@ -76,7 +76,7 @@ class RemoteURLToNote extends Entity
}
/**
* Create an instance of RemoteURLToNote or fill in the
* Create an instance of NoteToLink or fill in the
* properties of $obj with the associative array $args. Doesn't
* persist the result
*
@ -84,9 +84,9 @@ class RemoteURLToNote extends Entity
*/
public static function create(array $args, $obj = null)
{
$remoteURL = DB::find('remoteurl', ['id' => $args['remoteurl_id']]);
$note = DB::find('note', ['id' => $args['note_id']]);
Event::handle('NewRemoteURLFromNote', [$remoteURL, $note]);
$link = DB::find('link', ['id' => $args['link_id']]);
$note = DB::find('note', ['id' => $args['note_id']]);
Event::handle('NewLinkFromNote', [$link, $note]);
$obj = new self();
return parent::create($args, $obj);
}
@ -97,16 +97,16 @@ class RemoteURLToNote extends Entity
public static function schemaDef(): array
{
return [
'name' => 'remoteurl_to_note',
'name' => 'note_to_link',
'fields' => [
'remoteurl_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'remoteurl.id', 'multiplicity' => 'one to one', 'name' => 'remoteurl_to_note_remoteurl_id_fkey', 'not null' => true, 'description' => 'id of remoteurl'],
'note_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'Note.id', 'multiplicity' => 'one to one', 'name' => 'remoteurl_to_note_note_id_fkey', 'not null' => true, 'description' => 'id of the note it belongs to'],
'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'],
'link_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'link.id', 'multiplicity' => 'one to one', 'name' => 'note_to_link_link_id_fkey', 'not null' => true, 'description' => 'id of link'],
'note_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'Note.id', 'multiplicity' => 'one to one', 'name' => 'note_to_link_note_id_fkey', 'not null' => true, 'description' => 'id of the note it belongs to'],
'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'],
],
'primary key' => ['remoteurl_id', 'note_id'],
'primary key' => ['link_id', 'note_id'],
'indexes' => [
'remoteurl_id_idx' => ['remoteurl_id'],
'note_id_idx' => ['note_id'],
'link_id_idx' => ['link_id'],
'note_id_idx' => ['note_id'],
],
];
}