From 3f615371408b492022fa684873c85fd23931f73d Mon Sep 17 00:00:00 2001 From: Diogo Peralta Cordeiro Date: Sat, 14 Aug 2021 16:47:45 +0100 Subject: [PATCH] [ENTITY] Split Attachment in various new entities Remove Attachment Scope Fixed some minor bugs Scope will be implemented later in v3. It doesn't make sense to have the scope handling being per attachment. Different actors can post the same attachment with different scopes. The attachment controller will assume the highest level of scope applied to the attachment and the rest will be handled at the note level. Motivation: * Remove title from attachment, as it's part of the relation between attachment and note. * Remove actor from attachment, many actors may publish the same attachment. * Remove is_local from attachment, as it's part of the relation between attachment and note. * Remove remote_url from attachment, different urls can return the same attachment. Addition: * Attachment now has a lives attribute, it's a reference counter with a nicer name * GSActorToAttachment * GSActorToRemoteURL * RemoteURL * RemoteURLToNote * RemoteURLToAttachment * AttachmentToNote now has a title attribute --- components/Avatar/Avatar.php | 8 +- components/Avatar/Controller/Avatar.php | 8 +- components/Avatar/Entity/Avatar.php | 21 +- components/Posting/Posting.php | 39 +++- plugins/ImageEncoder/ImageEncoder.php | 6 +- .../imageEncoder/imageEncoderView.html.twig | 4 +- .../videoEncoder/videoEncoderView.html.twig | 2 +- src/Controller/Attachment.php | 15 +- src/Core/GSFile.php | 85 ++----- src/DataFixtures/CoreFixtures.php | 2 +- src/Entity/Attachment.php | 214 +++++++----------- src/Entity/AttachmentThumbnail.php | 2 +- src/Entity/AttachmentToNote.php | 13 ++ src/Entity/GSActorToAttachment.php | 95 ++++++++ src/Entity/GSActorToRemoteURL.php | 95 ++++++++ src/Entity/RemoteURL.php | 177 +++++++++++++++ src/Entity/RemoteURLToAttachment.php | 95 ++++++++ src/Entity/RemoteURLToNote.php | 95 ++++++++ templates/attachments/view.html.twig | 25 +- tests/Controller/AttachmentTest.php | 4 +- 20 files changed, 731 insertions(+), 274 deletions(-) create mode 100644 src/Entity/GSActorToAttachment.php create mode 100644 src/Entity/GSActorToRemoteURL.php create mode 100644 src/Entity/RemoteURL.php create mode 100644 src/Entity/RemoteURLToAttachment.php create mode 100644 src/Entity/RemoteURLToNote.php diff --git a/components/Avatar/Avatar.php b/components/Avatar/Avatar.php index 27a3392609..918c231f19 100644 --- a/components/Avatar/Avatar.php +++ b/components/Avatar/Avatar.php @@ -75,12 +75,6 @@ class Avatar extends Component return Event::next; } - public function onAttachmentRefCount(int $attachment_id, int &$dependencies): bool - { - $dependencies += DB::count('avatar', ['attachment_id' => $attachment_id]); - return Event::next; - } - // UTILS ---------------------------------- /** @@ -126,7 +120,7 @@ class Avatar extends Component { $res = Cache::get("avatar-file-info-{$gsactor_id}", function () use ($gsactor_id) { - return DB::dql('select f.id, f.filename, f.mimetype, f.title ' . + return DB::dql('select f.id, f.filename, f.mimetype ' . 'from App\Entity\Attachment f ' . 'join Component\Avatar\Entity\Avatar a with f.id = a.attachment_id ' . 'where a.gsactor_id = :gsactor_id', diff --git a/components/Avatar/Controller/Avatar.php b/components/Avatar/Controller/Avatar.php index adea96a18b..3002c50b50 100644 --- a/components/Avatar/Controller/Avatar.php +++ b/components/Avatar/Controller/Avatar.php @@ -29,7 +29,6 @@ use App\Core\GSFile; use App\Core\GSFile as M; use function App\Core\I18n\_m; use App\Core\Log; -use App\Core\Security; use App\Util\Common; use App\Util\Exception\ClientException; use App\Util\Exception\NotFoundException; @@ -106,12 +105,9 @@ class Avatar extends Controller } else { throw new ClientException('Invalid form'); } - $attachment = GSFile::validateAndStoreFileAsAttachment( + $attachment = GSFile::sanitizeAndStoreFileAsAttachment( $file, - dest_dir: Common::config('attachments', 'dir'), - actor_id: $gsactor_id, - title: Security::sanitize($file->getClientOriginalName()), - is_local: true + dest_dir: Common::config('attachments', 'dir') ); // Delete current avatar if there's one $avatar = DB::find('avatar', ['gsactor_id' => $gsactor_id]); diff --git a/components/Avatar/Entity/Avatar.php b/components/Avatar/Entity/Avatar.php index 9e20ab11c7..061799c0c4 100644 --- a/components/Avatar/Entity/Avatar.php +++ b/components/Avatar/Entity/Avatar.php @@ -122,24 +122,17 @@ class Avatar extends Entity } /** - * Delete this avatar and, if safe, the corresponding file and thumbnails, which this owns + * Delete this avatar and kill corresponding attachment * - * @param bool $cascade - * @param bool $flush + * @return bool */ - public function delete(bool $cascade = true, bool $flush = true): void + public function delete(): bool { DB::remove($this); - if ($cascade) { - $attachment = $this->getAttachment(); - // We can't use $attachment->isSafeDelete() because underlying findBy doesn't respect remove persistence - if ($attachment->refCount() - 1 === 0) { - $attachment->delete(cascade: true, flush: false); - } - } - if ($flush) { - DB::flush(); - } + $attachment = $this->getAttachment(); + $attachment->kill(); + DB::flush(); + return true; } public static function schemaDef(): array diff --git a/components/Posting/Posting.php b/components/Posting/Posting.php index 345fcc62cd..9597a898ec 100644 --- a/components/Posting/Posting.php +++ b/components/Posting/Posting.php @@ -28,13 +28,17 @@ use App\Core\Form; use App\Core\GSFile; use function App\Core\I18n\_m; use App\Core\Modules\Component; -use App\Core\Security; use App\Entity\Attachment; use App\Entity\AttachmentToNote; +use App\Entity\GSActorToAttachment; +use App\Entity\GSActorToRemoteURL; use App\Entity\Note; +use App\Entity\RemoteURL; +use App\Entity\RemoteURLToNote; use App\Util\Common; use App\Util\Exception\InvalidFormException; use App\Util\Exception\RedirectException; +use InvalidArgumentException; use Symfony\Component\Form\Extension\Core\Type\ChoiceType; use Symfony\Component\Form\Extension\Core\Type\FileType; use Symfony\Component\Form\Extension\Core\Type\SubmitType; @@ -115,31 +119,42 @@ END; $processed_attachments = []; foreach ($attachments as $f) { // where $f is a Symfony\Component\HttpFoundation\File\UploadedFile - $processed_attachments[] = GSFile::validateAndStoreFileAsAttachment( + $filesize = $f->getSize(); + Event::handle('EnforceQuota', [$actor_id, $filesize]); + $processed_attachments[] = GSFile::sanitizeAndStoreFileAsAttachment( $f, - dest_dir: Common::config('attachments', 'dir'), - actor_id: $actor_id, - title: Security::sanitize($f->getClientOriginalName()), - is_local: true + dest_dir: Common::config('attachments', 'dir') ); } - $matched_urls = []; - preg_match_all(self::URL_REGEX, $content, $matched_urls, PREG_SET_ORDER); - foreach ($matched_urls as $match) { - $processed_attachments[] = GSFile::validateAndStoreURLAsAttachment($match[0]); - } - DB::persist($note); // Need file and note ids for the next step DB::flush(); if ($processed_attachments != []) { foreach ($processed_attachments as $a) { + DB::persist(GSActorToAttachment::create(['attachment_id' => $a->getId(), 'gsactor_id' => $actor_id])); DB::persist(AttachmentToNote::create(['attachment_id' => $a->getId(), 'note_id' => $note->getId()])); } DB::flush(); } + + $matched_urls = []; + $processed_urls = false; + preg_match_all(self::URL_REGEX, $content, $matched_urls, PREG_SET_ORDER); + foreach ($matched_urls as $match) { + try { + $remoteurl_id = RemoteURL::getOrCreate($match[0])->getId(); + DB::persist(GSActorToRemoteURL::create(['remoteurl_id' => $remoteurl_id, 'gsactor_id' => $actor_id])); + DB::persist(RemoteURLToNote::create(['remoteurl_id' => $a->getId(), 'note_id' => $note->getId()])); + $processed_urls = true; + } catch (InvalidArgumentException) { + continue; + } + } + if ($processed_urls) { + DB::flush(); + } } /** diff --git a/plugins/ImageEncoder/ImageEncoder.php b/plugins/ImageEncoder/ImageEncoder.php index 3ca0b63bcc..6ce39cbc3b 100644 --- a/plugins/ImageEncoder/ImageEncoder.php +++ b/plugins/ImageEncoder/ImageEncoder.php @@ -71,7 +71,7 @@ class ImageEncoder extends Plugin * * @return bool */ - public function onAttachmentSanitization(SplFileInfo &$file, ?string &$mimetype, ?string &$title, ?int &$width, ?int &$height): bool + public function onAttachmentSanitization(SplFileInfo &$file, ?string &$mimetype, ?int &$width, ?int &$height): bool { $original_mimetype = $mimetype; if (GSFile::mimetypeMajor($original_mimetype) != 'image') { @@ -81,8 +81,8 @@ class ImageEncoder extends Plugin // Try to maintain original mimetype extension, otherwise default to preferred. $extension = image_type_to_extension($this->preferredType(), include_dot: true); - GSFile::titleToFilename( - title: $title, + GSFile::ensureFilenameWithProperExtension( + title: $file->getFilename(), mimetype: $original_mimetype, ext: $extension, force: false diff --git a/plugins/ImageEncoder/templates/imageEncoder/imageEncoderView.html.twig b/plugins/ImageEncoder/templates/imageEncoder/imageEncoderView.html.twig index 48af683f3b..e26773cbc2 100644 --- a/plugins/ImageEncoder/templates/imageEncoder/imageEncoderView.html.twig +++ b/plugins/ImageEncoder/templates/imageEncoder/imageEncoderView.html.twig @@ -1,7 +1,7 @@
{{ attachment.getTitle() }} + alt="{{ attachment.getFilename() }}">
{{ attachment.getTitle() }} + href="{{ path('attachment_show', {'id': attachment.getId()}) }}">{{ attachment.getFilename() }}
diff --git a/plugins/VideoEncoder/templates/videoEncoder/videoEncoderView.html.twig b/plugins/VideoEncoder/templates/videoEncoder/videoEncoderView.html.twig index ee35b23ea9..ca156c4660 100644 --- a/plugins/VideoEncoder/templates/videoEncoder/videoEncoderView.html.twig +++ b/plugins/VideoEncoder/templates/videoEncoder/videoEncoderView.html.twig @@ -3,7 +3,7 @@
{{ attachment.getTitle() }} + href="{{ path('attachment_show', {'id': attachment.getId()}) }}">{{ attachment.getFilename() }}
\ No newline at end of file diff --git a/src/Controller/Attachment.php b/src/Controller/Attachment.php index 8cec064cdf..31261ac45b 100644 --- a/src/Controller/Attachment.php +++ b/src/Controller/Attachment.php @@ -75,7 +75,6 @@ class Attachment extends Controller return $this->attachment($id, function ($res) use ($id, $attachment) { return [ '_template' => 'attachments/show.html.twig', - 'title' => $res['title'], 'download' => Router::url('attachment_download', ['id' => $id]), 'attachment' => $attachment, 'right_panel_vars' => ['attachment_id' => $id], @@ -91,12 +90,12 @@ class Attachment extends Controller */ public function attachment_view(Request $request, int $id) { - return $this->attachment($id, fn (array $res) => GSFile::sendFile($res['filepath'], $res['mimetype'], GSFile::titleToFilename($res['title'], $res['mimetype']), HeaderUtils::DISPOSITION_INLINE)); + return $this->attachment($id, fn (array $res) => GSFile::sendFile($res['filepath'], $res['mimetype'], GSFile::ensureFilenameWithProperExtension($res['filename'], $res['mimetype']) ?? $res['filename'], HeaderUtils::DISPOSITION_INLINE)); } public function attachment_download(Request $request, int $id) { - return $this->attachment($id, fn (array $res) => GSFile::sendFile($res['filepath'], $res['mimetype'], GSFile::titleToFilename($res['title'], $res['mimetype']), HeaderUtils::DISPOSITION_ATTACHMENT)); + return $this->attachment($id, fn (array $res) => GSFile::sendFile($res['filepath'], $res['mimetype'], GSFile::ensureFilenameWithProperExtension($res['filename'], $res['mimetype']) ?? $res['filename'], HeaderUtils::DISPOSITION_ATTACHMENT)); } /** @@ -116,14 +115,6 @@ class Attachment extends Controller { $attachment = DB::findOneBy('attachment', ['id' => $id]); - if (!is_null($attachment->getScope())) { - // @codeCoverageIgnoreStart - // && ($attachment->scope | VisibilityScope::PUBLIC) != 0 - // $user = Common::ensureLoggedIn(); - assert(false, 'Attachment scope not implemented'); - // @codeCoverageIgnoreEnd - } - $default_width = Common::config('thumbnail', 'width'); $default_height = Common::config('thumbnail', 'height'); $default_crop = Common::config('thumbnail', 'smart_crop'); @@ -140,7 +131,7 @@ class Attachment extends Controller $filename = $thumbnail->getFilename(); $path = $thumbnail->getPath(); - $mimetype = $attachment->getMimetype(); + $mimetype = $thumbnail->getMimetype(); return GSFile::sendFile(filepath: $path, mimetype: $mimetype, output_filename: $filename . '.' . MimeTypes::getDefault()->getExtensions($mimetype)[0], disposition: HeaderUtils::DISPOSITION_INLINE); } diff --git a/src/Core/GSFile.php b/src/Core/GSFile.php index 0463bd893b..4782a0086a 100644 --- a/src/Core/GSFile.php +++ b/src/Core/GSFile.php @@ -30,7 +30,6 @@ use App\Util\Exception\NoSuchFileException; use App\Util\Exception\NotFoundException; use App\Util\Exception\ServerException; use App\Util\Formatting; -use InvalidArgumentException; use SplFileInfo; use Symfony\Component\HttpFoundation\BinaryFileResponse; use Symfony\Component\HttpFoundation\HeaderUtils; @@ -51,7 +50,7 @@ use Symfony\Component\Mime\MimeTypes; class GSFile { /** - * Perform file validation (checks and normalization) and store the given file + * Perform file validation (checks and normalization), store the given file if needed and increment lives * * @param SplFileInfo $file * @param string $dest_dir @@ -63,11 +62,8 @@ class GSFile * * @return Attachment */ - public static function validateAndStoreFileAsAttachment(SplFileInfo $file, - string $dest_dir, - int $actor_id, - ?string $title = null, - bool $is_local = true): Attachment + public static function sanitizeAndStoreFileAsAttachment(SplFileInfo $file, + string $dest_dir): Attachment { if (!Formatting::startsWith($dest_dir, Common::config('storage', 'dir'))) { throw new \InvalidArgumentException("Attempted to store a file in a directory outside the GNU social files location: {$dest_dir}"); @@ -76,65 +72,27 @@ class GSFile $hash = null; Event::handle('HashFile', [$file->getPathname(), &$hash]); try { - return DB::findOneBy('attachment', ['file_hash' => $hash]); + $attachment = DB::findOneBy('attachment', ['filehash' => $hash]); + $attachment->livesIncrementAndGet(); } catch (NotFoundException) { // The following properly gets the mimetype with `file` or other // available methods, so should be safe $mimetype = $file->getMimeType(); $width = $height = null; - Event::handle('AttachmentSanitization', [&$file, &$mimetype, &$title, &$width, &$height]); - if ($is_local) { - $filesize = $file->getSize(); - Event::handle('EnforceQuota', [$actor_id, $filesize]); - } + Event::handle('AttachmentSanitization', [&$file, &$mimetype, &$width, &$height]); $attachment = Attachment::create([ - 'file_hash' => $hash, - 'gsactor_id' => $actor_id, - 'mimetype' => $mimetype, - 'title' => $title, - 'filename' => Formatting::removePrefix($dest_dir, Common::config('attachments', 'dir')) . $hash, - 'is_local' => $is_local, - 'size' => $file->getSize(), - 'width' => $width, - 'height' => $height, + 'filehash' => $hash, + 'mimetype' => $mimetype, + 'filename' => Formatting::removePrefix($dest_dir, Common::config('attachments', 'dir')) . $hash, + 'size' => $file->getSize(), + 'width' => $width, + 'height' => $height, ]); $file->move($dest_dir, $hash); DB::persist($attachment); Event::handle('AttachmentStoreNew', [&$attachment]); - return $attachment; - } - } - - /** - * Create an attachment for the given URL, fetching the mimetype - * - * @throws InvalidArgumentException - */ - public static function validateAndStoreURLAsAttachment(string $url): Attachment - { - if (Common::isValidHttpUrl($url)) { - $head = HTTPClient::head($url); - // This must come before getInfo given that Symfony HTTPClient is lazy (thus forcing curl exec) - $headers = $head->getHeaders(); - $url = $head->getInfo('url'); // The last effective url (after getHeaders so it follows redirects) - $url_hash = hash(Attachment::URLHASH_ALGO, $url); - try { - return DB::findOneBy('attachment', ['remote_url_hash' => $url_hash]); - } catch (NotFoundException) { - $headers = array_change_key_case($headers, CASE_LOWER); - $attachment = Attachment::create([ - 'remote_url' => $url, - 'remote_url_hash' => $url_hash, - 'mimetype' => $headers['content-type'][0], - 'is_local' => false, - ]); - DB::persist($attachment); - Event::handle('AttachmentStoreNew', [&$attachment]); - return $attachment; - } - } else { - throw new InvalidArgumentException(); } + return $attachment; } /** @@ -203,7 +161,7 @@ class GSFile $id, Cache::get("file-info-{$id}", function () use ($id) { - return DB::dql('select at.filename, at.mimetype, at.title ' . + return DB::dql('select at.filename, at.mimetype ' . 'from App\\Entity\\Attachment at ' . 'where at.id = :id', ['id' => $id]); @@ -255,16 +213,16 @@ class GSFile } /** - * Given an attachment title and mimetype allows to generate the most appropriate filename. + * Given an attachment filename and mimetype allows to generate the most appropriate filename. * - * @param string $title - * @param string $mimetype - * @param null|string $ext - * @param bool $force + * @param string $title Original filename with or without extension + * @param string $mimetype Original mimetype of the file + * @param null|string $ext Extension we believe to be best + * @param bool $force Should we force the extension we believe to be best? Defaults to false * * @return null|string */ - public static function titleToFilename(string $title, string $mimetype, ?string &$ext = null, bool $force = false): string | null + public static function ensureFilenameWithProperExtension(string $title, string $mimetype, ?string &$ext = null, bool $force = false): string | null { $valid_extensions = MimeTypes::getDefault()->getExtensions($mimetype); @@ -286,6 +244,9 @@ class GSFile if (!empty($valid_extensions)) { return "{$title}.{$valid_extensions[0]}"; } else { + if (!is_null($ext)) { + return ($title_without_extension ?? $title) . ".{$ext}"; + } return null; } } diff --git a/src/DataFixtures/CoreFixtures.php b/src/DataFixtures/CoreFixtures.php index 2051a39c40..2de0dc7a44 100644 --- a/src/DataFixtures/CoreFixtures.php +++ b/src/DataFixtures/CoreFixtures.php @@ -51,7 +51,7 @@ class CoreFixtures extends Fixture $copy_filepath = $filepath . '.copy'; copy($filepath, $copy_filepath); $file = new File($copy_filepath, checkPath: true); - GSFile::validateAndStoreFileAsAttachment($file, dest_dir: Common::config('attachments', 'dir') . 'test/', title: '1x1 JPEG image title', actor_id: $actors['taken_user']->getId()); + GSFile::sanitizeAndStoreFileAsAttachment($file, dest_dir: Common::config('attachments', 'dir') . 'test/'); $manager->flush(); } } diff --git a/src/Entity/Attachment.php b/src/Entity/Attachment.php index b5308eccb4..3c5226b5d6 100644 --- a/src/Entity/Attachment.php +++ b/src/Entity/Attachment.php @@ -23,7 +23,6 @@ namespace App\Entity; use App\Core\DB\DB; use App\Core\Entity; -use App\Core\Event; use App\Core\GSFile; use App\Core\Log; use App\Util\Common; @@ -40,6 +39,7 @@ use DateTimeInterface; * @author Mikael Nordfeldth * @copyright 2009-2014 Free Software Foundation, Inc http://www.fsf.org * @author Hugo Sales + * @author Diogo Peralta Cordeiro * @copyright 2020-2021 Free Software Foundation, Inc http://www.fsf.org * @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later */ @@ -48,16 +48,10 @@ class Attachment extends Entity // {{{ Autocode // @codeCoverageIgnoreStart private int $id; - private ?string $remote_url; - private ?string $remote_url_hash; - private ?string $file_hash; - private ?int $gsactor_id; + private int $lives = 1; + private ?string $filehash; private ?string $mimetype; - private ?string $title; private ?string $filename; - private ?bool $is_local; - private ?int $source; - private ?int $scope; private ?int $size; private ?int $width; private ?int $height; @@ -74,48 +68,32 @@ class Attachment extends Entity return $this->id; } - public function setRemoteUrl(?string $remote_url): self + /** + * @return int + */ + + public function getLives(): int { - $this->remote_url = $remote_url; + return $this->lives; + } + + /** + * @param int $lives + */ + public function setLives(int $lives): void + { + $this->lives = $lives; + } + + public function setFilehash(?string $filehash): self + { + $this->filehash = $filehash; return $this; } - public function getRemoteUrl(): ?string + public function getFilehash(): ?string { - return $this->remote_url; - } - - public function setRemoteUrlHash(?string $remote_url_hash): self - { - $this->remote_url_hash = $remote_url_hash; - return $this; - } - - public function getRemoteUrlHash(): ?string - { - return $this->remote_url_hash; - } - - public function setFileHash(?string $file_hash): self - { - $this->file_hash = $file_hash; - return $this; - } - - public function getFileHash(): ?string - { - return $this->file_hash; - } - - public function setGSActorId(?int $gsactor_id): self - { - $this->gsactor_id = $gsactor_id; - return $this; - } - - public function getGSActorId(): ?int - { - return $this->gsactor_id; + return $this->filehash; } public function setMimetype(?string $mimetype): self @@ -141,17 +119,6 @@ class Attachment extends Entity return is_null($mime) ? $mime : GSFile::mimetypeMinor($mime); } - public function setTitle(?string $title): self - { - $this->title = $title; - return $this; - } - - public function getTitle(): ?string - { - return $this->title; - } - public function setFilename(?string $filename): self { $this->filename = $filename; @@ -163,39 +130,6 @@ class Attachment extends Entity return $this->filename; } - public function setIsLocal(?bool $is_local): self - { - $this->is_local = $is_local; - return $this; - } - - public function getIsLocal(): ?bool - { - return $this->is_local; - } - - public function setSource(?int $source): self - { - $this->source = $source; - return $this; - } - - public function getSource(): ?int - { - return $this->source; - } - - public function setScope(?int $scope): self - { - $this->scope = $scope; - return $this; - } - - public function getScope(): ?int - { - return $this->scope; - } - public function setSize(?int $size): self { $this->size = $size; @@ -243,51 +177,63 @@ class Attachment extends Entity // @codeCoverageIgnoreEnd // }}} Autocode - const URLHASH_ALGO = 'sha256'; + /** + * @return int + */ + public function livesIncrementAndGet(): int + { + ++$this->lives; + return $this->lives; + } + + /** + * @return int + */ + public function livesDecrementAndGet(): int + { + --$this->lives; + return $this->lives; + } + const FILEHASH_ALGO = 'sha256'; - public function refCount(): int + public function kill(): bool { - $attachment_id = $this->getId(); - $dependencies = DB::count('attachment_to_note', ['attachment_id' => $attachment_id]); - Event::handle('AttachmentRefCount', [$attachment_id, &$dependencies]); - return $dependencies; + if ($this->livesDecrementAndGet() <= 0) { + return $this->delete(); + } + return true; } /** - * @depends Attachment->refCount() - * - * @return bool + * Attachment delete always removes dependencies, cleanups and flushes */ - public function isSafeDelete(): bool - { - return $this->refCount() === 0; - } - - /** - * Delete this attachment and optionally all the associated entities (avatar and/or thumbnails, which this owns) - */ - public function delete(bool $cascade = true, bool $flush = true): void + public function delete(): bool { + if ($this->getLives() > 0) { + Log::warning("Deleting file {$this->getId()} with {$this->getLives()} lives. Why are you killing it so young?"); + } + // Delete related files from storage $files = []; - if ($cascade) { - foreach ($this->getThumbnails() as $at) { - $files[] = $at->getPath(); - $at->delete(flush: false); - } + if (!is_null($filepath = $this->getPath())) { + $files[] = $filepath; + } + foreach ($this->getThumbnails() as $at) { + $files[] = $at->getPath(); + $at->delete(flush: false); } - $files[] = $this->getPath(); DB::remove($this); - if ($flush) { - DB::flush(); - } foreach ($files as $f) { if (file_exists($f)) { if (@unlink($f) === false) { - Log::warning("Failed deleting file for attachment with id={$this->id} at {$f}"); + Log::error("Failed deleting file for attachment with id={$this->getId()} at {$f}."); } + } else { + Log::warning("File for attachment with id={$this->getId()} at {$f} was already deleted when I was going to handle it."); } } + DB::flush(); + return true; } /** @@ -300,7 +246,8 @@ class Attachment extends Entity public function getPath() { - return Common::config('attachments', 'dir') . $this->getFilename(); + $filename = $this->getFilename(); + return is_null($filename) ? null : Common::config('attachments', 'dir') . $filename; } public function getAttachmentUrl() @@ -313,28 +260,23 @@ class Attachment extends Entity return [ 'name' => 'attachment', 'fields' => [ - 'id' => ['type' => 'serial', 'not null' => true], - 'remote_url' => ['type' => 'text', 'description' => 'URL after following possible redirections'], - 'remote_url_hash' => ['type' => 'varchar', 'length' => 64, 'description' => 'sha256 of destination URL (url field)'], - 'file_hash' => ['type' => 'varchar', 'length' => 64, 'description' => 'sha256 of the file contents, if the file is stored locally'], - 'gsactor_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'GSActor.id', 'multiplicity' => 'one to one', 'description' => 'If set, used so each actor can have a version of this file (for avatars, for instance)'], - 'mimetype' => ['type' => 'varchar', 'length' => 50, 'description' => 'mime type of resource'], - 'title' => ['type' => 'text', 'description' => 'title of resource when available'], - 'filename' => ['type' => 'varchar', 'length' => 191, 'description' => 'file name of resource when available'], - 'is_local' => ['type' => 'bool', 'description' => 'whether the file is stored locally'], - 'source' => ['type' => 'int', 'default' => null, 'description' => 'Source of the Attachment (upload, TFN, embed)'], - 'scope' => ['type' => 'int', 'default' => null, 'description' => 'visibility scope for this attachment'], - 'size' => ['type' => 'int', 'description' => 'size of resource when available'], - 'width' => ['type' => 'int', 'description' => 'width in pixels, if it can be described as such and data is available'], - 'height' => ['type' => 'int', 'description' => 'height in pixels, if it can be described as such and data is available'], - 'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'], + 'id' => ['type' => 'serial', 'not null' => true], + 'lives' => ['type' => 'int', 'not null' => true, 'description' => 'RefCount'], + 'filehash' => ['type' => 'varchar', 'length' => 64, 'description' => 'sha256 of the file contents, if the file is stored locally'], + 'mimetype' => ['type' => 'varchar', 'length' => 50, 'description' => 'mime type of resource'], + 'filename' => ['type' => 'varchar', 'length' => 191, 'description' => 'file name of resource when available'], + 'size' => ['type' => 'int', 'description' => 'size of resource when available'], + 'width' => ['type' => 'int', 'description' => 'width in pixels, if it can be described as such and data is available'], + 'height' => ['type' => 'int', 'description' => 'height in pixels, if it can be described as such and data is available'], + 'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'], ], 'primary key' => ['id'], 'unique keys' => [ - 'attachment_file_hash_uniq' => ['file_hash'], + 'attachment_filehash_uniq' => ['filehash'], + 'attachment_filename_uniq' => ['filename'], ], 'indexes' => [ - 'file_filehash_idx' => ['file_hash'], + 'file_filehash_idx' => ['filehash'], ], ]; } diff --git a/src/Entity/AttachmentThumbnail.php b/src/Entity/AttachmentThumbnail.php index f0798e294e..98d8b8f170 100644 --- a/src/Entity/AttachmentThumbnail.php +++ b/src/Entity/AttachmentThumbnail.php @@ -185,7 +185,7 @@ class AttachmentThumbnail extends Entity $thumbnail->setWidth($predicted_width); $thumbnail->setHeight($predicted_height); $ext = '.' . MimeTypes::getDefault()->getExtensions($temp->getMimeType())[0]; - $filename = "{$predicted_width}x{$predicted_height}{$ext}-" . $attachment->getFileHash(); + $filename = "{$predicted_width}x{$predicted_height}{$ext}-" . $attachment->getFilehash(); $thumbnail->setFilename($filename); $thumbnail->setMimetype($mimetype); DB::persist($thumbnail); diff --git a/src/Entity/AttachmentToNote.php b/src/Entity/AttachmentToNote.php index 0f57dfe0ec..84b17e126e 100644 --- a/src/Entity/AttachmentToNote.php +++ b/src/Entity/AttachmentToNote.php @@ -42,6 +42,7 @@ class AttachmentToNote extends Entity // @codeCoverageIgnoreStart private int $attachment_id; private int $note_id; + private ?string $title; private \DateTimeInterface $modified; public function setAttachmentId(int $attachment_id): self @@ -66,6 +67,17 @@ class AttachmentToNote extends Entity return $this->note_id; } + public function setTitle(?string $title): self + { + $this->title = $title; + return $this; + } + + public function getTitle(): ?string + { + return $this->title; + } + public function setModified(DateTimeInterface $modified): self { $this->modified = $modified; @@ -87,6 +99,7 @@ class AttachmentToNote extends Entity 'fields' => [ 'attachment_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'Attachment.id', 'multiplicity' => 'one to one', 'name' => 'attachment_to_note_attachment_id_fkey', 'not null' => true, 'description' => 'id of attachment'], 'note_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'Note.id', 'multiplicity' => 'one to one', 'name' => 'attachment_to_note_note_id_fkey', 'not null' => true, 'description' => 'id of the note it belongs to'], + 'title' => ['type' => 'text', 'description' => 'title of resource when available'], 'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'], ], 'primary key' => ['attachment_id', 'note_id'], diff --git a/src/Entity/GSActorToAttachment.php b/src/Entity/GSActorToAttachment.php new file mode 100644 index 0000000000..be09b6b723 --- /dev/null +++ b/src/Entity/GSActorToAttachment.php @@ -0,0 +1,95 @@ +. +// }}} + +namespace App\Entity; + +use App\Core\Entity; +use DateTimeInterface; + +/** + * Entity for relating an actor to an attachment + * + * @category DB + * @package GNUsocial + * + * @author Diogo Peralta Cordeiro + * @copyright 2021 Free Software Foundation, Inc http://www.fsf.org + * @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later + */ +class GSActorToAttachment extends Entity +{ + // {{{ Autocode + // @codeCoverageIgnoreStart + private int $attachment_id; + private int $gsactor_id; + private \DateTimeInterface $modified; + + public function setAttachmentId(int $attachment_id): self + { + $this->attachment_id = $attachment_id; + return $this; + } + + public function getAttachmentId(): int + { + return $this->attachment_id; + } + + public function setGSActorId(int $gsactor_id): self + { + $this->gsactor_id = $gsactor_id; + return $this; + } + + public function getGSActorId(): int + { + return $this->gsactor_id; + } + + public function setModified(DateTimeInterface $modified): self + { + $this->modified = $modified; + return $this; + } + + public function getModified(): DateTimeInterface + { + return $this->modified; + } + + // @codeCoverageIgnoreEnd + // }}} Autocode + + public static function schemaDef(): array + { + return [ + 'name' => 'gsactor_to_attachment', + 'fields' => [ + 'attachment_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'Attachment.id', 'multiplicity' => 'one to one', 'name' => 'attachment_to_note_attachment_id_fkey', 'not null' => true, 'description' => 'id of attachment'], + 'gsactor_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'GSActor.id', 'multiplicity' => 'one to one', 'name' => 'attachment_to_note_note_id_fkey', 'not null' => true, 'description' => 'id of the note it belongs to'], + 'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'], + ], + 'primary key' => ['attachment_id', 'gsactor_id'], + 'indexes' => [ + 'attachment_id_idx' => ['attachment_id'], + 'gsactor_id_idx' => ['gsactor_id'], + ], + ]; + } +} diff --git a/src/Entity/GSActorToRemoteURL.php b/src/Entity/GSActorToRemoteURL.php new file mode 100644 index 0000000000..1575fca46b --- /dev/null +++ b/src/Entity/GSActorToRemoteURL.php @@ -0,0 +1,95 @@ +. +// }}} + +namespace App\Entity; + +use App\Core\Entity; +use DateTimeInterface; + +/** + * Entity for relating an actor to a remote url + * + * @category DB + * @package GNUsocial + * + * @author Diogo Peralta Cordeiro + * @copyright 2021 Free Software Foundation, Inc http://www.fsf.org + * @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later + */ +class GSActorToRemoteURL extends Entity +{ + // {{{ Autocode + // @codeCoverageIgnoreStart + private int $remoteurl_id; + private int $gsactor_id; + private \DateTimeInterface $modified; + + public function setRemoteURLId(int $remoteurl_id): self + { + $this->remoteurl_id = $remoteurl_id; + return $this; + } + + public function getRemoteURLId(): int + { + return $this->remoteurl_id; + } + + public function setGSActorId(int $gsactor_id): self + { + $this->gsactor_id = $gsactor_id; + return $this; + } + + public function getGSActorId(): int + { + return $this->gsactor_id; + } + + public function setModified(DateTimeInterface $modified): self + { + $this->modified = $modified; + return $this; + } + + public function getModified(): DateTimeInterface + { + return $this->modified; + } + + // @codeCoverageIgnoreEnd + // }}} Autocode + + public static function schemaDef(): array + { + return [ + 'name' => 'gsactor_to_remoteurl', + 'fields' => [ + 'remoteurl_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'RemoteURL.id', 'multiplicity' => 'one to one', 'name' => 'attachment_to_note_attachment_id_fkey', 'not null' => true, 'description' => 'id of attachment'], + 'gsactor_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'GSActor.id', 'multiplicity' => 'one to one', 'name' => 'attachment_to_note_note_id_fkey', 'not null' => true, 'description' => 'id of the note it belongs to'], + 'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'], + ], + 'primary key' => ['remoteurl_id', 'gsactor_id'], + 'indexes' => [ + 'remoteurl_id_idx' => ['remoteurl_id'], + 'gsactor_id_idx' => ['gsactor_id'], + ], + ]; + } +} diff --git a/src/Entity/RemoteURL.php b/src/Entity/RemoteURL.php new file mode 100644 index 0000000000..6b28db3d93 --- /dev/null +++ b/src/Entity/RemoteURL.php @@ -0,0 +1,177 @@ +. +// }}} + +namespace App\Entity; + +use App\Core\DB\DB; +use App\Core\Entity; +use App\Core\Event; +use App\Core\HTTPClient; +use App\Util\Common; +use App\Util\Exception\DuplicateFoundException; +use App\Util\Exception\NotFoundException; +use DateTimeInterface; +use InvalidArgumentException; + +/** + * Entity for representing a RemoteURL + * + * @category DB + * @package GNUsocial + * + * @author Diogo Peralta Cordeiro + * @copyright 2021 Free Software Foundation, Inc http://www.fsf.org + * @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later + */ +class RemoteURL extends Entity +{ + // {{{ Autocode + // @codeCoverageIgnoreStart + private int $id; + private ?string $remote_url; + private ?string $remote_url_hash; + private ?string $mimetype; + private DateTimeInterface $modified; + + public function setId(int $id): self + { + $this->id = $id; + return $this; + } + + public function getId(): int + { + return $this->id; + } + + public function getRemoteUrl(): ?string + { + return $this->remote_url; + } + + public function setRemoteUrl(?string $remote_url): self + { + $this->remote_url = $remote_url; + return $this; + } + + public function setRemoteUrlHash(?string $remote_url_hash): self + { + $this->remote_url_hash = $remote_url_hash; + return $this; + } + + public function getRemoteUrlHash(): ?string + { + return $this->remote_url_hash; + } + + public function setMimetype(?string $mimetype): self + { + $this->mimetype = $mimetype; + return $this; + } + + public function getMimetype(): ?string + { + return $this->mimetype; + } + + public function getMimetypeMajor(): ?string + { + $mime = $this->getMimetype(); + return is_null($mime) ? $mime : GSFile::mimetypeMajor($mime); + } + + public function getMimetypeMinor(): ?string + { + $mime = $this->getMimetype(); + return is_null($mime) ? $mime : GSFile::mimetypeMinor($mime); + } + + public function setModified(DateTimeInterface $modified): self + { + $this->modified = $modified; + return $this; + } + + public function getModified(): DateTimeInterface + { + return $this->modified; + } + + // @codeCoverageIgnoreEnd + // }}} Autocode + + const URLHASH_ALGO = 'sha256'; + + /** + * Create an attachment for the given URL, fetching the mimetype + * + * @param string $url + * + * @throws DuplicateFoundException + * @throws InvalidArgumentException + * + * @return RemoteURL + */ + public static function getOrCreate(string $url): self + { + if (Common::isValidHttpUrl($url)) { + $head = HTTPClient::head($url); + // This must come before getInfo given that Symfony HTTPClient is lazy (thus forcing curl exec) + $headers = $head->getHeaders(); + $url = $head->getInfo('url'); // The last effective url (after getHeaders so it follows redirects) + $url_hash = hash(self::URLHASH_ALGO, $url); + try { + return DB::findOneBy('remoteurl', ['remote_url_hash' => $url_hash]); + } catch (NotFoundException) { + $headers = array_change_key_case($headers, CASE_LOWER); + $remoteurl = self::create([ + 'remote_url' => $url, + 'remote_url_hash' => $url_hash, + 'mimetype' => $headers['content-type'][0], + ]); + DB::persist($remoteurl); + Event::handle('RemoteURLStoreNew', [&$remoteurl]); + return $remoteurl; + } + } else { + throw new InvalidArgumentException(); + } + } + + public static function schemaDef(): array + { + return [ + 'name' => 'remoteurl', + 'fields' => [ + 'id' => ['type' => 'serial', 'not null' => true], + 'remote_url' => ['type' => 'text', 'description' => 'URL after following possible redirections'], + 'remote_url_hash' => ['type' => 'varchar', 'length' => 64, 'description' => 'sha256 of destination URL (url field)'], + 'mimetype' => ['type' => 'varchar', 'length' => 50, 'description' => 'mime type of resource'], + 'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'], + ], + 'primary key' => ['id'], + 'indexes' => [ + 'gsactor_remote_url_hash_idx' => ['remote_url_hash'], + ], + ]; + } +} diff --git a/src/Entity/RemoteURLToAttachment.php b/src/Entity/RemoteURLToAttachment.php new file mode 100644 index 0000000000..e6094e06d9 --- /dev/null +++ b/src/Entity/RemoteURLToAttachment.php @@ -0,0 +1,95 @@ +. +// }}} + +namespace App\Entity; + +use App\Core\Entity; +use DateTimeInterface; + +/** + * Entity for relating a remote url to an attachment + * + * @category DB + * @package GNUsocial + * + * @author Diogo Peralta Cordeiro + * @copyright 2021 Free Software Foundation, Inc http://www.fsf.org + * @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later + */ +class RemoteURLToAttachment extends Entity +{ + // {{{ Autocode + // @codeCoverageIgnoreStart + private int $attachment_id; + private int $remoteurl_id; + private \DateTimeInterface $modified; + + public function setAttachmentId(int $attachment_id): self + { + $this->attachment_id = $attachment_id; + return $this; + } + + public function getAttachmentId(): int + { + return $this->attachment_id; + } + + public function setRemoteURLId(int $remoteurl_id): self + { + $this->remoteurl_id = $remoteurl_id; + return $this; + } + + public function getRemoteURLId(): int + { + return $this->remoteurl_id; + } + + public function setModified(DateTimeInterface $modified): self + { + $this->modified = $modified; + return $this; + } + + public function getModified(): DateTimeInterface + { + return $this->modified; + } + + // @codeCoverageIgnoreEnd + // }}} Autocode + + public static function schemaDef(): array + { + return [ + 'name' => 'remoteurl_to_attachment', + 'fields' => [ + 'remoteurl_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'RemoteURL.id', 'multiplicity' => 'one to one', 'name' => 'attachment_to_note_note_id_fkey', 'not null' => true, 'description' => 'id of the note it belongs to'], + 'attachment_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'Attachment.id', 'multiplicity' => 'one to one', 'name' => 'attachment_to_note_attachment_id_fkey', 'not null' => true, 'description' => 'id of attachment'], + 'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'], + ], + 'primary key' => ['remoteurl_id', 'attachment_id'], + 'indexes' => [ + 'remoteurl_id_idx' => ['remoteurl_id'], + 'attachment_id_idx' => ['attachment_id'], + ], + ]; + } +} diff --git a/src/Entity/RemoteURLToNote.php b/src/Entity/RemoteURLToNote.php new file mode 100644 index 0000000000..a34ac71771 --- /dev/null +++ b/src/Entity/RemoteURLToNote.php @@ -0,0 +1,95 @@ +. +// }}} + +namespace App\Entity; + +use App\Core\Entity; +use DateTimeInterface; + +/** + * Entity for relating a RemoteURL to a post + * + * @category DB + * @package GNUsocial + * + * @author Diogo Peralta Cordeiro + * @copyright 2021 Free Software Foundation, Inc http://www.fsf.org + * @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later + */ +class RemoteURLToNote extends Entity +{ + // {{{ Autocode + // @codeCoverageIgnoreStart + private int $remoteurl_id; + private int $note_id; + private \DateTimeInterface $modified; + + public function setRemoteURLId(int $remoteurl_id): self + { + $this->remoteurl_id = $remoteurl_id; + return $this; + } + + public function getRemoteURLId(): int + { + return $this->remoteurl_id; + } + + public function setNoteId(int $note_id): self + { + $this->note_id = $note_id; + return $this; + } + + public function getNoteId(): int + { + return $this->note_id; + } + + public function setModified(DateTimeInterface $modified): self + { + $this->modified = $modified; + return $this; + } + + public function getModified(): DateTimeInterface + { + return $this->modified; + } + + // @codeCoverageIgnoreEnd + // }}} Autocode + + public static function schemaDef(): array + { + return [ + 'name' => 'remoteurl_to_note', + 'fields' => [ + 'remoteurl_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'remoteurl.id', 'multiplicity' => 'one to one', 'name' => 'remoteurl_to_note_remoteurl_id_fkey', 'not null' => true, 'description' => 'id of remoteurl'], + 'note_id' => ['type' => 'int', 'foreign key' => true, 'target' => 'Note.id', 'multiplicity' => 'one to one', 'name' => 'remoteurl_to_note_note_id_fkey', 'not null' => true, 'description' => 'id of the note it belongs to'], + 'modified' => ['type' => 'timestamp', 'not null' => true, 'default' => 'CURRENT_TIMESTAMP', 'description' => 'date this record was modified'], + ], + 'primary key' => ['remoteurl_id', 'note_id'], + 'indexes' => [ + 'remoteurl_id_idx' => ['remoteurl_id'], + 'note_id_idx' => ['note_id'], + ], + ]; + } +} diff --git a/templates/attachments/view.html.twig b/templates/attachments/view.html.twig index 40bf772a95..c444ebb7a3 100644 --- a/templates/attachments/view.html.twig +++ b/templates/attachments/view.html.twig @@ -1,17 +1,12 @@ {% set thumbnail_parameters = {'id': attachment.getId(), 'w': config('thumbnail','width'), 'h': config('thumbnail','height')} %} -{% if attachment.getIsLocal() %} - {% set handled = false %} - {% for block in handle_event('ViewAttachment' ~ attachment.getMimetypeMajor() | capitalize , {'attachment': attachment, 'thumbnail_parameters': thumbnail_parameters}) %} - {% set handled = true %} - {{ block | raw }} - {% endfor %} - {% if not handled %} - - {% endif %} -{% else %} - {% for block in handle_event('ViewRemoteAttachment', {'attachment': attachment, 'thumbnail_parameters': thumbnail_parameters}) %} - {{ block | raw }} - {% endfor %} +{% set handled = false %} +{% for block in handle_event('ViewAttachment' ~ attachment.getMimetypeMajor() | capitalize , {'attachment': attachment, 'thumbnail_parameters': thumbnail_parameters}) %} + {% set handled = true %} + {{ block | raw }} +{% endfor %} +{% if not handled %} + {% endif %} + diff --git a/tests/Controller/AttachmentTest.php b/tests/Controller/AttachmentTest.php index 866cd6a59d..0501bc1eaa 100644 --- a/tests/Controller/AttachmentTest.php +++ b/tests/Controller/AttachmentTest.php @@ -46,7 +46,7 @@ class AttachmentTest extends GNUsocialTestCase private function testAttachment(string $suffix) { $client = static::createClient(); - $attachment = DB::findOneBy('attachment', ['title' => '1x1 JPEG image title']); + $attachment = DB::findOneBy('attachment', ['filehash' => '5d8ee7ead51a28803b4ee5cb2306a0b90b6ba570f1e5bcc2209926f6ab08e7ea']); $crawler = $client->request('GET', "/attachment/{$attachment->getId()}{$suffix}"); } @@ -54,7 +54,7 @@ class AttachmentTest extends GNUsocialTestCase { $this->testAttachment(''); $this->assertResponseIsSuccessful(); - $this->assertSelectorTextContains('figure figcaption', '1x1 JPEG image title'); + $this->assertSelectorTextContains('figure figcaption', '5d8ee7ead51a28803b4ee5cb2306a0b90b6ba570f1e5bcc2209926f6ab08e7ea'); } public function testAttachmentView()