feature #28937 Improve Translator caching (rpkamp)

This PR was squashed before being merged into the 4.3-dev branch (closes #28937).

Discussion
----------

Improve Translator caching

| Q             | A
| ------------- | ---
| Branch?       | master
| Bug fix?      | no
| New feature?  | yes
| BC breaks?    | no
| Deprecations? | no
| Tests pass?   | yes
| Fixed tickets | #27600
| License       | MIT
| Doc PR        | N/A

Add DirectoryResources to MessageCatalogues when loaded.

So that when a file is added to one of the directories the cache for all MessageCatalogues will be invalidated.

All directories must be added to all MessageCatalogues because we can't predict for which locale files will be added to each individual directory.

Also, now that the translator keeps track of its own directories for caching the container no longer needs to it. This means that when a translation changes or is added the container no longer needs to be fully rebuilt, saving a considerable amount of time (compilation time went down from ~4 seconds to ~1 second on each translation change/add in our project).

Commits
-------

a5246589cf Improve Translator caching

src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php

@@ -1087,6 +1087,7 @@ class FrameworkExtension extends Extension
         // Discover translation directories
         $dirs = [];
         $transPaths = [];
+        $nonExistingDirs = [];
         if (class_exists('Symfony\Component\Validator\Validation')) {
             $r = new \ReflectionClass('Symfony\Component\Validator\Validation');
 
@@ -1105,18 +1106,21 @@ class FrameworkExtension extends Extension
         $defaultDir = $container->getParameterBag()->resolveValue($config['default_path']);
         $rootDir = $container->getParameter('kernel.root_dir');
         foreach ($container->getParameter('kernel.bundles_metadata') as $name => $bundle) {
-            if ($container->fileExists($dir = $bundle['path'].'/Resources/translations')) {
+            if (\is_dir($dir = $bundle['path'].'/Resources/translations')) {
                 $dirs[] = $dir;
+            } else {
+                $nonExistingDirs[] = $dir;
             }
-            if ($container->fileExists($dir = $rootDir.sprintf('/Resources/%s/translations', $name))) {
+            if (\is_dir($dir = $rootDir.sprintf('/Resources/%s/translations', $name))) {
                 @trigger_error(sprintf('Translations directory "%s" is deprecated since Symfony 4.2, use "%s" instead.', $dir, $defaultDir), E_USER_DEPRECATED);
-
                 $dirs[] = $dir;
+            } else {
+                $nonExistingDirs[] = $dir;
             }
         }
 
         foreach ($config['paths'] as $dir) {
-            if ($container->fileExists($dir)) {
+            if (\is_dir($dir)) {
                 $dirs[] = $transPaths[] = $dir;
             } else {
                 throw new \UnexpectedValueException(sprintf('%s defined in translator.paths does not exist or is not a directory', $dir));
@@ -1131,15 +1135,20 @@ class FrameworkExtension extends Extension
             $container->getDefinition('console.command.translation_update')->replaceArgument(6, $transPaths);
         }
 
-        if ($container->fileExists($defaultDir)) {
+        if (\is_dir($defaultDir)) {
             $dirs[] = $defaultDir;
+        } else {
+            $nonExistingDirs[] = $defaultDir;
         }
-        if ($container->fileExists($dir = $rootDir.'/Resources/translations')) {
+
+        if (\is_dir($dir = $rootDir.'/Resources/translations')) {
             if ($dir !== $defaultDir) {
                 @trigger_error(sprintf('Translations directory "%s" is deprecated since Symfony 4.2, use "%s" instead.', $dir, $defaultDir), E_USER_DEPRECATED);
             }
 
             $dirs[] = $dir;
+        } else {
+            $nonExistingDirs[] = $dir;
         }
 
         // Register translation resources
@@ -1166,7 +1175,10 @@ class FrameworkExtension extends Extension
 
             $options = array_merge(
                 $translator->getArgument(4),
-                ['resource_files' => $files]
+                [
+                    'resource_files' => $files,
+                    'scanned_directories' => \array_merge($dirs, $nonExistingDirs),
+                ]
             );
 
             $translator->replaceArgument(4, $options);

src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/FrameworkExtensionTest.php

@@ -25,6 +25,8 @@ use Symfony\Component\Cache\Adapter\DoctrineAdapter;
 use Symfony\Component\Cache\Adapter\FilesystemAdapter;
 use Symfony\Component\Cache\Adapter\ProxyAdapter;
 use Symfony\Component\Cache\Adapter\RedisAdapter;
+use Symfony\Component\Config\Resource\DirectoryResource;
+use Symfony\Component\Config\Resource\FileExistenceResource;
 use Symfony\Component\DependencyInjection\ChildDefinition;
 use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface;
 use Symfony\Component\DependencyInjection\Compiler\ResolveInstanceofConditionalsPass;
@@ -797,6 +799,26 @@ abstract class FrameworkExtensionTest extends TestCase
 
         $calls = $container->getDefinition('translator.default')->getMethodCalls();
         $this->assertEquals(['fr'], $calls[1][1][0]);
+
+        $nonExistingDirectories = array_filter(
+            $options['scanned_directories'],
+            function ($directory) {
+                return !file_exists($directory);
+            }
+        );
+
+        $this->assertNotEmpty($nonExistingDirectories, 'FrameworkBundle should pass non existing directories to Translator');
+
+        $resources = $container->getResources();
+        foreach ($resources as $resource) {
+            if ($resource instanceof DirectoryResource) {
+                $this->assertNotContains('translations', $resource->getResource());
+            }
+
+            if ($resource instanceof FileExistenceResource) {
+                $this->assertNotContains('translations', $resource->getResource());
+            }
+        }
     }
 
     /**

src/Symfony/Bundle/FrameworkBundle/Tests/Translation/TranslatorTest.php

@@ -14,6 +14,8 @@ namespace Symfony\Bundle\FrameworkBundle\Tests\Translation;
 use PHPUnit\Framework\TestCase;
 use Psr\Container\ContainerInterface;
 use Symfony\Bundle\FrameworkBundle\Translation\Translator;
+use Symfony\Component\Config\Resource\DirectoryResource;
+use Symfony\Component\Config\Resource\FileExistenceResource;
 use Symfony\Component\Filesystem\Filesystem;
 use Symfony\Component\Translation\Formatter\MessageFormatter;
 use Symfony\Component\Translation\MessageCatalogue;
@@ -223,6 +225,29 @@ class TranslatorTest extends TestCase
         ];
     }
 
+    public function testCatalogResourcesAreAddedForScannedDirectories()
+    {
+        $loader = new \Symfony\Component\Translation\Loader\YamlFileLoader();
+        $resourceFiles = [
+            'fr' => [
+                __DIR__.'/../Fixtures/Resources/translations/messages.fr.yml',
+            ],
+        ];
+
+        /** @var Translator $translator */
+        $translator = $this->getTranslator($loader, [
+            'resource_files' => $resourceFiles,
+            'scanned_directories' => [__DIR__, '/tmp/I/sure/hope/this/does/not/exist'],
+        ], 'yml');
+
+        $catalogue = $translator->getCatalogue('fr');
+
+        $resources = $catalogue->getResources();
+
+        $this->assertEquals(new DirectoryResource(__DIR__), $resources[1]);
+        $this->assertEquals(new FileExistenceResource('/tmp/I/sure/hope/this/does/not/exist'), $resources[2]);
+    }
+
     protected function getCatalogue($locale, $messages, $resources = [])
     {
         $catalogue = new MessageCatalogue($locale);

src/Symfony/Bundle/FrameworkBundle/Translation/Translator.php

@@ -12,6 +12,8 @@
 namespace Symfony\Bundle\FrameworkBundle\Translation;
 
 use Psr\Container\ContainerInterface;
+use Symfony\Component\Config\Resource\DirectoryResource;
+use Symfony\Component\Config\Resource\FileExistenceResource;
 use Symfony\Component\HttpKernel\CacheWarmer\WarmableInterface;
 use Symfony\Component\Translation\Exception\InvalidArgumentException;
 use Symfony\Component\Translation\Formatter\MessageFormatterInterface;
@@ -31,6 +33,7 @@ class Translator extends BaseTranslator implements WarmableInterface
         'cache_dir' => null,
         'debug' => false,
         'resource_files' => [],
+        'scanned_directories' => [],
     ];
 
     /**
@@ -48,6 +51,11 @@ class Translator extends BaseTranslator implements WarmableInterface
 
     private $resourceFiles;
 
+    /**
+     * @var string[]
+     */
+    private $scannedDirectories;
+
     /**
      * Constructor.
      *
@@ -78,6 +86,7 @@ class Translator extends BaseTranslator implements WarmableInterface
         $this->options = array_merge($this->options, $options);
         $this->resourceLocales = array_keys($this->options['resource_files']);
         $this->resourceFiles = $this->options['resource_files'];
+        $this->scannedDirectories = $this->options['scanned_directories'];
 
         parent::__construct($defaultLocale, $formatter, $this->options['cache_dir'], $this->options['debug']);
     }
@@ -120,6 +129,16 @@ class Translator extends BaseTranslator implements WarmableInterface
         parent::initializeCatalogue($locale);
     }
 
+    protected function doLoadCatalogue($locale): void
+    {
+        parent::doLoadCatalogue($locale);
+
+        foreach ($this->scannedDirectories as $directory) {
+            $resourceClass = file_exists($directory) ? DirectoryResource::class : FileExistenceResource::class;
+            $this->catalogues[$locale]->addResource(new $resourceClass($directory));
+        }
+    }
+
     protected function initialize()
     {
         if ($this->resourceFiles) {

src/Symfony/Component/Translation/Translator.php

@@ -395,7 +395,10 @@ EOF
         return $this->cacheDir.'/catalogue.'.$locale.'.'.strtr(substr(base64_encode(hash('sha256', serialize($this->fallbackLocales), true)), 0, 7), '/', '_').'.php';
     }
 
-    private function doLoadCatalogue($locale): void
+    /**
+     * @internal
+     */
+    protected function doLoadCatalogue($locale): void
     {
         $this->catalogues[$locale] = new MessageCatalogue($locale);