feature #36209 [HttpKernel] allow cache warmers to add to the list of preloaded classes and files (nicolas-grekas)

This PR was merged into the 5.1-dev branch.

Discussion
----------

[HttpKernel] allow cache warmers to add to the list of preloaded classes and files

| Q             | A
| ------------- | ---
| Branch?       | master
| Bug fix?      | no
| New feature?  | yes
| Deprecations? | yes
| Tickets       | -
| License       | MIT
| Doc PR        | -

This PR makes cache warmers responsible for returning a list of classes or files to preload. It does so by adding the following to `WarmableInterface::warmUp()`:
`@return string[] A list of classes or files to preload on PHP 7.4+`

Of course, this return value is properly implemented so that we can see what this provides in practice. Here are the benchmarks on a simple Hello World rendered with Twig:
- without preloading: 360 req/s
- with preloading in master: 560 req/s (+55%)
- with preloading and this PR: 630 req/s (+75%)

Commits
-------

8ab75d99d4 [HttpKernel] allow cache warmers to add to the list of preloaded classes and files

UPGRADE-5.1.md

@@ -59,6 +59,8 @@ HttpFoundation
 HttpKernel
 ----------
 
+ * Made `WarmableInterface::warmUp()` return a list of classes or files to preload on PHP 7.4+
+   not returning an array is deprecated
  * Deprecated support for `service:action` syntax to reference controllers. Use `serviceOrFqcn::method` instead.
 
 Mailer

UPGRADE-6.0.md

@@ -56,6 +56,7 @@ HttpFoundation
 HttpKernel
 ----------
 
+ * Made `WarmableInterface::warmUp()` return a list of classes or files to preload on PHP 7.4+
  * Removed support for `service:action` syntax to reference controllers. Use `serviceOrFqcn::method` instead.
 
 Messenger

src/Symfony/Bridge/Doctrine/CacheWarmer/ProxyCacheWarmer.php

@@ -43,9 +43,12 @@ class ProxyCacheWarmer implements CacheWarmerInterface
 
     /**
      * {@inheritdoc}
+     *
+     * @return string[] A list of files to preload on PHP 7.4+
      */
     public function warmUp(string $cacheDir)
     {
+        $files = [];
         foreach ($this->registry->getManagers() as $em) {
             // we need the directory no matter the proxy cache generation strategy
             if (!is_dir($proxyCacheDir = $em->getConfiguration()->getProxyDir())) {
@@ -64,6 +67,14 @@ class ProxyCacheWarmer implements CacheWarmerInterface
             $classes = $em->getMetadataFactory()->getAllMetadata();
 
             $em->getProxyFactory()->generateProxyClasses($classes);
+
+            foreach (scandir($proxyCacheDir) as $file) {
+                if (!is_dir($file = $proxyCacheDir.'/'.$file)) {
+                    $files[] = $file;
+                }
+            }
         }
+
+        return $files;
     }
 }

src/Symfony/Bundle/FrameworkBundle/CacheWarmer/AbstractPhpFileCacheWarmer.php

@@ -42,6 +42,8 @@ abstract class AbstractPhpFileCacheWarmer implements CacheWarmerInterface
 
     /**
      * {@inheritdoc}
+     *
+     * @return string[] A list of classes to preload on PHP 7.4+
      */
     public function warmUp(string $cacheDir)
     {
@@ -61,12 +63,15 @@ abstract class AbstractPhpFileCacheWarmer implements CacheWarmerInterface
         // so here we un-serialize the values first
         $values = array_map(function ($val) { return null !== $val ? unserialize($val) : null; }, $arrayAdapter->getValues());
 
-        $this->warmUpPhpArrayAdapter(new PhpArrayAdapter($this->phpArrayFile, new NullAdapter()), $values);
+        return $this->warmUpPhpArrayAdapter(new PhpArrayAdapter($this->phpArrayFile, new NullAdapter()), $values);
     }
 
+    /**
+     * @return string[] A list of classes to preload on PHP 7.4+
+     */
     protected function warmUpPhpArrayAdapter(PhpArrayAdapter $phpArrayAdapter, array $values)
     {
-        $phpArrayAdapter->warmUp($values);
+        return (array) $phpArrayAdapter->warmUp($values);
     }
 
     /**

src/Symfony/Bundle/FrameworkBundle/CacheWarmer/CachePoolClearerCacheWarmer.php

@@ -36,14 +36,18 @@ final class CachePoolClearerCacheWarmer implements CacheWarmerInterface
 
     /**
      * {@inheritdoc}
+     *
+     * @return string[]
      */
-    public function warmUp($cacheDirectory): void
+    public function warmUp($cacheDirectory): array
     {
         foreach ($this->pools as $pool) {
             if ($this->poolClearer->hasPool($pool)) {
                 $this->poolClearer->clearPool($pool);
             }
         }
+
+        return [];
     }
 
     /**

src/Symfony/Bundle/FrameworkBundle/CacheWarmer/RouterCacheWarmer.php

@@ -36,15 +36,15 @@ class RouterCacheWarmer implements CacheWarmerInterface, ServiceSubscriberInterf
 
     /**
      * {@inheritdoc}
+     *
+     * @return string[]
      */
     public function warmUp(string $cacheDir)
     {
         $router = $this->container->get('router');
 
         if ($router instanceof WarmableInterface) {
-            $router->warmUp($cacheDir);
-
-            return;
+            return (array) $router->warmUp($cacheDir);
         }
 
         throw new \LogicException(sprintf('The router "%s" cannot be warmed up because it does not implement "%s".', get_debug_type($router), WarmableInterface::class));

src/Symfony/Bundle/FrameworkBundle/CacheWarmer/TranslationsCacheWarmer.php

@@ -35,6 +35,8 @@ class TranslationsCacheWarmer implements CacheWarmerInterface, ServiceSubscriber
 
     /**
      * {@inheritdoc}
+     *
+     * @return string[]
      */
     public function warmUp(string $cacheDir)
     {
@@ -43,8 +45,10 @@ class TranslationsCacheWarmer implements CacheWarmerInterface, ServiceSubscriber
         }
 
         if ($this->translator instanceof WarmableInterface) {
-            $this->translator->warmUp($cacheDir);
+            return (array) $this->translator->warmUp($cacheDir);
         }
+
+        return [];
     }
 
     /**

src/Symfony/Bundle/FrameworkBundle/CacheWarmer/ValidatorCacheWarmer.php

@@ -68,10 +68,13 @@ class ValidatorCacheWarmer extends AbstractPhpFileCacheWarmer
         return true;
     }
 
+    /**
+     * @return string[] A list of classes to preload on PHP 7.4+
+     */
     protected function warmUpPhpArrayAdapter(PhpArrayAdapter $phpArrayAdapter, array $values)
     {
         // make sure we don't cache null values
-        parent::warmUpPhpArrayAdapter($phpArrayAdapter, array_filter($values));
+        return parent::warmUpPhpArrayAdapter($phpArrayAdapter, array_filter($values));
     }
 
     /**

src/Symfony/Bundle/FrameworkBundle/Command/CacheClearCommand.php

@@ -17,6 +17,7 @@ use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Style\SymfonyStyle;
+use Symfony\Component\DependencyInjection\Dumper\Preloader;
 use Symfony\Component\EventDispatcher\EventDispatcher;
 use Symfony\Component\Filesystem\Exception\IOException;
 use Symfony\Component\Filesystem\Filesystem;
@@ -117,7 +118,11 @@ EOF
                 $warmer = $kernel->getContainer()->get('cache_warmer');
                 // non optional warmers already ran during container compilation
                 $warmer->enableOnlyOptionalWarmers();
-                $warmer->warmUp($realCacheDir);
+                $preload = (array) $warmer->warmUp($warmupDir);
+
+                if (file_exists($preloadFile = $warmupDir.'/'.$kernel->getContainer()->getParameter('kernel.container_class').'.preload.php')) {
+                    Preloader::append($preloadFile, $preload);
+                }
             }
         } else {
             $fs->mkdir($warmupDir);
@@ -193,7 +198,11 @@ EOF
             $warmer = $kernel->getContainer()->get('cache_warmer');
             // non optional warmers already ran during container compilation
             $warmer->enableOnlyOptionalWarmers();
-            $warmer->warmUp($warmupDir);
+            $preload = (array) $warmer->warmUp($warmupDir);
+
+            if (file_exists($preloadFile = $warmupDir.'/'.$kernel->getContainer()->getParameter('kernel.container_class').'.preload.php')) {
+                Preloader::append($preloadFile, $preload);
+            }
         }
 
         // fix references to cached files with the real cache directory name

src/Symfony/Bundle/FrameworkBundle/Routing/Router.php

@@ -21,16 +21,11 @@ use Symfony\Component\DependencyInjection\ContainerInterface as SymfonyContainer
 use Symfony\Component\DependencyInjection\Exception\ParameterNotFoundException;
 use Symfony\Component\DependencyInjection\Exception\RuntimeException;
 use Symfony\Component\HttpKernel\CacheWarmer\WarmableInterface;
-use Symfony\Component\Routing\Annotation\Route;
 use Symfony\Component\Routing\RequestContext;
 use Symfony\Component\Routing\RouteCollection;
 use Symfony\Component\Routing\Router as BaseRouter;
 use Symfony\Contracts\Service\ServiceSubscriberInterface;
 
-// Help opcache.preload discover always-needed symbols
-class_exists(RedirectableCompiledUrlMatcher::class);
-class_exists(Route::class);
-
 /**
  * This Router creates the Loader only when the cache is empty.
  *
@@ -90,6 +85,8 @@ class Router extends BaseRouter implements WarmableInterface, ServiceSubscriberI
 
     /**
      * {@inheritdoc}
+     *
+     * @return string[] A list of classes to preload on PHP 7.4+
      */
     public function warmUp(string $cacheDir)
     {
@@ -101,6 +98,11 @@ class Router extends BaseRouter implements WarmableInterface, ServiceSubscriberI
         $this->getGenerator();
 
         $this->setOption('cache_dir', $currentDir);
+
+        return [
+            $this->getOption('generator_class'),
+            $this->getOption('matcher_class'),
+        ];
     }
 
     /**

src/Symfony/Bundle/FrameworkBundle/Translation/Translator.php

@@ -95,6 +95,8 @@ class Translator extends BaseTranslator implements WarmableInterface
 
     /**
      * {@inheritdoc}
+     *
+     * @return string[]
      */
     public function warmUp(string $cacheDir)
     {
@@ -113,6 +115,8 @@ class Translator extends BaseTranslator implements WarmableInterface
 
             $this->loadCatalogue($locale);
         }
+
+        return [];
     }
 
     public function addResource(string $format, $resource, string $locale, string $domain = null)

src/Symfony/Bundle/SecurityBundle/CacheWarmer/ExpressionCacheWarmer.php

@@ -34,10 +34,15 @@ class ExpressionCacheWarmer implements CacheWarmerInterface
         return true;
     }
 
+    /**
+     * @return string[]
+     */
     public function warmUp(string $cacheDir)
     {
         foreach ($this->expressions as $expression) {
             $this->expressionLanguage->parse($expression, ['token', 'user', 'object', 'subject', 'roles', 'request', 'trust_resolver']);
         }
+
+        return [];
     }
 }

src/Symfony/Bundle/TwigBundle/CacheWarmer/TemplateCacheWarmer.php

@@ -37,6 +37,8 @@ class TemplateCacheWarmer implements CacheWarmerInterface, ServiceSubscriberInte
 
     /**
      * {@inheritdoc}
+     *
+     * @return string[] A list of template files to preload on PHP 7.4+
      */
     public function warmUp(string $cacheDir)
     {
@@ -44,14 +46,22 @@ class TemplateCacheWarmer implements CacheWarmerInterface, ServiceSubscriberInte
             $this->twig = $this->container->get('twig');
         }
 
+        $files = [];
+
         foreach ($this->iterator as $template) {
             try {
-                $this->twig->load($template);
+                $template = $this->twig->load($template);
+
+                if (\is_callable([$template, 'unwrap'])) {
+                    $files[] = (new \ReflectionClass($template->unwrap()))->getFileName();
+                }
             } catch (Error $e) {
                 // problem during compilation, give up
                 // might be a syntax error or a non-Twig template
             }
         }
+
+        return $files;
     }
 
     /**

src/Symfony/Component/Cache/Adapter/PhpArrayAdapter.php

@@ -291,6 +291,8 @@ class PhpArrayAdapter implements AdapterInterface, CacheInterface, PruneableInte
      * Store an array of cached values.
      *
      * @param array $values The cached values
+     *
+     * @return string[] A list of classes to preload on PHP 7.4+
      */
     public function warmUp(array $values)
     {
@@ -314,6 +316,7 @@ class PhpArrayAdapter implements AdapterInterface, CacheInterface, PruneableInte
             }
         }
 
+        $preload = [];
         $dumpedValues = '';
         $dumpedMap = [];
         $dump = <<<'EOF'
@@ -334,7 +337,7 @@ EOF;
                 $value = "'N;'";
             } elseif (\is_object($value) || \is_array($value)) {
                 try {
-                    $value = VarExporter::export($value, $isStaticValue);
+                    $value = VarExporter::export($value, $isStaticValue, $preload);
                 } catch (\Exception $e) {
                     throw new InvalidArgumentException(sprintf('Cache key "%s" has non-serializable "%s" value.', $key, get_debug_type($value)), 0, $e);
                 }
@@ -376,6 +379,8 @@ EOF;
         unset(self::$valuesCache[$this->file]);
 
         $this->initialize();
+
+        return $preload;
     }
 
     /**

src/Symfony/Component/DependencyInjection/CHANGELOG.md

@@ -12,6 +12,7 @@ CHANGELOG
  * updated the signature of method `DeprecateTrait::deprecate()` to `DeprecateTrait::deprecation(string $package, string $version, string $message)`
  * deprecated the `Psr\Container\ContainerInterface` and `Symfony\Component\DependencyInjection\ContainerInterface` aliases of the `service_container` service,
    configure them explicitly instead
+ * added class `Symfony\Component\DependencyInjection\Dumper\Preloader` to help with preloading on PHP 7.4+
 
 5.0.0
 -----

src/Symfony/Component/DependencyInjection/Dumper/Preloader.php

@@ -13,12 +13,31 @@ namespace Symfony\Component\DependencyInjection\Dumper;
 
 /**
  * @author Nicolas Grekas <p@tchwork.com>
- *
- * @internal
  */
-class Preloader
+final class Preloader
 {
-    public static function preload(array $classes)
+    public static function append(string $file, array $list): void
+    {
+        if (!file_exists($file)) {
+            throw new \LogicException(sprintf('File "%s" does not exist.', $file));
+        }
+
+        $cacheDir = \dirname($file);
+        $classes = [];
+
+        foreach ($list as $item) {
+            if (0 === strpos($item, $cacheDir)) {
+                file_put_contents($file, sprintf("require __DIR__.%s;\n", var_export(substr($item, \strlen($cacheDir)), true)), FILE_APPEND);
+                continue;
+            }
+
+            $classes[] = sprintf("\$classes[] = %s;\n", var_export($item, true));
+        }
+
+        file_put_contents($file, sprintf("\n\$classes = [];\n%sPreloader::preload(\$classes);\n", implode('', $classes)), FILE_APPEND);
+    }
+
+    public static function preload(array $classes): void
     {
         set_error_handler(function ($t, $m, $f, $l) {
             if (error_reporting() & $t) {

src/Symfony/Component/HttpKernel/CHANGELOG.md

@@ -4,6 +4,8 @@ CHANGELOG
 5.1.0
 -----
 
+ * made `WarmableInterface::warmUp()` return a list of classes or files to preload on PHP 7.4+;
+   not returning an array is deprecated
  * deprecated support for `service:action` syntax to reference controllers, use `serviceOrFqcn::method` instead
  * allowed using public aliases to reference controllers
  * added session usage reporting when the `_stateless` attribute of the request is set to `true`

src/Symfony/Component/HttpKernel/CacheWarmer/CacheWarmerAggregate.php

@@ -45,6 +45,8 @@ class CacheWarmerAggregate implements CacheWarmerInterface
 
     /**
      * Warms up the cache.
+     *
+     * @return string[] A list of classes or files to preload on PHP 7.4+
      */
     public function warmUp(string $cacheDir)
     {
@@ -83,6 +85,7 @@ class CacheWarmerAggregate implements CacheWarmerInterface
             });
         }
 
+        $preload = [];
         try {
             foreach ($this->warmers as $warmer) {
                 if (!$this->optionalsEnabled && $warmer->isOptional()) {
@@ -92,7 +95,7 @@ class CacheWarmerAggregate implements CacheWarmerInterface
                     continue;
                 }
 
-                $warmer->warmUp($cacheDir);
+                $preload[] = array_values((array) $warmer->warmUp($cacheDir));
             }
         } finally {
             if ($collectDeprecations) {
@@ -106,6 +109,8 @@ class CacheWarmerAggregate implements CacheWarmerInterface
                 file_put_contents($this->deprecationLogsFilepath, serialize(array_values($collectedLogs)));
             }
         }
+
+        return array_values(array_unique(array_merge([], ...$preload)));
     }
 
     /**

src/Symfony/Component/HttpKernel/CacheWarmer/WarmableInterface.php

@@ -20,6 +20,8 @@ interface WarmableInterface
 {
     /**
      * Warms up the cache.
+     *
+     * @return string[] A list of classes or files to preload on PHP 7.4+
      */
     public function warmUp(string $cacheDir);
 }

src/Symfony/Component/HttpKernel/Kernel.php

@@ -22,6 +22,7 @@ use Symfony\Component\DependencyInjection\Compiler\PassConfig;
 use Symfony\Component\DependencyInjection\ContainerBuilder;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 use Symfony\Component\DependencyInjection\Dumper\PhpDumper;
+use Symfony\Component\DependencyInjection\Dumper\Preloader;
 use Symfony\Component\DependencyInjection\Loader\ClosureLoader;
 use Symfony\Component\DependencyInjection\Loader\DirectoryLoader;
 use Symfony\Component\DependencyInjection\Loader\GlobFileLoader;
@@ -551,7 +552,11 @@ abstract class Kernel implements KernelInterface, RebootableInterface, Terminabl
         }
 
         if ($this->container->has('cache_warmer')) {
-            $this->container->get('cache_warmer')->warmUp($this->container->getParameter('kernel.cache_dir'));
+            $preload = (array) $this->container->get('cache_warmer')->warmUp($this->container->getParameter('kernel.cache_dir'));
+
+            if (method_exists(Preloader::class, 'append') && file_exists($preloadFile = $cacheDir.'/'.$class.'.preload.php')) {
+                Preloader::append($preloadFile, $preload);
+            }
         }
     }
 

src/Symfony/Component/HttpKernel/Tests/CacheWarmer/CacheWarmerTest.php

@@ -54,9 +54,14 @@ class TestCacheWarmer extends CacheWarmer
         $this->file = $file;
     }
 
+    /**
+     * @return string[]
+     */
     public function warmUp(string $cacheDir)
     {
         $this->writeCacheFile($this->file, 'content');
+
+        return [];
     }
 
     public function isOptional(): bool

src/Symfony/Component/Translation/DataCollectorTranslator.php

@@ -81,12 +81,16 @@ class DataCollectorTranslator implements TranslatorInterface, TranslatorBagInter
 
     /**
      * {@inheritdoc}
+     *
+     * @return string[]
      */
     public function warmUp(string $cacheDir)
     {
         if ($this->translator instanceof WarmableInterface) {
-            $this->translator->warmUp($cacheDir);
+            return (array) $this->translator->warmUp($cacheDir);
         }
+
+        return [];
     }
 
     /**

src/Symfony/Component/VarExporter/CHANGELOG.md

@@ -1,6 +1,11 @@
 CHANGELOG
 =========
 
+5.1.0
+-----
+
+ * added argument `array &$foundClasses` to `VarExporter::export()` to ease with preloading exported values
+
 4.2.0
 -----
 

src/Symfony/Component/VarExporter/VarExporter.php

@@ -34,12 +34,13 @@ final class VarExporter
      *
      * @param mixed $value          The value to export
      * @param bool  &$isStaticValue Set to true after execution if the provided value is static, false otherwise
+     * @param bool  &$classes       Classes found in the value are added to this list as both keys and values
      *
      * @return string The value exported as PHP code
      *
      * @throws ExceptionInterface When the provided value cannot be serialized
      */
-    public static function export($value, bool &$isStaticValue = null): string
+    public static function export($value, bool &$isStaticValue = null, array &$foundClasses = []): string
     {
         $isStaticValue = true;
 
@@ -71,7 +72,9 @@ final class VarExporter
         $values = [];
         $states = [];
         foreach ($objectsPool as $i => $v) {
-            list(, $classes[], $values[], $wakeup) = $objectsPool[$v];
+            [, $class, $values[], $wakeup] = $objectsPool[$v];
+            $foundClasses[$class] = $classes[] = $class;
+
             if (0 < $wakeup) {
                 $states[$wakeup] = $i;
             } elseif (0 > $wakeup) {