diogo
/
symfony
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bug #17377 Fix performance (PHP5) and memory (PHP7) issues when using token_get_all (nicolas-grekas, peteward) 

This PR was merged into the 2.3 branch.

Discussion
----------

Fix performance (PHP5) and memory (PHP7) issues when using token_get_all

| Q             | A
| ------------- | ---
| Bug fix?      | yes
| New feature?  | no
| BC breaks?    | no
| Deprecations? | no
| Tests pass?   | yes
| Fixed tickets | #16868
| License       | MIT
| Doc PR        | -

Although it's not the case anymore on PHP 7, on PHP 5, `is_array` checks are much slower than `isset` checks.

Also from @peteward in #17384:
>  New PHP7 memory manager will not release small buckets to OS automatically in cases exposed by `token_get_all()`. This function call addition specifically for PHP7 will reclaim this memory to keep the footprint down of long processe

> See above ticket and suggested actions by PHP internals team for long-running tasks (https://bugs.php.net/70098) - I think `cache:clear/warmup` on a heavy app justifies this.

> We're running on cloud-based hosting platforms under memory limitations (Platform.sh). When memory is exceeded we're into swap and the cache clearing process goes from seconds to minutes for the initial deployment, which really slows our development workflow and also causes holding page delays.

Commits
-------

e555aad Add gc_mem_caches() call for PHP7 after itoken_get_all() as new memory manager will not release small buckets to OS automatically
d1f72d8 Fix perf and mem issue when using token_get_all
This commit is contained in:
Fabien Potencier 2016-01-15 17:52:13 +01:00
parent 54442313ae e555aade12
commit ac3111fe8a
6 changed files with 64 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/Symfony/Bundle/FrameworkBundle/Translation/PhpExtractor.php
View File

@ -61,6 +61,11 @@ class PhpExtractor implements ExtractorInterface
$files = $finder->files()->name('*.php')->in($directory);
foreach ($files as $file) {
$this->parseTokens(token_get_all(file_get_contents($file)), $catalog);
if (PHP_VERSION_ID >= 70000) {
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
gc_mem_caches();
}
}
}
@ -81,7 +86,7 @@ class PhpExtractor implements ExtractorInterface
*/
protected function normalizeToken($token)
{
if (is_array($token)) {
if (isset($token[1]) && 'b"' !== $token) {
return $token[1];
}
@ -95,7 +100,7 @@ class PhpExtractor implements ExtractorInterface
{
for (; $tokenIterator->valid(); $tokenIterator->next()) {
$t = $tokenIterator->current();
if (!is_array($t) || ($t[0] !== T_WHITESPACE)) {
if (T_WHITESPACE !== $t[0]) {
break;
}
}
@ -112,7 +117,7 @@ class PhpExtractor implements ExtractorInterface
for (; $tokenIterator->valid(); $tokenIterator->next()) {
$t = $tokenIterator->current();
if (!is_array($t)) {
if (!isset($t[1])) {
break;
}

27
src/Symfony/Component/ClassLoader/ClassCollectionLoader.php
View File

@ -149,8 +149,9 @@ class ClassCollectionLoader
$inNamespace = false;
$tokens = token_get_all($source);
for (reset($tokens); false !== $token = current($tokens); next($tokens)) {
if (is_string($token)) {
for ($i = 0; isset($tokens[$i]); ++$i) {
$token = $tokens[$i];
if (!isset($token[1]) || 'b"' === $token) {
$rawChunk .= $token;
} elseif (in_array($token[0], array(T_COMMENT, T_DOC_COMMENT))) {
// strip comments
@ -162,12 +163,12 @@ class ClassCollectionLoader
$rawChunk .= $token[1];
// namespace name and whitespaces
while (($t = next($tokens)) && is_array($t) && in_array($t[0], array(T_WHITESPACE, T_NS_SEPARATOR, T_STRING))) {
$rawChunk .= $t[1];
while (isset($tokens[++$i][1]) && in_array($tokens[$i][0], array(T_WHITESPACE, T_NS_SEPARATOR, T_STRING))) {
$rawChunk .= $tokens[$i][1];
}
if ('{' === $t) {
if ('{' === $tokens[$i]) {
$inNamespace = false;
prev($tokens);
--$i;
} else {
$rawChunk = rtrim($rawChunk)."\n{";
$inNamespace = true;
@ -175,8 +176,8 @@ class ClassCollectionLoader
} elseif (T_START_HEREDOC === $token[0]) {
$output .= self::compressCode($rawChunk).$token[1];
do {
$token = next($tokens);
$output .= is_string($token) ? $token : $token[1];
$token = $tokens[++$i];
$output .= isset($token[1]) && 'b"' !== $token ? $token[1] : $token;
} while ($token[0] !== T_END_HEREDOC);
$output .= "\n";
$rawChunk = '';
@ -192,7 +193,15 @@ class ClassCollectionLoader
$rawChunk .= "}\n";
}
return $output.self::compressCode($rawChunk);
$output .= self::compressCode($rawChunk);
if (PHP_VERSION_ID >= 70000) {
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
unset($tokens, $rawChunk);
gc_mem_caches();
}
return $output;
}
/**

22
src/Symfony/Component/ClassLoader/ClassMapGenerator.php
View File

@ -72,6 +72,11 @@ class ClassMapGenerator
$classes = self::findClasses($path);
if (PHP_VERSION_ID >= 70000) {
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
gc_mem_caches();
}
foreach ($classes as $class) {
$map[$class] = $path;
}
@ -95,10 +100,10 @@ class ClassMapGenerator
$classes = array();
$namespace = '';
for ($i = 0, $max = count($tokens); $i < $max; ++$i) {
for ($i = 0; isset($tokens[$i]); ++$i) {
$token = $tokens[$i];
if (is_string($token)) {
if (!isset($token[1])) {
continue;
}
@ -108,9 +113,9 @@ class ClassMapGenerator
case T_NAMESPACE:
$namespace = '';
// If there is a namespace, extract it
while (($t = $tokens[++$i]) && is_array($t)) {
if (in_array($t[0], array(T_STRING, T_NS_SEPARATOR))) {
$namespace .= $t[1];
while (isset($tokens[++$i][1])) {
if (in_array($tokens[$i][0], array(T_STRING, T_NS_SEPARATOR))) {
$namespace .= $tokens[$i][1];
}
}
$namespace .= '\\';
@ -121,7 +126,7 @@ class ClassMapGenerator
// Skip usage of ::class constant
$isClassConstant = false;
for ($j = $i - 1; $j > 0; --$j) {
if (is_string($tokens[$j])) {
if (!isset($tokens[$j][1])) {
break;
}
@ -138,10 +143,11 @@ class ClassMapGenerator
}
// Find the classname
while (($t = $tokens[++$i]) && is_array($t)) {
while (isset($tokens[++$i][1])) {
$t = $tokens[$i];
if (T_STRING === $t[0]) {
$class .= $t[1];
} elseif ($class !== '' && T_WHITESPACE == $t[0]) {
} elseif ('' !== $class && T_WHITESPACE === $t[0]) {
break;
}
}

15
src/Symfony/Component/HttpKernel/Kernel.php
View File

@ -699,14 +699,15 @@ abstract class Kernel implements KernelInterface, TerminableInterface
$output = '';
$tokens = token_get_all($source);
$ignoreSpace = false;
for (reset($tokens); false !== $token = current($tokens); next($tokens)) {
if (is_string($token)) {
for ($i = 0; isset($tokens[$i]); ++$i) {
$token = $tokens[$i];
if (!isset($token[1]) || 'b"' === $token) {
$rawChunk .= $token;
} elseif (T_START_HEREDOC === $token[0]) {
$output .= $rawChunk.$token[1];
do {
$token = next($tokens);
$output .= $token[1];
$token = $tokens[++$i];
$output .= isset($token[1]) && 'b"' !== $token ? $token[1] : $token;
} while ($token[0] !== T_END_HEREDOC);
$rawChunk = '';
} elseif (T_WHITESPACE === $token[0]) {
@ -732,6 +733,12 @@ abstract class Kernel implements KernelInterface, TerminableInterface
$output .= $rawChunk;
if (PHP_VERSION_ID >= 70000) {
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
unset($tokens, $rawChunk);
gc_mem_caches();
}
return $output;
}

4
src/Symfony/Component/HttpKernel/Tests/KernelTest.php
View File

@ -307,7 +307,7 @@ modified';
$heredoc = <<<HD
Heredoc should not be modified
Heredoc should not be modified {$a[1+$b]}
HD;
@ -343,7 +343,7 @@ modified';
$heredoc = <<<HD
Heredoc should not be modified
Heredoc should not be modified {$a[1+$b]}
HD;

18
src/Symfony/Component/Routing/Loader/AnnotationFileLoader.php
View File

@ -64,6 +64,10 @@ class AnnotationFileLoader extends FileLoader
$collection->addResource(new FileResource($path));
$collection->addCollection($this->loader->load($class, $type));
}
if (PHP_VERSION_ID >= 70000) {
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
gc_mem_caches();
}
return $collection;
}
@ -88,10 +92,10 @@ class AnnotationFileLoader extends FileLoader
$class = false;
$namespace = false;
$tokens = token_get_all(file_get_contents($file));
for ($i = 0, $count = count($tokens); $i < $count; ++$i) {
for ($i = 0; isset($tokens[$i]); ++$i) {
$token = $tokens[$i];
if (!is_array($token)) {
if (!isset($token[1])) {
continue;
}
@ -100,11 +104,11 @@ class AnnotationFileLoader extends FileLoader
}
if (true === $namespace && T_STRING === $token[0]) {
$namespace = '';
do {
$namespace .= $token[1];
$token = $tokens[++$i];
} while ($i < $count && is_array($token) && in_array($token[0], array(T_NS_SEPARATOR, T_STRING)));
$namespace = $token[1];
while (isset($tokens[++$i][1]) && in_array($tokens[$i][0], array(T_NS_SEPARATOR, T_STRING))) {
$namespace .= $tokens[$i][1];
}
$token = $tokens[$i];
}
if (T_CLASS === $token[0]) {