bug #17377 Fix performance (PHP5) and memory (PHP7) issues when using token_get_all (nicolas-grekas, peteward)
This PR was merged into the 2.3 branch. Discussion ---------- Fix performance (PHP5) and memory (PHP7) issues when using token_get_all | Q | A | ------------- | --- | Bug fix? | yes | New feature? | no | BC breaks? | no | Deprecations? | no | Tests pass? | yes | Fixed tickets | #16868 | License | MIT | Doc PR | - Although it's not the case anymore on PHP 7, on PHP 5, `is_array` checks are much slower than `isset` checks. Also from @peteward in #17384: > New PHP7 memory manager will not release small buckets to OS automatically in cases exposed by `token_get_all()`. This function call addition specifically for PHP7 will reclaim this memory to keep the footprint down of long processe > See above ticket and suggested actions by PHP internals team for long-running tasks (https://bugs.php.net/70098) - I think `cache:clear/warmup` on a heavy app justifies this. > We're running on cloud-based hosting platforms under memory limitations (Platform.sh). When memory is exceeded we're into swap and the cache clearing process goes from seconds to minutes for the initial deployment, which really slows our development workflow and also causes holding page delays. Commits -------e555aad
Add gc_mem_caches() call for PHP7 after itoken_get_all() as new memory manager will not release small buckets to OS automaticallyd1f72d8
Fix perf and mem issue when using token_get_all
This commit is contained in:
commit
ac3111fe8a
@ -61,6 +61,11 @@ class PhpExtractor implements ExtractorInterface
|
|||||||
$files = $finder->files()->name('*.php')->in($directory);
|
$files = $finder->files()->name('*.php')->in($directory);
|
||||||
foreach ($files as $file) {
|
foreach ($files as $file) {
|
||||||
$this->parseTokens(token_get_all(file_get_contents($file)), $catalog);
|
$this->parseTokens(token_get_all(file_get_contents($file)), $catalog);
|
||||||
|
|
||||||
|
if (PHP_VERSION_ID >= 70000) {
|
||||||
|
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
|
||||||
|
gc_mem_caches();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -81,7 +86,7 @@ class PhpExtractor implements ExtractorInterface
|
|||||||
*/
|
*/
|
||||||
protected function normalizeToken($token)
|
protected function normalizeToken($token)
|
||||||
{
|
{
|
||||||
if (is_array($token)) {
|
if (isset($token[1]) && 'b"' !== $token) {
|
||||||
return $token[1];
|
return $token[1];
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -95,7 +100,7 @@ class PhpExtractor implements ExtractorInterface
|
|||||||
{
|
{
|
||||||
for (; $tokenIterator->valid(); $tokenIterator->next()) {
|
for (; $tokenIterator->valid(); $tokenIterator->next()) {
|
||||||
$t = $tokenIterator->current();
|
$t = $tokenIterator->current();
|
||||||
if (!is_array($t) || ($t[0] !== T_WHITESPACE)) {
|
if (T_WHITESPACE !== $t[0]) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -112,7 +117,7 @@ class PhpExtractor implements ExtractorInterface
|
|||||||
|
|
||||||
for (; $tokenIterator->valid(); $tokenIterator->next()) {
|
for (; $tokenIterator->valid(); $tokenIterator->next()) {
|
||||||
$t = $tokenIterator->current();
|
$t = $tokenIterator->current();
|
||||||
if (!is_array($t)) {
|
if (!isset($t[1])) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -149,8 +149,9 @@ class ClassCollectionLoader
|
|||||||
$inNamespace = false;
|
$inNamespace = false;
|
||||||
$tokens = token_get_all($source);
|
$tokens = token_get_all($source);
|
||||||
|
|
||||||
for (reset($tokens); false !== $token = current($tokens); next($tokens)) {
|
for ($i = 0; isset($tokens[$i]); ++$i) {
|
||||||
if (is_string($token)) {
|
$token = $tokens[$i];
|
||||||
|
if (!isset($token[1]) || 'b"' === $token) {
|
||||||
$rawChunk .= $token;
|
$rawChunk .= $token;
|
||||||
} elseif (in_array($token[0], array(T_COMMENT, T_DOC_COMMENT))) {
|
} elseif (in_array($token[0], array(T_COMMENT, T_DOC_COMMENT))) {
|
||||||
// strip comments
|
// strip comments
|
||||||
@ -162,12 +163,12 @@ class ClassCollectionLoader
|
|||||||
$rawChunk .= $token[1];
|
$rawChunk .= $token[1];
|
||||||
|
|
||||||
// namespace name and whitespaces
|
// namespace name and whitespaces
|
||||||
while (($t = next($tokens)) && is_array($t) && in_array($t[0], array(T_WHITESPACE, T_NS_SEPARATOR, T_STRING))) {
|
while (isset($tokens[++$i][1]) && in_array($tokens[$i][0], array(T_WHITESPACE, T_NS_SEPARATOR, T_STRING))) {
|
||||||
$rawChunk .= $t[1];
|
$rawChunk .= $tokens[$i][1];
|
||||||
}
|
}
|
||||||
if ('{' === $t) {
|
if ('{' === $tokens[$i]) {
|
||||||
$inNamespace = false;
|
$inNamespace = false;
|
||||||
prev($tokens);
|
--$i;
|
||||||
} else {
|
} else {
|
||||||
$rawChunk = rtrim($rawChunk)."\n{";
|
$rawChunk = rtrim($rawChunk)."\n{";
|
||||||
$inNamespace = true;
|
$inNamespace = true;
|
||||||
@ -175,8 +176,8 @@ class ClassCollectionLoader
|
|||||||
} elseif (T_START_HEREDOC === $token[0]) {
|
} elseif (T_START_HEREDOC === $token[0]) {
|
||||||
$output .= self::compressCode($rawChunk).$token[1];
|
$output .= self::compressCode($rawChunk).$token[1];
|
||||||
do {
|
do {
|
||||||
$token = next($tokens);
|
$token = $tokens[++$i];
|
||||||
$output .= is_string($token) ? $token : $token[1];
|
$output .= isset($token[1]) && 'b"' !== $token ? $token[1] : $token;
|
||||||
} while ($token[0] !== T_END_HEREDOC);
|
} while ($token[0] !== T_END_HEREDOC);
|
||||||
$output .= "\n";
|
$output .= "\n";
|
||||||
$rawChunk = '';
|
$rawChunk = '';
|
||||||
@ -192,7 +193,15 @@ class ClassCollectionLoader
|
|||||||
$rawChunk .= "}\n";
|
$rawChunk .= "}\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
return $output.self::compressCode($rawChunk);
|
$output .= self::compressCode($rawChunk);
|
||||||
|
|
||||||
|
if (PHP_VERSION_ID >= 70000) {
|
||||||
|
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
|
||||||
|
unset($tokens, $rawChunk);
|
||||||
|
gc_mem_caches();
|
||||||
|
}
|
||||||
|
|
||||||
|
return $output;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -72,6 +72,11 @@ class ClassMapGenerator
|
|||||||
|
|
||||||
$classes = self::findClasses($path);
|
$classes = self::findClasses($path);
|
||||||
|
|
||||||
|
if (PHP_VERSION_ID >= 70000) {
|
||||||
|
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
|
||||||
|
gc_mem_caches();
|
||||||
|
}
|
||||||
|
|
||||||
foreach ($classes as $class) {
|
foreach ($classes as $class) {
|
||||||
$map[$class] = $path;
|
$map[$class] = $path;
|
||||||
}
|
}
|
||||||
@ -95,10 +100,10 @@ class ClassMapGenerator
|
|||||||
$classes = array();
|
$classes = array();
|
||||||
|
|
||||||
$namespace = '';
|
$namespace = '';
|
||||||
for ($i = 0, $max = count($tokens); $i < $max; ++$i) {
|
for ($i = 0; isset($tokens[$i]); ++$i) {
|
||||||
$token = $tokens[$i];
|
$token = $tokens[$i];
|
||||||
|
|
||||||
if (is_string($token)) {
|
if (!isset($token[1])) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -108,9 +113,9 @@ class ClassMapGenerator
|
|||||||
case T_NAMESPACE:
|
case T_NAMESPACE:
|
||||||
$namespace = '';
|
$namespace = '';
|
||||||
// If there is a namespace, extract it
|
// If there is a namespace, extract it
|
||||||
while (($t = $tokens[++$i]) && is_array($t)) {
|
while (isset($tokens[++$i][1])) {
|
||||||
if (in_array($t[0], array(T_STRING, T_NS_SEPARATOR))) {
|
if (in_array($tokens[$i][0], array(T_STRING, T_NS_SEPARATOR))) {
|
||||||
$namespace .= $t[1];
|
$namespace .= $tokens[$i][1];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$namespace .= '\\';
|
$namespace .= '\\';
|
||||||
@ -121,7 +126,7 @@ class ClassMapGenerator
|
|||||||
// Skip usage of ::class constant
|
// Skip usage of ::class constant
|
||||||
$isClassConstant = false;
|
$isClassConstant = false;
|
||||||
for ($j = $i - 1; $j > 0; --$j) {
|
for ($j = $i - 1; $j > 0; --$j) {
|
||||||
if (is_string($tokens[$j])) {
|
if (!isset($tokens[$j][1])) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -138,10 +143,11 @@ class ClassMapGenerator
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Find the classname
|
// Find the classname
|
||||||
while (($t = $tokens[++$i]) && is_array($t)) {
|
while (isset($tokens[++$i][1])) {
|
||||||
|
$t = $tokens[$i];
|
||||||
if (T_STRING === $t[0]) {
|
if (T_STRING === $t[0]) {
|
||||||
$class .= $t[1];
|
$class .= $t[1];
|
||||||
} elseif ($class !== '' && T_WHITESPACE == $t[0]) {
|
} elseif ('' !== $class && T_WHITESPACE === $t[0]) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -699,14 +699,15 @@ abstract class Kernel implements KernelInterface, TerminableInterface
|
|||||||
$output = '';
|
$output = '';
|
||||||
$tokens = token_get_all($source);
|
$tokens = token_get_all($source);
|
||||||
$ignoreSpace = false;
|
$ignoreSpace = false;
|
||||||
for (reset($tokens); false !== $token = current($tokens); next($tokens)) {
|
for ($i = 0; isset($tokens[$i]); ++$i) {
|
||||||
if (is_string($token)) {
|
$token = $tokens[$i];
|
||||||
|
if (!isset($token[1]) || 'b"' === $token) {
|
||||||
$rawChunk .= $token;
|
$rawChunk .= $token;
|
||||||
} elseif (T_START_HEREDOC === $token[0]) {
|
} elseif (T_START_HEREDOC === $token[0]) {
|
||||||
$output .= $rawChunk.$token[1];
|
$output .= $rawChunk.$token[1];
|
||||||
do {
|
do {
|
||||||
$token = next($tokens);
|
$token = $tokens[++$i];
|
||||||
$output .= $token[1];
|
$output .= isset($token[1]) && 'b"' !== $token ? $token[1] : $token;
|
||||||
} while ($token[0] !== T_END_HEREDOC);
|
} while ($token[0] !== T_END_HEREDOC);
|
||||||
$rawChunk = '';
|
$rawChunk = '';
|
||||||
} elseif (T_WHITESPACE === $token[0]) {
|
} elseif (T_WHITESPACE === $token[0]) {
|
||||||
@ -732,6 +733,12 @@ abstract class Kernel implements KernelInterface, TerminableInterface
|
|||||||
|
|
||||||
$output .= $rawChunk;
|
$output .= $rawChunk;
|
||||||
|
|
||||||
|
if (PHP_VERSION_ID >= 70000) {
|
||||||
|
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
|
||||||
|
unset($tokens, $rawChunk);
|
||||||
|
gc_mem_caches();
|
||||||
|
}
|
||||||
|
|
||||||
return $output;
|
return $output;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -307,7 +307,7 @@ modified';
|
|||||||
$heredoc = <<<HD
|
$heredoc = <<<HD
|
||||||
|
|
||||||
|
|
||||||
Heredoc should not be modified
|
Heredoc should not be modified {$a[1+$b]}
|
||||||
|
|
||||||
|
|
||||||
HD;
|
HD;
|
||||||
@ -343,7 +343,7 @@ modified';
|
|||||||
$heredoc = <<<HD
|
$heredoc = <<<HD
|
||||||
|
|
||||||
|
|
||||||
Heredoc should not be modified
|
Heredoc should not be modified {$a[1+$b]}
|
||||||
|
|
||||||
|
|
||||||
HD;
|
HD;
|
||||||
|
@ -64,6 +64,10 @@ class AnnotationFileLoader extends FileLoader
|
|||||||
$collection->addResource(new FileResource($path));
|
$collection->addResource(new FileResource($path));
|
||||||
$collection->addCollection($this->loader->load($class, $type));
|
$collection->addCollection($this->loader->load($class, $type));
|
||||||
}
|
}
|
||||||
|
if (PHP_VERSION_ID >= 70000) {
|
||||||
|
// PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
|
||||||
|
gc_mem_caches();
|
||||||
|
}
|
||||||
|
|
||||||
return $collection;
|
return $collection;
|
||||||
}
|
}
|
||||||
@ -88,10 +92,10 @@ class AnnotationFileLoader extends FileLoader
|
|||||||
$class = false;
|
$class = false;
|
||||||
$namespace = false;
|
$namespace = false;
|
||||||
$tokens = token_get_all(file_get_contents($file));
|
$tokens = token_get_all(file_get_contents($file));
|
||||||
for ($i = 0, $count = count($tokens); $i < $count; ++$i) {
|
for ($i = 0; isset($tokens[$i]); ++$i) {
|
||||||
$token = $tokens[$i];
|
$token = $tokens[$i];
|
||||||
|
|
||||||
if (!is_array($token)) {
|
if (!isset($token[1])) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -100,11 +104,11 @@ class AnnotationFileLoader extends FileLoader
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (true === $namespace && T_STRING === $token[0]) {
|
if (true === $namespace && T_STRING === $token[0]) {
|
||||||
$namespace = '';
|
$namespace = $token[1];
|
||||||
do {
|
while (isset($tokens[++$i][1]) && in_array($tokens[$i][0], array(T_NS_SEPARATOR, T_STRING))) {
|
||||||
$namespace .= $token[1];
|
$namespace .= $tokens[$i][1];
|
||||||
$token = $tokens[++$i];
|
}
|
||||||
} while ($i < $count && is_array($token) && in_array($token[0], array(T_NS_SEPARATOR, T_STRING)));
|
$token = $tokens[$i];
|
||||||
}
|
}
|
||||||
|
|
||||||
if (T_CLASS === $token[0]) {
|
if (T_CLASS === $token[0]) {
|
||||||
|
Reference in New Issue
Block a user