feature #37733 [PhpUnitBridge] Add ability to set a baseline for deprecation testing (alexpott)

This PR was submitted for the master branch but it was squashed and merged into the 5.x branch instead.

Discussion
----------

[PhpUnitBridge] Add ability to set a baseline for deprecation testing

| Q             | A
| ------------- | ---
| Branch?       | master
| Bug fix?      | no
| New feature?  | yes <!-- please update src/**/CHANGELOG.md files -->
| Deprecations? | no <!-- please update UPGRADE-*.md and src/**/CHANGELOG.md files -->
| Tickets       | Fix #37715, #34496
| License       | MIT
| Doc PR        | symfony/symfony-docs#... <!-- required for new features -->

This PR allows you set new options for `SYMFONY_DEPRECATIONS_HELPER` env var:
* `generateBaseline` - if this is set to TRUE any deprecations that occur will be written to the file defined in the `baselineFile` option
* `baselineFile` a path to a file that contains a json encoded array of deprecations that will be skipped.

### Questions
* If you set `generateBaseline` without also setting `baselineFile` an exception is thrown. We could use a default filename if one is not provided (like PHPStan).
* How much error checking should we do around the `baselineFile` variable - should we check if it is readable or should we rely on `file_get_contents`()?

### Still @todo
Add proper end-to-end testing using a .phpt test

Commits
-------

483236f34a [PhpUnitBridge] Add ability to set a baseline for deprecation testing
This commit is contained in:
Fabien Potencier 2020-10-07 13:07:15 +02:00
commit 559ebe35fe
7 changed files with 574 additions and 7 deletions

View File

@ -129,6 +129,9 @@ class DeprecationErrorHandler
if ($deprecation->isMuted()) { if ($deprecation->isMuted()) {
return null; return null;
} }
if ($this->getConfiguration()->isBaselineDeprecation($deprecation)) {
return null;
}
$group = 'other'; $group = 'other';
if ($deprecation->originatesFromAnObject()) { if ($deprecation->originatesFromAnObject()) {
@ -207,6 +210,10 @@ class DeprecationErrorHandler
$isFailingAtShutdown = !$configuration->tolerates($this->deprecationGroups); $isFailingAtShutdown = !$configuration->tolerates($this->deprecationGroups);
$this->displayDeprecations($groups, $configuration, $isFailingAtShutdown); $this->displayDeprecations($groups, $configuration, $isFailingAtShutdown);
if ($configuration->isGeneratingBaseline()) {
$configuration->writeBaseline();
}
if ($isFailing || $isFailingAtShutdown) { if ($isFailing || $isFailingAtShutdown) {
exit(1); exit(1);
} }

View File

@ -37,12 +37,28 @@ class Configuration
private $verboseOutput; private $verboseOutput;
/** /**
* @param int[] $thresholds A hash associating groups to thresholds * @var bool
* @param string $regex Will be matched against messages, to decide
* whether to display a stack trace
* @param bool[] $verboseOutput Keyed by groups
*/ */
private function __construct(array $thresholds = [], $regex = '', $verboseOutput = []) private $generateBaseline = false;
/**
* @var string
*/
private $baselineFile = '';
/**
* @var array
*/
private $baselineDeprecations = [];
/**
* @param int[] $thresholds A hash associating groups to thresholds
* @param string $regex Will be matched against messages, to decide whether to display a stack trace
* @param bool[] $verboseOutput Keyed by groups
* @param bool $generateBaseline Whether to generate or update the baseline file
* @param string $baselineFile The path to the baseline file
*/
private function __construct(array $thresholds = [], $regex = '', $verboseOutput = [], $generateBaseline = false, $baselineFile = '')
{ {
$groups = ['total', 'indirect', 'direct', 'self']; $groups = ['total', 'indirect', 'direct', 'self'];
@ -87,6 +103,22 @@ class Configuration
} }
$this->verboseOutput[$group] = (bool) $status; $this->verboseOutput[$group] = (bool) $status;
} }
if ($generateBaseline && !$baselineFile) {
throw new \InvalidArgumentException('You cannot use the "generateBaseline" configuration option without providing a "baselineFile" configuration option.');
}
$this->generateBaseline = $generateBaseline;
$this->baselineFile = $baselineFile;
if ($this->baselineFile && !$this->generateBaseline) {
if (is_file($this->baselineFile)) {
$map = json_decode(file_get_contents($this->baselineFile));
foreach ($map as $baseline_deprecation) {
$this->baselineDeprecations[$baseline_deprecation->location][$baseline_deprecation->message] = $baseline_deprecation->count;
}
} else {
throw new \InvalidArgumentException(sprintf('The baselineFile "%s" does not exist.', $this->baselineFile));
}
}
} }
/** /**
@ -125,6 +157,61 @@ class Configuration
return true; return true;
} }
/**
* @return bool
*/
public function isBaselineDeprecation(Deprecation $deprecation)
{
if ($deprecation->originatesFromAnObject()) {
$location = $deprecation->originatingClass().'::'.$deprecation->originatingMethod();
} else {
$location = 'procedural code';
}
$message = $deprecation->getMessage();
$result = isset($this->baselineDeprecations[$location][$message]) && $this->baselineDeprecations[$location][$message] > 0;
if ($this->generateBaseline) {
if ($result) {
++$this->baselineDeprecations[$location][$message];
} else {
$this->baselineDeprecations[$location][$message] = 1;
$result = true;
}
} elseif ($result) {
--$this->baselineDeprecations[$location][$message];
}
return $result;
}
/**
* @return bool
*/
public function isGeneratingBaseline()
{
return $this->generateBaseline;
}
public function getBaselineFile()
{
return $this->baselineFile;
}
public function writeBaseline()
{
$map = [];
foreach ($this->baselineDeprecations as $location => $messages) {
foreach ($messages as $message => $count) {
$map[] = [
'location' => $location,
'message' => $message,
'count' => $count,
];
}
}
file_put_contents($this->baselineFile, json_encode($map, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES));
}
/** /**
* @param string $message * @param string $message
* *
@ -161,7 +248,7 @@ class Configuration
{ {
parse_str($serializedConfiguration, $normalizedConfiguration); parse_str($serializedConfiguration, $normalizedConfiguration);
foreach (array_keys($normalizedConfiguration) as $key) { foreach (array_keys($normalizedConfiguration) as $key) {
if (!\in_array($key, ['max', 'disabled', 'verbose', 'quiet'], true)) { if (!\in_array($key, ['max', 'disabled', 'verbose', 'quiet', 'generateBaseline', 'baselineFile'], true)) {
throw new \InvalidArgumentException(sprintf('Unknown configuration option "%s".', $key)); throw new \InvalidArgumentException(sprintf('Unknown configuration option "%s".', $key));
} }
} }
@ -171,6 +258,8 @@ class Configuration
'disabled' => false, 'disabled' => false,
'verbose' => true, 'verbose' => true,
'quiet' => [], 'quiet' => [],
'generateBaseline' => false,
'baselineFile' => '',
]; ];
if ('' === $normalizedConfiguration['disabled'] || filter_var($normalizedConfiguration['disabled'], \FILTER_VALIDATE_BOOLEAN)) { if ('' === $normalizedConfiguration['disabled'] || filter_var($normalizedConfiguration['disabled'], \FILTER_VALIDATE_BOOLEAN)) {
@ -188,7 +277,13 @@ class Configuration
} }
} }
return new self($normalizedConfiguration['max'], '', $verboseOutput); return new self(
isset($normalizedConfiguration['max']) ? $normalizedConfiguration['max'] : [],
'',
$verboseOutput,
filter_var($normalizedConfiguration['generateBaseline'], \FILTER_VALIDATE_BOOLEAN),
$normalizedConfiguration['baselineFile']
);
} }
/** /**

View File

@ -13,10 +13,13 @@ namespace Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler;
use PHPUnit\Framework\TestCase; use PHPUnit\Framework\TestCase;
use Symfony\Bridge\PhpUnit\DeprecationErrorHandler\Configuration; use Symfony\Bridge\PhpUnit\DeprecationErrorHandler\Configuration;
use Symfony\Bridge\PhpUnit\DeprecationErrorHandler\Deprecation;
use Symfony\Bridge\PhpUnit\DeprecationErrorHandler\DeprecationGroup; use Symfony\Bridge\PhpUnit\DeprecationErrorHandler\DeprecationGroup;
class ConfigurationTest extends TestCase class ConfigurationTest extends TestCase
{ {
private $files;
public function testItThrowsOnStringishValue() public function testItThrowsOnStringishValue()
{ {
$this->expectException(\InvalidArgumentException::class); $this->expectException(\InvalidArgumentException::class);
@ -244,4 +247,169 @@ class ConfigurationTest extends TestCase
return $groups; return $groups;
} }
public function testBaselineGenerationEmptyFile()
{
$filename = $this->createFile();
$configuration = Configuration::fromUrlEncodedString('generateBaseline=true&baselineFile=' . urlencode($filename));
$this->assertTrue($configuration->isGeneratingBaseline());
$trace = debug_backtrace();
$this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 1', $trace, '')));
$this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 2', $trace, '')));
$this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 1', $trace, '')));
$configuration->writeBaseline();
$this->assertEquals($filename, $configuration->getBaselineFile());
$expected_baseline = [
[
'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
'message' => 'Test message 1',
'count' => 2,
],
[
'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
'message' => 'Test message 2',
'count' => 1,
],
];
$this->assertEquals(json_encode($expected_baseline, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES), file_get_contents($filename));
}
public function testBaselineGenerationNoFile()
{
$filename = $this->createFile();
$configuration = Configuration::fromUrlEncodedString('generateBaseline=true&baselineFile=' . urlencode($filename));
$this->assertTrue($configuration->isGeneratingBaseline());
$trace = debug_backtrace();
$this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 1', $trace, '')));
$this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 2', $trace, '')));
$this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 2', $trace, '')));
$this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 1', $trace, '')));
$configuration->writeBaseline();
$this->assertEquals($filename, $configuration->getBaselineFile());
$expected_baseline = [
[
'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
'message' => 'Test message 1',
'count' => 2,
],
[
'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
'message' => 'Test message 2',
'count' => 2,
],
];
$this->assertEquals(json_encode($expected_baseline, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES), file_get_contents($filename));
}
public function testExistingBaseline()
{
$filename = $this->createFile();
$baseline = [
[
'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
'message' => 'Test message 1',
'count' => 1,
],
[
'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
'message' => 'Test message 2',
'count' => 1,
],
];
file_put_contents($filename, json_encode($baseline));
$configuration = Configuration::fromUrlEncodedString('baselineFile=' . urlencode($filename));
$this->assertFalse($configuration->isGeneratingBaseline());
$trace = debug_backtrace();
$this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 1', $trace, '')));
$this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 2', $trace, '')));
$this->assertFalse($configuration->isBaselineDeprecation(new Deprecation('Test message 3', $trace, '')));
$this->assertEquals($filename, $configuration->getBaselineFile());
}
public function testExistingBaselineAndGeneration()
{
$filename = $this->createFile();
$baseline = [
[
'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
'message' => 'Test message 1',
'count' => 1,
],
[
'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
'message' => 'Test message 2',
'count' => 1,
],
];
file_put_contents($filename, json_encode($baseline));
$configuration = Configuration::fromUrlEncodedString('generateBaseline=true&baselineFile=' . urlencode($filename));
$this->assertTrue($configuration->isGeneratingBaseline());
$trace = debug_backtrace();
$this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 2', $trace, '')));
$this->assertTrue($configuration->isBaselineDeprecation(new Deprecation('Test message 3', $trace, '')));
$configuration->writeBaseline();
$this->assertEquals($filename, $configuration->getBaselineFile());
$expected_baseline = [
[
'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
'message' => 'Test message 2',
'count' => 1,
],
[
'location' => 'Symfony\Bridge\PhpUnit\Tests\DeprecationErrorHandler\ConfigurationTest::runTest',
'message' => 'Test message 3',
'count' => 1,
],
];
$this->assertEquals(json_encode($expected_baseline, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES), file_get_contents($filename));
}
public function testBaselineArgumentException()
{
$this->expectException(\InvalidArgumentException::class);
$this->expectExceptionMessage('You cannot use the "generateBaseline" configuration option without providing a "baselineFile" configuration option.');
Configuration::fromUrlEncodedString('generateBaseline=true');
}
public function testBaselineFileException()
{
$filename = $this->createFile();
unlink($filename);
$this->expectException(\InvalidArgumentException::class);
$this->expectExceptionMessage(sprintf('The baselineFile "%s" does not exist.', $filename));
Configuration::fromUrlEncodedString('baselineFile=' . urlencode($filename));
}
public function testBaselineFileWriteError()
{
$filename = $this->createFile();
chmod($filename, 0444);
$this->expectError();
$this->expectErrorMessageMatches('/failed to open stream: Permission denied/');
$configuration = Configuration::fromUrlEncodedString('generateBaseline=true&baselineFile=' . urlencode($filename));
$configuration->writeBaseline();
}
protected function setUp(): void
{
$this->files = [];
}
protected function tearDown(): void
{
foreach ($this->files as $file) {
if (file_exists($file)) {
unlink($file);
}
}
}
private function createFile() {
$filename = tempnam(sys_get_temp_dir(), 'sf-');
$this->files[] = $filename;
return $filename;
}
} }

View File

@ -0,0 +1,80 @@
--TEST--
Test DeprecationErrorHandler in baseline mode
--FILE--
<?php
$filename = tempnam(sys_get_temp_dir(), 'sf-');
$baseline = [[
'location' => 'FooTestCase::testLegacyFoo',
'message' => 'silenced foo deprecation',
'count' => 1,
],
[
'location' => 'FooTestCase::testNonLegacyBar',
'message' => 'silenced bar deprecation',
'count' => 1,
],
[
'location' => 'procedural code',
'message' => 'root deprecation',
'count' => 1,
]];
file_put_contents($filename, json_encode($baseline, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES));
$k = 'SYMFONY_DEPRECATIONS_HELPER';
unset($_SERVER[$k], $_ENV[$k]);
putenv($k.'='.$_SERVER[$k] = $_ENV[$k] = 'baselineFile=' . urlencode($filename));
putenv('ANSICON');
putenv('ConEmuANSI');
putenv('TERM');
$vendor = __DIR__;
while (!file_exists($vendor.'/vendor')) {
$vendor = dirname($vendor);
}
define('PHPUNIT_COMPOSER_INSTALL', $vendor.'/vendor/autoload.php');
require PHPUNIT_COMPOSER_INSTALL;
require_once __DIR__.'/../../bootstrap.php';
@trigger_error('root deprecation', E_USER_DEPRECATED);
eval(<<<'EOPHP'
namespace PHPUnit\Util;
class Test
{
public static function getGroups()
{
return array();
}
}
EOPHP
);
class PHPUnit_Util_Test
{
public static function getGroups()
{
return array();
}
}
class FooTestCase
{
public function testLegacyFoo()
{
@trigger_error('silenced foo deprecation', E_USER_DEPRECATED);
}
public function testNonLegacyBar()
{
@trigger_error('silenced bar deprecation', E_USER_DEPRECATED);
}
}
$foo = new FooTestCase();
$foo->testLegacyFoo();
$foo->testNonLegacyBar();
print "Cannot test baselineFile contents because it is generated in a shutdown function registered by another shutdown function."
?>
--EXPECT--
Cannot test baselineFile contents because it is generated in a shutdown function registered by another shutdown function.

View File

@ -0,0 +1,74 @@
--TEST--
Test DeprecationErrorHandler in baseline mode
--FILE--
<?php
$filename = tempnam(sys_get_temp_dir(), 'sf-');
$baseline = [[
'location' => 'FooTestCase::testLegacyFoo',
'message' => 'silenced foo deprecation',
'count' => 1,
]];
file_put_contents($filename, json_encode($baseline, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES));
$k = 'SYMFONY_DEPRECATIONS_HELPER';
unset($_SERVER[$k], $_ENV[$k]);
putenv($k.'='.$_SERVER[$k] = $_ENV[$k] = 'baselineFile=' . urlencode($filename));
putenv('ANSICON');
putenv('ConEmuANSI');
putenv('TERM');
$vendor = __DIR__;
while (!file_exists($vendor.'/vendor')) {
$vendor = dirname($vendor);
}
define('PHPUNIT_COMPOSER_INSTALL', $vendor.'/vendor/autoload.php');
require PHPUNIT_COMPOSER_INSTALL;
require_once __DIR__.'/../../bootstrap.php';
@trigger_error('root deprecation', E_USER_DEPRECATED);
eval(<<<'EOPHP'
namespace PHPUnit\Util;
class Test
{
public static function getGroups()
{
return array();
}
}
EOPHP
);
class PHPUnit_Util_Test
{
public static function getGroups()
{
return array();
}
}
class FooTestCase
{
public function testLegacyFoo()
{
@trigger_error('silenced foo deprecation', E_USER_DEPRECATED);
}
public function testNonLegacyBar()
{
@trigger_error('silenced bar deprecation', E_USER_DEPRECATED);
}
}
$foo = new FooTestCase();
$foo->testLegacyFoo();
$foo->testNonLegacyBar();
?>
--EXPECTF--
Other deprecation notices (2)
1x: root deprecation
1x: silenced bar deprecation
1x in FooTestCase::testNonLegacyBar

View File

@ -0,0 +1,79 @@
--TEST--
Test DeprecationErrorHandler in baseline mode
--FILE--
<?php
$filename = tempnam(sys_get_temp_dir(), 'sf-');
$baseline = [[
'location' => 'FooTestCase::testLegacyFoo',
'message' => 'silenced foo deprecation',
'count' => 1,
]];
file_put_contents($filename, json_encode($baseline, \JSON_PRETTY_PRINT | \JSON_UNESCAPED_SLASHES));
$k = 'SYMFONY_DEPRECATIONS_HELPER';
unset($_SERVER[$k], $_ENV[$k]);
putenv($k.'='.$_SERVER[$k] = $_ENV[$k] = 'baselineFile=' . urlencode($filename));
putenv('ANSICON');
putenv('ConEmuANSI');
putenv('TERM');
$vendor = __DIR__;
while (!file_exists($vendor.'/vendor')) {
$vendor = dirname($vendor);
}
define('PHPUNIT_COMPOSER_INSTALL', $vendor.'/vendor/autoload.php');
require PHPUNIT_COMPOSER_INSTALL;
require_once __DIR__.'/../../bootstrap.php';
@trigger_error('root deprecation', E_USER_DEPRECATED);
eval(<<<'EOPHP'
namespace PHPUnit\Util;
class Test
{
public static function getGroups()
{
return array();
}
}
EOPHP
);
class PHPUnit_Util_Test
{
public static function getGroups()
{
return array();
}
}
class FooTestCase
{
public function testLegacyFoo()
{
@trigger_error('silenced foo deprecation', E_USER_DEPRECATED);
// This will cause a deprecation because the baseline only expects 1
// deprecation.
@trigger_error('silenced foo deprecation', E_USER_DEPRECATED);
}
public function testNonLegacyBar()
{
@trigger_error('silenced bar deprecation', E_USER_DEPRECATED);
}
}
$foo = new FooTestCase();
$foo->testLegacyFoo();
$foo->testNonLegacyBar();
?>
--EXPECTF--
Legacy deprecation notices (1)
Other deprecation notices (2)
1x: root deprecation
1x: silenced bar deprecation
1x in FooTestCase::testNonLegacyBar

View File

@ -0,0 +1,64 @@
--TEST--
Test DeprecationErrorHandler in baseline generation mode
--FILE--
<?php
$filename = tempnam(sys_get_temp_dir(), 'sf-');
$k = 'SYMFONY_DEPRECATIONS_HELPER';
unset($_SERVER[$k], $_ENV[$k]);
putenv($k.'='.$_SERVER[$k] = $_ENV[$k] = 'generateBaseline=true&baselineFile=' . urlencode($filename));
putenv('ANSICON');
putenv('ConEmuANSI');
putenv('TERM');
$vendor = __DIR__;
while (!file_exists($vendor.'/vendor')) {
$vendor = dirname($vendor);
}
define('PHPUNIT_COMPOSER_INSTALL', $vendor.'/vendor/autoload.php');
require PHPUNIT_COMPOSER_INSTALL;
require_once __DIR__.'/../../bootstrap.php';
@trigger_error('root deprecation', E_USER_DEPRECATED);
eval(<<<'EOPHP'
namespace PHPUnit\Util;
class Test
{
public static function getGroups()
{
return array();
}
}
EOPHP
);
class PHPUnit_Util_Test
{
public static function getGroups()
{
return array();
}
}
class FooTestCase
{
public function testLegacyFoo()
{
@trigger_error('silenced foo deprecation', E_USER_DEPRECATED);
}
public function testNonLegacyBar()
{
@trigger_error('silenced bar deprecation', E_USER_DEPRECATED);
}
}
$foo = new FooTestCase();
$foo->testLegacyFoo();
$foo->testNonLegacyBar();
print "Cannot test baselineFile contents because it is generated in a shutdown function registered by another shutdown function."
?>
--EXPECT--
Cannot test baselineFile contents because it is generated in a shutdown function registered by another shutdown function.