diff --git a/composer.lock b/composer.lock index 0c2de243897..047822b72c5 100644 --- a/composer.lock +++ b/composer.lock @@ -68,16 +68,16 @@ }, { "name": "nikic/php-parser", - "version": "v5.6.1", + "version": "v5.6.2", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "f103601b29efebd7ff4a1ca7b3eeea9e3336a2a2" + "reference": "3a454ca033b9e06b63282ce19562e892747449bb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/f103601b29efebd7ff4a1ca7b3eeea9e3336a2a2", - "reference": "f103601b29efebd7ff4a1ca7b3eeea9e3336a2a2", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/3a454ca033b9e06b63282ce19562e892747449bb", + "reference": "3a454ca033b9e06b63282ce19562e892747449bb", "shasum": "" }, "require": { @@ -120,9 +120,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.6.1" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.6.2" }, - "time": "2025-08-13T20:13:15+00:00" + "time": "2025-10-21T19:32:17+00:00" }, { "name": "phar-io/manifest", diff --git a/src/Event/Value/Test/Phpt.php b/src/Event/Value/Test/Phpt.php index 65a3aec8268..0652be42cd3 100644 --- a/src/Event/Value/Test/Phpt.php +++ b/src/Event/Value/Test/Phpt.php @@ -16,6 +16,22 @@ */ final readonly class Phpt extends Test { + /** + * @var positive-int + */ + private int $repeatAttemptNumber; + + /** + * @param non-empty-string $file + * @param positive-int $repeatAttemptNumber + */ + public function __construct(string $file, int $repeatAttemptNumber = 1) + { + parent::__construct($file); + + $this->repeatAttemptNumber = $repeatAttemptNumber; + } + public function isPhpt(): true { return true; @@ -26,7 +42,7 @@ public function isPhpt(): true */ public function id(): string { - return $this->file(); + return $this->name(); } /** @@ -34,6 +50,10 @@ public function id(): string */ public function name(): string { - return $this->file(); + if ($this->repeatAttemptNumber === 1) { + return $this->file(); + } + + return $this->file() . " (repeat attempt #{$this->repeatAttemptNumber})"; } } diff --git a/src/Event/Value/Test/TestMethod.php b/src/Event/Value/Test/TestMethod.php index 4c972645436..c4c769b3d7b 100644 --- a/src/Event/Value/Test/TestMethod.php +++ b/src/Event/Value/Test/TestMethod.php @@ -39,22 +39,29 @@ private MetadataCollection $metadata; private TestDataCollection $testData; + /** + * @var positive-int + */ + private int $repeatAttemptNumber; + /** * @param class-string $className * @param non-empty-string $methodName * @param non-empty-string $file * @param non-negative-int $line + * @param positive-int $repeatAttemptNumber */ - public function __construct(string $className, string $methodName, string $file, int $line, TestDox $testDox, MetadataCollection $metadata, TestDataCollection $testData) + public function __construct(string $className, string $methodName, string $file, int $line, TestDox $testDox, MetadataCollection $metadata, TestDataCollection $testData, int $repeatAttemptNumber = 1) { parent::__construct($file); - $this->className = $className; - $this->methodName = $methodName; - $this->line = $line; - $this->testDox = $testDox; - $this->metadata = $metadata; - $this->testData = $testData; + $this->className = $className; + $this->methodName = $methodName; + $this->line = $line; + $this->testDox = $testDox; + $this->metadata = $metadata; + $this->testData = $testData; + $this->repeatAttemptNumber = $repeatAttemptNumber; } /** @@ -129,7 +136,13 @@ public function nameWithClass(): string public function name(): string { if (!$this->testData->hasDataFromDataProvider()) { - return $this->methodName; + $name = $this->methodName; + + if ($this->repeatAttemptNumber !== 1) { + $name .= " (repeat attempt #{$this->repeatAttemptNumber})"; + } + + return $name; } $dataSetName = $this->testData->dataFromDataProvider()->dataSetName(); @@ -146,6 +159,12 @@ public function name(): string ); } - return $this->methodName . $dataSetName; + $name = $this->methodName . $dataSetName; + + if ($this->repeatAttemptNumber !== 1) { + $name .= " (repeat attempt #{$this->repeatAttemptNumber})"; + } + + return $name; } } diff --git a/src/Event/Value/Test/TestMethodBuilder.php b/src/Event/Value/Test/TestMethodBuilder.php index dc1a32ef8ab..8218eddecef 100644 --- a/src/Event/Value/Test/TestMethodBuilder.php +++ b/src/Event/Value/Test/TestMethodBuilder.php @@ -45,6 +45,7 @@ public static function fromTestCase(TestCase $testCase, bool $useTestCaseForTest $testDox, MetadataRegistry::parser()->forClassAndMethod($testCase::class, $methodName), self::dataFor($testCase), + $testCase->repeatAttemptNumber(), ); } diff --git a/src/Event/Value/TestSuite/TestSuiteBuilder.php b/src/Event/Value/TestSuite/TestSuiteBuilder.php index 3192636baa9..6d6db20ba41 100644 --- a/src/Event/Value/TestSuite/TestSuiteBuilder.php +++ b/src/Event/Value/TestSuite/TestSuiteBuilder.php @@ -18,6 +18,7 @@ use PHPUnit\Event\Code\TestCollection; use PHPUnit\Event\RuntimeException; use PHPUnit\Framework\DataProviderTestSuite; +use PHPUnit\Framework\RepeatTestSuite; use PHPUnit\Framework\TestCase; use PHPUnit\Framework\TestSuite as FrameworkTestSuite; use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; @@ -107,7 +108,9 @@ private static function process(FrameworkTestSuite $testSuite, array &$tests): v continue; } - if ($test instanceof TestCase || $test instanceof PhptTestCase) { + if ($test instanceof TestCase || + $test instanceof PhptTestCase || + $test instanceof RepeatTestSuite) { $tests[] = $test->valueObjectForEvents(); } } diff --git a/src/Framework/AbstractRepeatTestSuite.php b/src/Framework/AbstractRepeatTestSuite.php new file mode 100644 index 00000000000..2f8a704284a --- /dev/null +++ b/src/Framework/AbstractRepeatTestSuite.php @@ -0,0 +1,62 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\Framework; + +use function count; +use PHPUnit\Event; +use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; + +/** + * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit + * + * @internal This class is not covered by the backward compatibility promise for PHPUnit + * + * @template T of TestCase|PhptTestCase + */ +abstract readonly class AbstractRepeatTestSuite implements Reorderable, Test +{ + /** + * @var non-empty-list + */ + protected array $tests; + + /** + * @param non-empty-list $tests + */ + public function __construct(array $tests) + { + $this->tests = $tests; + } + + final public function count(): int + { + return count($this->tests); + } + + final public function sortId(): string + { + return $this->tests[0]->sortId(); + } + + final public function provides(): array + { + return $this->tests[0]->provides(); + } + + final public function requires(): array + { + return $this->tests[0]->requires(); + } + + final public function valueObjectForEvents(): Event\Code\Phpt|Event\Code\TestMethod + { + return $this->tests[0]->valueObjectForEvents(); + } +} diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php new file mode 100644 index 00000000000..a94f1da3e44 --- /dev/null +++ b/src/Framework/RepeatTestSuite.php @@ -0,0 +1,66 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\Framework; + +use PHPUnit\Metadata\Api\ProvidedData; + +/** + * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit + * + * @internal This class is not covered by the backward compatibility promise for PHPUnit + * + * @extends AbstractRepeatTestSuite + */ +final readonly class RepeatTestSuite extends AbstractRepeatTestSuite +{ + public function run(): void + { + $defectOccurred = false; + + foreach ($this->tests as $test) { + if ($defectOccurred) { + $test->markSkippedForErrorInPreviousRepetition(); + + continue; + } + + $test->run(); + + if ($test->status()->isFailure() || $test->status()->isError() || $test->status()->isSkipped()) { + $defectOccurred = true; + } + } + } + + public function name(): string + { + return $this->tests[0]::class . '::' . $this->tests[0]->nameWithDataSet(); + } + + /** + * @param array $data + */ + public function setData(int|string $dataName, array $data): void + { + foreach ($this->tests as $test) { + $test->setData($dataName, $data); + } + } + + /** + * @param list $dependencies + */ + public function setDependencies(array $dependencies): void + { + foreach ($this->tests as $test) { + $test->setDependencies($dependencies); + } + } +} diff --git a/src/Framework/TestBuilder.php b/src/Framework/TestBuilder.php index 3eb4b1a02de..8d87048704f 100644 --- a/src/Framework/TestBuilder.php +++ b/src/Framework/TestBuilder.php @@ -11,6 +11,7 @@ use function array_merge; use function assert; +use function range; use PHPUnit\Metadata\Api\DataProvider; use PHPUnit\Metadata\Api\Groups; use PHPUnit\Metadata\Api\ProvidedData; @@ -36,10 +37,11 @@ * @param ReflectionClass $theClass * @param non-empty-string $methodName * @param list $groups + * @param positive-int $repeatTimes * * @throws InvalidDataProviderException */ - public function build(ReflectionClass $theClass, string $methodName, array $groups = []): Test + public function build(ReflectionClass $theClass, string $methodName, array $groups = [], int $repeatTimes = 1): Test { $className = $theClass->getName(); @@ -56,37 +58,20 @@ public function build(ReflectionClass $theClass, string $methodName, array $grou } if ($data !== null) { - return $this->buildDataProviderTestSuite( - $methodName, - $className, - $data, - $this->shouldTestMethodBeRunInSeparateProcess($className, $methodName), - $this->shouldGlobalStateBePreserved($className, $methodName), - $this->backupSettings($className, $methodName), - $groups, - ); + return $this->buildDataProviderTestSuite($methodName, $className, $data, $groups, $repeatTimes); } - $test = new $className($methodName); - - $this->configureTestCase( - $test, - $this->shouldTestMethodBeRunInSeparateProcess($className, $methodName), - $this->shouldGlobalStateBePreserved($className, $methodName), - $this->backupSettings($className, $methodName), - ); - - return $test; + return $this->createTest($className, $methodName, $repeatTimes); } /** - * @param non-empty-string $methodName - * @param class-string $className - * @param array $data - * @param array{backupGlobals: ?true, backupGlobalsExcludeList: list, backupStaticProperties: ?true, backupStaticPropertiesExcludeList: array>} $backupSettings - * @param list $groups + * @param non-empty-string $methodName + * @param class-string $className + * @param array $data + * @param list $groups + * @param positive-int $repeatTimes */ - private function buildDataProviderTestSuite(string $methodName, string $className, array $data, bool $runTestInSeparateProcess, ?bool $preserveGlobalState, array $backupSettings, array $groups): DataProviderTestSuite + private function buildDataProviderTestSuite(string $methodName, string $className, array $data, array $groups, int $repeatTimes = 1): DataProviderTestSuite { $dataProviderTestSuite = DataProviderTestSuite::empty( $className . '::' . $methodName, @@ -98,21 +83,52 @@ private function buildDataProviderTestSuite(string $methodName, string $classNam ); foreach ($data as $_dataName => $_data) { - $_test = new $className($methodName); + $_test = $this->createTest($className, $methodName, $repeatTimes); $_test->setData($_dataName, $_data->value()); + $dataProviderTestSuite->addTest($_test, $groups); + } + + return $dataProviderTestSuite; + } + + /** + * @param class-string $className + * @param non-empty-string $methodName + * @param positive-int $repeatTimes + */ + private function createTest(string $className, string $methodName, int $repeatTimes): RepeatTestSuite|TestCase + { + if ($repeatTimes === 1) { + $test = new $className($methodName); + $this->configureTestCase( - $_test, - $runTestInSeparateProcess, - $preserveGlobalState, - $backupSettings, + $test, + $this->shouldTestMethodBeRunInSeparateProcess($className, $methodName), + $this->shouldGlobalStateBePreserved($className, $methodName), + $this->backupSettings($className, $methodName), ); + } else { + $tests = []; - $dataProviderTestSuite->addTest($_test, $groups); + foreach (range(1, $repeatTimes) as $i) { + $_test = new $className($methodName, $repeatTimes, $i); + + $this->configureTestCase( + $_test, + $this->shouldTestMethodBeRunInSeparateProcess($className, $methodName), + $this->shouldGlobalStateBePreserved($className, $methodName), + $this->backupSettings($className, $methodName), + ); + + $tests[] = $_test; + } + + $test = new RepeatTestSuite($tests); } - return $dataProviderTestSuite; + return $test; } /** diff --git a/src/Framework/TestCase.php b/src/Framework/TestCase.php index 57a33425aaa..d8f31f795e7 100644 --- a/src/Framework/TestCase.php +++ b/src/Framework/TestCase.php @@ -59,6 +59,7 @@ use AssertionError; use DeepCopy\DeepCopy; use PHPUnit\Event; +use PHPUnit\Event\InvalidArgumentException; use PHPUnit\Event\NoPreviousThrowableException; use PHPUnit\Framework\Constraint\Exception as ExceptionConstraint; use PHPUnit\Framework\Constraint\ExceptionCode; @@ -227,16 +228,35 @@ abstract class TestCase extends Assert implements Reorderable, SelfDescribing, T private mixed $errorLogCapture = false; private false|string $previousErrorLogTarget = false; + /** + * @var positive-int + */ + private readonly int $repeatTimes; + + /** + * @var positive-int + */ + private readonly int $repeatAttemptNumber; + /** * @param non-empty-string $name + * @param positive-int $repeatTimes + * @param positive-int $repeatAttemptNumber * * @internal This method is not covered by the backward compatibility promise for PHPUnit */ - final public function __construct(string $name) + final public function __construct(string $name, int $repeatTimes = 1, int $repeatAttemptNumber = 1) { $this->methodName = $name; $this->status = TestStatus::unknown(); + if ($repeatTimes < $repeatAttemptNumber) { + throw new InvalidArgumentException("Given repeat attempt number \"{$repeatAttemptNumber}\" must be inferior or equal to repeat times \"{$repeatTimes}\""); + } + + $this->repeatTimes = $repeatTimes; + $this->repeatAttemptNumber = $repeatAttemptNumber; + if (is_callable($this->sortId(), true)) { $this->providedTests = [new ExecutionOrderDependency($this->sortId())]; } @@ -643,7 +663,7 @@ final public function runBare(): void } } - if (!isset($e) && !isset($_e)) { + if (!isset($e) && !isset($_e) && $this->repeatAttemptNumber === $this->repeatTimes) { $emitter->testPassed( $this->valueObjectForEvents(), ); @@ -965,6 +985,31 @@ final public function wasPrepared(): bool return $this->wasPrepared; } + /** + * @return positive-int + * + * @internal This method is not covered by the backward compatibility promise for PHPUnit + */ + final public function repeatAttemptNumber(): int + { + return $this->repeatAttemptNumber; + } + + /** + * @internal This method is not covered by the backward compatibility promise for PHPUnit + */ + final public function markSkippedForErrorInPreviousRepetition(): void + { + $message = "Test repetition #{$this->repeatAttemptNumber} failure"; + + Event\Facade::emitter()->testSkipped( + $this->valueObjectForEvents(), + $message, + ); + + $this->status = TestStatus::skipped($message); + } + /** * Returns a matcher that matches when the method is executed * zero or more times. @@ -1486,6 +1531,12 @@ private function handleDependencies(): bool $returnValue = $passedTests->returnValue($dependencyTarget); + if ($this->repeatTimes > 1 && $returnValue !== null) { + $this->markSkippedForRepeatAndReturningDependency($dependency); + + return false; + } + if ($dependency->deepClone()) { $deepCopy = new DeepCopy; $deepCopy->skipUncloneable(false); @@ -1543,6 +1594,30 @@ private function markSkippedForMissingDependency(ExecutionOrderDependency $depen $this->status = TestStatus::skipped($message); } + /** + * @throws Exception + * @throws NoPreviousThrowableException + */ + private function markSkippedForRepeatAndReturningDependency(ExecutionOrderDependency $dependency): void + { + $message = sprintf( + 'This test depends on "%s" which returns a value. Such test cannot be run in repeat mode', + $dependency->targetIsClass() ? $dependency->getTargetClassName() : $dependency->getTarget(), + ); + + Event\Facade::emitter()->testTriggeredPhpunitWarning( + $this->valueObjectForEvents(), + $message, + ); + + Event\Facade::emitter()->testSkipped( + $this->valueObjectForEvents(), + $message, + ); + + $this->status = TestStatus::skipped($message); + } + private function startOutputBuffering(): void { ob_start(); diff --git a/src/Framework/TestSuite.php b/src/Framework/TestSuite.php index acf9e2b763e..b1f1a01247a 100644 --- a/src/Framework/TestSuite.php +++ b/src/Framework/TestSuite.php @@ -10,6 +10,7 @@ namespace PHPUnit\Framework; use const PHP_EOL; +use function array_map; use function array_merge; use function array_pop; use function array_reverse; @@ -21,6 +22,7 @@ use function is_callable; use function is_file; use function is_subclass_of; +use function range; use function sprintf; use function str_ends_with; use function str_starts_with; @@ -37,6 +39,7 @@ use PHPUnit\Metadata\MetadataCollection; use PHPUnit\Runner\Exception as RunnerException; use PHPUnit\Runner\Filter\Factory; +use PHPUnit\Runner\Phpt\RepeatTestSuite as PhptRepeatTestSuite; use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; use PHPUnit\Runner\TestSuiteLoader; use PHPUnit\TestRunner\TestResult\Facade as TestResultFacade; @@ -96,8 +99,9 @@ public static function empty(string $name): static /** * @param ReflectionClass $class * @param list $groups + * @param positive-int $repeatTimes */ - public static function fromClassReflector(ReflectionClass $class, array $groups = []): static + public static function fromClassReflector(ReflectionClass $class, array $groups = [], int $repeatTimes = 1): static { $testSuite = new static($class->getName()); @@ -118,7 +122,7 @@ public static function fromClassReflector(ReflectionClass $class, array $groups continue; } - $testSuite->addTestMethod($class, $method, $groups); + $testSuite->addTestMethod($class, $method, $groups, $repeatTimes); } if ($testSuite->isEmpty()) { @@ -188,10 +192,11 @@ public function addTest(Test $test, array $groups = []): void * * @param ReflectionClass $testClass * @param list $groups + * @param positive-int $repeatTimes * * @throws Exception */ - public function addTestSuite(ReflectionClass $testClass, array $groups = []): void + public function addTestSuite(ReflectionClass $testClass, array $groups = [], int $repeatTimes = 1): void { if ($testClass->isAbstract()) { throw new Exception( @@ -212,7 +217,7 @@ public function addTestSuite(ReflectionClass $testClass, array $groups = []): vo ); } - $this->addTest(self::fromClassReflector($testClass, $groups), $groups); + $this->addTest(self::fromClassReflector($testClass, $groups, $repeatTimes), $groups); } /** @@ -224,18 +229,31 @@ public function addTestSuite(ReflectionClass $testClass, array $groups = []): vo * leaving the current test run untouched. * * @param list $groups + * @param positive-int $repeatTimes * * @throws Exception */ - public function addTestFile(string $filename, array $groups = []): void + public function addTestFile(string $filename, array $groups = [], int $repeatTimes = 1): void { try { if (str_ends_with($filename, '.phpt') && is_file($filename)) { - $this->addTest(new PhptTestCase($filename)); + if ($repeatTimes === 1) { + $test = new PhptTestCase($filename); + } else { + $test = new PhptRepeatTestSuite( + array_map( + static fn () => new PhptTestCase($filename), + range(1, $repeatTimes), + ), + ); + } + + $this->addTest($test); } else { $this->addTestSuite( (new TestSuiteLoader)->load($filename), $groups, + $repeatTimes, ); } } catch (RunnerException $e) { @@ -249,13 +267,14 @@ public function addTestFile(string $filename, array $groups = []): void * Wrapper for addTestFile() that adds multiple test files. * * @param iterable $fileNames + * @param positive-int $repeatTimes * * @throws Exception */ - public function addTestFiles(iterable $fileNames): void + public function addTestFiles(iterable $fileNames, int $repeatTimes = 1): void { foreach ($fileNames as $filename) { - $this->addTestFile((string) $filename); + $this->addTestFile((string) $filename, [], $repeatTimes); } } @@ -503,16 +522,17 @@ public function isForTestClass(): bool /** * @param ReflectionClass $class * @param list $groups + * @param positive-int $repeatTimes * * @throws Exception */ - protected function addTestMethod(ReflectionClass $class, ReflectionMethod $method, array $groups): void + protected function addTestMethod(ReflectionClass $class, ReflectionMethod $method, array $groups, int $repeatTimes): void { $className = $class->getName(); $methodName = $method->getName(); try { - $test = (new TestBuilder)->build($class, $methodName, $groups); + $test = (new TestBuilder)->build($class, $methodName, $groups, $repeatTimes); } catch (InvalidDataProviderException $e) { if ($e->getProviderLabel() === null) { $message = sprintf( @@ -550,7 +570,7 @@ protected function addTestMethod(ReflectionClass $class, ReflectionMethod $metho return; } - if ($test instanceof TestCase || $test instanceof DataProviderTestSuite) { + if ($test instanceof TestCase || $test instanceof DataProviderTestSuite || $test instanceof RepeatTestSuite) { $test->setDependencies( Dependencies::dependencies($class->getName(), $methodName), ); diff --git a/src/Runner/Filter/NameFilterIterator.php b/src/Runner/Filter/NameFilterIterator.php index a2fc69b3d36..da5074f9e9e 100644 --- a/src/Runner/Filter/NameFilterIterator.php +++ b/src/Runner/Filter/NameFilterIterator.php @@ -13,8 +13,10 @@ use function preg_match; use function sprintf; use function substr; +use PHPUnit\Framework\RepeatTestSuite; use PHPUnit\Framework\Test; use PHPUnit\Framework\TestSuite; +use PHPUnit\Runner\Phpt\RepeatTestSuite as PhptRepeatTestSuite; use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; use RecursiveFilterIterator; use RecursiveIterator; @@ -56,11 +58,15 @@ public function accept(): bool return true; } - if ($test instanceof PhptTestCase) { + if ($test instanceof PhptTestCase || $test instanceof PhptRepeatTestSuite) { return false; } - $name = $test::class . '::' . $test->nameWithDataSet(); + if ($test instanceof RepeatTestSuite) { + $name = $test->name(); + } else { + $name = $test::class . '::' . $test->nameWithDataSet(); + } $accepted = @preg_match($this->regularExpression, $name, $matches) === 1; diff --git a/src/Runner/Phpt/RepeatTestSuite.php b/src/Runner/Phpt/RepeatTestSuite.php new file mode 100644 index 00000000000..8aaa5155bf9 --- /dev/null +++ b/src/Runner/Phpt/RepeatTestSuite.php @@ -0,0 +1,46 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\Runner\Phpt; + +use PHPUnit\Event\Facade as EventFacade; +use PHPUnit\Framework\AbstractRepeatTestSuite; +use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; + +/** + * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit + * + * @internal This class is not covered by the backward compatibility promise for PHPUnit + * + * @extends AbstractRepeatTestSuite + */ +final readonly class RepeatTestSuite extends AbstractRepeatTestSuite +{ + public function run(): void + { + $defectOccurred = false; + + foreach ($this->tests as $test) { + if ($defectOccurred) { + EventFacade::emitter()->testSkipped( + $this->valueObjectForEvents(), + 'Test repetition failure', + ); + + continue; + } + + $test->run(); + + if (!$test->passed()) { + $defectOccurred = true; + } + } + } +} diff --git a/src/Runner/Phpt/TestCase.php b/src/Runner/Phpt/TestCase.php index ab75397b20a..02b02609ef0 100644 --- a/src/Runner/Phpt/TestCase.php +++ b/src/Runner/Phpt/TestCase.php @@ -70,19 +70,27 @@ * * @see https://qa.php.net/phpt_details.php */ -final readonly class TestCase implements Reorderable, SelfDescribing, Test +final class TestCase implements Reorderable, SelfDescribing, Test { /** * @var non-empty-string */ - private string $filename; + private readonly string $filename; + private bool $passed = false; + + /** + * @var positive-int + */ + private int $repeatAttemptNumber; /** * @param non-empty-string $filename + * @param positive-int $repeatAttemptNumber */ - public function __construct(string $filename) + public function __construct(string $filename, int $repeatAttemptNumber = 1) { - $this->filename = $filename; + $this->filename = $filename; + $this->repeatAttemptNumber = $repeatAttemptNumber; } public function count(): int @@ -242,6 +250,8 @@ public function run(): void $emitter->testPassed($this->valueObjectForEvents()); } + $this->passed = $passed; + $this->runClean($sections, CodeCoverage::instance()->isActive()); $emitter->testFinished($this->valueObjectForEvents(), 1); @@ -255,6 +265,11 @@ public function getName(): string return $this->toString(); } + public function passed(): bool + { + return $this->passed; + } + /** * Returns a string representation of the test case. */ @@ -289,7 +304,7 @@ public function requires(): array */ public function valueObjectForEvents(): Phpt { - return new Phpt($this->filename); + return new Phpt($this->filename, $this->repeatAttemptNumber); } /** diff --git a/src/TextUI/Configuration/Cli/Builder.php b/src/TextUI/Configuration/Cli/Builder.php index 1accafe3a8f..674c474317f 100644 --- a/src/TextUI/Configuration/Cli/Builder.php +++ b/src/TextUI/Configuration/Cli/Builder.php @@ -158,6 +158,7 @@ final class Builder 'log-events-verbose-text=', 'version', 'debug', + 'repeat=', 'with-telemetry', 'extension=', ]; @@ -312,6 +313,7 @@ public function fromParameters(array $parameters): Configuration $printerTestDox = null; $printerTestDoxSummary = null; $debug = false; + $repeatTimes = 1; $withTelemetry = false; $extensions = []; @@ -1211,6 +1213,17 @@ public function fromParameters(array $parameters): Configuration break; + case '--repeat': + $repeatTimes = (int) $option[1]; + + if ($repeatTimes < 1) { + throw new Exception( + 'The value for the --repeat option must be a positive integer', + ); + } + + break; + case '--with-telemetry': $withTelemetry = true; @@ -1363,6 +1376,7 @@ public function fromParameters(array $parameters): Configuration $debug, $withTelemetry, $extensions, + $repeatTimes, ); } diff --git a/src/TextUI/Configuration/Cli/Configuration.php b/src/TextUI/Configuration/Cli/Configuration.php index e637281c698..62547b8c098 100644 --- a/src/TextUI/Configuration/Cli/Configuration.php +++ b/src/TextUI/Configuration/Cli/Configuration.php @@ -177,6 +177,7 @@ private ?string $logEventsVerboseText; private bool $debug; private bool $withTelemetry; + private int $repeatTimes; /** * @var ?non-empty-list @@ -194,8 +195,9 @@ * @param ?non-empty-list $testSuffixes * @param ?non-empty-list $coverageFilter * @param ?non-empty-list $extensions + * @param positive-int $repeatTimes */ - public function __construct(array $arguments, ?bool $all, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticProperties, ?bool $beStrictAboutChangesToGlobalState, ?string $bootstrap, ?string $cacheDirectory, ?bool $cacheResult, bool $checkPhpConfiguration, bool $checkVersion, ?string $colors, null|int|string $columns, ?string $configurationFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $pathCoverage, bool $warmCoverageCache, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?bool $failOnAllIssues, ?bool $failOnDeprecation, ?bool $failOnPhpunitDeprecation, ?bool $failOnPhpunitNotice, ?bool $failOnPhpunitWarning, ?bool $failOnEmptyTestSuite, ?bool $failOnIncomplete, ?bool $failOnNotice, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?bool $doNotFailOnDeprecation, ?bool $doNotFailOnPhpunitDeprecation, ?bool $doNotFailOnPhpunitNotice, ?bool $doNotFailOnPhpunitWarning, ?bool $doNotFailOnEmptyTestSuite, ?bool $doNotFailOnIncomplete, ?bool $doNotFailOnNotice, ?bool $doNotFailOnRisky, ?bool $doNotFailOnSkipped, ?bool $doNotFailOnWarning, ?bool $stopOnDefect, ?bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnNotice, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $filter, ?string $excludeFilter, ?string $generateBaseline, ?string $useBaseline, bool $ignoreBaseline, bool $generateConfiguration, bool $migrateConfiguration, ?array $groups, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?string $otrLogfile, ?bool $includeGitInformation, bool $listGroups, bool $listSuites, bool $listTestFiles, bool $listTests, ?string $listTestsXml, ?bool $noCoverage, ?bool $noExtensions, ?bool $noOutput, ?bool $noProgress, ?bool $noResults, ?bool $noLogging, ?bool $processIsolation, ?int $randomOrderSeed, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?string $teamcityLogfile, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?array $testSuffixes, ?string $testSuite, ?string $excludeTestSuite, bool $useDefaultConfiguration, ?bool $displayDetailsOnAllIssues, ?bool $displayDetailsOnIncompleteTests, ?bool $displayDetailsOnSkippedTests, ?bool $displayDetailsOnTestsThatTriggerDeprecations, ?bool $displayDetailsOnPhpunitDeprecations, ?bool $displayDetailsOnPhpunitNotices, ?bool $displayDetailsOnTestsThatTriggerErrors, ?bool $displayDetailsOnTestsThatTriggerNotices, ?bool $displayDetailsOnTestsThatTriggerWarnings, bool $version, ?array $coverageFilter, ?string $logEventsText, ?string $logEventsVerboseText, ?bool $printerTeamCity, ?bool $testdoxPrinter, ?bool $testdoxPrinterSummary, bool $debug, bool $withTelemetry, ?array $extensions) + public function __construct(array $arguments, ?bool $all, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticProperties, ?bool $beStrictAboutChangesToGlobalState, ?string $bootstrap, ?string $cacheDirectory, ?bool $cacheResult, bool $checkPhpConfiguration, bool $checkVersion, ?string $colors, null|int|string $columns, ?string $configurationFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $pathCoverage, bool $warmCoverageCache, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?bool $failOnAllIssues, ?bool $failOnDeprecation, ?bool $failOnPhpunitDeprecation, ?bool $failOnPhpunitNotice, ?bool $failOnPhpunitWarning, ?bool $failOnEmptyTestSuite, ?bool $failOnIncomplete, ?bool $failOnNotice, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?bool $doNotFailOnDeprecation, ?bool $doNotFailOnPhpunitDeprecation, ?bool $doNotFailOnPhpunitNotice, ?bool $doNotFailOnPhpunitWarning, ?bool $doNotFailOnEmptyTestSuite, ?bool $doNotFailOnIncomplete, ?bool $doNotFailOnNotice, ?bool $doNotFailOnRisky, ?bool $doNotFailOnSkipped, ?bool $doNotFailOnWarning, ?bool $stopOnDefect, ?bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnNotice, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $filter, ?string $excludeFilter, ?string $generateBaseline, ?string $useBaseline, bool $ignoreBaseline, bool $generateConfiguration, bool $migrateConfiguration, ?array $groups, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?string $otrLogfile, ?bool $includeGitInformation, bool $listGroups, bool $listSuites, bool $listTestFiles, bool $listTests, ?string $listTestsXml, ?bool $noCoverage, ?bool $noExtensions, ?bool $noOutput, ?bool $noProgress, ?bool $noResults, ?bool $noLogging, ?bool $processIsolation, ?int $randomOrderSeed, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?string $teamcityLogfile, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?array $testSuffixes, ?string $testSuite, ?string $excludeTestSuite, bool $useDefaultConfiguration, ?bool $displayDetailsOnAllIssues, ?bool $displayDetailsOnIncompleteTests, ?bool $displayDetailsOnSkippedTests, ?bool $displayDetailsOnTestsThatTriggerDeprecations, ?bool $displayDetailsOnPhpunitDeprecations, ?bool $displayDetailsOnPhpunitNotices, ?bool $displayDetailsOnTestsThatTriggerErrors, ?bool $displayDetailsOnTestsThatTriggerNotices, ?bool $displayDetailsOnTestsThatTriggerWarnings, bool $version, ?array $coverageFilter, ?string $logEventsText, ?string $logEventsVerboseText, ?bool $printerTeamCity, ?bool $testdoxPrinter, ?bool $testdoxPrinterSummary, bool $debug, bool $withTelemetry, ?array $extensions, int $repeatTimes) { $this->arguments = $arguments; $this->all = $all; @@ -322,6 +324,7 @@ public function __construct(array $arguments, ?bool $all, ?string $atLeastVersio $this->debug = $debug; $this->withTelemetry = $withTelemetry; $this->extensions = $extensions; + $this->repeatTimes = $repeatTimes; } /** @@ -2578,6 +2581,14 @@ public function debug(): bool return $this->debug; } + /** + * @return positive-int + */ + public function repeatTimes(): int + { + return $this->repeatTimes; + } + public function withTelemetry(): bool { return $this->withTelemetry; diff --git a/src/TextUI/Configuration/Configuration.php b/src/TextUI/Configuration/Configuration.php index 82631e5e675..268de53fb14 100644 --- a/src/TextUI/Configuration/Configuration.php +++ b/src/TextUI/Configuration/Configuration.php @@ -194,6 +194,11 @@ */ private ?string $generateBaseline; private bool $debug; + + /** + * @var positive-int + */ + private int $repeatTimes; private bool $withTelemetry; /** @@ -213,9 +218,10 @@ * @param list $excludeGroups * @param non-empty-list $testSuffixes * @param null|non-empty-string $generateBaseline + * @param positive-int $repeatTimes * @param non-negative-int $shortenArraysForExportThreshold */ - public function __construct(array $cliArguments, ?string $configurationFile, ?string $bootstrap, array $bootstrapForTestSuite, bool $cacheResult, ?string $cacheDirectory, ?string $coverageCacheDirectory, Source $source, string $testResultCacheFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4j, int $coverageCrap4jThreshold, ?string $coverageHtml, int $coverageHtmlLowUpperBound, int $coverageHtmlHighLowerBound, string $coverageHtmlColorSuccessLow, string $coverageHtmlColorSuccessMedium, string $coverageHtmlColorSuccessHigh, string $coverageHtmlColorWarning, string $coverageHtmlColorDanger, ?string $coverageHtmlCustomCssFile, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, bool $coverageTextShowUncoveredFiles, bool $coverageTextShowOnlySummary, ?string $coverageXml, bool $pathCoverage, bool $ignoreDeprecatedCodeUnitsFromCodeCoverage, bool $disableCodeCoverageIgnore, bool $failOnAllIssues, bool $failOnDeprecation, bool $failOnPhpunitDeprecation, bool $failOnPhpunitNotice, bool $failOnPhpunitWarning, bool $failOnEmptyTestSuite, bool $failOnIncomplete, bool $failOnNotice, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $doNotFailOnDeprecation, bool $doNotFailOnPhpunitDeprecation, bool $doNotFailOnPhpunitNotice, bool $doNotFailOnPhpunitWarning, bool $doNotFailOnEmptyTestSuite, bool $doNotFailOnIncomplete, bool $doNotFailOnNotice, bool $doNotFailOnRisky, bool $doNotFailOnSkipped, bool $doNotFailOnWarning, bool $stopOnDefect, bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, bool $stopOnError, bool $stopOnFailure, bool $stopOnIncomplete, bool $stopOnNotice, bool $stopOnRisky, bool $stopOnSkipped, bool $stopOnWarning, bool $outputToStandardErrorStream, int $columns, bool $noExtensions, ?string $pharExtensionDirectory, array $extensionBootstrappers, bool $backupGlobals, bool $backupStaticProperties, bool $beStrictAboutChangesToGlobalState, bool $colors, bool $processIsolation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, bool $reportUselessTests, bool $strictCoverage, bool $disallowTestOutput, bool $displayDetailsOnAllIssues, bool $displayDetailsOnIncompleteTests, bool $displayDetailsOnSkippedTests, bool $displayDetailsOnTestsThatTriggerDeprecations, bool $displayDetailsOnPhpunitDeprecations, bool $displayDetailsOnPhpunitNotices, bool $displayDetailsOnTestsThatTriggerErrors, bool $displayDetailsOnTestsThatTriggerNotices, bool $displayDetailsOnTestsThatTriggerWarnings, bool $reverseDefectList, bool $requireCoverageMetadata, bool $noProgress, bool $noResults, bool $noOutput, int $executionOrder, int $executionOrderDefects, bool $resolveDependencies, ?string $logfileTeamcity, ?string $logfileJunit, ?string $logfileOtr, bool $includeGitInformationInOtrLogfile, ?string $logfileTestdoxHtml, ?string $logfileTestdoxText, ?string $logEventsText, ?string $logEventsVerboseText, bool $teamCityOutput, bool $testDoxOutput, bool $testDoxOutputSummary, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, ?string $filter, ?string $excludeFilter, array $groups, array $excludeGroups, int $randomOrderSeed, bool $includeUncoveredFiles, TestSuiteCollection $testSuite, string $includeTestSuite, string $excludeTestSuite, ?string $defaultTestSuite, bool $ignoreTestSelectionInXmlConfiguration, array $testSuffixes, Php $php, bool $controlGarbageCollector, int $numberOfTestsBeforeGarbageCollection, ?string $generateBaseline, bool $debug, bool $withTelemetry, int $shortenArraysForExportThreshold) + public function __construct(array $cliArguments, ?string $configurationFile, ?string $bootstrap, array $bootstrapForTestSuite, bool $cacheResult, ?string $cacheDirectory, ?string $coverageCacheDirectory, Source $source, string $testResultCacheFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4j, int $coverageCrap4jThreshold, ?string $coverageHtml, int $coverageHtmlLowUpperBound, int $coverageHtmlHighLowerBound, string $coverageHtmlColorSuccessLow, string $coverageHtmlColorSuccessMedium, string $coverageHtmlColorSuccessHigh, string $coverageHtmlColorWarning, string $coverageHtmlColorDanger, ?string $coverageHtmlCustomCssFile, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, bool $coverageTextShowUncoveredFiles, bool $coverageTextShowOnlySummary, ?string $coverageXml, bool $pathCoverage, bool $ignoreDeprecatedCodeUnitsFromCodeCoverage, bool $disableCodeCoverageIgnore, bool $failOnAllIssues, bool $failOnDeprecation, bool $failOnPhpunitDeprecation, bool $failOnPhpunitNotice, bool $failOnPhpunitWarning, bool $failOnEmptyTestSuite, bool $failOnIncomplete, bool $failOnNotice, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $doNotFailOnDeprecation, bool $doNotFailOnPhpunitDeprecation, bool $doNotFailOnPhpunitNotice, bool $doNotFailOnPhpunitWarning, bool $doNotFailOnEmptyTestSuite, bool $doNotFailOnIncomplete, bool $doNotFailOnNotice, bool $doNotFailOnRisky, bool $doNotFailOnSkipped, bool $doNotFailOnWarning, bool $stopOnDefect, bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, bool $stopOnError, bool $stopOnFailure, bool $stopOnIncomplete, bool $stopOnNotice, bool $stopOnRisky, bool $stopOnSkipped, bool $stopOnWarning, bool $outputToStandardErrorStream, int $columns, bool $noExtensions, ?string $pharExtensionDirectory, array $extensionBootstrappers, bool $backupGlobals, bool $backupStaticProperties, bool $beStrictAboutChangesToGlobalState, bool $colors, bool $processIsolation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, bool $reportUselessTests, bool $strictCoverage, bool $disallowTestOutput, bool $displayDetailsOnAllIssues, bool $displayDetailsOnIncompleteTests, bool $displayDetailsOnSkippedTests, bool $displayDetailsOnTestsThatTriggerDeprecations, bool $displayDetailsOnPhpunitDeprecations, bool $displayDetailsOnPhpunitNotices, bool $displayDetailsOnTestsThatTriggerErrors, bool $displayDetailsOnTestsThatTriggerNotices, bool $displayDetailsOnTestsThatTriggerWarnings, bool $reverseDefectList, bool $requireCoverageMetadata, bool $noProgress, bool $noResults, bool $noOutput, int $executionOrder, int $executionOrderDefects, bool $resolveDependencies, ?string $logfileTeamcity, ?string $logfileJunit, ?string $logfileOtr, bool $includeGitInformationInOtrLogfile, ?string $logfileTestdoxHtml, ?string $logfileTestdoxText, ?string $logEventsText, ?string $logEventsVerboseText, bool $teamCityOutput, bool $testDoxOutput, bool $testDoxOutputSummary, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, ?string $filter, ?string $excludeFilter, array $groups, array $excludeGroups, int $randomOrderSeed, bool $includeUncoveredFiles, TestSuiteCollection $testSuite, string $includeTestSuite, string $excludeTestSuite, ?string $defaultTestSuite, bool $ignoreTestSelectionInXmlConfiguration, array $testSuffixes, Php $php, bool $controlGarbageCollector, int $numberOfTestsBeforeGarbageCollection, ?string $generateBaseline, bool $debug, int $repeatTimes, bool $withTelemetry, int $shortenArraysForExportThreshold) { $this->cliArguments = $cliArguments; $this->configurationFile = $configurationFile; @@ -345,6 +351,7 @@ public function __construct(array $cliArguments, ?string $configurationFile, ?st $this->numberOfTestsBeforeGarbageCollection = $numberOfTestsBeforeGarbageCollection; $this->generateBaseline = $generateBaseline; $this->debug = $debug; + $this->repeatTimes = $repeatTimes; $this->withTelemetry = $withTelemetry; $this->shortenArraysForExportThreshold = $shortenArraysForExportThreshold; } @@ -1514,6 +1521,14 @@ public function debug(): bool return $this->debug; } + /** + * @return positive-int + */ + public function repeatTimes(): int + { + return $this->repeatTimes; + } + public function withTelemetry(): bool { return $this->withTelemetry; diff --git a/src/TextUI/Configuration/Merger.php b/src/TextUI/Configuration/Merger.php index bdc22be7d24..b6b7ec74926 100644 --- a/src/TextUI/Configuration/Merger.php +++ b/src/TextUI/Configuration/Merger.php @@ -1064,6 +1064,7 @@ public function merge(CliConfiguration $cliConfiguration, XmlConfiguration $xmlC $xmlConfiguration->phpunit()->numberOfTestsBeforeGarbageCollection(), $generateBaseline, $cliConfiguration->debug(), + $cliConfiguration->repeatTimes(), $cliConfiguration->withTelemetry(), $xmlConfiguration->phpunit()->shortenArraysForExportThreshold(), ); diff --git a/src/TextUI/Configuration/TestSuiteBuilder.php b/src/TextUI/Configuration/TestSuiteBuilder.php index a474b02f548..a39abf933e6 100644 --- a/src/TextUI/Configuration/TestSuiteBuilder.php +++ b/src/TextUI/Configuration/TestSuiteBuilder.php @@ -58,11 +58,13 @@ public function build(Configuration $configuration): TestSuite $testSuite = $this->testSuiteFromPath( $arguments[0], $configuration->testSuffixes(), + $configuration->repeatTimes(), ); } else { $testSuite = $this->testSuiteFromPathList( $arguments, $configuration->testSuffixes(), + $configuration->repeatTimes(), ); } } @@ -77,6 +79,7 @@ public function build(Configuration $configuration): TestSuite $configuration->testSuite(), $configuration->ignoreTestSelectionInXmlConfiguration() ? [] : $configuration->includeTestSuites(), $configuration->ignoreTestSelectionInXmlConfiguration() ? [] : $configuration->excludeTestSuites(), + $configuration->repeatTimes(), ); } @@ -88,17 +91,18 @@ public function build(Configuration $configuration): TestSuite /** * @param non-empty-string $path * @param list $suffixes + * @param positive-int $repeatTimes * * @throws \PHPUnit\Framework\Exception */ - private function testSuiteFromPath(string $path, array $suffixes, ?TestSuite $suite = null): TestSuite + private function testSuiteFromPath(string $path, array $suffixes, int $repeatTimes, ?TestSuite $suite = null): TestSuite { if (str_ends_with($path, '.phpt') && is_file($path)) { if ($suite === null) { $suite = TestSuite::empty($path); } - $suite->addTestFile($path); + $suite->addTestFile($path, [], $repeatTimes); return $suite; } @@ -110,7 +114,7 @@ private function testSuiteFromPath(string $path, array $suffixes, ?TestSuite $su $suite = TestSuite::empty('CLI Arguments'); } - $suite->addTestFiles($files); + $suite->addTestFiles($files, $repeatTimes); return $suite; } @@ -124,10 +128,10 @@ private function testSuiteFromPath(string $path, array $suffixes, ?TestSuite $su } if ($suite === null) { - return TestSuite::fromClassReflector($testClass); + return TestSuite::fromClassReflector($testClass, [], $repeatTimes); } - $suite->addTestSuite($testClass); + $suite->addTestSuite($testClass, [], $repeatTimes); return $suite; } @@ -135,15 +139,16 @@ private function testSuiteFromPath(string $path, array $suffixes, ?TestSuite $su /** * @param list $paths * @param list $suffixes + * @param positive-int $repeatTimes * * @throws \PHPUnit\Framework\Exception */ - private function testSuiteFromPathList(array $paths, array $suffixes): TestSuite + private function testSuiteFromPathList(array $paths, array $suffixes, int $repeatTimes): TestSuite { $suite = TestSuite::empty('CLI Arguments'); foreach ($paths as $path) { - $this->testSuiteFromPath($path, $suffixes, $suite); + $this->testSuiteFromPath($path, $suffixes, $repeatTimes, $suite); } return $suite; diff --git a/src/TextUI/Configuration/Xml/TestSuiteMapper.php b/src/TextUI/Configuration/Xml/TestSuiteMapper.php index 66bd611c1c6..db3457b0579 100644 --- a/src/TextUI/Configuration/Xml/TestSuiteMapper.php +++ b/src/TextUI/Configuration/Xml/TestSuiteMapper.php @@ -36,12 +36,13 @@ * @param non-empty-string $xmlConfigurationFile * @param list $includeTestSuites * @param list $excludeTestSuites + * @param positive-int $repeatTimes * * @throws RuntimeException * @throws TestDirectoryNotFoundException * @throws TestFileNotFoundException */ - public function map(string $xmlConfigurationFile, TestSuiteCollection $configuredTestSuites, array $includeTestSuites, array $excludeTestSuites): TestSuiteObject + public function map(string $xmlConfigurationFile, TestSuiteCollection $configuredTestSuites, array $includeTestSuites, array $excludeTestSuites, int $repeatTimes = 1): TestSuiteObject { try { $result = TestSuiteObject::empty($xmlConfigurationFile); @@ -101,7 +102,7 @@ public function map(string $xmlConfigurationFile, TestSuiteCollection $configure $processed[$file] = $testSuiteName; $empty = false; - $testSuite->addTestFile($file, $groups); + $testSuite->addTestFile($file, $groups, $repeatTimes); } } @@ -130,7 +131,7 @@ public function map(string $xmlConfigurationFile, TestSuiteCollection $configure $processed[$file->path()] = $testSuiteName; $empty = false; - $testSuite->addTestFile($file->path(), $file->groups()); + $testSuite->addTestFile($file->path(), $file->groups(), $repeatTimes); } if (!$empty) { diff --git a/src/TextUI/Help.php b/src/TextUI/Help.php index 8a4f802abdf..ba92315a469 100644 --- a/src/TextUI/Help.php +++ b/src/TextUI/Help.php @@ -276,6 +276,7 @@ private function elements(): array ['spacer' => ''], ['arg' => '--debug', 'desc' => 'Replace default progress and result output with debugging information'], + ['arg' => '--repeat ', 'desc' => 'Runs the test(s) repeatedly'], ['arg' => '--with-telemetry', 'desc' => 'Include telemetry information in debugging information output'], ], diff --git a/tests/end-to-end/_files/output-cli-help-color.txt b/tests/end-to-end/_files/output-cli-help-color.txt index 4e68ba81c2c..429335baac3 100644 --- a/tests/end-to-end/_files/output-cli-help-color.txt +++ b/tests/end-to-end/_files/output-cli-help-color.txt @@ -176,6 +176,7 @@ --debug  Replace default progress and result output with debugging information + --repeat   Runs the test(s) repeatedly --with-telemetry  Include telemetry information in debugging information output diff --git a/tests/end-to-end/_files/output-cli-usage.txt b/tests/end-to-end/_files/output-cli-usage.txt index 932879ec51a..82a67fa5c6e 100644 --- a/tests/end-to-end/_files/output-cli-usage.txt +++ b/tests/end-to-end/_files/output-cli-usage.txt @@ -117,6 +117,7 @@ Reporting: --testdox-summary Repeat TestDox output for tests with errors, failures, or issues --debug Replace default progress and result output with debugging information + --repeat Runs the test(s) repeatedly --with-telemetry Include telemetry information in debugging information output Logging: diff --git a/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php b/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php new file mode 100644 index 00000000000..6dbc00eff0f --- /dev/null +++ b/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php @@ -0,0 +1,31 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +use PHPUnit\Framework\Attributes\Depends; +use PHPUnit\Framework\TestCase; + +final class DependentOfTestFailedInRepetitionTest extends TestCase +{ + public function test1(): void + { + static $cout = 0; + + if ($cout++ > 0) { + $this->assertFalse(true); + } + + $this->assertTrue(true); + } + + #[Depends('test1')] + public function test2(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/repeat/_files/DependentOfTestWhichReturnsSomethingTest.php b/tests/end-to-end/repeat/_files/DependentOfTestWhichReturnsSomethingTest.php new file mode 100644 index 00000000000..b24ac4c65f1 --- /dev/null +++ b/tests/end-to-end/repeat/_files/DependentOfTestWhichReturnsSomethingTest.php @@ -0,0 +1,27 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +use PHPUnit\Framework\Attributes\Depends; +use PHPUnit\Framework\TestCase; + +final class DependentOfTestWhichReturnsSomethingTest extends TestCase +{ + public function test1(): string + { + $this->assertTrue(true); + + return 'foo'; + } + + #[Depends('test1')] + public function test2(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/repeat/_files/RepeatDependentTest.php b/tests/end-to-end/repeat/_files/RepeatDependentTest.php new file mode 100644 index 00000000000..86ffa44654a --- /dev/null +++ b/tests/end-to-end/repeat/_files/RepeatDependentTest.php @@ -0,0 +1,36 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +use PHPUnit\Framework\Attributes\Depends; +use PHPUnit\Framework\TestCase; + +final class RepeatDependentTest extends TestCase +{ + public function test1(): void + { + $this->assertTrue(true); + } + + #[Depends('test1')] + public function testDepends1(): void + { + $this->assertTrue(true); + } + + public function test2(): void + { + $this->assertTrue(false); + } + + #[Depends('test2')] + public function testDepends2(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/repeat/_files/RepeatInIsolationTest.php b/tests/end-to-end/repeat/_files/RepeatInIsolationTest.php new file mode 100644 index 00000000000..f5c6328dc99 --- /dev/null +++ b/tests/end-to-end/repeat/_files/RepeatInIsolationTest.php @@ -0,0 +1,47 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +use PHPUnit\Framework\Attributes\RunTestsInSeparateProcesses; +use PHPUnit\Framework\TestCase; + +#[RunTestsInSeparateProcesses] +final class RepeatInIsolationTest extends TestCase +{ + private Closure $notCloneableFixture; + private int $counter; + + public function __clone() + { + if (isset($this->notCloneableFixture)) { + throw new LogicException('Cannot clone RepeatInIsolationTest because it contains a non-cloneable fixture.'); + } + } + + protected function setUp(): void + { + $this->counter = 0; + $this->notCloneableFixture = static fn () => true; + } + + public function test1(): void + { + $this->assertTrue(($this->notCloneableFixture)()); + + $this->counter++; + $this->assertSame(1, $this->counter); + } + + public function test2(): void + { + $this->assertTrue(($this->notCloneableFixture)()); + + $this->counter++; + $this->assertSame(1, $this->counter); + } +} diff --git a/tests/end-to-end/repeat/_files/RepeatTest.php b/tests/end-to-end/repeat/_files/RepeatTest.php new file mode 100644 index 00000000000..4d960902f13 --- /dev/null +++ b/tests/end-to-end/repeat/_files/RepeatTest.php @@ -0,0 +1,45 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +use PHPUnit\Framework\TestCase; + +final class RepeatTest extends TestCase +{ + private Closure $notCloneableFixture; + private int $counter; + + public function __clone() + { + if (isset($this->notCloneableFixture)) { + throw new LogicException('Cannot clone RepeatInIsolationTest because it contains a non-cloneable fixture.'); + } + } + + protected function setUp(): void + { + $this->counter = 0; + $this->notCloneableFixture = static fn () => true; + } + + public function test1(): void + { + $this->assertTrue(($this->notCloneableFixture)()); + + $this->counter++; + $this->assertSame(1, $this->counter); + } + + public function test2(): void + { + $this->assertTrue(($this->notCloneableFixture)()); + + $this->counter++; + $this->assertSame(1, $this->counter); + } +} diff --git a/tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php b/tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php new file mode 100644 index 00000000000..fa071e5f112 --- /dev/null +++ b/tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php @@ -0,0 +1,55 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\Depends; +use PHPUnit\Framework\TestCase; + +final class RepeatWithDataProviderAndDependsTest extends TestCase +{ + public static function provide1(): iterable + { + yield [true]; + + yield [true]; + } + + public static function provide2(): iterable + { + yield [true]; + + yield [false]; + + yield [true]; + } + + #[DataProvider('provide1')] + public function test1(bool $bool): void + { + $this->assertTrue($bool); + } + + #[Depends('test1')] + public function testDependsOn1(): void + { + $this->assertTrue(true); + } + + #[DataProvider('provide2')] + public function test2(bool $bool): void + { + $this->assertTrue($bool); + } + + #[Depends('test2')] + public function testDependsOn2(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/repeat/_files/RepeatWithDataProviderFailingTest.php b/tests/end-to-end/repeat/_files/RepeatWithDataProviderFailingTest.php new file mode 100644 index 00000000000..6d5f4ca4a6d --- /dev/null +++ b/tests/end-to-end/repeat/_files/RepeatWithDataProviderFailingTest.php @@ -0,0 +1,29 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; + +final class RepeatWithDataProviderFailingTest extends TestCase +{ + public static function provide(): iterable + { + yield [true]; + + yield [false]; + + yield [true]; + } + + #[DataProvider('provide')] + public function test1(bool $bool): void + { + $this->assertTrue($bool); + } +} diff --git a/tests/end-to-end/repeat/_files/RepeatWithDataProviderTest.php b/tests/end-to-end/repeat/_files/RepeatWithDataProviderTest.php new file mode 100644 index 00000000000..f24b199d5f9 --- /dev/null +++ b/tests/end-to-end/repeat/_files/RepeatWithDataProviderTest.php @@ -0,0 +1,27 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; + +final class RepeatWithDataProviderTest extends TestCase +{ + public static function provide(): iterable + { + yield [true]; + + yield [true]; + } + + #[DataProvider('provide')] + public function test1(bool $bool): void + { + $this->assertTrue($bool); + } +} diff --git a/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php b/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php new file mode 100644 index 00000000000..6bfd9318015 --- /dev/null +++ b/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php @@ -0,0 +1,45 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +use PHPUnit\Framework\TestCase; + +final class RepeatWithFailuresTest extends TestCase +{ + public function test1(): void + { + $this->assertFalse(true); + } + + public function test2(): void + { + static $cout = 0; + + if ($cout++ > 0) { + $this->assertFalse(true); + } + + $this->assertTrue(true); + } + + public function test3(): void + { + static $cout = 0; + + if ($cout++ > 1) { + $this->assertFalse(true); + } + + $this->assertTrue(true); + } + + public function test4(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/repeat/_files/directory/RepeatInDirectoryTest.php b/tests/end-to-end/repeat/_files/directory/RepeatInDirectoryTest.php new file mode 100644 index 00000000000..855c136c031 --- /dev/null +++ b/tests/end-to-end/repeat/_files/directory/RepeatInDirectoryTest.php @@ -0,0 +1,20 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace directory; + +use PHPUnit\Framework\TestCase; + +final class RepeatInDirectoryTest extends TestCase +{ + public function test1(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/repeat/_files/phpt/failure.phpt b/tests/end-to-end/repeat/_files/phpt/failure.phpt new file mode 100644 index 00000000000..9911b3ec52b --- /dev/null +++ b/tests/end-to-end/repeat/_files/phpt/failure.phpt @@ -0,0 +1,8 @@ +--TEST-- +Repeat option +--FILE-- + + + + + RepeatTest.php + + + diff --git a/tests/end-to-end/repeat/dependent-on-test-which-returns-something.phpt b/tests/end-to-end/repeat/dependent-on-test-which-returns-something.phpt new file mode 100644 index 00000000000..bc9f4406c90 --- /dev/null +++ b/tests/end-to-end/repeat/dependent-on-test-which-returns-something.phpt @@ -0,0 +1,31 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +..SS 4 / 4 (100%) + +Time: %s, Memory: %s MB + +1 test triggered 1 PHPUnit warning: + +1) DependentOfTestWhichReturnsSomethingTest::test2 +This test depends on "DependentOfTestWhichReturnsSomethingTest::test1" which returns a value. Such test cannot be run in repeat mode + +%s/tests/end-to-end/repeat/_files/DependentOfTestWhichReturnsSomethingTest.php:%d + +OK, but there were issues! +Tests: 4, Assertions: 2, PHPUnit Warnings: 1, Skipped: 2. diff --git a/tests/end-to-end/repeat/dependent-test-random.phpt b/tests/end-to-end/repeat/dependent-test-random.phpt new file mode 100644 index 00000000000..e1b6ed15096 --- /dev/null +++ b/tests/end-to-end/repeat/dependent-test-random.phpt @@ -0,0 +1,17 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTREGEX-- +(FSSS....)|(FS..SS..)|(FS....SS)|(..FSSS..)|(..FS..SS)|(....FSSS) diff --git a/tests/end-to-end/repeat/dependent-test-with-failure.phpt b/tests/end-to-end/repeat/dependent-test-with-failure.phpt new file mode 100644 index 00000000000..0be2ded7a5e --- /dev/null +++ b/tests/end-to-end/repeat/dependent-test-with-failure.phpt @@ -0,0 +1,31 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.FSS 4 / 4 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) DependentOfTestFailedInRepetitionTest::test1 (repeat attempt #2) +Failed asserting that true is false. + +%s/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php:%d + +FAILURES! +Tests: 4, Assertions: 2, Failures: 1, Skipped: 2. diff --git a/tests/end-to-end/repeat/dependent-test.phpt b/tests/end-to-end/repeat/dependent-test.phpt new file mode 100644 index 00000000000..9f8b2b120c5 --- /dev/null +++ b/tests/end-to-end/repeat/dependent-test.phpt @@ -0,0 +1,31 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +....FSSS 8 / 8 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) RepeatDependentTest::test2 +Failed asserting that false is true. + +%s/tests/end-to-end/repeat/_files/RepeatDependentTest.php:%d + +FAILURES! +Tests: 8, Assertions: 5, Failures: 1, Skipped: 3. diff --git a/tests/end-to-end/repeat/error-skips-next-repetitions.phpt b/tests/end-to-end/repeat/error-skips-next-repetitions.phpt new file mode 100644 index 00000000000..67069463f2d --- /dev/null +++ b/tests/end-to-end/repeat/error-skips-next-repetitions.phpt @@ -0,0 +1,41 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +FSS.FS..F... 12 / 12 (100%) + +Time: %s, Memory: %s MB + +There were 3 failures: + +1) RepeatWithFailuresTest::test1 +Failed asserting that true is false. + +%s/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php:%d + +2) RepeatWithFailuresTest::test2 (repeat attempt #2) +Failed asserting that true is false. + +%s/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php:%d + +3) RepeatWithFailuresTest::test3 (repeat attempt #3) +Failed asserting that true is false. + +%s/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php:%d + +FAILURES! +Tests: 12, Assertions: 9, Failures: 3, Skipped: 3. diff --git a/tests/end-to-end/repeat/error-stop-on-failure-debug.phpt b/tests/end-to-end/repeat/error-stop-on-failure-debug.phpt new file mode 100644 index 00000000000..bba6848f8ea --- /dev/null +++ b/tests/end-to-end/repeat/error-stop-on-failure-debug.phpt @@ -0,0 +1,41 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using PHP %s) +Test Runner Configured +Event Facade Sealed +Test Suite Loaded (%d tests) +Test Runner Started +Test Suite Sorted +Test Suite Filtered (3 tests) +Test Runner Execution Started (3 tests) +Test Suite Started (RepeatWithFailuresTest, 3 tests) +Test Preparation Started (RepeatWithFailuresTest::test2) +Test Prepared (RepeatWithFailuresTest::test2) +Test Finished (RepeatWithFailuresTest::test2) +Test Preparation Started (RepeatWithFailuresTest::test2) +Test Prepared (RepeatWithFailuresTest::test2) +Test Failed (RepeatWithFailuresTest::test2) +Failed asserting that true is false. +Test Finished (RepeatWithFailuresTest::test2) +Test Skipped (RepeatWithFailuresTest::test2) +Test repetition #3 failure +Test Suite Finished (RepeatWithFailuresTest, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 1) diff --git a/tests/end-to-end/repeat/error-stop-on-failure.phpt b/tests/end-to-end/repeat/error-stop-on-failure.phpt new file mode 100644 index 00000000000..6570ffa60b7 --- /dev/null +++ b/tests/end-to-end/repeat/error-stop-on-failure.phpt @@ -0,0 +1,34 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.FS 3 / 3 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) RepeatWithFailuresTest::test2 (repeat attempt #2) +Failed asserting that true is false. + +%s/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php:%d + +FAILURES! +Tests: 3, Assertions: 2, Failures: 1, Skipped: 1. diff --git a/tests/end-to-end/repeat/filter.phpt b/tests/end-to-end/repeat/filter.phpt new file mode 100644 index 00000000000..4286ed131ea --- /dev/null +++ b/tests/end-to-end/repeat/filter.phpt @@ -0,0 +1,25 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.. 2 / 2 (100%) + +Time: %s, Memory: %s MB + +OK (2 tests, 4 assertions) diff --git a/tests/end-to-end/repeat/in-directory.phpt b/tests/end-to-end/repeat/in-directory.phpt new file mode 100644 index 00000000000..5194f0eae18 --- /dev/null +++ b/tests/end-to-end/repeat/in-directory.phpt @@ -0,0 +1,23 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.. 2 / 2 (100%) + +Time: %s, Memory: %s MB + +OK (2 tests, 2 assertions) diff --git a/tests/end-to-end/repeat/isolation.phpt b/tests/end-to-end/repeat/isolation.phpt new file mode 100644 index 00000000000..174aa18f6b5 --- /dev/null +++ b/tests/end-to-end/repeat/isolation.phpt @@ -0,0 +1,23 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.... 4 / 4 (100%) + +Time: %s, Memory: %s MB + +OK (4 tests, 8 assertions) diff --git a/tests/end-to-end/repeat/no-cli-argument-filtered.phpt b/tests/end-to-end/repeat/no-cli-argument-filtered.phpt new file mode 100644 index 00000000000..6d06792e3e5 --- /dev/null +++ b/tests/end-to-end/repeat/no-cli-argument-filtered.phpt @@ -0,0 +1,26 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s +Configuration: %s/tests/end-to-end/repeat/_files/phpunit.xml + +.. 2 / 2 (100%) + +Time: %s, Memory: %s MB + +OK (2 tests, 4 assertions) diff --git a/tests/end-to-end/repeat/no-cli-argument.phpt b/tests/end-to-end/repeat/no-cli-argument.phpt new file mode 100644 index 00000000000..7ece5490e4c --- /dev/null +++ b/tests/end-to-end/repeat/no-cli-argument.phpt @@ -0,0 +1,24 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s +Configuration: %s/tests/end-to-end/repeat/_files/phpunit.xml + +.... 4 / 4 (100%) + +Time: %s, Memory: %s MB + +OK (4 tests, 8 assertions) diff --git a/tests/end-to-end/repeat/phpt-failure-and-success.phpt b/tests/end-to-end/repeat/phpt-failure-and-success.phpt new file mode 100644 index 00000000000..09e91a43ec9 --- /dev/null +++ b/tests/end-to-end/repeat/phpt-failure-and-success.phpt @@ -0,0 +1,36 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +FS.. 4 / 4 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) %s/phpunit/tests/end-to-end/repeat/_files/phpt/failure.phpt +Failed asserting that string matches format description. +--- Expected ++++ Actual +@@ @@ +-ko ++ok + +%s/phpunit/tests/end-to-end/repeat/_files/phpt/failure.phpt:%d + +FAILURES! +Tests: 4, Assertions: 3, Failures: 1, Skipped: 1. diff --git a/tests/end-to-end/repeat/phpt-failure.phpt b/tests/end-to-end/repeat/phpt-failure.phpt new file mode 100644 index 00000000000..d185d82d1ff --- /dev/null +++ b/tests/end-to-end/repeat/phpt-failure.phpt @@ -0,0 +1,37 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +FS 2 / 2 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) %s/tests/end-to-end/repeat/_files/phpt/failure.phpt +Failed asserting that string matches format description. +--- Expected ++++ Actual +@@ @@ +-ko ++ok + +%s/tests/end-to-end/repeat/_files/phpt/failure.phpt:%d + +FAILURES! +Tests: 2, Assertions: 1, Failures: 1, Skipped: 1. + diff --git a/tests/end-to-end/repeat/phpt-success.phpt b/tests/end-to-end/repeat/phpt-success.phpt new file mode 100644 index 00000000000..5bfbebd7a13 --- /dev/null +++ b/tests/end-to-end/repeat/phpt-success.phpt @@ -0,0 +1,23 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.. 2 / 2 (100%) + +Time: %s, Memory: %s MB + +OK (2 tests, 2 assertions) diff --git a/tests/end-to-end/repeat/success-debug.phpt b/tests/end-to-end/repeat/success-debug.phpt new file mode 100644 index 00000000000..ef17fee52bf --- /dev/null +++ b/tests/end-to-end/repeat/success-debug.phpt @@ -0,0 +1,53 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using PHP %s) +Test Runner Configured +Event Facade Sealed +Test Suite Loaded (4 tests) +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (4 tests) +Test Suite Started (RepeatTest, 4 tests) +Test Preparation Started (RepeatTest::test1) +Before Test Method Called (RepeatTest::setUp) +Before Test Method Finished: +- RepeatTest::setUp +Test Prepared (RepeatTest::test1) +Test Finished (RepeatTest::test1) +Test Preparation Started (RepeatTest::test1) +Before Test Method Called (RepeatTest::setUp) +Before Test Method Finished: +- RepeatTest::setUp +Test Prepared (RepeatTest::test1) +Test Passed (RepeatTest::test1) +Test Finished (RepeatTest::test1) +Test Preparation Started (RepeatTest::test2) +Before Test Method Called (RepeatTest::setUp) +Before Test Method Finished: +- RepeatTest::setUp +Test Prepared (RepeatTest::test2) +Test Finished (RepeatTest::test2) +Test Preparation Started (RepeatTest::test2) +Before Test Method Called (RepeatTest::setUp) +Before Test Method Finished: +- RepeatTest::setUp +Test Prepared (RepeatTest::test2) +Test Passed (RepeatTest::test2) +Test Finished (RepeatTest::test2) +Test Suite Finished (RepeatTest, 4 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/repeat/success.phpt b/tests/end-to-end/repeat/success.phpt new file mode 100644 index 00000000000..76f478290cf --- /dev/null +++ b/tests/end-to-end/repeat/success.phpt @@ -0,0 +1,23 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: PHP 8.4.13 + +.... 4 / 4 (100%) + +Time: %s, Memory: %s MB + +OK (4 tests, 8 assertions) diff --git a/tests/end-to-end/repeat/test-with-data-provider-and-depends.phpt b/tests/end-to-end/repeat/test-with-data-provider-and-depends.phpt new file mode 100644 index 00000000000..2158e1ad5f7 --- /dev/null +++ b/tests/end-to-end/repeat/test-with-data-provider-and-depends.phpt @@ -0,0 +1,31 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +........FS..SS 14 / 14 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) RepeatWithDataProviderAndDependsTest::test2#1 with data (false) +Failed asserting that false is true. + +%s/tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php:%d + +FAILURES! +Tests: 14, Assertions: 11, Failures: 1, Skipped: 3. diff --git a/tests/end-to-end/repeat/test-with-data-provider-failing.phpt b/tests/end-to-end/repeat/test-with-data-provider-failing.phpt new file mode 100644 index 00000000000..e8bf8c1e94a --- /dev/null +++ b/tests/end-to-end/repeat/test-with-data-provider-failing.phpt @@ -0,0 +1,31 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +..FS.. 6 / 6 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) RepeatWithDataProviderFailingTest::test1#1 with data (false) +Failed asserting that false is true. + +%s/tests/end-to-end/repeat/_files/RepeatWithDataProviderFailingTest.php:%d + +FAILURES! +Tests: 6, Assertions: 5, Failures: 1, Skipped: 1. diff --git a/tests/end-to-end/repeat/test-with-data-provider.phpt b/tests/end-to-end/repeat/test-with-data-provider.phpt new file mode 100644 index 00000000000..9d996e38366 --- /dev/null +++ b/tests/end-to-end/repeat/test-with-data-provider.phpt @@ -0,0 +1,23 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.... 4 / 4 (100%) + +Time: %s, Memory: %s MB + +OK (4 tests, 4 assertions) diff --git a/tests/end-to-end/repeat/wonrg-repeat-value.phpt b/tests/end-to-end/repeat/wonrg-repeat-value.phpt new file mode 100644 index 00000000000..fe7de94af24 --- /dev/null +++ b/tests/end-to-end/repeat/wonrg-repeat-value.phpt @@ -0,0 +1,17 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +The value for the --repeat option must be a positive integer