From bb47770704f5734cab4eab32266fca02c55904b1 Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Mon, 20 Oct 2025 19:36:50 +0200 Subject: [PATCH 01/21] Add basic phpt test with repeat option --- tests/end-to-end/repeat/_files/RepeatTest.php | 16 +++++++++++++ tests/end-to-end/repeat/repeat.phpt | 23 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 tests/end-to-end/repeat/_files/RepeatTest.php create mode 100644 tests/end-to-end/repeat/repeat.phpt diff --git a/tests/end-to-end/repeat/_files/RepeatTest.php b/tests/end-to-end/repeat/_files/RepeatTest.php new file mode 100644 index 0000000000..582bfce92c --- /dev/null +++ b/tests/end-to-end/repeat/_files/RepeatTest.php @@ -0,0 +1,16 @@ +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.... 4 / 4 (100%) + +Time: 00:00, Memory: 16.00 MB + +OK (4 tests, 4 assertions) From 6b5f2b2a8fffbc46db6f0109996da3b061bbf77f Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Mon, 20 Oct 2025 21:45:19 +0200 Subject: [PATCH 02/21] implement naive simple --repeat functionality --- src/Framework/TestSuite.php | 21 ++++++++++++------- src/TextUI/Configuration/Cli/Builder.php | 8 +++++++ .../Configuration/Cli/Configuration.php | 9 +++++++- src/TextUI/Configuration/Configuration.php | 9 +++++++- src/TextUI/Configuration/Merger.php | 1 + src/TextUI/Configuration/TestSuiteBuilder.php | 12 ++++++----- 6 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/Framework/TestSuite.php b/src/Framework/TestSuite.php index acf9e2b763..af1e72b4b4 100644 --- a/src/Framework/TestSuite.php +++ b/src/Framework/TestSuite.php @@ -97,7 +97,7 @@ public static function empty(string $name): static * @param ReflectionClass $class * @param list $groups */ - public static function fromClassReflector(ReflectionClass $class, array $groups = []): static + public static function fromClassReflector(ReflectionClass $class, array $groups = [], int $repeat = 1): static { $testSuite = new static($class->getName()); @@ -118,7 +118,7 @@ public static function fromClassReflector(ReflectionClass $class, array $groups continue; } - $testSuite->addTestMethod($class, $method, $groups); + $testSuite->addTestMethod($class, $method, $groups, $repeat); } if ($testSuite->isEmpty()) { @@ -146,7 +146,7 @@ final private function __construct(string $name) * * @param list $groups */ - public function addTest(Test $test, array $groups = []): void + public function addTest(Test $test, array $groups = [], int $repeat = 1): void { if ($test instanceof self) { $this->tests[] = $test; @@ -158,7 +158,13 @@ public function addTest(Test $test, array $groups = []): void assert($test instanceof TestCase || $test instanceof PhptTestCase); - $this->tests[] = $test; + if ($test instanceof PhptTestCase) { + $this->tests[] = $test; + } else { + for ($i = 0; $i < $repeat; $i++) { + $this->tests[] = $test; + } + } $this->clearCaches(); @@ -191,7 +197,7 @@ public function addTest(Test $test, array $groups = []): void * * @throws Exception */ - public function addTestSuite(ReflectionClass $testClass, array $groups = []): void + public function addTestSuite(ReflectionClass $testClass, array $groups = [], int $repeat = 1): void { if ($testClass->isAbstract()) { throw new Exception( @@ -212,7 +218,7 @@ public function addTestSuite(ReflectionClass $testClass, array $groups = []): vo ); } - $this->addTest(self::fromClassReflector($testClass, $groups), $groups); + $this->addTest(self::fromClassReflector($testClass, $groups), $groups, $repeat); } /** @@ -506,7 +512,7 @@ public function isForTestClass(): bool * * @throws Exception */ - protected function addTestMethod(ReflectionClass $class, ReflectionMethod $method, array $groups): void + protected function addTestMethod(ReflectionClass $class, ReflectionMethod $method, array $groups, int $repeat): void { $className = $class->getName(); $methodName = $method->getName(); @@ -562,6 +568,7 @@ protected function addTestMethod(ReflectionClass $class, ReflectionMethod $metho $groups, (new Groups)->groups($class->getName(), $methodName), ), + $repeat, ); } diff --git a/src/TextUI/Configuration/Cli/Builder.php b/src/TextUI/Configuration/Cli/Builder.php index 1accafe3a8..01b28397ce 100644 --- a/src/TextUI/Configuration/Cli/Builder.php +++ b/src/TextUI/Configuration/Cli/Builder.php @@ -158,6 +158,7 @@ final class Builder 'log-events-verbose-text=', 'version', 'debug', + 'repeat=', 'with-telemetry', 'extension=', ]; @@ -312,6 +313,7 @@ public function fromParameters(array $parameters): Configuration $printerTestDox = null; $printerTestDoxSummary = null; $debug = false; + $repeat = 1; $withTelemetry = false; $extensions = []; @@ -1211,6 +1213,11 @@ public function fromParameters(array $parameters): Configuration break; + case '--repeat': + $repeat = (int) $option[1]; // todo: superior to 1 + + break; + case '--with-telemetry': $withTelemetry = true; @@ -1363,6 +1370,7 @@ public function fromParameters(array $parameters): Configuration $debug, $withTelemetry, $extensions, + $repeat, ); } diff --git a/src/TextUI/Configuration/Cli/Configuration.php b/src/TextUI/Configuration/Cli/Configuration.php index e637281c69..79da9bfc8e 100644 --- a/src/TextUI/Configuration/Cli/Configuration.php +++ b/src/TextUI/Configuration/Cli/Configuration.php @@ -177,6 +177,7 @@ private ?string $logEventsVerboseText; private bool $debug; private bool $withTelemetry; + private int $repeat; /** * @var ?non-empty-list @@ -195,7 +196,7 @@ * @param ?non-empty-list $coverageFilter * @param ?non-empty-list $extensions */ - public function __construct(array $arguments, ?bool $all, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticProperties, ?bool $beStrictAboutChangesToGlobalState, ?string $bootstrap, ?string $cacheDirectory, ?bool $cacheResult, bool $checkPhpConfiguration, bool $checkVersion, ?string $colors, null|int|string $columns, ?string $configurationFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $pathCoverage, bool $warmCoverageCache, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?bool $failOnAllIssues, ?bool $failOnDeprecation, ?bool $failOnPhpunitDeprecation, ?bool $failOnPhpunitNotice, ?bool $failOnPhpunitWarning, ?bool $failOnEmptyTestSuite, ?bool $failOnIncomplete, ?bool $failOnNotice, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?bool $doNotFailOnDeprecation, ?bool $doNotFailOnPhpunitDeprecation, ?bool $doNotFailOnPhpunitNotice, ?bool $doNotFailOnPhpunitWarning, ?bool $doNotFailOnEmptyTestSuite, ?bool $doNotFailOnIncomplete, ?bool $doNotFailOnNotice, ?bool $doNotFailOnRisky, ?bool $doNotFailOnSkipped, ?bool $doNotFailOnWarning, ?bool $stopOnDefect, ?bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnNotice, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $filter, ?string $excludeFilter, ?string $generateBaseline, ?string $useBaseline, bool $ignoreBaseline, bool $generateConfiguration, bool $migrateConfiguration, ?array $groups, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?string $otrLogfile, ?bool $includeGitInformation, bool $listGroups, bool $listSuites, bool $listTestFiles, bool $listTests, ?string $listTestsXml, ?bool $noCoverage, ?bool $noExtensions, ?bool $noOutput, ?bool $noProgress, ?bool $noResults, ?bool $noLogging, ?bool $processIsolation, ?int $randomOrderSeed, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?string $teamcityLogfile, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?array $testSuffixes, ?string $testSuite, ?string $excludeTestSuite, bool $useDefaultConfiguration, ?bool $displayDetailsOnAllIssues, ?bool $displayDetailsOnIncompleteTests, ?bool $displayDetailsOnSkippedTests, ?bool $displayDetailsOnTestsThatTriggerDeprecations, ?bool $displayDetailsOnPhpunitDeprecations, ?bool $displayDetailsOnPhpunitNotices, ?bool $displayDetailsOnTestsThatTriggerErrors, ?bool $displayDetailsOnTestsThatTriggerNotices, ?bool $displayDetailsOnTestsThatTriggerWarnings, bool $version, ?array $coverageFilter, ?string $logEventsText, ?string $logEventsVerboseText, ?bool $printerTeamCity, ?bool $testdoxPrinter, ?bool $testdoxPrinterSummary, bool $debug, bool $withTelemetry, ?array $extensions) + public function __construct(array $arguments, ?bool $all, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticProperties, ?bool $beStrictAboutChangesToGlobalState, ?string $bootstrap, ?string $cacheDirectory, ?bool $cacheResult, bool $checkPhpConfiguration, bool $checkVersion, ?string $colors, null|int|string $columns, ?string $configurationFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $pathCoverage, bool $warmCoverageCache, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?bool $failOnAllIssues, ?bool $failOnDeprecation, ?bool $failOnPhpunitDeprecation, ?bool $failOnPhpunitNotice, ?bool $failOnPhpunitWarning, ?bool $failOnEmptyTestSuite, ?bool $failOnIncomplete, ?bool $failOnNotice, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?bool $doNotFailOnDeprecation, ?bool $doNotFailOnPhpunitDeprecation, ?bool $doNotFailOnPhpunitNotice, ?bool $doNotFailOnPhpunitWarning, ?bool $doNotFailOnEmptyTestSuite, ?bool $doNotFailOnIncomplete, ?bool $doNotFailOnNotice, ?bool $doNotFailOnRisky, ?bool $doNotFailOnSkipped, ?bool $doNotFailOnWarning, ?bool $stopOnDefect, ?bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnNotice, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $filter, ?string $excludeFilter, ?string $generateBaseline, ?string $useBaseline, bool $ignoreBaseline, bool $generateConfiguration, bool $migrateConfiguration, ?array $groups, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?string $otrLogfile, ?bool $includeGitInformation, bool $listGroups, bool $listSuites, bool $listTestFiles, bool $listTests, ?string $listTestsXml, ?bool $noCoverage, ?bool $noExtensions, ?bool $noOutput, ?bool $noProgress, ?bool $noResults, ?bool $noLogging, ?bool $processIsolation, ?int $randomOrderSeed, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?string $teamcityLogfile, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?array $testSuffixes, ?string $testSuite, ?string $excludeTestSuite, bool $useDefaultConfiguration, ?bool $displayDetailsOnAllIssues, ?bool $displayDetailsOnIncompleteTests, ?bool $displayDetailsOnSkippedTests, ?bool $displayDetailsOnTestsThatTriggerDeprecations, ?bool $displayDetailsOnPhpunitDeprecations, ?bool $displayDetailsOnPhpunitNotices, ?bool $displayDetailsOnTestsThatTriggerErrors, ?bool $displayDetailsOnTestsThatTriggerNotices, ?bool $displayDetailsOnTestsThatTriggerWarnings, bool $version, ?array $coverageFilter, ?string $logEventsText, ?string $logEventsVerboseText, ?bool $printerTeamCity, ?bool $testdoxPrinter, ?bool $testdoxPrinterSummary, bool $debug, bool $withTelemetry, ?array $extensions, int $repeat) { $this->arguments = $arguments; $this->all = $all; @@ -322,6 +323,7 @@ public function __construct(array $arguments, ?bool $all, ?string $atLeastVersio $this->debug = $debug; $this->withTelemetry = $withTelemetry; $this->extensions = $extensions; + $this->repeat = $repeat; } /** @@ -2578,6 +2580,11 @@ public function debug(): bool return $this->debug; } + public function repeat(): int + { + return $this->repeat; + } + public function withTelemetry(): bool { return $this->withTelemetry; diff --git a/src/TextUI/Configuration/Configuration.php b/src/TextUI/Configuration/Configuration.php index 82631e5e67..6f356ca0bb 100644 --- a/src/TextUI/Configuration/Configuration.php +++ b/src/TextUI/Configuration/Configuration.php @@ -194,6 +194,7 @@ */ private ?string $generateBaseline; private bool $debug; + private int $repeat; private bool $withTelemetry; /** @@ -215,7 +216,7 @@ * @param null|non-empty-string $generateBaseline * @param non-negative-int $shortenArraysForExportThreshold */ - public function __construct(array $cliArguments, ?string $configurationFile, ?string $bootstrap, array $bootstrapForTestSuite, bool $cacheResult, ?string $cacheDirectory, ?string $coverageCacheDirectory, Source $source, string $testResultCacheFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4j, int $coverageCrap4jThreshold, ?string $coverageHtml, int $coverageHtmlLowUpperBound, int $coverageHtmlHighLowerBound, string $coverageHtmlColorSuccessLow, string $coverageHtmlColorSuccessMedium, string $coverageHtmlColorSuccessHigh, string $coverageHtmlColorWarning, string $coverageHtmlColorDanger, ?string $coverageHtmlCustomCssFile, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, bool $coverageTextShowUncoveredFiles, bool $coverageTextShowOnlySummary, ?string $coverageXml, bool $pathCoverage, bool $ignoreDeprecatedCodeUnitsFromCodeCoverage, bool $disableCodeCoverageIgnore, bool $failOnAllIssues, bool $failOnDeprecation, bool $failOnPhpunitDeprecation, bool $failOnPhpunitNotice, bool $failOnPhpunitWarning, bool $failOnEmptyTestSuite, bool $failOnIncomplete, bool $failOnNotice, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $doNotFailOnDeprecation, bool $doNotFailOnPhpunitDeprecation, bool $doNotFailOnPhpunitNotice, bool $doNotFailOnPhpunitWarning, bool $doNotFailOnEmptyTestSuite, bool $doNotFailOnIncomplete, bool $doNotFailOnNotice, bool $doNotFailOnRisky, bool $doNotFailOnSkipped, bool $doNotFailOnWarning, bool $stopOnDefect, bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, bool $stopOnError, bool $stopOnFailure, bool $stopOnIncomplete, bool $stopOnNotice, bool $stopOnRisky, bool $stopOnSkipped, bool $stopOnWarning, bool $outputToStandardErrorStream, int $columns, bool $noExtensions, ?string $pharExtensionDirectory, array $extensionBootstrappers, bool $backupGlobals, bool $backupStaticProperties, bool $beStrictAboutChangesToGlobalState, bool $colors, bool $processIsolation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, bool $reportUselessTests, bool $strictCoverage, bool $disallowTestOutput, bool $displayDetailsOnAllIssues, bool $displayDetailsOnIncompleteTests, bool $displayDetailsOnSkippedTests, bool $displayDetailsOnTestsThatTriggerDeprecations, bool $displayDetailsOnPhpunitDeprecations, bool $displayDetailsOnPhpunitNotices, bool $displayDetailsOnTestsThatTriggerErrors, bool $displayDetailsOnTestsThatTriggerNotices, bool $displayDetailsOnTestsThatTriggerWarnings, bool $reverseDefectList, bool $requireCoverageMetadata, bool $noProgress, bool $noResults, bool $noOutput, int $executionOrder, int $executionOrderDefects, bool $resolveDependencies, ?string $logfileTeamcity, ?string $logfileJunit, ?string $logfileOtr, bool $includeGitInformationInOtrLogfile, ?string $logfileTestdoxHtml, ?string $logfileTestdoxText, ?string $logEventsText, ?string $logEventsVerboseText, bool $teamCityOutput, bool $testDoxOutput, bool $testDoxOutputSummary, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, ?string $filter, ?string $excludeFilter, array $groups, array $excludeGroups, int $randomOrderSeed, bool $includeUncoveredFiles, TestSuiteCollection $testSuite, string $includeTestSuite, string $excludeTestSuite, ?string $defaultTestSuite, bool $ignoreTestSelectionInXmlConfiguration, array $testSuffixes, Php $php, bool $controlGarbageCollector, int $numberOfTestsBeforeGarbageCollection, ?string $generateBaseline, bool $debug, bool $withTelemetry, int $shortenArraysForExportThreshold) + public function __construct(array $cliArguments, ?string $configurationFile, ?string $bootstrap, array $bootstrapForTestSuite, bool $cacheResult, ?string $cacheDirectory, ?string $coverageCacheDirectory, Source $source, string $testResultCacheFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4j, int $coverageCrap4jThreshold, ?string $coverageHtml, int $coverageHtmlLowUpperBound, int $coverageHtmlHighLowerBound, string $coverageHtmlColorSuccessLow, string $coverageHtmlColorSuccessMedium, string $coverageHtmlColorSuccessHigh, string $coverageHtmlColorWarning, string $coverageHtmlColorDanger, ?string $coverageHtmlCustomCssFile, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, bool $coverageTextShowUncoveredFiles, bool $coverageTextShowOnlySummary, ?string $coverageXml, bool $pathCoverage, bool $ignoreDeprecatedCodeUnitsFromCodeCoverage, bool $disableCodeCoverageIgnore, bool $failOnAllIssues, bool $failOnDeprecation, bool $failOnPhpunitDeprecation, bool $failOnPhpunitNotice, bool $failOnPhpunitWarning, bool $failOnEmptyTestSuite, bool $failOnIncomplete, bool $failOnNotice, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $doNotFailOnDeprecation, bool $doNotFailOnPhpunitDeprecation, bool $doNotFailOnPhpunitNotice, bool $doNotFailOnPhpunitWarning, bool $doNotFailOnEmptyTestSuite, bool $doNotFailOnIncomplete, bool $doNotFailOnNotice, bool $doNotFailOnRisky, bool $doNotFailOnSkipped, bool $doNotFailOnWarning, bool $stopOnDefect, bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, bool $stopOnError, bool $stopOnFailure, bool $stopOnIncomplete, bool $stopOnNotice, bool $stopOnRisky, bool $stopOnSkipped, bool $stopOnWarning, bool $outputToStandardErrorStream, int $columns, bool $noExtensions, ?string $pharExtensionDirectory, array $extensionBootstrappers, bool $backupGlobals, bool $backupStaticProperties, bool $beStrictAboutChangesToGlobalState, bool $colors, bool $processIsolation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, bool $reportUselessTests, bool $strictCoverage, bool $disallowTestOutput, bool $displayDetailsOnAllIssues, bool $displayDetailsOnIncompleteTests, bool $displayDetailsOnSkippedTests, bool $displayDetailsOnTestsThatTriggerDeprecations, bool $displayDetailsOnPhpunitDeprecations, bool $displayDetailsOnPhpunitNotices, bool $displayDetailsOnTestsThatTriggerErrors, bool $displayDetailsOnTestsThatTriggerNotices, bool $displayDetailsOnTestsThatTriggerWarnings, bool $reverseDefectList, bool $requireCoverageMetadata, bool $noProgress, bool $noResults, bool $noOutput, int $executionOrder, int $executionOrderDefects, bool $resolveDependencies, ?string $logfileTeamcity, ?string $logfileJunit, ?string $logfileOtr, bool $includeGitInformationInOtrLogfile, ?string $logfileTestdoxHtml, ?string $logfileTestdoxText, ?string $logEventsText, ?string $logEventsVerboseText, bool $teamCityOutput, bool $testDoxOutput, bool $testDoxOutputSummary, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, ?string $filter, ?string $excludeFilter, array $groups, array $excludeGroups, int $randomOrderSeed, bool $includeUncoveredFiles, TestSuiteCollection $testSuite, string $includeTestSuite, string $excludeTestSuite, ?string $defaultTestSuite, bool $ignoreTestSelectionInXmlConfiguration, array $testSuffixes, Php $php, bool $controlGarbageCollector, int $numberOfTestsBeforeGarbageCollection, ?string $generateBaseline, bool $debug, int $repeat, bool $withTelemetry, int $shortenArraysForExportThreshold) { $this->cliArguments = $cliArguments; $this->configurationFile = $configurationFile; @@ -345,6 +346,7 @@ public function __construct(array $cliArguments, ?string $configurationFile, ?st $this->numberOfTestsBeforeGarbageCollection = $numberOfTestsBeforeGarbageCollection; $this->generateBaseline = $generateBaseline; $this->debug = $debug; + $this->repeat = $repeat; $this->withTelemetry = $withTelemetry; $this->shortenArraysForExportThreshold = $shortenArraysForExportThreshold; } @@ -1514,6 +1516,11 @@ public function debug(): bool return $this->debug; } + public function repeat(): int + { + return $this->repeat; + } + public function withTelemetry(): bool { return $this->withTelemetry; diff --git a/src/TextUI/Configuration/Merger.php b/src/TextUI/Configuration/Merger.php index bdc22be7d2..6ff3a70f34 100644 --- a/src/TextUI/Configuration/Merger.php +++ b/src/TextUI/Configuration/Merger.php @@ -1064,6 +1064,7 @@ public function merge(CliConfiguration $cliConfiguration, XmlConfiguration $xmlC $xmlConfiguration->phpunit()->numberOfTestsBeforeGarbageCollection(), $generateBaseline, $cliConfiguration->debug(), + $cliConfiguration->repeat(), $cliConfiguration->withTelemetry(), $xmlConfiguration->phpunit()->shortenArraysForExportThreshold(), ); diff --git a/src/TextUI/Configuration/TestSuiteBuilder.php b/src/TextUI/Configuration/TestSuiteBuilder.php index a474b02f54..5570779a63 100644 --- a/src/TextUI/Configuration/TestSuiteBuilder.php +++ b/src/TextUI/Configuration/TestSuiteBuilder.php @@ -58,11 +58,13 @@ public function build(Configuration $configuration): TestSuite $testSuite = $this->testSuiteFromPath( $arguments[0], $configuration->testSuffixes(), + $configuration->repeat(), ); } else { $testSuite = $this->testSuiteFromPathList( $arguments, $configuration->testSuffixes(), + $configuration->repeat(), ); } } @@ -91,7 +93,7 @@ public function build(Configuration $configuration): TestSuite * * @throws \PHPUnit\Framework\Exception */ - private function testSuiteFromPath(string $path, array $suffixes, ?TestSuite $suite = null): TestSuite + private function testSuiteFromPath(string $path, array $suffixes, int $repeat, ?TestSuite $suite = null): TestSuite { if (str_ends_with($path, '.phpt') && is_file($path)) { if ($suite === null) { @@ -124,10 +126,10 @@ private function testSuiteFromPath(string $path, array $suffixes, ?TestSuite $su } if ($suite === null) { - return TestSuite::fromClassReflector($testClass); + return TestSuite::fromClassReflector($testClass, repeat: $repeat); } - $suite->addTestSuite($testClass); + $suite->addTestSuite($testClass, repeat: $repeat); return $suite; } @@ -138,12 +140,12 @@ private function testSuiteFromPath(string $path, array $suffixes, ?TestSuite $su * * @throws \PHPUnit\Framework\Exception */ - private function testSuiteFromPathList(array $paths, array $suffixes): TestSuite + private function testSuiteFromPathList(array $paths, array $suffixes, int $repeat): TestSuite { $suite = TestSuite::empty('CLI Arguments'); foreach ($paths as $path) { - $this->testSuiteFromPath($path, $suffixes, $suite); + $this->testSuiteFromPath($path, $suffixes, $repeat, $suite); } return $suite; From 6286d21ee7e67ff920407726a18715ae5eb680a3 Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Thu, 23 Oct 2025 19:40:24 +0200 Subject: [PATCH 03/21] Repeat tests using RepeatTestSuite --- src/Framework/RepeatTestSuite.php | 80 +++++++++++++++++++ src/Framework/TestCase.php | 12 +++ src/Framework/TestSuite.php | 6 +- .../repeat/_files/RepeatWithErrorsTest.php | 38 +++++++++ ...eat-with-error-skips-next-repetitions.phpt | 41 ++++++++++ .../end-to-end/repeat/repeat-with-filter.phpt | 25 ++++++ .../{repeat.phpt => simple-repeat.phpt} | 2 +- 7 files changed, 199 insertions(+), 5 deletions(-) create mode 100644 src/Framework/RepeatTestSuite.php create mode 100644 tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php create mode 100644 tests/end-to-end/repeat/repeat-with-error-skips-next-repetitions.phpt create mode 100644 tests/end-to-end/repeat/repeat-with-filter.phpt rename tests/end-to-end/repeat/{repeat.phpt => simple-repeat.phpt} (94%) diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php new file mode 100644 index 0000000000..8384cdc671 --- /dev/null +++ b/src/Framework/RepeatTestSuite.php @@ -0,0 +1,80 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\Framework; + +/** + * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit + * + * @internal This class is not covered by the backward compatibility promise for PHPUnit + */ +final class RepeatTestSuite implements Test, Reorderable +{ + /** + * @var non-empty-list + */ + private array $tests; + + /** + * @param positive-int $times + */ + public function __construct(TestCase $test, int $times) + { + $tests = []; + for ($i = 0; $i < $times; $i++) { + $tests[] = clone $test; + } + + $this->tests = $tests; + } + + public function count(): int + { + return count($this->tests); + } + + public function run(): void + { + $defectOccurred = false; + + foreach ($this->tests as $test) { + if ($defectOccurred) { + $test->markSkippedForErrorInPreviousRepetition(); + + continue; + } + + $test->run(); + + if ($test->status()->isFailure() || $test->status()->isError()) { + $defectOccurred = true; + } + } + } + + public function sortId(): string + { + return $this->tests[0]->sortId(); + } + + public function provides(): array + { + return $this->tests[0]->provides(); + } + + public function requires(): array + { + return $this->tests[0]->requires(); + } + + public function nameWithDataSet(): string + { + return $this->tests[0]->nameWithDataSet(); + } +} diff --git a/src/Framework/TestCase.php b/src/Framework/TestCase.php index 57a33425aa..80fcefa740 100644 --- a/src/Framework/TestCase.php +++ b/src/Framework/TestCase.php @@ -1543,6 +1543,18 @@ private function markSkippedForMissingDependency(ExecutionOrderDependency $depen $this->status = TestStatus::skipped($message); } + public function markSkippedForErrorInPreviousRepetition(): void + { + $message = 'Test repetition failure'; + + Event\Facade::emitter()->testSkipped( + $this->valueObjectForEvents(), + $message, + ); + + $this->status = TestStatus::skipped($message); + } + private function startOutputBuffering(): void { ob_start(); diff --git a/src/Framework/TestSuite.php b/src/Framework/TestSuite.php index af1e72b4b4..ce6794e00f 100644 --- a/src/Framework/TestSuite.php +++ b/src/Framework/TestSuite.php @@ -158,12 +158,10 @@ public function addTest(Test $test, array $groups = [], int $repeat = 1): void assert($test instanceof TestCase || $test instanceof PhptTestCase); - if ($test instanceof PhptTestCase) { + if ($test instanceof PhptTestCase || $repeat === 1) { $this->tests[] = $test; } else { - for ($i = 0; $i < $repeat; $i++) { - $this->tests[] = $test; - } + $this->tests[] = new RepeatTestSuite($test, $repeat); } $this->clearCaches(); diff --git a/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php b/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php new file mode 100644 index 0000000000..016c75fb12 --- /dev/null +++ b/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php @@ -0,0 +1,38 @@ + 0) { + self::assertFalse(true); + } + + self::assertTrue(true); + } + + public function test3(): void + { + static $cout = 0; + + if ($cout++ > 1) { + self::assertFalse(true); + } + + self::assertTrue(true); + } + + public function test4(): void + { + self::assertTrue(true); + } +} diff --git a/tests/end-to-end/repeat/repeat-with-error-skips-next-repetitions.phpt b/tests/end-to-end/repeat/repeat-with-error-skips-next-repetitions.phpt new file mode 100644 index 0000000000..71b573edd3 --- /dev/null +++ b/tests/end-to-end/repeat/repeat-with-error-skips-next-repetitions.phpt @@ -0,0 +1,41 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +FSS.FS..F... 12 / 12 (100%) + +Time: %s, Memory: %s MB + +There were 3 failures: + +1) RepeatWithErrorsTest::test1 +Failed asserting that true is false. + +/home/niko/works/github.com/sebastianbergmann/phpunit/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:9 + +2) RepeatWithErrorsTest::test2 +Failed asserting that true is false. + +/home/niko/works/github.com/sebastianbergmann/phpunit/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:17 + +3) RepeatWithErrorsTest::test3 +Failed asserting that true is false. + +/home/niko/works/github.com/sebastianbergmann/phpunit/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:28 + +FAILURES! +Tests: 12, Assertions: 9, Failures: 3, Skipped: 3. diff --git a/tests/end-to-end/repeat/repeat-with-filter.phpt b/tests/end-to-end/repeat/repeat-with-filter.phpt new file mode 100644 index 0000000000..a4e76310d4 --- /dev/null +++ b/tests/end-to-end/repeat/repeat-with-filter.phpt @@ -0,0 +1,25 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.. 2 / 2 (100%) + +Time: %s, Memory: %s MB + +OK (2 tests, 2 assertions) diff --git a/tests/end-to-end/repeat/repeat.phpt b/tests/end-to-end/repeat/simple-repeat.phpt similarity index 94% rename from tests/end-to-end/repeat/repeat.phpt rename to tests/end-to-end/repeat/simple-repeat.phpt index 181f51a303..7ce4f596b1 100644 --- a/tests/end-to-end/repeat/repeat.phpt +++ b/tests/end-to-end/repeat/simple-repeat.phpt @@ -18,6 +18,6 @@ Runtime: %s .... 4 / 4 (100%) -Time: 00:00, Memory: 16.00 MB +Time: %s, Memory: %s MB OK (4 tests, 4 assertions) From 3ef773896c4a157a97bbf530b5c893bb7f51a49a Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Thu, 23 Oct 2025 19:54:10 +0200 Subject: [PATCH 04/21] Handle dependent tests with --repeat --- src/Framework/RepeatTestSuite.php | 4 +++ src/Runner/TestResult/PassedTests.php | 5 +++ .../DependentOfTestFailedInRepetitionTest.php | 24 ++++++++++++++ .../repeat/_files/RepeatDependentTest.php | 29 +++++++++++++++++ .../repeat-dependent-test-with-failure.phpt | 31 +++++++++++++++++++ .../repeat/repeat-dependent-test.phpt | 31 +++++++++++++++++++ ...eat-with-error-skips-next-repetitions.phpt | 6 ++-- 7 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php create mode 100644 tests/end-to-end/repeat/_files/RepeatDependentTest.php create mode 100644 tests/end-to-end/repeat/repeat-dependent-test-with-failure.phpt create mode 100644 tests/end-to-end/repeat/repeat-dependent-test.phpt diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php index 8384cdc671..3eb946f052 100644 --- a/src/Framework/RepeatTestSuite.php +++ b/src/Framework/RepeatTestSuite.php @@ -9,6 +9,8 @@ */ namespace PHPUnit\Framework; +use PHPUnit\TestRunner\TestResult\PassedTests; + /** * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit * @@ -54,6 +56,8 @@ public function run(): void if ($test->status()->isFailure() || $test->status()->isError()) { $defectOccurred = true; + + PassedTests::instance()->testMethodDidNotPass($test::class . '::' . $test->name()); } } } diff --git a/src/Runner/TestResult/PassedTests.php b/src/Runner/TestResult/PassedTests.php index 2f611461d6..a79342f84d 100644 --- a/src/Runner/TestResult/PassedTests.php +++ b/src/Runner/TestResult/PassedTests.php @@ -84,6 +84,11 @@ public function import(self $other): void ); } + public function testMethodDidNotPass(string $method): void + { + unset($this->passedTestMethods[$method]); + } + /** * @param class-string $className */ diff --git a/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php b/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php new file mode 100644 index 0000000000..038c091daa --- /dev/null +++ b/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php @@ -0,0 +1,24 @@ + 0) { + self::assertFalse(true); + } + + self::assertTrue(true); + } + + #[Depends('test1')] + public function test2(): void + { + self::assertTrue(true); + } +} diff --git a/tests/end-to-end/repeat/_files/RepeatDependentTest.php b/tests/end-to-end/repeat/_files/RepeatDependentTest.php new file mode 100644 index 0000000000..62860c94f5 --- /dev/null +++ b/tests/end-to-end/repeat/_files/RepeatDependentTest.php @@ -0,0 +1,29 @@ +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.FSS 4 / 4 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) DependentOfTestFailedInRepetitionTest::test1 +Failed asserting that true is false. + +%s/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php:%d + +FAILURES! +Tests: 4, Assertions: 2, Failures: 1, Skipped: 2. diff --git a/tests/end-to-end/repeat/repeat-dependent-test.phpt b/tests/end-to-end/repeat/repeat-dependent-test.phpt new file mode 100644 index 0000000000..ae0a5a73f1 --- /dev/null +++ b/tests/end-to-end/repeat/repeat-dependent-test.phpt @@ -0,0 +1,31 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +....FSSS 8 / 8 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) RepeatDependentTest::test2 +Failed asserting that false is true. + +%s/tests/end-to-end/repeat/_files/RepeatDependentTest.php:21 + +FAILURES! +Tests: 8, Assertions: 5, Failures: 1, Skipped: 3. diff --git a/tests/end-to-end/repeat/repeat-with-error-skips-next-repetitions.phpt b/tests/end-to-end/repeat/repeat-with-error-skips-next-repetitions.phpt index 71b573edd3..9d5c096381 100644 --- a/tests/end-to-end/repeat/repeat-with-error-skips-next-repetitions.phpt +++ b/tests/end-to-end/repeat/repeat-with-error-skips-next-repetitions.phpt @@ -25,17 +25,17 @@ There were 3 failures: 1) RepeatWithErrorsTest::test1 Failed asserting that true is false. -/home/niko/works/github.com/sebastianbergmann/phpunit/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:9 +%s/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:9 2) RepeatWithErrorsTest::test2 Failed asserting that true is false. -/home/niko/works/github.com/sebastianbergmann/phpunit/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:17 +%s/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:17 3) RepeatWithErrorsTest::test3 Failed asserting that true is false. -/home/niko/works/github.com/sebastianbergmann/phpunit/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:28 +%s/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:28 FAILURES! Tests: 12, Assertions: 9, Failures: 3, Skipped: 3. From ec603b7d7ece43e613e2267aaab8202f31861b4f Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Thu, 23 Oct 2025 21:24:34 +0200 Subject: [PATCH 05/21] Handle data providers with --repeat --- .../Value/TestSuite/TestSuiteBuilder.php | 5 ++- src/Framework/RepeatTestSuite.php | 6 +++ src/Framework/TestBuilder.php | 7 +-- src/Framework/TestSuite.php | 2 +- .../RepeatWithDataProviderAndDependsTest.php | 45 +++++++++++++++++++ .../RepeatWithDataProviderFailingTest.php | 20 +++++++++ .../_files/RepeatWithDataProviderTest.php | 19 ++++++++ ...t-test-with-data-provider-and-depends.phpt | 31 +++++++++++++ ...epeat-test-with-data-provider-failing.phpt | 31 +++++++++++++ .../repeat-test-with-data-provider.phpt | 23 ++++++++++ 10 files changed, 184 insertions(+), 5 deletions(-) create mode 100644 tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php create mode 100644 tests/end-to-end/repeat/_files/RepeatWithDataProviderFailingTest.php create mode 100644 tests/end-to-end/repeat/_files/RepeatWithDataProviderTest.php create mode 100644 tests/end-to-end/repeat/repeat-test-with-data-provider-and-depends.phpt create mode 100644 tests/end-to-end/repeat/repeat-test-with-data-provider-failing.phpt create mode 100644 tests/end-to-end/repeat/repeat-test-with-data-provider.phpt diff --git a/src/Event/Value/TestSuite/TestSuiteBuilder.php b/src/Event/Value/TestSuite/TestSuiteBuilder.php index 3192636baa..56f85a858f 100644 --- a/src/Event/Value/TestSuite/TestSuiteBuilder.php +++ b/src/Event/Value/TestSuite/TestSuiteBuilder.php @@ -9,6 +9,7 @@ */ namespace PHPUnit\Event\TestSuite; +use PHPUnit\Framework\RepeatTestSuite; use function assert; use function class_exists; use function count; @@ -107,7 +108,9 @@ private static function process(FrameworkTestSuite $testSuite, array &$tests): v continue; } - if ($test instanceof TestCase || $test instanceof PhptTestCase) { + if ($test instanceof TestCase || + $test instanceof PhptTestCase || + $test instanceof RepeatTestSuite) { $tests[] = $test->valueObjectForEvents(); } } diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php index 3eb946f052..8cc3d0d488 100644 --- a/src/Framework/RepeatTestSuite.php +++ b/src/Framework/RepeatTestSuite.php @@ -10,6 +10,7 @@ namespace PHPUnit\Framework; use PHPUnit\TestRunner\TestResult\PassedTests; +use PHPUnit\Event; /** * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit @@ -81,4 +82,9 @@ public function nameWithDataSet(): string { return $this->tests[0]->nameWithDataSet(); } + + public function valueObjectForEvents(): Event\Code\TestMethod + { + return $this->tests[0]->valueObjectForEvents(); + } } diff --git a/src/Framework/TestBuilder.php b/src/Framework/TestBuilder.php index 3eb4b1a02d..f383fa174e 100644 --- a/src/Framework/TestBuilder.php +++ b/src/Framework/TestBuilder.php @@ -39,7 +39,7 @@ * * @throws InvalidDataProviderException */ - public function build(ReflectionClass $theClass, string $methodName, array $groups = []): Test + public function build(ReflectionClass $theClass, string $methodName, array $groups = [], int $repeat = 1): Test { $className = $theClass->getName(); @@ -64,6 +64,7 @@ public function build(ReflectionClass $theClass, string $methodName, array $grou $this->shouldGlobalStateBePreserved($className, $methodName), $this->backupSettings($className, $methodName), $groups, + $repeat, ); } @@ -86,7 +87,7 @@ public function build(ReflectionClass $theClass, string $methodName, array $grou * @param array{backupGlobals: ?true, backupGlobalsExcludeList: list, backupStaticProperties: ?true, backupStaticPropertiesExcludeList: array>} $backupSettings * @param list $groups */ - private function buildDataProviderTestSuite(string $methodName, string $className, array $data, bool $runTestInSeparateProcess, ?bool $preserveGlobalState, array $backupSettings, array $groups): DataProviderTestSuite + private function buildDataProviderTestSuite(string $methodName, string $className, array $data, bool $runTestInSeparateProcess, ?bool $preserveGlobalState, array $backupSettings, array $groups, int $repeat = 1): DataProviderTestSuite { $dataProviderTestSuite = DataProviderTestSuite::empty( $className . '::' . $methodName, @@ -109,7 +110,7 @@ private function buildDataProviderTestSuite(string $methodName, string $classNam $backupSettings, ); - $dataProviderTestSuite->addTest($_test, $groups); + $dataProviderTestSuite->addTest($_test, $groups, $repeat); } return $dataProviderTestSuite; diff --git a/src/Framework/TestSuite.php b/src/Framework/TestSuite.php index ce6794e00f..830c95f791 100644 --- a/src/Framework/TestSuite.php +++ b/src/Framework/TestSuite.php @@ -516,7 +516,7 @@ protected function addTestMethod(ReflectionClass $class, ReflectionMethod $metho $methodName = $method->getName(); try { - $test = (new TestBuilder)->build($class, $methodName, $groups); + $test = (new TestBuilder)->build($class, $methodName, $groups, $repeat); } catch (InvalidDataProviderException $e) { if ($e->getProviderLabel() === null) { $message = sprintf( diff --git a/tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php b/tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php new file mode 100644 index 0000000000..bc05bf457e --- /dev/null +++ b/tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php @@ -0,0 +1,45 @@ +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +........FS..SS 14 / 14 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) RepeatWithDataProviderAndDependsTest::test2#1 with data (false) +Failed asserting that false is true. + +%s/tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php:%d + +FAILURES! +Tests: 14, Assertions: 11, Failures: 1, Skipped: 3. diff --git a/tests/end-to-end/repeat/repeat-test-with-data-provider-failing.phpt b/tests/end-to-end/repeat/repeat-test-with-data-provider-failing.phpt new file mode 100644 index 0000000000..e8bf8c1e94 --- /dev/null +++ b/tests/end-to-end/repeat/repeat-test-with-data-provider-failing.phpt @@ -0,0 +1,31 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +..FS.. 6 / 6 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) RepeatWithDataProviderFailingTest::test1#1 with data (false) +Failed asserting that false is true. + +%s/tests/end-to-end/repeat/_files/RepeatWithDataProviderFailingTest.php:%d + +FAILURES! +Tests: 6, Assertions: 5, Failures: 1, Skipped: 1. diff --git a/tests/end-to-end/repeat/repeat-test-with-data-provider.phpt b/tests/end-to-end/repeat/repeat-test-with-data-provider.phpt new file mode 100644 index 0000000000..9d996e3836 --- /dev/null +++ b/tests/end-to-end/repeat/repeat-test-with-data-provider.phpt @@ -0,0 +1,23 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.... 4 / 4 (100%) + +Time: %s, Memory: %s MB + +OK (4 tests, 4 assertions) From 1f5c46ca14235351fac1ded77ab245fd02215469 Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Thu, 23 Oct 2025 22:26:59 +0200 Subject: [PATCH 06/21] Repeat phpt tests --- src/Framework/RepeatTestSuite.php | 77 ++++++++++++++----- src/Framework/TestSuite.php | 6 +- src/Runner/Phpt/TestCase.php | 13 +++- src/TextUI/Configuration/TestSuiteBuilder.php | 2 +- .../repeat/_files/phpt/failure.phpt | 8 ++ .../repeat/_files/phpt/success.phpt | 8 ++ ....phpt => dependent-test-with-failure.phpt} | 0 ...ependent-test.phpt => dependent-test.phpt} | 0 ...phpt => error-skips-next-repetitions.phpt} | 0 .../{repeat-with-filter.phpt => filter.phpt} | 0 .../repeat/phpt-failure-and-success.phpt | 23 ++++++ tests/end-to-end/repeat/phpt-failure.phpt | 37 +++++++++ tests/end-to-end/repeat/phpt-success.phpt | 23 ++++++ ... test-with-data-provider-and-depends.phpt} | 0 ...t => test-with-data-provider-failing.phpt} | 0 ...ider.phpt => test-with-data-provider.phpt} | 0 16 files changed, 173 insertions(+), 24 deletions(-) create mode 100644 tests/end-to-end/repeat/_files/phpt/failure.phpt create mode 100644 tests/end-to-end/repeat/_files/phpt/success.phpt rename tests/end-to-end/repeat/{repeat-dependent-test-with-failure.phpt => dependent-test-with-failure.phpt} (100%) rename tests/end-to-end/repeat/{repeat-dependent-test.phpt => dependent-test.phpt} (100%) rename tests/end-to-end/repeat/{repeat-with-error-skips-next-repetitions.phpt => error-skips-next-repetitions.phpt} (100%) rename tests/end-to-end/repeat/{repeat-with-filter.phpt => filter.phpt} (100%) create mode 100644 tests/end-to-end/repeat/phpt-failure-and-success.phpt create mode 100644 tests/end-to-end/repeat/phpt-failure.phpt create mode 100644 tests/end-to-end/repeat/phpt-success.phpt rename tests/end-to-end/repeat/{repeat-test-with-data-provider-and-depends.phpt => test-with-data-provider-and-depends.phpt} (100%) rename tests/end-to-end/repeat/{repeat-test-with-data-provider-failing.phpt => test-with-data-provider-failing.phpt} (100%) rename tests/end-to-end/repeat/{repeat-test-with-data-provider.phpt => test-with-data-provider.phpt} (100%) diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php index 8cc3d0d488..475f5cd783 100644 --- a/src/Framework/RepeatTestSuite.php +++ b/src/Framework/RepeatTestSuite.php @@ -9,6 +9,8 @@ */ namespace PHPUnit\Framework; +use PHPUnit\Event\Facade as EventFacade; +use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; use PHPUnit\TestRunner\TestResult\PassedTests; use PHPUnit\Event; @@ -20,14 +22,14 @@ final class RepeatTestSuite implements Test, Reorderable { /** - * @var non-empty-list + * @var non-empty-list|non-empty-list */ private array $tests; /** * @param positive-int $times */ - public function __construct(TestCase $test, int $times) + public function __construct(TestCase|PhptTestCase $test, int $times) { $tests = []; for ($i = 0; $i < $times; $i++) { @@ -43,6 +45,40 @@ public function count(): int } public function run(): void + { + if ($this->isPhptTestCase()) { + $this->runPhptTestCase(); + } else { + $this->runTestCase(); + } + } + + public function sortId(): string + { + return $this->tests[0]->sortId(); + } + + public function provides(): array + { + return $this->tests[0]->provides(); + } + + public function requires(): array + { + return $this->tests[0]->requires(); + } + + public function nameWithDataSet(): string + { + return $this->tests[0]->nameWithDataSet(); + } + + public function valueObjectForEvents(): Event\Code\TestMethod|Event\Code\Phpt + { + return $this->tests[0]->valueObjectForEvents(); + } + + private function runTestCase(): void { $defectOccurred = false; @@ -63,28 +99,33 @@ public function run(): void } } - public function sortId(): string + private function runPhptTestCase(): void { - return $this->tests[0]->sortId(); - } + $defectOccurred = false; - public function provides(): array - { - return $this->tests[0]->provides(); - } + foreach ($this->tests as $test) { + if ($defectOccurred) { + EventFacade::emitter()->testSkipped( + $this->valueObjectForEvents(), + 'Test repetition failure', + ); - public function requires(): array - { - return $this->tests[0]->requires(); - } + continue; + } - public function nameWithDataSet(): string - { - return $this->tests[0]->nameWithDataSet(); + $test->run(); + + if (!$test->passed()) { + $defectOccurred = true; + } + } } - public function valueObjectForEvents(): Event\Code\TestMethod + /** + * @phpstan-assert-if-true non-empty-list $this->tests + */ + private function isPhptTestCase(): bool { - return $this->tests[0]->valueObjectForEvents(); + return $this->tests[0] instanceof PhptTestCase; } } diff --git a/src/Framework/TestSuite.php b/src/Framework/TestSuite.php index 830c95f791..eb8c0c6bb3 100644 --- a/src/Framework/TestSuite.php +++ b/src/Framework/TestSuite.php @@ -158,7 +158,7 @@ public function addTest(Test $test, array $groups = [], int $repeat = 1): void assert($test instanceof TestCase || $test instanceof PhptTestCase); - if ($test instanceof PhptTestCase || $repeat === 1) { + if ($repeat === 1) { $this->tests[] = $test; } else { $this->tests[] = new RepeatTestSuite($test, $repeat); @@ -231,11 +231,11 @@ public function addTestSuite(ReflectionClass $testClass, array $groups = [], int * * @throws Exception */ - public function addTestFile(string $filename, array $groups = []): void + public function addTestFile(string $filename, array $groups = [], int $repeat = 1): void { try { if (str_ends_with($filename, '.phpt') && is_file($filename)) { - $this->addTest(new PhptTestCase($filename)); + $this->addTest(new PhptTestCase($filename), [], $repeat); } else { $this->addTestSuite( (new TestSuiteLoader)->load($filename), diff --git a/src/Runner/Phpt/TestCase.php b/src/Runner/Phpt/TestCase.php index ab75397b20..cc9c818623 100644 --- a/src/Runner/Phpt/TestCase.php +++ b/src/Runner/Phpt/TestCase.php @@ -70,12 +70,14 @@ * * @see https://qa.php.net/phpt_details.php */ -final readonly class TestCase implements Reorderable, SelfDescribing, Test +final class TestCase implements Reorderable, SelfDescribing, Test { /** * @var non-empty-string */ - private string $filename; + private readonly string $filename; + + private bool $passed = false; /** * @param non-empty-string $filename @@ -242,6 +244,8 @@ public function run(): void $emitter->testPassed($this->valueObjectForEvents()); } + $this->passed = $passed; + $this->runClean($sections, CodeCoverage::instance()->isActive()); $emitter->testFinished($this->valueObjectForEvents(), 1); @@ -255,6 +259,11 @@ public function getName(): string return $this->toString(); } + public function passed(): bool + { + return $this->passed; + } + /** * Returns a string representation of the test case. */ diff --git a/src/TextUI/Configuration/TestSuiteBuilder.php b/src/TextUI/Configuration/TestSuiteBuilder.php index 5570779a63..cab2213526 100644 --- a/src/TextUI/Configuration/TestSuiteBuilder.php +++ b/src/TextUI/Configuration/TestSuiteBuilder.php @@ -100,7 +100,7 @@ private function testSuiteFromPath(string $path, array $suffixes, int $repeat, ? $suite = TestSuite::empty($path); } - $suite->addTestFile($path); + $suite->addTestFile($path, [], $repeat); return $suite; } diff --git a/tests/end-to-end/repeat/_files/phpt/failure.phpt b/tests/end-to-end/repeat/_files/phpt/failure.phpt new file mode 100644 index 0000000000..9911b3ec52 --- /dev/null +++ b/tests/end-to-end/repeat/_files/phpt/failure.phpt @@ -0,0 +1,8 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.. 2 / 2 (100%) + +Time: %s, Memory: %s MB + +OK (2 tests, 2 assertions) diff --git a/tests/end-to-end/repeat/phpt-failure.phpt b/tests/end-to-end/repeat/phpt-failure.phpt new file mode 100644 index 0000000000..9fa046c299 --- /dev/null +++ b/tests/end-to-end/repeat/phpt-failure.phpt @@ -0,0 +1,37 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +FS 2 / 2 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) %s/tests/end-to-end/repeat/_files/phpt/failure.phpt +Failed asserting that string matches format description. +--- Expected ++++ Actual +@@ @@ +-ko ++ok + +%s/tests/end-to-end/repeat/_files/phpt/failure.phpt:8 + +FAILURES! +Tests: 2, Assertions: 1, Failures: 1, Skipped: 1. + diff --git a/tests/end-to-end/repeat/phpt-success.phpt b/tests/end-to-end/repeat/phpt-success.phpt new file mode 100644 index 0000000000..5bfbebd7a1 --- /dev/null +++ b/tests/end-to-end/repeat/phpt-success.phpt @@ -0,0 +1,23 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.. 2 / 2 (100%) + +Time: %s, Memory: %s MB + +OK (2 tests, 2 assertions) diff --git a/tests/end-to-end/repeat/repeat-test-with-data-provider-and-depends.phpt b/tests/end-to-end/repeat/test-with-data-provider-and-depends.phpt similarity index 100% rename from tests/end-to-end/repeat/repeat-test-with-data-provider-and-depends.phpt rename to tests/end-to-end/repeat/test-with-data-provider-and-depends.phpt diff --git a/tests/end-to-end/repeat/repeat-test-with-data-provider-failing.phpt b/tests/end-to-end/repeat/test-with-data-provider-failing.phpt similarity index 100% rename from tests/end-to-end/repeat/repeat-test-with-data-provider-failing.phpt rename to tests/end-to-end/repeat/test-with-data-provider-failing.phpt diff --git a/tests/end-to-end/repeat/repeat-test-with-data-provider.phpt b/tests/end-to-end/repeat/test-with-data-provider.phpt similarity index 100% rename from tests/end-to-end/repeat/repeat-test-with-data-provider.phpt rename to tests/end-to-end/repeat/test-with-data-provider.phpt From eb4829f5c8cf05a7362fb3198f64c799ccde8bbc Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Thu, 23 Oct 2025 22:35:11 +0200 Subject: [PATCH 07/21] Handle test suite loaded from path with --repeat --- src/Framework/TestSuite.php | 7 +++--- src/TextUI/Configuration/TestSuiteBuilder.php | 2 +- .../directory/RepeatInDirectoryTest.php | 13 +++++++++++ tests/end-to-end/repeat/in-directory.phpt | 23 +++++++++++++++++++ .../repeat/phpt-failure-and-success.phpt | 17 ++++++++++++-- 5 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 tests/end-to-end/repeat/_files/directory/RepeatInDirectoryTest.php create mode 100644 tests/end-to-end/repeat/in-directory.phpt diff --git a/src/Framework/TestSuite.php b/src/Framework/TestSuite.php index eb8c0c6bb3..cb84f560b8 100644 --- a/src/Framework/TestSuite.php +++ b/src/Framework/TestSuite.php @@ -216,7 +216,7 @@ public function addTestSuite(ReflectionClass $testClass, array $groups = [], int ); } - $this->addTest(self::fromClassReflector($testClass, $groups), $groups, $repeat); + $this->addTest(self::fromClassReflector($testClass, $groups, $repeat), $groups, $repeat); } /** @@ -240,6 +240,7 @@ public function addTestFile(string $filename, array $groups = [], int $repeat = $this->addTestSuite( (new TestSuiteLoader)->load($filename), $groups, + $repeat, ); } } catch (RunnerException $e) { @@ -256,10 +257,10 @@ public function addTestFile(string $filename, array $groups = [], int $repeat = * * @throws Exception */ - public function addTestFiles(iterable $fileNames): void + public function addTestFiles(iterable $fileNames, int $repeat = 1): void { foreach ($fileNames as $filename) { - $this->addTestFile((string) $filename); + $this->addTestFile((string) $filename, [], $repeat); } } diff --git a/src/TextUI/Configuration/TestSuiteBuilder.php b/src/TextUI/Configuration/TestSuiteBuilder.php index cab2213526..eaf2294e5b 100644 --- a/src/TextUI/Configuration/TestSuiteBuilder.php +++ b/src/TextUI/Configuration/TestSuiteBuilder.php @@ -112,7 +112,7 @@ private function testSuiteFromPath(string $path, array $suffixes, int $repeat, ? $suite = TestSuite::empty('CLI Arguments'); } - $suite->addTestFiles($files); + $suite->addTestFiles($files, $repeat); return $suite; } diff --git a/tests/end-to-end/repeat/_files/directory/RepeatInDirectoryTest.php b/tests/end-to-end/repeat/_files/directory/RepeatInDirectoryTest.php new file mode 100644 index 0000000000..fee416b189 --- /dev/null +++ b/tests/end-to-end/repeat/_files/directory/RepeatInDirectoryTest.php @@ -0,0 +1,13 @@ +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.. 2 / 2 (100%) + +Time: %s, Memory: %s MB + +OK (2 tests, 2 assertions) diff --git a/tests/end-to-end/repeat/phpt-failure-and-success.phpt b/tests/end-to-end/repeat/phpt-failure-and-success.phpt index 3b5c5c7b53..1431412512 100644 --- a/tests/end-to-end/repeat/phpt-failure-and-success.phpt +++ b/tests/end-to-end/repeat/phpt-failure-and-success.phpt @@ -16,8 +16,21 @@ PHPUnit %s by Sebastian Bergmann and contributors. Runtime: %s -.. 2 / 2 (100%) +FS.. 4 / 4 (100%) Time: %s, Memory: %s MB -OK (2 tests, 2 assertions) +There was 1 failure: + +1) /home/niko/works/github.com/sebastianbergmann/phpunit/tests/end-to-end/repeat/_files/phpt/failure.phpt +Failed asserting that string matches format description. +--- Expected ++++ Actual +@@ @@ +-ko ++ok + +/home/niko/works/github.com/sebastianbergmann/phpunit/tests/end-to-end/repeat/_files/phpt/failure.phpt:8 + +FAILURES! +Tests: 4, Assertions: 3, Failures: 1, Skipped: 1. From e079b71ac6ded029f9a61f6ef3a8a3e6d2b6cdd6 Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Thu, 23 Oct 2025 22:41:34 +0200 Subject: [PATCH 08/21] --repeat value should be a positive integer --- src/Framework/TestBuilder.php | 2 ++ src/Framework/TestSuite.php | 6 ++++++ src/TextUI/Configuration/Cli/Builder.php | 8 +++++++- src/TextUI/Configuration/Cli/Configuration.php | 4 ++++ src/TextUI/Configuration/Configuration.php | 7 +++++++ src/TextUI/Configuration/TestSuiteBuilder.php | 2 ++ tests/end-to-end/repeat/wonrg-repeat-value.phpt | 17 +++++++++++++++++ 7 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tests/end-to-end/repeat/wonrg-repeat-value.phpt diff --git a/src/Framework/TestBuilder.php b/src/Framework/TestBuilder.php index f383fa174e..2cf0796929 100644 --- a/src/Framework/TestBuilder.php +++ b/src/Framework/TestBuilder.php @@ -36,6 +36,7 @@ * @param ReflectionClass $theClass * @param non-empty-string $methodName * @param list $groups + * @param positive-int $repeat * * @throws InvalidDataProviderException */ @@ -86,6 +87,7 @@ public function build(ReflectionClass $theClass, string $methodName, array $grou * @param array $data * @param array{backupGlobals: ?true, backupGlobalsExcludeList: list, backupStaticProperties: ?true, backupStaticPropertiesExcludeList: array>} $backupSettings * @param list $groups + * @param positive-int $repeat */ private function buildDataProviderTestSuite(string $methodName, string $className, array $data, bool $runTestInSeparateProcess, ?bool $preserveGlobalState, array $backupSettings, array $groups, int $repeat = 1): DataProviderTestSuite { diff --git a/src/Framework/TestSuite.php b/src/Framework/TestSuite.php index cb84f560b8..fc2ddf9037 100644 --- a/src/Framework/TestSuite.php +++ b/src/Framework/TestSuite.php @@ -96,6 +96,7 @@ public static function empty(string $name): static /** * @param ReflectionClass $class * @param list $groups + * @param positive-int $repeat */ public static function fromClassReflector(ReflectionClass $class, array $groups = [], int $repeat = 1): static { @@ -145,6 +146,7 @@ final private function __construct(string $name) * Adds a test to the suite. * * @param list $groups + * @param positive-int $repeat */ public function addTest(Test $test, array $groups = [], int $repeat = 1): void { @@ -192,6 +194,7 @@ public function addTest(Test $test, array $groups = [], int $repeat = 1): void * * @param ReflectionClass $testClass * @param list $groups + * @param positive-int $repeat * * @throws Exception */ @@ -228,6 +231,7 @@ public function addTestSuite(ReflectionClass $testClass, array $groups = [], int * leaving the current test run untouched. * * @param list $groups + * @param positive-int $repeat * * @throws Exception */ @@ -254,6 +258,7 @@ public function addTestFile(string $filename, array $groups = [], int $repeat = * Wrapper for addTestFile() that adds multiple test files. * * @param iterable $fileNames + * @param positive-int $repeat * * @throws Exception */ @@ -508,6 +513,7 @@ public function isForTestClass(): bool /** * @param ReflectionClass $class * @param list $groups + * @param positive-int $repeat * * @throws Exception */ diff --git a/src/TextUI/Configuration/Cli/Builder.php b/src/TextUI/Configuration/Cli/Builder.php index 01b28397ce..14d86f523b 100644 --- a/src/TextUI/Configuration/Cli/Builder.php +++ b/src/TextUI/Configuration/Cli/Builder.php @@ -1214,7 +1214,13 @@ public function fromParameters(array $parameters): Configuration break; case '--repeat': - $repeat = (int) $option[1]; // todo: superior to 1 + $repeat = (int) $option[1]; + + if ($repeat < 1) { + throw new Exception( + 'The value for the --repeat option must be a positive integer', + ); + } break; diff --git a/src/TextUI/Configuration/Cli/Configuration.php b/src/TextUI/Configuration/Cli/Configuration.php index 79da9bfc8e..995308491f 100644 --- a/src/TextUI/Configuration/Cli/Configuration.php +++ b/src/TextUI/Configuration/Cli/Configuration.php @@ -195,6 +195,7 @@ * @param ?non-empty-list $testSuffixes * @param ?non-empty-list $coverageFilter * @param ?non-empty-list $extensions + * @param positive-int $repeat */ public function __construct(array $arguments, ?bool $all, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticProperties, ?bool $beStrictAboutChangesToGlobalState, ?string $bootstrap, ?string $cacheDirectory, ?bool $cacheResult, bool $checkPhpConfiguration, bool $checkVersion, ?string $colors, null|int|string $columns, ?string $configurationFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $pathCoverage, bool $warmCoverageCache, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?bool $failOnAllIssues, ?bool $failOnDeprecation, ?bool $failOnPhpunitDeprecation, ?bool $failOnPhpunitNotice, ?bool $failOnPhpunitWarning, ?bool $failOnEmptyTestSuite, ?bool $failOnIncomplete, ?bool $failOnNotice, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?bool $doNotFailOnDeprecation, ?bool $doNotFailOnPhpunitDeprecation, ?bool $doNotFailOnPhpunitNotice, ?bool $doNotFailOnPhpunitWarning, ?bool $doNotFailOnEmptyTestSuite, ?bool $doNotFailOnIncomplete, ?bool $doNotFailOnNotice, ?bool $doNotFailOnRisky, ?bool $doNotFailOnSkipped, ?bool $doNotFailOnWarning, ?bool $stopOnDefect, ?bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnNotice, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $filter, ?string $excludeFilter, ?string $generateBaseline, ?string $useBaseline, bool $ignoreBaseline, bool $generateConfiguration, bool $migrateConfiguration, ?array $groups, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?string $otrLogfile, ?bool $includeGitInformation, bool $listGroups, bool $listSuites, bool $listTestFiles, bool $listTests, ?string $listTestsXml, ?bool $noCoverage, ?bool $noExtensions, ?bool $noOutput, ?bool $noProgress, ?bool $noResults, ?bool $noLogging, ?bool $processIsolation, ?int $randomOrderSeed, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?string $teamcityLogfile, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?array $testSuffixes, ?string $testSuite, ?string $excludeTestSuite, bool $useDefaultConfiguration, ?bool $displayDetailsOnAllIssues, ?bool $displayDetailsOnIncompleteTests, ?bool $displayDetailsOnSkippedTests, ?bool $displayDetailsOnTestsThatTriggerDeprecations, ?bool $displayDetailsOnPhpunitDeprecations, ?bool $displayDetailsOnPhpunitNotices, ?bool $displayDetailsOnTestsThatTriggerErrors, ?bool $displayDetailsOnTestsThatTriggerNotices, ?bool $displayDetailsOnTestsThatTriggerWarnings, bool $version, ?array $coverageFilter, ?string $logEventsText, ?string $logEventsVerboseText, ?bool $printerTeamCity, ?bool $testdoxPrinter, ?bool $testdoxPrinterSummary, bool $debug, bool $withTelemetry, ?array $extensions, int $repeat) { @@ -2580,6 +2581,9 @@ public function debug(): bool return $this->debug; } + /** + * @return positive-int + */ public function repeat(): int { return $this->repeat; diff --git a/src/TextUI/Configuration/Configuration.php b/src/TextUI/Configuration/Configuration.php index 6f356ca0bb..a542731411 100644 --- a/src/TextUI/Configuration/Configuration.php +++ b/src/TextUI/Configuration/Configuration.php @@ -194,6 +194,9 @@ */ private ?string $generateBaseline; private bool $debug; + /** + * @var positive-int + */ private int $repeat; private bool $withTelemetry; @@ -215,6 +218,7 @@ * @param non-empty-list $testSuffixes * @param null|non-empty-string $generateBaseline * @param non-negative-int $shortenArraysForExportThreshold + * @param positive-int $repeat */ public function __construct(array $cliArguments, ?string $configurationFile, ?string $bootstrap, array $bootstrapForTestSuite, bool $cacheResult, ?string $cacheDirectory, ?string $coverageCacheDirectory, Source $source, string $testResultCacheFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4j, int $coverageCrap4jThreshold, ?string $coverageHtml, int $coverageHtmlLowUpperBound, int $coverageHtmlHighLowerBound, string $coverageHtmlColorSuccessLow, string $coverageHtmlColorSuccessMedium, string $coverageHtmlColorSuccessHigh, string $coverageHtmlColorWarning, string $coverageHtmlColorDanger, ?string $coverageHtmlCustomCssFile, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, bool $coverageTextShowUncoveredFiles, bool $coverageTextShowOnlySummary, ?string $coverageXml, bool $pathCoverage, bool $ignoreDeprecatedCodeUnitsFromCodeCoverage, bool $disableCodeCoverageIgnore, bool $failOnAllIssues, bool $failOnDeprecation, bool $failOnPhpunitDeprecation, bool $failOnPhpunitNotice, bool $failOnPhpunitWarning, bool $failOnEmptyTestSuite, bool $failOnIncomplete, bool $failOnNotice, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $doNotFailOnDeprecation, bool $doNotFailOnPhpunitDeprecation, bool $doNotFailOnPhpunitNotice, bool $doNotFailOnPhpunitWarning, bool $doNotFailOnEmptyTestSuite, bool $doNotFailOnIncomplete, bool $doNotFailOnNotice, bool $doNotFailOnRisky, bool $doNotFailOnSkipped, bool $doNotFailOnWarning, bool $stopOnDefect, bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, bool $stopOnError, bool $stopOnFailure, bool $stopOnIncomplete, bool $stopOnNotice, bool $stopOnRisky, bool $stopOnSkipped, bool $stopOnWarning, bool $outputToStandardErrorStream, int $columns, bool $noExtensions, ?string $pharExtensionDirectory, array $extensionBootstrappers, bool $backupGlobals, bool $backupStaticProperties, bool $beStrictAboutChangesToGlobalState, bool $colors, bool $processIsolation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, bool $reportUselessTests, bool $strictCoverage, bool $disallowTestOutput, bool $displayDetailsOnAllIssues, bool $displayDetailsOnIncompleteTests, bool $displayDetailsOnSkippedTests, bool $displayDetailsOnTestsThatTriggerDeprecations, bool $displayDetailsOnPhpunitDeprecations, bool $displayDetailsOnPhpunitNotices, bool $displayDetailsOnTestsThatTriggerErrors, bool $displayDetailsOnTestsThatTriggerNotices, bool $displayDetailsOnTestsThatTriggerWarnings, bool $reverseDefectList, bool $requireCoverageMetadata, bool $noProgress, bool $noResults, bool $noOutput, int $executionOrder, int $executionOrderDefects, bool $resolveDependencies, ?string $logfileTeamcity, ?string $logfileJunit, ?string $logfileOtr, bool $includeGitInformationInOtrLogfile, ?string $logfileTestdoxHtml, ?string $logfileTestdoxText, ?string $logEventsText, ?string $logEventsVerboseText, bool $teamCityOutput, bool $testDoxOutput, bool $testDoxOutputSummary, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, ?string $filter, ?string $excludeFilter, array $groups, array $excludeGroups, int $randomOrderSeed, bool $includeUncoveredFiles, TestSuiteCollection $testSuite, string $includeTestSuite, string $excludeTestSuite, ?string $defaultTestSuite, bool $ignoreTestSelectionInXmlConfiguration, array $testSuffixes, Php $php, bool $controlGarbageCollector, int $numberOfTestsBeforeGarbageCollection, ?string $generateBaseline, bool $debug, int $repeat, bool $withTelemetry, int $shortenArraysForExportThreshold) { @@ -1516,6 +1520,9 @@ public function debug(): bool return $this->debug; } + /** + * @return positive-int + */ public function repeat(): int { return $this->repeat; diff --git a/src/TextUI/Configuration/TestSuiteBuilder.php b/src/TextUI/Configuration/TestSuiteBuilder.php index eaf2294e5b..4461d3a4b6 100644 --- a/src/TextUI/Configuration/TestSuiteBuilder.php +++ b/src/TextUI/Configuration/TestSuiteBuilder.php @@ -90,6 +90,7 @@ public function build(Configuration $configuration): TestSuite /** * @param non-empty-string $path * @param list $suffixes + * @param positive-int $repeat * * @throws \PHPUnit\Framework\Exception */ @@ -137,6 +138,7 @@ private function testSuiteFromPath(string $path, array $suffixes, int $repeat, ? /** * @param list $paths * @param list $suffixes + * @param positive-int $repeat * * @throws \PHPUnit\Framework\Exception */ diff --git a/tests/end-to-end/repeat/wonrg-repeat-value.phpt b/tests/end-to-end/repeat/wonrg-repeat-value.phpt new file mode 100644 index 0000000000..fe7de94af2 --- /dev/null +++ b/tests/end-to-end/repeat/wonrg-repeat-value.phpt @@ -0,0 +1,17 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +The value for the --repeat option must be a positive integer From 6a9faddec1f09dd123b893e71364520a96430b3c Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Thu, 23 Oct 2025 22:47:16 +0200 Subject: [PATCH 09/21] cs fixer --- .../Value/TestSuite/TestSuiteBuilder.php | 2 +- src/Framework/RepeatTestSuite.php | 12 +++--- src/Framework/TestBuilder.php | 4 +- src/Framework/TestCase.php | 24 +++++------ src/Framework/TestSuite.php | 12 +++--- src/Runner/Phpt/TestCase.php | 1 - src/TextUI/Configuration/Configuration.php | 5 ++- src/TextUI/Configuration/TestSuiteBuilder.php | 4 +- .../DependentOfTestFailedInRepetitionTest.php | 17 +++++--- .../repeat/_files/RepeatDependentTest.php | 19 ++++++--- tests/end-to-end/repeat/_files/RepeatTest.php | 15 +++++-- .../RepeatWithDataProviderAndDependsTest.php | 42 ++++++++++++------- .../RepeatWithDataProviderFailingTest.php | 25 +++++++---- .../_files/RepeatWithDataProviderTest.php | 24 +++++++---- .../repeat/_files/RepeatWithErrorsTest.php | 23 ++++++---- .../directory/RepeatInDirectoryTest.php | 13 ++++-- 16 files changed, 153 insertions(+), 89 deletions(-) diff --git a/src/Event/Value/TestSuite/TestSuiteBuilder.php b/src/Event/Value/TestSuite/TestSuiteBuilder.php index 56f85a858f..6d6db20ba4 100644 --- a/src/Event/Value/TestSuite/TestSuiteBuilder.php +++ b/src/Event/Value/TestSuite/TestSuiteBuilder.php @@ -9,7 +9,6 @@ */ namespace PHPUnit\Event\TestSuite; -use PHPUnit\Framework\RepeatTestSuite; use function assert; use function class_exists; use function count; @@ -19,6 +18,7 @@ use PHPUnit\Event\Code\TestCollection; use PHPUnit\Event\RuntimeException; use PHPUnit\Framework\DataProviderTestSuite; +use PHPUnit\Framework\RepeatTestSuite; use PHPUnit\Framework\TestCase; use PHPUnit\Framework\TestSuite as FrameworkTestSuite; use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php index 475f5cd783..978ba4d348 100644 --- a/src/Framework/RepeatTestSuite.php +++ b/src/Framework/RepeatTestSuite.php @@ -9,29 +9,31 @@ */ namespace PHPUnit\Framework; +use function count; +use PHPUnit\Event; use PHPUnit\Event\Facade as EventFacade; use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; use PHPUnit\TestRunner\TestResult\PassedTests; -use PHPUnit\Event; /** * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit * * @internal This class is not covered by the backward compatibility promise for PHPUnit */ -final class RepeatTestSuite implements Test, Reorderable +final class RepeatTestSuite implements Reorderable, Test { /** - * @var non-empty-list|non-empty-list + * @var non-empty-list|non-empty-list */ private array $tests; /** * @param positive-int $times */ - public function __construct(TestCase|PhptTestCase $test, int $times) + public function __construct(PhptTestCase|TestCase $test, int $times) { $tests = []; + for ($i = 0; $i < $times; $i++) { $tests[] = clone $test; } @@ -73,7 +75,7 @@ public function nameWithDataSet(): string return $this->tests[0]->nameWithDataSet(); } - public function valueObjectForEvents(): Event\Code\TestMethod|Event\Code\Phpt + public function valueObjectForEvents(): Event\Code\Phpt|Event\Code\TestMethod { return $this->tests[0]->valueObjectForEvents(); } diff --git a/src/Framework/TestBuilder.php b/src/Framework/TestBuilder.php index 2cf0796929..ddccd33e3f 100644 --- a/src/Framework/TestBuilder.php +++ b/src/Framework/TestBuilder.php @@ -36,7 +36,7 @@ * @param ReflectionClass $theClass * @param non-empty-string $methodName * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeat * * @throws InvalidDataProviderException */ @@ -87,7 +87,7 @@ public function build(ReflectionClass $theClass, string $methodName, array $grou * @param array $data * @param array{backupGlobals: ?true, backupGlobalsExcludeList: list, backupStaticProperties: ?true, backupStaticPropertiesExcludeList: array>} $backupSettings * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeat */ private function buildDataProviderTestSuite(string $methodName, string $className, array $data, bool $runTestInSeparateProcess, ?bool $preserveGlobalState, array $backupSettings, array $groups, int $repeat = 1): DataProviderTestSuite { diff --git a/src/Framework/TestCase.php b/src/Framework/TestCase.php index 80fcefa740..43fdcc6ac4 100644 --- a/src/Framework/TestCase.php +++ b/src/Framework/TestCase.php @@ -965,6 +965,18 @@ final public function wasPrepared(): bool return $this->wasPrepared; } + public function markSkippedForErrorInPreviousRepetition(): void + { + $message = 'Test repetition failure'; + + Event\Facade::emitter()->testSkipped( + $this->valueObjectForEvents(), + $message, + ); + + $this->status = TestStatus::skipped($message); + } + /** * Returns a matcher that matches when the method is executed * zero or more times. @@ -1543,18 +1555,6 @@ private function markSkippedForMissingDependency(ExecutionOrderDependency $depen $this->status = TestStatus::skipped($message); } - public function markSkippedForErrorInPreviousRepetition(): void - { - $message = 'Test repetition failure'; - - Event\Facade::emitter()->testSkipped( - $this->valueObjectForEvents(), - $message, - ); - - $this->status = TestStatus::skipped($message); - } - private function startOutputBuffering(): void { ob_start(); diff --git a/src/Framework/TestSuite.php b/src/Framework/TestSuite.php index fc2ddf9037..30c7b84f00 100644 --- a/src/Framework/TestSuite.php +++ b/src/Framework/TestSuite.php @@ -96,7 +96,7 @@ public static function empty(string $name): static /** * @param ReflectionClass $class * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeat */ public static function fromClassReflector(ReflectionClass $class, array $groups = [], int $repeat = 1): static { @@ -146,7 +146,7 @@ final private function __construct(string $name) * Adds a test to the suite. * * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeat */ public function addTest(Test $test, array $groups = [], int $repeat = 1): void { @@ -194,7 +194,7 @@ public function addTest(Test $test, array $groups = [], int $repeat = 1): void * * @param ReflectionClass $testClass * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeat * * @throws Exception */ @@ -231,7 +231,7 @@ public function addTestSuite(ReflectionClass $testClass, array $groups = [], int * leaving the current test run untouched. * * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeat * * @throws Exception */ @@ -258,7 +258,7 @@ public function addTestFile(string $filename, array $groups = [], int $repeat = * Wrapper for addTestFile() that adds multiple test files. * * @param iterable $fileNames - * @param positive-int $repeat + * @param positive-int $repeat * * @throws Exception */ @@ -513,7 +513,7 @@ public function isForTestClass(): bool /** * @param ReflectionClass $class * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeat * * @throws Exception */ diff --git a/src/Runner/Phpt/TestCase.php b/src/Runner/Phpt/TestCase.php index cc9c818623..564cc2c567 100644 --- a/src/Runner/Phpt/TestCase.php +++ b/src/Runner/Phpt/TestCase.php @@ -76,7 +76,6 @@ final class TestCase implements Reorderable, SelfDescribing, Test * @var non-empty-string */ private readonly string $filename; - private bool $passed = false; /** diff --git a/src/TextUI/Configuration/Configuration.php b/src/TextUI/Configuration/Configuration.php index a542731411..565525e81c 100644 --- a/src/TextUI/Configuration/Configuration.php +++ b/src/TextUI/Configuration/Configuration.php @@ -194,6 +194,7 @@ */ private ?string $generateBaseline; private bool $debug; + /** * @var positive-int */ @@ -217,8 +218,8 @@ * @param list $excludeGroups * @param non-empty-list $testSuffixes * @param null|non-empty-string $generateBaseline - * @param non-negative-int $shortenArraysForExportThreshold * @param positive-int $repeat + * @param non-negative-int $shortenArraysForExportThreshold */ public function __construct(array $cliArguments, ?string $configurationFile, ?string $bootstrap, array $bootstrapForTestSuite, bool $cacheResult, ?string $cacheDirectory, ?string $coverageCacheDirectory, Source $source, string $testResultCacheFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4j, int $coverageCrap4jThreshold, ?string $coverageHtml, int $coverageHtmlLowUpperBound, int $coverageHtmlHighLowerBound, string $coverageHtmlColorSuccessLow, string $coverageHtmlColorSuccessMedium, string $coverageHtmlColorSuccessHigh, string $coverageHtmlColorWarning, string $coverageHtmlColorDanger, ?string $coverageHtmlCustomCssFile, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, bool $coverageTextShowUncoveredFiles, bool $coverageTextShowOnlySummary, ?string $coverageXml, bool $pathCoverage, bool $ignoreDeprecatedCodeUnitsFromCodeCoverage, bool $disableCodeCoverageIgnore, bool $failOnAllIssues, bool $failOnDeprecation, bool $failOnPhpunitDeprecation, bool $failOnPhpunitNotice, bool $failOnPhpunitWarning, bool $failOnEmptyTestSuite, bool $failOnIncomplete, bool $failOnNotice, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $doNotFailOnDeprecation, bool $doNotFailOnPhpunitDeprecation, bool $doNotFailOnPhpunitNotice, bool $doNotFailOnPhpunitWarning, bool $doNotFailOnEmptyTestSuite, bool $doNotFailOnIncomplete, bool $doNotFailOnNotice, bool $doNotFailOnRisky, bool $doNotFailOnSkipped, bool $doNotFailOnWarning, bool $stopOnDefect, bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, bool $stopOnError, bool $stopOnFailure, bool $stopOnIncomplete, bool $stopOnNotice, bool $stopOnRisky, bool $stopOnSkipped, bool $stopOnWarning, bool $outputToStandardErrorStream, int $columns, bool $noExtensions, ?string $pharExtensionDirectory, array $extensionBootstrappers, bool $backupGlobals, bool $backupStaticProperties, bool $beStrictAboutChangesToGlobalState, bool $colors, bool $processIsolation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, bool $reportUselessTests, bool $strictCoverage, bool $disallowTestOutput, bool $displayDetailsOnAllIssues, bool $displayDetailsOnIncompleteTests, bool $displayDetailsOnSkippedTests, bool $displayDetailsOnTestsThatTriggerDeprecations, bool $displayDetailsOnPhpunitDeprecations, bool $displayDetailsOnPhpunitNotices, bool $displayDetailsOnTestsThatTriggerErrors, bool $displayDetailsOnTestsThatTriggerNotices, bool $displayDetailsOnTestsThatTriggerWarnings, bool $reverseDefectList, bool $requireCoverageMetadata, bool $noProgress, bool $noResults, bool $noOutput, int $executionOrder, int $executionOrderDefects, bool $resolveDependencies, ?string $logfileTeamcity, ?string $logfileJunit, ?string $logfileOtr, bool $includeGitInformationInOtrLogfile, ?string $logfileTestdoxHtml, ?string $logfileTestdoxText, ?string $logEventsText, ?string $logEventsVerboseText, bool $teamCityOutput, bool $testDoxOutput, bool $testDoxOutputSummary, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, ?string $filter, ?string $excludeFilter, array $groups, array $excludeGroups, int $randomOrderSeed, bool $includeUncoveredFiles, TestSuiteCollection $testSuite, string $includeTestSuite, string $excludeTestSuite, ?string $defaultTestSuite, bool $ignoreTestSelectionInXmlConfiguration, array $testSuffixes, Php $php, bool $controlGarbageCollector, int $numberOfTestsBeforeGarbageCollection, ?string $generateBaseline, bool $debug, int $repeat, bool $withTelemetry, int $shortenArraysForExportThreshold) { @@ -350,7 +351,7 @@ public function __construct(array $cliArguments, ?string $configurationFile, ?st $this->numberOfTestsBeforeGarbageCollection = $numberOfTestsBeforeGarbageCollection; $this->generateBaseline = $generateBaseline; $this->debug = $debug; - $this->repeat = $repeat; + $this->repeat = $repeat; $this->withTelemetry = $withTelemetry; $this->shortenArraysForExportThreshold = $shortenArraysForExportThreshold; } diff --git a/src/TextUI/Configuration/TestSuiteBuilder.php b/src/TextUI/Configuration/TestSuiteBuilder.php index 4461d3a4b6..80eb56391c 100644 --- a/src/TextUI/Configuration/TestSuiteBuilder.php +++ b/src/TextUI/Configuration/TestSuiteBuilder.php @@ -90,7 +90,7 @@ public function build(Configuration $configuration): TestSuite /** * @param non-empty-string $path * @param list $suffixes - * @param positive-int $repeat + * @param positive-int $repeat * * @throws \PHPUnit\Framework\Exception */ @@ -138,7 +138,7 @@ private function testSuiteFromPath(string $path, array $suffixes, int $repeat, ? /** * @param list $paths * @param list $suffixes - * @param positive-int $repeat + * @param positive-int $repeat * * @throws \PHPUnit\Framework\Exception */ diff --git a/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php b/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php index 038c091daa..6dbc00eff0 100644 --- a/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php +++ b/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php @@ -1,5 +1,12 @@ - + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ use PHPUnit\Framework\Attributes\Depends; use PHPUnit\Framework\TestCase; @@ -10,15 +17,15 @@ public function test1(): void static $cout = 0; if ($cout++ > 0) { - self::assertFalse(true); + $this->assertFalse(true); } - self::assertTrue(true); + $this->assertTrue(true); } #[Depends('test1')] public function test2(): void { - self::assertTrue(true); + $this->assertTrue(true); } } diff --git a/tests/end-to-end/repeat/_files/RepeatDependentTest.php b/tests/end-to-end/repeat/_files/RepeatDependentTest.php index 62860c94f5..86ffa44654 100644 --- a/tests/end-to-end/repeat/_files/RepeatDependentTest.php +++ b/tests/end-to-end/repeat/_files/RepeatDependentTest.php @@ -1,5 +1,12 @@ - + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ use PHPUnit\Framework\Attributes\Depends; use PHPUnit\Framework\TestCase; @@ -7,23 +14,23 @@ final class RepeatDependentTest extends TestCase { public function test1(): void { - self::assertTrue(true); + $this->assertTrue(true); } #[Depends('test1')] public function testDepends1(): void { - self::assertTrue(true); + $this->assertTrue(true); } public function test2(): void { - self::assertTrue(false); + $this->assertTrue(false); } #[Depends('test2')] public function testDepends2(): void { - self::assertTrue(true); + $this->assertTrue(true); } } diff --git a/tests/end-to-end/repeat/_files/RepeatTest.php b/tests/end-to-end/repeat/_files/RepeatTest.php index 582bfce92c..2963f8b51c 100644 --- a/tests/end-to-end/repeat/_files/RepeatTest.php +++ b/tests/end-to-end/repeat/_files/RepeatTest.php @@ -1,16 +1,23 @@ - + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ use PHPUnit\Framework\TestCase; final class RepeatTest extends TestCase { public function test1(): void { - self::assertTrue(true); + $this->assertTrue(true); } public function test2(): void { - self::assertTrue(true); + $this->assertTrue(true); } } diff --git a/tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php b/tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php index bc05bf457e..fa071e5f11 100644 --- a/tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php +++ b/tests/end-to-end/repeat/_files/RepeatWithDataProviderAndDependsTest.php @@ -1,45 +1,55 @@ - + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\Depends; use PHPUnit\Framework\TestCase; final class RepeatWithDataProviderAndDependsTest extends TestCase { - #[DataProvider('provide1')] - public function test1(bool $bool): void + public static function provide1(): iterable { - self::assertTrue($bool); + yield [true]; + + yield [true]; } - public static function provide1(): iterable + public static function provide2(): iterable { yield [true]; + + yield [false]; + yield [true]; } + #[DataProvider('provide1')] + public function test1(bool $bool): void + { + $this->assertTrue($bool); + } + #[Depends('test1')] public function testDependsOn1(): void { - self::assertTrue(true); + $this->assertTrue(true); } #[DataProvider('provide2')] public function test2(bool $bool): void { - self::assertTrue($bool); - } - - public static function provide2(): iterable - { - yield [true]; - yield [false]; - yield [true]; + $this->assertTrue($bool); } #[Depends('test2')] public function testDependsOn2(): void { - self::assertTrue(true); + $this->assertTrue(true); } } diff --git a/tests/end-to-end/repeat/_files/RepeatWithDataProviderFailingTest.php b/tests/end-to-end/repeat/_files/RepeatWithDataProviderFailingTest.php index 7eb2554141..6d5f4ca4a6 100644 --- a/tests/end-to-end/repeat/_files/RepeatWithDataProviderFailingTest.php +++ b/tests/end-to-end/repeat/_files/RepeatWithDataProviderFailingTest.php @@ -1,20 +1,29 @@ - + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; final class RepeatWithDataProviderFailingTest extends TestCase { - #[DataProvider('provide')] - public function test1(bool $bool): void - { - self::assertTrue($bool); - } - public static function provide(): iterable { yield [true]; + yield [false]; + yield [true]; } + + #[DataProvider('provide')] + public function test1(bool $bool): void + { + $this->assertTrue($bool); + } } diff --git a/tests/end-to-end/repeat/_files/RepeatWithDataProviderTest.php b/tests/end-to-end/repeat/_files/RepeatWithDataProviderTest.php index ee0aaa26f1..f24b199d5f 100644 --- a/tests/end-to-end/repeat/_files/RepeatWithDataProviderTest.php +++ b/tests/end-to-end/repeat/_files/RepeatWithDataProviderTest.php @@ -1,19 +1,27 @@ - + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; final class RepeatWithDataProviderTest extends TestCase { - #[DataProvider('provide')] - public function test1(bool $bool): void - { - self::assertTrue($bool); - } - public static function provide(): iterable { yield [true]; + yield [true]; } + + #[DataProvider('provide')] + public function test1(bool $bool): void + { + $this->assertTrue($bool); + } } diff --git a/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php b/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php index 016c75fb12..5ec8d246a7 100644 --- a/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php +++ b/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php @@ -1,12 +1,19 @@ - + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ use PHPUnit\Framework\TestCase; final class RepeatWithErrorsTest extends TestCase { public function test1(): void { - self::assertFalse(true); + $this->assertFalse(true); } public function test2(): void @@ -14,10 +21,10 @@ public function test2(): void static $cout = 0; if ($cout++ > 0) { - self::assertFalse(true); + $this->assertFalse(true); } - self::assertTrue(true); + $this->assertTrue(true); } public function test3(): void @@ -25,14 +32,14 @@ public function test3(): void static $cout = 0; if ($cout++ > 1) { - self::assertFalse(true); + $this->assertFalse(true); } - self::assertTrue(true); + $this->assertTrue(true); } public function test4(): void { - self::assertTrue(true); + $this->assertTrue(true); } } diff --git a/tests/end-to-end/repeat/_files/directory/RepeatInDirectoryTest.php b/tests/end-to-end/repeat/_files/directory/RepeatInDirectoryTest.php index fee416b189..855c136c03 100644 --- a/tests/end-to-end/repeat/_files/directory/RepeatInDirectoryTest.php +++ b/tests/end-to-end/repeat/_files/directory/RepeatInDirectoryTest.php @@ -1,5 +1,12 @@ - + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ namespace directory; use PHPUnit\Framework\TestCase; @@ -8,6 +15,6 @@ final class RepeatInDirectoryTest extends TestCase { public function test1(): void { - self::assertTrue(true); + $this->assertTrue(true); } } From c6ec4c0023b77e283925d98bd137d6e9fb014613 Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Thu, 23 Oct 2025 23:22:10 +0200 Subject: [PATCH 10/21] add @internal tag on markSkippedForErrorInPreviousRepetition --- src/Framework/RepeatTestSuite.php | 2 +- src/Framework/TestCase.php | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php index 978ba4d348..5f1d48f60a 100644 --- a/src/Framework/RepeatTestSuite.php +++ b/src/Framework/RepeatTestSuite.php @@ -20,7 +20,7 @@ * * @internal This class is not covered by the backward compatibility promise for PHPUnit */ -final class RepeatTestSuite implements Reorderable, Test +final readonly class RepeatTestSuite implements Reorderable, Test { /** * @var non-empty-list|non-empty-list diff --git a/src/Framework/TestCase.php b/src/Framework/TestCase.php index 43fdcc6ac4..fdd128bc89 100644 --- a/src/Framework/TestCase.php +++ b/src/Framework/TestCase.php @@ -965,6 +965,9 @@ final public function wasPrepared(): bool return $this->wasPrepared; } + /** + * @internal This method is not covered by the backward compatibility promise for PHPUnit + */ public function markSkippedForErrorInPreviousRepetition(): void { $message = 'Test repetition failure'; From 638b334da7826679e1b6a8c3b8025b311747e4ba Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Fri, 24 Oct 2025 19:39:59 +0200 Subject: [PATCH 11/21] rename $repeat into $repeatTimes --- src/Framework/TestBuilder.php | 12 +++--- src/Framework/TestSuite.php | 42 +++++++++---------- src/TextUI/Configuration/Cli/Builder.php | 8 ++-- .../Configuration/Cli/Configuration.php | 12 +++--- src/TextUI/Configuration/Configuration.php | 12 +++--- src/TextUI/Configuration/Merger.php | 2 +- src/TextUI/Configuration/TestSuiteBuilder.php | 22 +++++----- 7 files changed, 55 insertions(+), 55 deletions(-) diff --git a/src/Framework/TestBuilder.php b/src/Framework/TestBuilder.php index ddccd33e3f..ff5e9c3e74 100644 --- a/src/Framework/TestBuilder.php +++ b/src/Framework/TestBuilder.php @@ -36,11 +36,11 @@ * @param ReflectionClass $theClass * @param non-empty-string $methodName * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeatTimes * * @throws InvalidDataProviderException */ - public function build(ReflectionClass $theClass, string $methodName, array $groups = [], int $repeat = 1): Test + public function build(ReflectionClass $theClass, string $methodName, array $groups = [], int $repeatTimes = 1): Test { $className = $theClass->getName(); @@ -65,7 +65,7 @@ public function build(ReflectionClass $theClass, string $methodName, array $grou $this->shouldGlobalStateBePreserved($className, $methodName), $this->backupSettings($className, $methodName), $groups, - $repeat, + $repeatTimes, ); } @@ -87,9 +87,9 @@ public function build(ReflectionClass $theClass, string $methodName, array $grou * @param array $data * @param array{backupGlobals: ?true, backupGlobalsExcludeList: list, backupStaticProperties: ?true, backupStaticPropertiesExcludeList: array>} $backupSettings * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeatTimes */ - private function buildDataProviderTestSuite(string $methodName, string $className, array $data, bool $runTestInSeparateProcess, ?bool $preserveGlobalState, array $backupSettings, array $groups, int $repeat = 1): DataProviderTestSuite + private function buildDataProviderTestSuite(string $methodName, string $className, array $data, bool $runTestInSeparateProcess, ?bool $preserveGlobalState, array $backupSettings, array $groups, int $repeatTimes = 1): DataProviderTestSuite { $dataProviderTestSuite = DataProviderTestSuite::empty( $className . '::' . $methodName, @@ -112,7 +112,7 @@ private function buildDataProviderTestSuite(string $methodName, string $classNam $backupSettings, ); - $dataProviderTestSuite->addTest($_test, $groups, $repeat); + $dataProviderTestSuite->addTest($_test, $groups, $repeatTimes); } return $dataProviderTestSuite; diff --git a/src/Framework/TestSuite.php b/src/Framework/TestSuite.php index 30c7b84f00..d248d74677 100644 --- a/src/Framework/TestSuite.php +++ b/src/Framework/TestSuite.php @@ -96,9 +96,9 @@ public static function empty(string $name): static /** * @param ReflectionClass $class * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeatTimes */ - public static function fromClassReflector(ReflectionClass $class, array $groups = [], int $repeat = 1): static + public static function fromClassReflector(ReflectionClass $class, array $groups = [], int $repeatTimes = 1): static { $testSuite = new static($class->getName()); @@ -119,7 +119,7 @@ public static function fromClassReflector(ReflectionClass $class, array $groups continue; } - $testSuite->addTestMethod($class, $method, $groups, $repeat); + $testSuite->addTestMethod($class, $method, $groups, $repeatTimes); } if ($testSuite->isEmpty()) { @@ -146,9 +146,9 @@ final private function __construct(string $name) * Adds a test to the suite. * * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeatTimes */ - public function addTest(Test $test, array $groups = [], int $repeat = 1): void + public function addTest(Test $test, array $groups = [], int $repeatTimes = 1): void { if ($test instanceof self) { $this->tests[] = $test; @@ -160,10 +160,10 @@ public function addTest(Test $test, array $groups = [], int $repeat = 1): void assert($test instanceof TestCase || $test instanceof PhptTestCase); - if ($repeat === 1) { + if ($repeatTimes === 1) { $this->tests[] = $test; } else { - $this->tests[] = new RepeatTestSuite($test, $repeat); + $this->tests[] = new RepeatTestSuite($test, $repeatTimes); } $this->clearCaches(); @@ -194,11 +194,11 @@ public function addTest(Test $test, array $groups = [], int $repeat = 1): void * * @param ReflectionClass $testClass * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeatTimes * * @throws Exception */ - public function addTestSuite(ReflectionClass $testClass, array $groups = [], int $repeat = 1): void + public function addTestSuite(ReflectionClass $testClass, array $groups = [], int $repeatTimes = 1): void { if ($testClass->isAbstract()) { throw new Exception( @@ -219,7 +219,7 @@ public function addTestSuite(ReflectionClass $testClass, array $groups = [], int ); } - $this->addTest(self::fromClassReflector($testClass, $groups, $repeat), $groups, $repeat); + $this->addTest(self::fromClassReflector($testClass, $groups, $repeatTimes), $groups, $repeatTimes); } /** @@ -231,20 +231,20 @@ public function addTestSuite(ReflectionClass $testClass, array $groups = [], int * leaving the current test run untouched. * * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeatTimes * * @throws Exception */ - public function addTestFile(string $filename, array $groups = [], int $repeat = 1): void + public function addTestFile(string $filename, array $groups = [], int $repeatTimes = 1): void { try { if (str_ends_with($filename, '.phpt') && is_file($filename)) { - $this->addTest(new PhptTestCase($filename), [], $repeat); + $this->addTest(new PhptTestCase($filename), [], $repeatTimes); } else { $this->addTestSuite( (new TestSuiteLoader)->load($filename), $groups, - $repeat, + $repeatTimes, ); } } catch (RunnerException $e) { @@ -258,14 +258,14 @@ public function addTestFile(string $filename, array $groups = [], int $repeat = * Wrapper for addTestFile() that adds multiple test files. * * @param iterable $fileNames - * @param positive-int $repeat + * @param positive-int $repeatTimes * * @throws Exception */ - public function addTestFiles(iterable $fileNames, int $repeat = 1): void + public function addTestFiles(iterable $fileNames, int $repeatTimes = 1): void { foreach ($fileNames as $filename) { - $this->addTestFile((string) $filename, [], $repeat); + $this->addTestFile((string) $filename, [], $repeatTimes); } } @@ -513,17 +513,17 @@ public function isForTestClass(): bool /** * @param ReflectionClass $class * @param list $groups - * @param positive-int $repeat + * @param positive-int $repeatTimes * * @throws Exception */ - protected function addTestMethod(ReflectionClass $class, ReflectionMethod $method, array $groups, int $repeat): void + protected function addTestMethod(ReflectionClass $class, ReflectionMethod $method, array $groups, int $repeatTimes): void { $className = $class->getName(); $methodName = $method->getName(); try { - $test = (new TestBuilder)->build($class, $methodName, $groups, $repeat); + $test = (new TestBuilder)->build($class, $methodName, $groups, $repeatTimes); } catch (InvalidDataProviderException $e) { if ($e->getProviderLabel() === null) { $message = sprintf( @@ -573,7 +573,7 @@ protected function addTestMethod(ReflectionClass $class, ReflectionMethod $metho $groups, (new Groups)->groups($class->getName(), $methodName), ), - $repeat, + $repeatTimes, ); } diff --git a/src/TextUI/Configuration/Cli/Builder.php b/src/TextUI/Configuration/Cli/Builder.php index 14d86f523b..674c474317 100644 --- a/src/TextUI/Configuration/Cli/Builder.php +++ b/src/TextUI/Configuration/Cli/Builder.php @@ -313,7 +313,7 @@ public function fromParameters(array $parameters): Configuration $printerTestDox = null; $printerTestDoxSummary = null; $debug = false; - $repeat = 1; + $repeatTimes = 1; $withTelemetry = false; $extensions = []; @@ -1214,9 +1214,9 @@ public function fromParameters(array $parameters): Configuration break; case '--repeat': - $repeat = (int) $option[1]; + $repeatTimes = (int) $option[1]; - if ($repeat < 1) { + if ($repeatTimes < 1) { throw new Exception( 'The value for the --repeat option must be a positive integer', ); @@ -1376,7 +1376,7 @@ public function fromParameters(array $parameters): Configuration $debug, $withTelemetry, $extensions, - $repeat, + $repeatTimes, ); } diff --git a/src/TextUI/Configuration/Cli/Configuration.php b/src/TextUI/Configuration/Cli/Configuration.php index 995308491f..62547b8c09 100644 --- a/src/TextUI/Configuration/Cli/Configuration.php +++ b/src/TextUI/Configuration/Cli/Configuration.php @@ -177,7 +177,7 @@ private ?string $logEventsVerboseText; private bool $debug; private bool $withTelemetry; - private int $repeat; + private int $repeatTimes; /** * @var ?non-empty-list @@ -195,9 +195,9 @@ * @param ?non-empty-list $testSuffixes * @param ?non-empty-list $coverageFilter * @param ?non-empty-list $extensions - * @param positive-int $repeat + * @param positive-int $repeatTimes */ - public function __construct(array $arguments, ?bool $all, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticProperties, ?bool $beStrictAboutChangesToGlobalState, ?string $bootstrap, ?string $cacheDirectory, ?bool $cacheResult, bool $checkPhpConfiguration, bool $checkVersion, ?string $colors, null|int|string $columns, ?string $configurationFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $pathCoverage, bool $warmCoverageCache, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?bool $failOnAllIssues, ?bool $failOnDeprecation, ?bool $failOnPhpunitDeprecation, ?bool $failOnPhpunitNotice, ?bool $failOnPhpunitWarning, ?bool $failOnEmptyTestSuite, ?bool $failOnIncomplete, ?bool $failOnNotice, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?bool $doNotFailOnDeprecation, ?bool $doNotFailOnPhpunitDeprecation, ?bool $doNotFailOnPhpunitNotice, ?bool $doNotFailOnPhpunitWarning, ?bool $doNotFailOnEmptyTestSuite, ?bool $doNotFailOnIncomplete, ?bool $doNotFailOnNotice, ?bool $doNotFailOnRisky, ?bool $doNotFailOnSkipped, ?bool $doNotFailOnWarning, ?bool $stopOnDefect, ?bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnNotice, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $filter, ?string $excludeFilter, ?string $generateBaseline, ?string $useBaseline, bool $ignoreBaseline, bool $generateConfiguration, bool $migrateConfiguration, ?array $groups, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?string $otrLogfile, ?bool $includeGitInformation, bool $listGroups, bool $listSuites, bool $listTestFiles, bool $listTests, ?string $listTestsXml, ?bool $noCoverage, ?bool $noExtensions, ?bool $noOutput, ?bool $noProgress, ?bool $noResults, ?bool $noLogging, ?bool $processIsolation, ?int $randomOrderSeed, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?string $teamcityLogfile, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?array $testSuffixes, ?string $testSuite, ?string $excludeTestSuite, bool $useDefaultConfiguration, ?bool $displayDetailsOnAllIssues, ?bool $displayDetailsOnIncompleteTests, ?bool $displayDetailsOnSkippedTests, ?bool $displayDetailsOnTestsThatTriggerDeprecations, ?bool $displayDetailsOnPhpunitDeprecations, ?bool $displayDetailsOnPhpunitNotices, ?bool $displayDetailsOnTestsThatTriggerErrors, ?bool $displayDetailsOnTestsThatTriggerNotices, ?bool $displayDetailsOnTestsThatTriggerWarnings, bool $version, ?array $coverageFilter, ?string $logEventsText, ?string $logEventsVerboseText, ?bool $printerTeamCity, ?bool $testdoxPrinter, ?bool $testdoxPrinterSummary, bool $debug, bool $withTelemetry, ?array $extensions, int $repeat) + public function __construct(array $arguments, ?bool $all, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticProperties, ?bool $beStrictAboutChangesToGlobalState, ?string $bootstrap, ?string $cacheDirectory, ?bool $cacheResult, bool $checkPhpConfiguration, bool $checkVersion, ?string $colors, null|int|string $columns, ?string $configurationFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $pathCoverage, bool $warmCoverageCache, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?bool $failOnAllIssues, ?bool $failOnDeprecation, ?bool $failOnPhpunitDeprecation, ?bool $failOnPhpunitNotice, ?bool $failOnPhpunitWarning, ?bool $failOnEmptyTestSuite, ?bool $failOnIncomplete, ?bool $failOnNotice, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?bool $doNotFailOnDeprecation, ?bool $doNotFailOnPhpunitDeprecation, ?bool $doNotFailOnPhpunitNotice, ?bool $doNotFailOnPhpunitWarning, ?bool $doNotFailOnEmptyTestSuite, ?bool $doNotFailOnIncomplete, ?bool $doNotFailOnNotice, ?bool $doNotFailOnRisky, ?bool $doNotFailOnSkipped, ?bool $doNotFailOnWarning, ?bool $stopOnDefect, ?bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnNotice, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $filter, ?string $excludeFilter, ?string $generateBaseline, ?string $useBaseline, bool $ignoreBaseline, bool $generateConfiguration, bool $migrateConfiguration, ?array $groups, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?string $otrLogfile, ?bool $includeGitInformation, bool $listGroups, bool $listSuites, bool $listTestFiles, bool $listTests, ?string $listTestsXml, ?bool $noCoverage, ?bool $noExtensions, ?bool $noOutput, ?bool $noProgress, ?bool $noResults, ?bool $noLogging, ?bool $processIsolation, ?int $randomOrderSeed, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?string $teamcityLogfile, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?array $testSuffixes, ?string $testSuite, ?string $excludeTestSuite, bool $useDefaultConfiguration, ?bool $displayDetailsOnAllIssues, ?bool $displayDetailsOnIncompleteTests, ?bool $displayDetailsOnSkippedTests, ?bool $displayDetailsOnTestsThatTriggerDeprecations, ?bool $displayDetailsOnPhpunitDeprecations, ?bool $displayDetailsOnPhpunitNotices, ?bool $displayDetailsOnTestsThatTriggerErrors, ?bool $displayDetailsOnTestsThatTriggerNotices, ?bool $displayDetailsOnTestsThatTriggerWarnings, bool $version, ?array $coverageFilter, ?string $logEventsText, ?string $logEventsVerboseText, ?bool $printerTeamCity, ?bool $testdoxPrinter, ?bool $testdoxPrinterSummary, bool $debug, bool $withTelemetry, ?array $extensions, int $repeatTimes) { $this->arguments = $arguments; $this->all = $all; @@ -324,7 +324,7 @@ public function __construct(array $arguments, ?bool $all, ?string $atLeastVersio $this->debug = $debug; $this->withTelemetry = $withTelemetry; $this->extensions = $extensions; - $this->repeat = $repeat; + $this->repeatTimes = $repeatTimes; } /** @@ -2584,9 +2584,9 @@ public function debug(): bool /** * @return positive-int */ - public function repeat(): int + public function repeatTimes(): int { - return $this->repeat; + return $this->repeatTimes; } public function withTelemetry(): bool diff --git a/src/TextUI/Configuration/Configuration.php b/src/TextUI/Configuration/Configuration.php index 565525e81c..268de53fb1 100644 --- a/src/TextUI/Configuration/Configuration.php +++ b/src/TextUI/Configuration/Configuration.php @@ -198,7 +198,7 @@ /** * @var positive-int */ - private int $repeat; + private int $repeatTimes; private bool $withTelemetry; /** @@ -218,10 +218,10 @@ * @param list $excludeGroups * @param non-empty-list $testSuffixes * @param null|non-empty-string $generateBaseline - * @param positive-int $repeat + * @param positive-int $repeatTimes * @param non-negative-int $shortenArraysForExportThreshold */ - public function __construct(array $cliArguments, ?string $configurationFile, ?string $bootstrap, array $bootstrapForTestSuite, bool $cacheResult, ?string $cacheDirectory, ?string $coverageCacheDirectory, Source $source, string $testResultCacheFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4j, int $coverageCrap4jThreshold, ?string $coverageHtml, int $coverageHtmlLowUpperBound, int $coverageHtmlHighLowerBound, string $coverageHtmlColorSuccessLow, string $coverageHtmlColorSuccessMedium, string $coverageHtmlColorSuccessHigh, string $coverageHtmlColorWarning, string $coverageHtmlColorDanger, ?string $coverageHtmlCustomCssFile, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, bool $coverageTextShowUncoveredFiles, bool $coverageTextShowOnlySummary, ?string $coverageXml, bool $pathCoverage, bool $ignoreDeprecatedCodeUnitsFromCodeCoverage, bool $disableCodeCoverageIgnore, bool $failOnAllIssues, bool $failOnDeprecation, bool $failOnPhpunitDeprecation, bool $failOnPhpunitNotice, bool $failOnPhpunitWarning, bool $failOnEmptyTestSuite, bool $failOnIncomplete, bool $failOnNotice, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $doNotFailOnDeprecation, bool $doNotFailOnPhpunitDeprecation, bool $doNotFailOnPhpunitNotice, bool $doNotFailOnPhpunitWarning, bool $doNotFailOnEmptyTestSuite, bool $doNotFailOnIncomplete, bool $doNotFailOnNotice, bool $doNotFailOnRisky, bool $doNotFailOnSkipped, bool $doNotFailOnWarning, bool $stopOnDefect, bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, bool $stopOnError, bool $stopOnFailure, bool $stopOnIncomplete, bool $stopOnNotice, bool $stopOnRisky, bool $stopOnSkipped, bool $stopOnWarning, bool $outputToStandardErrorStream, int $columns, bool $noExtensions, ?string $pharExtensionDirectory, array $extensionBootstrappers, bool $backupGlobals, bool $backupStaticProperties, bool $beStrictAboutChangesToGlobalState, bool $colors, bool $processIsolation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, bool $reportUselessTests, bool $strictCoverage, bool $disallowTestOutput, bool $displayDetailsOnAllIssues, bool $displayDetailsOnIncompleteTests, bool $displayDetailsOnSkippedTests, bool $displayDetailsOnTestsThatTriggerDeprecations, bool $displayDetailsOnPhpunitDeprecations, bool $displayDetailsOnPhpunitNotices, bool $displayDetailsOnTestsThatTriggerErrors, bool $displayDetailsOnTestsThatTriggerNotices, bool $displayDetailsOnTestsThatTriggerWarnings, bool $reverseDefectList, bool $requireCoverageMetadata, bool $noProgress, bool $noResults, bool $noOutput, int $executionOrder, int $executionOrderDefects, bool $resolveDependencies, ?string $logfileTeamcity, ?string $logfileJunit, ?string $logfileOtr, bool $includeGitInformationInOtrLogfile, ?string $logfileTestdoxHtml, ?string $logfileTestdoxText, ?string $logEventsText, ?string $logEventsVerboseText, bool $teamCityOutput, bool $testDoxOutput, bool $testDoxOutputSummary, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, ?string $filter, ?string $excludeFilter, array $groups, array $excludeGroups, int $randomOrderSeed, bool $includeUncoveredFiles, TestSuiteCollection $testSuite, string $includeTestSuite, string $excludeTestSuite, ?string $defaultTestSuite, bool $ignoreTestSelectionInXmlConfiguration, array $testSuffixes, Php $php, bool $controlGarbageCollector, int $numberOfTestsBeforeGarbageCollection, ?string $generateBaseline, bool $debug, int $repeat, bool $withTelemetry, int $shortenArraysForExportThreshold) + public function __construct(array $cliArguments, ?string $configurationFile, ?string $bootstrap, array $bootstrapForTestSuite, bool $cacheResult, ?string $cacheDirectory, ?string $coverageCacheDirectory, Source $source, string $testResultCacheFile, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4j, int $coverageCrap4jThreshold, ?string $coverageHtml, int $coverageHtmlLowUpperBound, int $coverageHtmlHighLowerBound, string $coverageHtmlColorSuccessLow, string $coverageHtmlColorSuccessMedium, string $coverageHtmlColorSuccessHigh, string $coverageHtmlColorWarning, string $coverageHtmlColorDanger, ?string $coverageHtmlCustomCssFile, ?string $coverageOpenClover, ?string $coveragePhp, ?string $coverageText, bool $coverageTextShowUncoveredFiles, bool $coverageTextShowOnlySummary, ?string $coverageXml, bool $pathCoverage, bool $ignoreDeprecatedCodeUnitsFromCodeCoverage, bool $disableCodeCoverageIgnore, bool $failOnAllIssues, bool $failOnDeprecation, bool $failOnPhpunitDeprecation, bool $failOnPhpunitNotice, bool $failOnPhpunitWarning, bool $failOnEmptyTestSuite, bool $failOnIncomplete, bool $failOnNotice, bool $failOnRisky, bool $failOnSkipped, bool $failOnWarning, bool $doNotFailOnDeprecation, bool $doNotFailOnPhpunitDeprecation, bool $doNotFailOnPhpunitNotice, bool $doNotFailOnPhpunitWarning, bool $doNotFailOnEmptyTestSuite, bool $doNotFailOnIncomplete, bool $doNotFailOnNotice, bool $doNotFailOnRisky, bool $doNotFailOnSkipped, bool $doNotFailOnWarning, bool $stopOnDefect, bool $stopOnDeprecation, ?string $specificDeprecationToStopOn, bool $stopOnError, bool $stopOnFailure, bool $stopOnIncomplete, bool $stopOnNotice, bool $stopOnRisky, bool $stopOnSkipped, bool $stopOnWarning, bool $outputToStandardErrorStream, int $columns, bool $noExtensions, ?string $pharExtensionDirectory, array $extensionBootstrappers, bool $backupGlobals, bool $backupStaticProperties, bool $beStrictAboutChangesToGlobalState, bool $colors, bool $processIsolation, bool $enforceTimeLimit, int $defaultTimeLimit, int $timeoutForSmallTests, int $timeoutForMediumTests, int $timeoutForLargeTests, bool $reportUselessTests, bool $strictCoverage, bool $disallowTestOutput, bool $displayDetailsOnAllIssues, bool $displayDetailsOnIncompleteTests, bool $displayDetailsOnSkippedTests, bool $displayDetailsOnTestsThatTriggerDeprecations, bool $displayDetailsOnPhpunitDeprecations, bool $displayDetailsOnPhpunitNotices, bool $displayDetailsOnTestsThatTriggerErrors, bool $displayDetailsOnTestsThatTriggerNotices, bool $displayDetailsOnTestsThatTriggerWarnings, bool $reverseDefectList, bool $requireCoverageMetadata, bool $noProgress, bool $noResults, bool $noOutput, int $executionOrder, int $executionOrderDefects, bool $resolveDependencies, ?string $logfileTeamcity, ?string $logfileJunit, ?string $logfileOtr, bool $includeGitInformationInOtrLogfile, ?string $logfileTestdoxHtml, ?string $logfileTestdoxText, ?string $logEventsText, ?string $logEventsVerboseText, bool $teamCityOutput, bool $testDoxOutput, bool $testDoxOutputSummary, ?array $testsCovering, ?array $testsUsing, ?array $testsRequiringPhpExtension, ?string $filter, ?string $excludeFilter, array $groups, array $excludeGroups, int $randomOrderSeed, bool $includeUncoveredFiles, TestSuiteCollection $testSuite, string $includeTestSuite, string $excludeTestSuite, ?string $defaultTestSuite, bool $ignoreTestSelectionInXmlConfiguration, array $testSuffixes, Php $php, bool $controlGarbageCollector, int $numberOfTestsBeforeGarbageCollection, ?string $generateBaseline, bool $debug, int $repeatTimes, bool $withTelemetry, int $shortenArraysForExportThreshold) { $this->cliArguments = $cliArguments; $this->configurationFile = $configurationFile; @@ -351,7 +351,7 @@ public function __construct(array $cliArguments, ?string $configurationFile, ?st $this->numberOfTestsBeforeGarbageCollection = $numberOfTestsBeforeGarbageCollection; $this->generateBaseline = $generateBaseline; $this->debug = $debug; - $this->repeat = $repeat; + $this->repeatTimes = $repeatTimes; $this->withTelemetry = $withTelemetry; $this->shortenArraysForExportThreshold = $shortenArraysForExportThreshold; } @@ -1524,9 +1524,9 @@ public function debug(): bool /** * @return positive-int */ - public function repeat(): int + public function repeatTimes(): int { - return $this->repeat; + return $this->repeatTimes; } public function withTelemetry(): bool diff --git a/src/TextUI/Configuration/Merger.php b/src/TextUI/Configuration/Merger.php index 6ff3a70f34..b6b7ec7492 100644 --- a/src/TextUI/Configuration/Merger.php +++ b/src/TextUI/Configuration/Merger.php @@ -1064,7 +1064,7 @@ public function merge(CliConfiguration $cliConfiguration, XmlConfiguration $xmlC $xmlConfiguration->phpunit()->numberOfTestsBeforeGarbageCollection(), $generateBaseline, $cliConfiguration->debug(), - $cliConfiguration->repeat(), + $cliConfiguration->repeatTimes(), $cliConfiguration->withTelemetry(), $xmlConfiguration->phpunit()->shortenArraysForExportThreshold(), ); diff --git a/src/TextUI/Configuration/TestSuiteBuilder.php b/src/TextUI/Configuration/TestSuiteBuilder.php index 80eb56391c..fef6c4f216 100644 --- a/src/TextUI/Configuration/TestSuiteBuilder.php +++ b/src/TextUI/Configuration/TestSuiteBuilder.php @@ -58,13 +58,13 @@ public function build(Configuration $configuration): TestSuite $testSuite = $this->testSuiteFromPath( $arguments[0], $configuration->testSuffixes(), - $configuration->repeat(), + $configuration->repeatTimes(), ); } else { $testSuite = $this->testSuiteFromPathList( $arguments, $configuration->testSuffixes(), - $configuration->repeat(), + $configuration->repeatTimes(), ); } } @@ -90,18 +90,18 @@ public function build(Configuration $configuration): TestSuite /** * @param non-empty-string $path * @param list $suffixes - * @param positive-int $repeat + * @param positive-int $repeatTimes * * @throws \PHPUnit\Framework\Exception */ - private function testSuiteFromPath(string $path, array $suffixes, int $repeat, ?TestSuite $suite = null): TestSuite + private function testSuiteFromPath(string $path, array $suffixes, int $repeatTimes, ?TestSuite $suite = null): TestSuite { if (str_ends_with($path, '.phpt') && is_file($path)) { if ($suite === null) { $suite = TestSuite::empty($path); } - $suite->addTestFile($path, [], $repeat); + $suite->addTestFile($path, [], $repeatTimes); return $suite; } @@ -113,7 +113,7 @@ private function testSuiteFromPath(string $path, array $suffixes, int $repeat, ? $suite = TestSuite::empty('CLI Arguments'); } - $suite->addTestFiles($files, $repeat); + $suite->addTestFiles($files, $repeatTimes); return $suite; } @@ -127,10 +127,10 @@ private function testSuiteFromPath(string $path, array $suffixes, int $repeat, ? } if ($suite === null) { - return TestSuite::fromClassReflector($testClass, repeat: $repeat); + return TestSuite::fromClassReflector($testClass, [], $repeatTimes); } - $suite->addTestSuite($testClass, repeat: $repeat); + $suite->addTestSuite($testClass, [], $repeatTimes); return $suite; } @@ -138,16 +138,16 @@ private function testSuiteFromPath(string $path, array $suffixes, int $repeat, ? /** * @param list $paths * @param list $suffixes - * @param positive-int $repeat + * @param positive-int $repeatTimes * * @throws \PHPUnit\Framework\Exception */ - private function testSuiteFromPathList(array $paths, array $suffixes, int $repeat): TestSuite + private function testSuiteFromPathList(array $paths, array $suffixes, int $repeatTimes): TestSuite { $suite = TestSuite::empty('CLI Arguments'); foreach ($paths as $path) { - $this->testSuiteFromPath($path, $suffixes, $repeat, $suite); + $this->testSuiteFromPath($path, $suffixes, $repeatTimes, $suite); } return $suite; From ab52255d1ff27d20f8ac8e12e6d881e60cfba225 Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Fri, 24 Oct 2025 19:40:39 +0200 Subject: [PATCH 12/21] fix tests --- tests/end-to-end/repeat/dependent-test.phpt | 2 +- tests/end-to-end/repeat/error-skips-next-repetitions.phpt | 6 +++--- tests/end-to-end/repeat/phpt-failure-and-success.phpt | 4 ++-- tests/end-to-end/repeat/phpt-failure.phpt | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/end-to-end/repeat/dependent-test.phpt b/tests/end-to-end/repeat/dependent-test.phpt index ae0a5a73f1..9f8b2b120c 100644 --- a/tests/end-to-end/repeat/dependent-test.phpt +++ b/tests/end-to-end/repeat/dependent-test.phpt @@ -25,7 +25,7 @@ There was 1 failure: 1) RepeatDependentTest::test2 Failed asserting that false is true. -%s/tests/end-to-end/repeat/_files/RepeatDependentTest.php:21 +%s/tests/end-to-end/repeat/_files/RepeatDependentTest.php:%d FAILURES! Tests: 8, Assertions: 5, Failures: 1, Skipped: 3. diff --git a/tests/end-to-end/repeat/error-skips-next-repetitions.phpt b/tests/end-to-end/repeat/error-skips-next-repetitions.phpt index 9d5c096381..8d236c54aa 100644 --- a/tests/end-to-end/repeat/error-skips-next-repetitions.phpt +++ b/tests/end-to-end/repeat/error-skips-next-repetitions.phpt @@ -25,17 +25,17 @@ There were 3 failures: 1) RepeatWithErrorsTest::test1 Failed asserting that true is false. -%s/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:9 +%s/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:%d 2) RepeatWithErrorsTest::test2 Failed asserting that true is false. -%s/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:17 +%s/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:%d 3) RepeatWithErrorsTest::test3 Failed asserting that true is false. -%s/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:28 +%s/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:%d FAILURES! Tests: 12, Assertions: 9, Failures: 3, Skipped: 3. diff --git a/tests/end-to-end/repeat/phpt-failure-and-success.phpt b/tests/end-to-end/repeat/phpt-failure-and-success.phpt index 1431412512..09e91a43ec 100644 --- a/tests/end-to-end/repeat/phpt-failure-and-success.phpt +++ b/tests/end-to-end/repeat/phpt-failure-and-success.phpt @@ -22,7 +22,7 @@ Time: %s, Memory: %s MB There was 1 failure: -1) /home/niko/works/github.com/sebastianbergmann/phpunit/tests/end-to-end/repeat/_files/phpt/failure.phpt +1) %s/phpunit/tests/end-to-end/repeat/_files/phpt/failure.phpt Failed asserting that string matches format description. --- Expected +++ Actual @@ -30,7 +30,7 @@ Failed asserting that string matches format description. -ko +ok -/home/niko/works/github.com/sebastianbergmann/phpunit/tests/end-to-end/repeat/_files/phpt/failure.phpt:8 +%s/phpunit/tests/end-to-end/repeat/_files/phpt/failure.phpt:%d FAILURES! Tests: 4, Assertions: 3, Failures: 1, Skipped: 1. diff --git a/tests/end-to-end/repeat/phpt-failure.phpt b/tests/end-to-end/repeat/phpt-failure.phpt index 9fa046c299..d185d82d1f 100644 --- a/tests/end-to-end/repeat/phpt-failure.phpt +++ b/tests/end-to-end/repeat/phpt-failure.phpt @@ -30,7 +30,7 @@ Failed asserting that string matches format description. -ko +ok -%s/tests/end-to-end/repeat/_files/phpt/failure.phpt:8 +%s/tests/end-to-end/repeat/_files/phpt/failure.phpt:%d FAILURES! Tests: 2, Assertions: 1, Failures: 1, Skipped: 1. From 1773e584e3372fee41f726794462e8f53f3dd36a Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Fri, 24 Oct 2025 19:40:55 +0200 Subject: [PATCH 13/21] add --repeat to the cli help --- src/TextUI/Help.php | 1 + tests/end-to-end/_files/output-cli-help-color.txt | 1 + tests/end-to-end/_files/output-cli-usage.txt | 1 + 3 files changed, 3 insertions(+) diff --git a/src/TextUI/Help.php b/src/TextUI/Help.php index 8a4f802abd..ba92315a46 100644 --- a/src/TextUI/Help.php +++ b/src/TextUI/Help.php @@ -276,6 +276,7 @@ private function elements(): array ['spacer' => ''], ['arg' => '--debug', 'desc' => 'Replace default progress and result output with debugging information'], + ['arg' => '--repeat ', 'desc' => 'Runs the test(s) repeatedly'], ['arg' => '--with-telemetry', 'desc' => 'Include telemetry information in debugging information output'], ], diff --git a/tests/end-to-end/_files/output-cli-help-color.txt b/tests/end-to-end/_files/output-cli-help-color.txt index 4e68ba81c2..429335baac 100644 --- a/tests/end-to-end/_files/output-cli-help-color.txt +++ b/tests/end-to-end/_files/output-cli-help-color.txt @@ -176,6 +176,7 @@ --debug  Replace default progress and result output with debugging information + --repeat   Runs the test(s) repeatedly --with-telemetry  Include telemetry information in debugging information output diff --git a/tests/end-to-end/_files/output-cli-usage.txt b/tests/end-to-end/_files/output-cli-usage.txt index 932879ec51..82a67fa5c6 100644 --- a/tests/end-to-end/_files/output-cli-usage.txt +++ b/tests/end-to-end/_files/output-cli-usage.txt @@ -117,6 +117,7 @@ Reporting: --testdox-summary Repeat TestDox output for tests with errors, failures, or issues --debug Replace default progress and result output with debugging information + --repeat Runs the test(s) repeatedly --with-telemetry Include telemetry information in debugging information output Logging: From 32e3acb011153316982f183cd9004d791218e501 Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Mon, 27 Oct 2025 15:48:51 +0100 Subject: [PATCH 14/21] handle when no cli arguments --- composer.lock | 12 ++++----- src/Framework/RepeatTestSuite.php | 27 +++++++++++-------- src/Runner/Filter/NameFilterIterator.php | 9 +++++-- src/TextUI/Configuration/TestSuiteBuilder.php | 1 + .../Configuration/Xml/TestSuiteMapper.php | 9 ++++--- tests/end-to-end/repeat/_files/phpunit.xml | 11 ++++++++ .../repeat/no-cli-argument-filtered.phpt | 26 ++++++++++++++++++ tests/end-to-end/repeat/no-cli-argument.phpt | 24 +++++++++++++++++ 8 files changed, 96 insertions(+), 23 deletions(-) create mode 100644 tests/end-to-end/repeat/_files/phpunit.xml create mode 100644 tests/end-to-end/repeat/no-cli-argument-filtered.phpt create mode 100644 tests/end-to-end/repeat/no-cli-argument.phpt diff --git a/composer.lock b/composer.lock index 0c2de24389..047822b72c 100644 --- a/composer.lock +++ b/composer.lock @@ -68,16 +68,16 @@ }, { "name": "nikic/php-parser", - "version": "v5.6.1", + "version": "v5.6.2", "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "f103601b29efebd7ff4a1ca7b3eeea9e3336a2a2" + "reference": "3a454ca033b9e06b63282ce19562e892747449bb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/f103601b29efebd7ff4a1ca7b3eeea9e3336a2a2", - "reference": "f103601b29efebd7ff4a1ca7b3eeea9e3336a2a2", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/3a454ca033b9e06b63282ce19562e892747449bb", + "reference": "3a454ca033b9e06b63282ce19562e892747449bb", "shasum": "" }, "require": { @@ -120,9 +120,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v5.6.1" + "source": "https://github.com/nikic/PHP-Parser/tree/v5.6.2" }, - "time": "2025-08-13T20:13:15+00:00" + "time": "2025-10-21T19:32:17+00:00" }, { "name": "phar-io/manifest", diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php index 5f1d48f60a..1712e7fb5b 100644 --- a/src/Framework/RepeatTestSuite.php +++ b/src/Framework/RepeatTestSuite.php @@ -10,6 +10,7 @@ namespace PHPUnit\Framework; use function count; +use LogicException; use PHPUnit\Event; use PHPUnit\Event\Facade as EventFacade; use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; @@ -35,7 +36,7 @@ public function __construct(PhptTestCase|TestCase $test, int $times) $tests = []; for ($i = 0; $i < $times; $i++) { - $tests[] = clone $test; + $tests[] = $test; } $this->tests = $tests; @@ -70,9 +71,13 @@ public function requires(): array return $this->tests[0]->requires(); } - public function nameWithDataSet(): string + public function name(): string { - return $this->tests[0]->nameWithDataSet(); + if ($this->isPhptTestCase()) { + throw new LogicException('Cannot call RepeatTestSuite::nameWithDataSet() on a PhptTestCase.'); + } + + return $this->tests[0]::class . '::' . $this->tests[0]->nameWithDataSet(); } public function valueObjectForEvents(): Event\Code\Phpt|Event\Code\TestMethod @@ -80,6 +85,14 @@ public function valueObjectForEvents(): Event\Code\Phpt|Event\Code\TestMethod return $this->tests[0]->valueObjectForEvents(); } + /** + * @phpstan-assert-if-true non-empty-list $this->tests + */ + public function isPhptTestCase(): bool + { + return $this->tests[0] instanceof PhptTestCase; + } + private function runTestCase(): void { $defectOccurred = false; @@ -122,12 +135,4 @@ private function runPhptTestCase(): void } } } - - /** - * @phpstan-assert-if-true non-empty-list $this->tests - */ - private function isPhptTestCase(): bool - { - return $this->tests[0] instanceof PhptTestCase; - } } diff --git a/src/Runner/Filter/NameFilterIterator.php b/src/Runner/Filter/NameFilterIterator.php index a2fc69b3d3..ce6696b4db 100644 --- a/src/Runner/Filter/NameFilterIterator.php +++ b/src/Runner/Filter/NameFilterIterator.php @@ -13,6 +13,7 @@ use function preg_match; use function sprintf; use function substr; +use PHPUnit\Framework\RepeatTestSuite; use PHPUnit\Framework\Test; use PHPUnit\Framework\TestSuite; use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; @@ -56,11 +57,15 @@ public function accept(): bool return true; } - if ($test instanceof PhptTestCase) { + if ($test instanceof PhptTestCase || ($test instanceof RepeatTestSuite && $test->isPhptTestCase())) { return false; } - $name = $test::class . '::' . $test->nameWithDataSet(); + if ($test instanceof RepeatTestSuite) { + $name = $test->name(); + } else { + $name = $test::class . '::' . $test->nameWithDataSet(); + } $accepted = @preg_match($this->regularExpression, $name, $matches) === 1; diff --git a/src/TextUI/Configuration/TestSuiteBuilder.php b/src/TextUI/Configuration/TestSuiteBuilder.php index fef6c4f216..a39abf933e 100644 --- a/src/TextUI/Configuration/TestSuiteBuilder.php +++ b/src/TextUI/Configuration/TestSuiteBuilder.php @@ -79,6 +79,7 @@ public function build(Configuration $configuration): TestSuite $configuration->testSuite(), $configuration->ignoreTestSelectionInXmlConfiguration() ? [] : $configuration->includeTestSuites(), $configuration->ignoreTestSelectionInXmlConfiguration() ? [] : $configuration->excludeTestSuites(), + $configuration->repeatTimes(), ); } diff --git a/src/TextUI/Configuration/Xml/TestSuiteMapper.php b/src/TextUI/Configuration/Xml/TestSuiteMapper.php index 66bd611c1c..b7afcd89d7 100644 --- a/src/TextUI/Configuration/Xml/TestSuiteMapper.php +++ b/src/TextUI/Configuration/Xml/TestSuiteMapper.php @@ -36,12 +36,13 @@ * @param non-empty-string $xmlConfigurationFile * @param list $includeTestSuites * @param list $excludeTestSuites + * @param positive-int $repeatTimes * * @throws RuntimeException * @throws TestDirectoryNotFoundException * @throws TestFileNotFoundException */ - public function map(string $xmlConfigurationFile, TestSuiteCollection $configuredTestSuites, array $includeTestSuites, array $excludeTestSuites): TestSuiteObject + public function map(string $xmlConfigurationFile, TestSuiteCollection $configuredTestSuites, array $includeTestSuites, array $excludeTestSuites, int $repeatTimes = 1): TestSuiteObject { try { $result = TestSuiteObject::empty($xmlConfigurationFile); @@ -101,7 +102,7 @@ public function map(string $xmlConfigurationFile, TestSuiteCollection $configure $processed[$file] = $testSuiteName; $empty = false; - $testSuite->addTestFile($file, $groups); + $testSuite->addTestFile($file, $groups, $repeatTimes); } } @@ -130,11 +131,11 @@ public function map(string $xmlConfigurationFile, TestSuiteCollection $configure $processed[$file->path()] = $testSuiteName; $empty = false; - $testSuite->addTestFile($file->path(), $file->groups()); + $testSuite->addTestFile($file->path(), $file->groups(), $repeatTimes); } if (!$empty) { - $result->addTest($testSuite); + $result->addTest($testSuite, [], $repeatTimes); } } diff --git a/tests/end-to-end/repeat/_files/phpunit.xml b/tests/end-to-end/repeat/_files/phpunit.xml new file mode 100644 index 0000000000..3d1bc9c153 --- /dev/null +++ b/tests/end-to-end/repeat/_files/phpunit.xml @@ -0,0 +1,11 @@ + + + + + RepeatTest.php + + + diff --git a/tests/end-to-end/repeat/no-cli-argument-filtered.phpt b/tests/end-to-end/repeat/no-cli-argument-filtered.phpt new file mode 100644 index 0000000000..0a2e0ad3ba --- /dev/null +++ b/tests/end-to-end/repeat/no-cli-argument-filtered.phpt @@ -0,0 +1,26 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s +Configuration: %s/tests/end-to-end/repeat/_files/phpunit.xml + +.. 2 / 2 (100%) + +Time: %s, Memory: %s MB + +OK (2 tests, 2 assertions) diff --git a/tests/end-to-end/repeat/no-cli-argument.phpt b/tests/end-to-end/repeat/no-cli-argument.phpt new file mode 100644 index 0000000000..5679d3cd5a --- /dev/null +++ b/tests/end-to-end/repeat/no-cli-argument.phpt @@ -0,0 +1,24 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s +Configuration: %s/tests/end-to-end/repeat/_files/phpunit.xml + +.... 4 / 4 (100%) + +Time: %s, Memory: %s MB + +OK (4 tests, 4 assertions) From a34f96615be7e49706454fe483416d278846fe5b Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Mon, 27 Oct 2025 16:57:02 +0100 Subject: [PATCH 15/21] add test which uses #[RunTestsInSeparateProcesses] --- src/Framework/RepeatTestSuite.php | 2 +- .../repeat/_files/RepeatInIsolationTest.php | 47 +++++++++++++++++++ tests/end-to-end/repeat/_files/RepeatTest.php | 26 +++++++++- tests/end-to-end/repeat/filter.phpt | 2 +- tests/end-to-end/repeat/isolation.phpt | 23 +++++++++ .../repeat/no-cli-argument-filtered.phpt | 2 +- tests/end-to-end/repeat/no-cli-argument.phpt | 2 +- tests/end-to-end/repeat/simple-repeat.phpt | 2 +- 8 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 tests/end-to-end/repeat/_files/RepeatInIsolationTest.php create mode 100644 tests/end-to-end/repeat/isolation.phpt diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php index 1712e7fb5b..ca03f8712f 100644 --- a/src/Framework/RepeatTestSuite.php +++ b/src/Framework/RepeatTestSuite.php @@ -36,7 +36,7 @@ public function __construct(PhptTestCase|TestCase $test, int $times) $tests = []; for ($i = 0; $i < $times; $i++) { - $tests[] = $test; + $tests[] = clone $test; } $this->tests = $tests; diff --git a/tests/end-to-end/repeat/_files/RepeatInIsolationTest.php b/tests/end-to-end/repeat/_files/RepeatInIsolationTest.php new file mode 100644 index 0000000000..f5c6328dc9 --- /dev/null +++ b/tests/end-to-end/repeat/_files/RepeatInIsolationTest.php @@ -0,0 +1,47 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +use PHPUnit\Framework\Attributes\RunTestsInSeparateProcesses; +use PHPUnit\Framework\TestCase; + +#[RunTestsInSeparateProcesses] +final class RepeatInIsolationTest extends TestCase +{ + private Closure $notCloneableFixture; + private int $counter; + + public function __clone() + { + if (isset($this->notCloneableFixture)) { + throw new LogicException('Cannot clone RepeatInIsolationTest because it contains a non-cloneable fixture.'); + } + } + + protected function setUp(): void + { + $this->counter = 0; + $this->notCloneableFixture = static fn () => true; + } + + public function test1(): void + { + $this->assertTrue(($this->notCloneableFixture)()); + + $this->counter++; + $this->assertSame(1, $this->counter); + } + + public function test2(): void + { + $this->assertTrue(($this->notCloneableFixture)()); + + $this->counter++; + $this->assertSame(1, $this->counter); + } +} diff --git a/tests/end-to-end/repeat/_files/RepeatTest.php b/tests/end-to-end/repeat/_files/RepeatTest.php index 2963f8b51c..4d960902f1 100644 --- a/tests/end-to-end/repeat/_files/RepeatTest.php +++ b/tests/end-to-end/repeat/_files/RepeatTest.php @@ -11,13 +11,35 @@ final class RepeatTest extends TestCase { + private Closure $notCloneableFixture; + private int $counter; + + public function __clone() + { + if (isset($this->notCloneableFixture)) { + throw new LogicException('Cannot clone RepeatInIsolationTest because it contains a non-cloneable fixture.'); + } + } + + protected function setUp(): void + { + $this->counter = 0; + $this->notCloneableFixture = static fn () => true; + } + public function test1(): void { - $this->assertTrue(true); + $this->assertTrue(($this->notCloneableFixture)()); + + $this->counter++; + $this->assertSame(1, $this->counter); } public function test2(): void { - $this->assertTrue(true); + $this->assertTrue(($this->notCloneableFixture)()); + + $this->counter++; + $this->assertSame(1, $this->counter); } } diff --git a/tests/end-to-end/repeat/filter.phpt b/tests/end-to-end/repeat/filter.phpt index a4e76310d4..4286ed131e 100644 --- a/tests/end-to-end/repeat/filter.phpt +++ b/tests/end-to-end/repeat/filter.phpt @@ -22,4 +22,4 @@ Runtime: %s Time: %s, Memory: %s MB -OK (2 tests, 2 assertions) +OK (2 tests, 4 assertions) diff --git a/tests/end-to-end/repeat/isolation.phpt b/tests/end-to-end/repeat/isolation.phpt new file mode 100644 index 0000000000..174aa18f6b --- /dev/null +++ b/tests/end-to-end/repeat/isolation.phpt @@ -0,0 +1,23 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.... 4 / 4 (100%) + +Time: %s, Memory: %s MB + +OK (4 tests, 8 assertions) diff --git a/tests/end-to-end/repeat/no-cli-argument-filtered.phpt b/tests/end-to-end/repeat/no-cli-argument-filtered.phpt index 0a2e0ad3ba..6d06792e3e 100644 --- a/tests/end-to-end/repeat/no-cli-argument-filtered.phpt +++ b/tests/end-to-end/repeat/no-cli-argument-filtered.phpt @@ -23,4 +23,4 @@ Configuration: %s/tests/end-to-end/repeat/_files/phpunit.xml Time: %s, Memory: %s MB -OK (2 tests, 2 assertions) +OK (2 tests, 4 assertions) diff --git a/tests/end-to-end/repeat/no-cli-argument.phpt b/tests/end-to-end/repeat/no-cli-argument.phpt index 5679d3cd5a..7ece5490e4 100644 --- a/tests/end-to-end/repeat/no-cli-argument.phpt +++ b/tests/end-to-end/repeat/no-cli-argument.phpt @@ -21,4 +21,4 @@ Configuration: %s/tests/end-to-end/repeat/_files/phpunit.xml Time: %s, Memory: %s MB -OK (4 tests, 4 assertions) +OK (4 tests, 8 assertions) diff --git a/tests/end-to-end/repeat/simple-repeat.phpt b/tests/end-to-end/repeat/simple-repeat.phpt index 7ce4f596b1..fcf90d7571 100644 --- a/tests/end-to-end/repeat/simple-repeat.phpt +++ b/tests/end-to-end/repeat/simple-repeat.phpt @@ -20,4 +20,4 @@ Runtime: %s Time: %s, Memory: %s MB -OK (4 tests, 4 assertions) +OK (4 tests, 8 assertions) From 62ea6452ab4e107160600427751e1e9fe69cfa0f Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Tue, 28 Oct 2025 08:43:54 +0100 Subject: [PATCH 16/21] Trigger test Passed event only when last repetition passed --- src/Framework/RepeatTestSuite.php | 18 ++++---- src/Framework/TestBuilder.php | 4 +- src/Framework/TestCase.php | 22 ++++++++-- src/Runner/TestResult/PassedTests.php | 5 --- tests/end-to-end/repeat/simple-repeat.phpt | 48 ++++++++++++++++++---- 5 files changed, 70 insertions(+), 27 deletions(-) diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php index ca03f8712f..dc70b3ba60 100644 --- a/src/Framework/RepeatTestSuite.php +++ b/src/Framework/RepeatTestSuite.php @@ -14,7 +14,6 @@ use PHPUnit\Event; use PHPUnit\Event\Facade as EventFacade; use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; -use PHPUnit\TestRunner\TestResult\PassedTests; /** * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit @@ -33,10 +32,15 @@ */ public function __construct(PhptTestCase|TestCase $test, int $times) { - $tests = []; - - for ($i = 0; $i < $times; $i++) { - $tests[] = clone $test; + $tests = []; + $tests[] = $test; + + for ($i = 1; $i < $times; $i++) { + if ($test instanceof PhptTestCase) { + $tests[] = clone $test; + } else { + $tests[] = $test->newRepeatInstance(); + } } $this->tests = $tests; @@ -87,6 +91,8 @@ public function valueObjectForEvents(): Event\Code\Phpt|Event\Code\TestMethod /** * @phpstan-assert-if-true non-empty-list $this->tests + * + * @phpstan-assert-if-false non-empty-list $this->tests */ public function isPhptTestCase(): bool { @@ -108,8 +114,6 @@ private function runTestCase(): void if ($test->status()->isFailure() || $test->status()->isError()) { $defectOccurred = true; - - PassedTests::instance()->testMethodDidNotPass($test::class . '::' . $test->name()); } } } diff --git a/src/Framework/TestBuilder.php b/src/Framework/TestBuilder.php index ff5e9c3e74..ef5f3c5e4c 100644 --- a/src/Framework/TestBuilder.php +++ b/src/Framework/TestBuilder.php @@ -69,7 +69,7 @@ public function build(ReflectionClass $theClass, string $methodName, array $grou ); } - $test = new $className($methodName); + $test = new $className($methodName, $repeatTimes); $this->configureTestCase( $test, @@ -101,7 +101,7 @@ private function buildDataProviderTestSuite(string $methodName, string $classNam ); foreach ($data as $_dataName => $_data) { - $_test = new $className($methodName); + $_test = new $className($methodName, $repeatTimes); $_test->setData($_dataName, $_data->value()); diff --git a/src/Framework/TestCase.php b/src/Framework/TestCase.php index fdd128bc89..5346526752 100644 --- a/src/Framework/TestCase.php +++ b/src/Framework/TestCase.php @@ -226,16 +226,19 @@ abstract class TestCase extends Assert implements Reorderable, SelfDescribing, T */ private mixed $errorLogCapture = false; private false|string $previousErrorLogTarget = false; + private readonly int $repeatTimes; + private int $currentRepeat = 1; /** * @param non-empty-string $name * * @internal This method is not covered by the backward compatibility promise for PHPUnit */ - final public function __construct(string $name) + final public function __construct(string $name, int $repeatTimes = 1) { - $this->methodName = $name; - $this->status = TestStatus::unknown(); + $this->methodName = $name; + $this->status = TestStatus::unknown(); + $this->repeatTimes = $repeatTimes; if (is_callable($this->sortId(), true)) { $this->providedTests = [new ExecutionOrderDependency($this->sortId())]; @@ -300,6 +303,17 @@ protected function tearDown(): void { } + /** + * @internal This method is not covered by the backward compatibility promise for PHPUnit + */ + public function newRepeatInstance(): static + { + $clone = clone $this; + $clone->currentRepeat++; + + return $clone; + } + /** * Returns a string representation of the test case. * @@ -643,7 +657,7 @@ final public function runBare(): void } } - if (!isset($e) && !isset($_e)) { + if (!isset($e) && !isset($_e) && $this->currentRepeat === $this->repeatTimes) { $emitter->testPassed( $this->valueObjectForEvents(), ); diff --git a/src/Runner/TestResult/PassedTests.php b/src/Runner/TestResult/PassedTests.php index a79342f84d..2f611461d6 100644 --- a/src/Runner/TestResult/PassedTests.php +++ b/src/Runner/TestResult/PassedTests.php @@ -84,11 +84,6 @@ public function import(self $other): void ); } - public function testMethodDidNotPass(string $method): void - { - unset($this->passedTestMethods[$method]); - } - /** * @param class-string $className */ diff --git a/tests/end-to-end/repeat/simple-repeat.phpt b/tests/end-to-end/repeat/simple-repeat.phpt index fcf90d7571..ef17fee52b 100644 --- a/tests/end-to-end/repeat/simple-repeat.phpt +++ b/tests/end-to-end/repeat/simple-repeat.phpt @@ -4,6 +4,7 @@ Repeat option run($_SERVER['argv']); --EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -.... 4 / 4 (100%) - -Time: %s, Memory: %s MB - -OK (4 tests, 8 assertions) +PHPUnit Started (PHPUnit %s using PHP %s) +Test Runner Configured +Event Facade Sealed +Test Suite Loaded (4 tests) +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (4 tests) +Test Suite Started (RepeatTest, 4 tests) +Test Preparation Started (RepeatTest::test1) +Before Test Method Called (RepeatTest::setUp) +Before Test Method Finished: +- RepeatTest::setUp +Test Prepared (RepeatTest::test1) +Test Finished (RepeatTest::test1) +Test Preparation Started (RepeatTest::test1) +Before Test Method Called (RepeatTest::setUp) +Before Test Method Finished: +- RepeatTest::setUp +Test Prepared (RepeatTest::test1) +Test Passed (RepeatTest::test1) +Test Finished (RepeatTest::test1) +Test Preparation Started (RepeatTest::test2) +Before Test Method Called (RepeatTest::setUp) +Before Test Method Finished: +- RepeatTest::setUp +Test Prepared (RepeatTest::test2) +Test Finished (RepeatTest::test2) +Test Preparation Started (RepeatTest::test2) +Before Test Method Called (RepeatTest::setUp) +Before Test Method Finished: +- RepeatTest::setUp +Test Prepared (RepeatTest::test2) +Test Passed (RepeatTest::test2) +Test Finished (RepeatTest::test2) +Test Suite Finished (RepeatTest, 4 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) From 5cc4fae06745614d7a852961f5784147ee862b06 Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Wed, 29 Oct 2025 08:39:46 +0100 Subject: [PATCH 17/21] Create RepeatTestSuite in TestBuilder --- src/Framework/RepeatTestSuite.php | 44 +++++++--- src/Framework/TestBuilder.php | 81 +++++++++++-------- src/Framework/TestCase.php | 11 +-- src/Framework/TestSuite.php | 29 ++++--- .../Configuration/Xml/TestSuiteMapper.php | 2 +- 5 files changed, 103 insertions(+), 64 deletions(-) diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php index dc70b3ba60..8fb33c1cc8 100644 --- a/src/Framework/RepeatTestSuite.php +++ b/src/Framework/RepeatTestSuite.php @@ -9,6 +9,7 @@ */ namespace PHPUnit\Framework; +use PHPUnit\Metadata\Api\ProvidedData; use function count; use LogicException; use PHPUnit\Event; @@ -28,21 +29,10 @@ private array $tests; /** - * @param positive-int $times + * @param non-empty-list|non-empty-list $tests */ - public function __construct(PhptTestCase|TestCase $test, int $times) + public function __construct(array $tests) { - $tests = []; - $tests[] = $test; - - for ($i = 1; $i < $times; $i++) { - if ($test instanceof PhptTestCase) { - $tests[] = clone $test; - } else { - $tests[] = $test->newRepeatInstance(); - } - } - $this->tests = $tests; } @@ -89,6 +79,34 @@ public function valueObjectForEvents(): Event\Code\Phpt|Event\Code\TestMethod return $this->tests[0]->valueObjectForEvents(); } + /** + * @param array $data + */ + public function setData(int|string $dataName, array $data): void + { + if ($this->isPhptTestCase()) { + throw new LogicException('Cannot call RepeatTestSuite::setData() on a PhptTestCase.'); + } + + foreach ($this->tests as $test) { + $test->setData($dataName, $data); + } + } + + /** + * @param list $dependencies + */ + public function setDependencies(array $dependencies): void + { + if ($this->isPhptTestCase()) { + throw new LogicException('Cannot call RepeatTestSuite::setDependencies() on a PhptTestCase.'); + } + + foreach ($this->tests as $test) { + $test->setDependencies($dependencies); + } + } + /** * @phpstan-assert-if-true non-empty-list $this->tests * diff --git a/src/Framework/TestBuilder.php b/src/Framework/TestBuilder.php index ef5f3c5e4c..8d87048704 100644 --- a/src/Framework/TestBuilder.php +++ b/src/Framework/TestBuilder.php @@ -11,6 +11,7 @@ use function array_merge; use function assert; +use function range; use PHPUnit\Metadata\Api\DataProvider; use PHPUnit\Metadata\Api\Groups; use PHPUnit\Metadata\Api\ProvidedData; @@ -57,39 +58,20 @@ public function build(ReflectionClass $theClass, string $methodName, array $grou } if ($data !== null) { - return $this->buildDataProviderTestSuite( - $methodName, - $className, - $data, - $this->shouldTestMethodBeRunInSeparateProcess($className, $methodName), - $this->shouldGlobalStateBePreserved($className, $methodName), - $this->backupSettings($className, $methodName), - $groups, - $repeatTimes, - ); + return $this->buildDataProviderTestSuite($methodName, $className, $data, $groups, $repeatTimes); } - $test = new $className($methodName, $repeatTimes); - - $this->configureTestCase( - $test, - $this->shouldTestMethodBeRunInSeparateProcess($className, $methodName), - $this->shouldGlobalStateBePreserved($className, $methodName), - $this->backupSettings($className, $methodName), - ); - - return $test; + return $this->createTest($className, $methodName, $repeatTimes); } /** - * @param non-empty-string $methodName - * @param class-string $className - * @param array $data - * @param array{backupGlobals: ?true, backupGlobalsExcludeList: list, backupStaticProperties: ?true, backupStaticPropertiesExcludeList: array>} $backupSettings - * @param list $groups - * @param positive-int $repeatTimes + * @param non-empty-string $methodName + * @param class-string $className + * @param array $data + * @param list $groups + * @param positive-int $repeatTimes */ - private function buildDataProviderTestSuite(string $methodName, string $className, array $data, bool $runTestInSeparateProcess, ?bool $preserveGlobalState, array $backupSettings, array $groups, int $repeatTimes = 1): DataProviderTestSuite + private function buildDataProviderTestSuite(string $methodName, string $className, array $data, array $groups, int $repeatTimes = 1): DataProviderTestSuite { $dataProviderTestSuite = DataProviderTestSuite::empty( $className . '::' . $methodName, @@ -101,21 +83,52 @@ private function buildDataProviderTestSuite(string $methodName, string $classNam ); foreach ($data as $_dataName => $_data) { - $_test = new $className($methodName, $repeatTimes); + $_test = $this->createTest($className, $methodName, $repeatTimes); $_test->setData($_dataName, $_data->value()); + $dataProviderTestSuite->addTest($_test, $groups); + } + + return $dataProviderTestSuite; + } + + /** + * @param class-string $className + * @param non-empty-string $methodName + * @param positive-int $repeatTimes + */ + private function createTest(string $className, string $methodName, int $repeatTimes): RepeatTestSuite|TestCase + { + if ($repeatTimes === 1) { + $test = new $className($methodName); + $this->configureTestCase( - $_test, - $runTestInSeparateProcess, - $preserveGlobalState, - $backupSettings, + $test, + $this->shouldTestMethodBeRunInSeparateProcess($className, $methodName), + $this->shouldGlobalStateBePreserved($className, $methodName), + $this->backupSettings($className, $methodName), ); + } else { + $tests = []; + + foreach (range(1, $repeatTimes) as $i) { + $_test = new $className($methodName, $repeatTimes, $i); + + $this->configureTestCase( + $_test, + $this->shouldTestMethodBeRunInSeparateProcess($className, $methodName), + $this->shouldGlobalStateBePreserved($className, $methodName), + $this->backupSettings($className, $methodName), + ); - $dataProviderTestSuite->addTest($_test, $groups, $repeatTimes); + $tests[] = $_test; + } + + $test = new RepeatTestSuite($tests); } - return $dataProviderTestSuite; + return $test; } /** diff --git a/src/Framework/TestCase.php b/src/Framework/TestCase.php index 5346526752..8e74d2c19f 100644 --- a/src/Framework/TestCase.php +++ b/src/Framework/TestCase.php @@ -227,18 +227,19 @@ abstract class TestCase extends Assert implements Reorderable, SelfDescribing, T private mixed $errorLogCapture = false; private false|string $previousErrorLogTarget = false; private readonly int $repeatTimes; - private int $currentRepeat = 1; + private int $currentRepeat; /** * @param non-empty-string $name * * @internal This method is not covered by the backward compatibility promise for PHPUnit */ - final public function __construct(string $name, int $repeatTimes = 1) + final public function __construct(string $name, int $repeatTimes = 1, int $currentRepeat = 1) { - $this->methodName = $name; - $this->status = TestStatus::unknown(); - $this->repeatTimes = $repeatTimes; + $this->methodName = $name; + $this->status = TestStatus::unknown(); + $this->repeatTimes = $repeatTimes; + $this->currentRepeat = $currentRepeat; if (is_callable($this->sortId(), true)) { $this->providedTests = [new ExecutionOrderDependency($this->sortId())]; diff --git a/src/Framework/TestSuite.php b/src/Framework/TestSuite.php index d248d74677..10fde3c207 100644 --- a/src/Framework/TestSuite.php +++ b/src/Framework/TestSuite.php @@ -10,6 +10,7 @@ namespace PHPUnit\Framework; use const PHP_EOL; +use function array_map; use function array_merge; use function array_pop; use function array_reverse; @@ -21,6 +22,7 @@ use function is_callable; use function is_file; use function is_subclass_of; +use function range; use function sprintf; use function str_ends_with; use function str_starts_with; @@ -146,9 +148,8 @@ final private function __construct(string $name) * Adds a test to the suite. * * @param list $groups - * @param positive-int $repeatTimes */ - public function addTest(Test $test, array $groups = [], int $repeatTimes = 1): void + public function addTest(Test $test, array $groups = []): void { if ($test instanceof self) { $this->tests[] = $test; @@ -160,11 +161,7 @@ public function addTest(Test $test, array $groups = [], int $repeatTimes = 1): v assert($test instanceof TestCase || $test instanceof PhptTestCase); - if ($repeatTimes === 1) { - $this->tests[] = $test; - } else { - $this->tests[] = new RepeatTestSuite($test, $repeatTimes); - } + $this->tests[] = $test; $this->clearCaches(); @@ -219,7 +216,7 @@ public function addTestSuite(ReflectionClass $testClass, array $groups = [], int ); } - $this->addTest(self::fromClassReflector($testClass, $groups, $repeatTimes), $groups, $repeatTimes); + $this->addTest(self::fromClassReflector($testClass, $groups, $repeatTimes), $groups); } /** @@ -239,7 +236,18 @@ public function addTestFile(string $filename, array $groups = [], int $repeatTim { try { if (str_ends_with($filename, '.phpt') && is_file($filename)) { - $this->addTest(new PhptTestCase($filename), [], $repeatTimes); + if ($repeatTimes === 1) { + $test = new PhptTestCase($filename); + } else { + $test = new RepeatTestSuite( + array_map( + static fn () => new PhptTestCase($filename), + range(1, $repeatTimes), + ), + ); + } + + $this->addTest($test); } else { $this->addTestSuite( (new TestSuiteLoader)->load($filename), @@ -561,7 +569,7 @@ protected function addTestMethod(ReflectionClass $class, ReflectionMethod $metho return; } - if ($test instanceof TestCase || $test instanceof DataProviderTestSuite) { + if ($test instanceof TestCase || $test instanceof DataProviderTestSuite || $test instanceof RepeatTestSuite && !$test->isPhptTestCase()) { $test->setDependencies( Dependencies::dependencies($class->getName(), $methodName), ); @@ -573,7 +581,6 @@ protected function addTestMethod(ReflectionClass $class, ReflectionMethod $metho $groups, (new Groups)->groups($class->getName(), $methodName), ), - $repeatTimes, ); } diff --git a/src/TextUI/Configuration/Xml/TestSuiteMapper.php b/src/TextUI/Configuration/Xml/TestSuiteMapper.php index b7afcd89d7..db3457b057 100644 --- a/src/TextUI/Configuration/Xml/TestSuiteMapper.php +++ b/src/TextUI/Configuration/Xml/TestSuiteMapper.php @@ -135,7 +135,7 @@ public function map(string $xmlConfigurationFile, TestSuiteCollection $configure } if (!$empty) { - $result->addTest($testSuite, [], $repeatTimes); + $result->addTest($testSuite); } } From 20aa69cf84dbe7244fb0087671efd705922a1992 Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Wed, 29 Oct 2025 13:40:21 +0100 Subject: [PATCH 18/21] Split regular RepeatTestSuite and PhptRepeatTestSuite --- src/Framework/AbstractRepeatTestSuite.php | 62 +++++++++++ src/Framework/RepeatTestSuite.php | 124 +++------------------- src/Framework/TestSuite.php | 5 +- src/Runner/Filter/NameFilterIterator.php | 3 +- src/Runner/Phpt/RepeatTestSuite.php | 46 ++++++++ 5 files changed, 128 insertions(+), 112 deletions(-) create mode 100644 src/Framework/AbstractRepeatTestSuite.php create mode 100644 src/Runner/Phpt/RepeatTestSuite.php diff --git a/src/Framework/AbstractRepeatTestSuite.php b/src/Framework/AbstractRepeatTestSuite.php new file mode 100644 index 0000000000..2f8a704284 --- /dev/null +++ b/src/Framework/AbstractRepeatTestSuite.php @@ -0,0 +1,62 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\Framework; + +use function count; +use PHPUnit\Event; +use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; + +/** + * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit + * + * @internal This class is not covered by the backward compatibility promise for PHPUnit + * + * @template T of TestCase|PhptTestCase + */ +abstract readonly class AbstractRepeatTestSuite implements Reorderable, Test +{ + /** + * @var non-empty-list + */ + protected array $tests; + + /** + * @param non-empty-list $tests + */ + public function __construct(array $tests) + { + $this->tests = $tests; + } + + final public function count(): int + { + return count($this->tests); + } + + final public function sortId(): string + { + return $this->tests[0]->sortId(); + } + + final public function provides(): array + { + return $this->tests[0]->provides(); + } + + final public function requires(): array + { + return $this->tests[0]->requires(); + } + + final public function valueObjectForEvents(): Event\Code\Phpt|Event\Code\TestMethod + { + return $this->tests[0]->valueObjectForEvents(); + } +} diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php index 8fb33c1cc8..572650720f 100644 --- a/src/Framework/RepeatTestSuite.php +++ b/src/Framework/RepeatTestSuite.php @@ -10,73 +10,38 @@ namespace PHPUnit\Framework; use PHPUnit\Metadata\Api\ProvidedData; -use function count; -use LogicException; -use PHPUnit\Event; -use PHPUnit\Event\Facade as EventFacade; -use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; /** * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit * * @internal This class is not covered by the backward compatibility promise for PHPUnit + * + * @extends AbstractRepeatTestSuite */ -final readonly class RepeatTestSuite implements Reorderable, Test +final readonly class RepeatTestSuite extends AbstractRepeatTestSuite { - /** - * @var non-empty-list|non-empty-list - */ - private array $tests; - - /** - * @param non-empty-list|non-empty-list $tests - */ - public function __construct(array $tests) - { - $this->tests = $tests; - } - - public function count(): int - { - return count($this->tests); - } - public function run(): void { - if ($this->isPhptTestCase()) { - $this->runPhptTestCase(); - } else { - $this->runTestCase(); - } - } + $defectOccurred = false; - public function sortId(): string - { - return $this->tests[0]->sortId(); - } + foreach ($this->tests as $test) { + if ($defectOccurred) { + $test->markSkippedForErrorInPreviousRepetition(); - public function provides(): array - { - return $this->tests[0]->provides(); - } + continue; + } - public function requires(): array - { - return $this->tests[0]->requires(); - } + $test->run(); - public function name(): string - { - if ($this->isPhptTestCase()) { - throw new LogicException('Cannot call RepeatTestSuite::nameWithDataSet() on a PhptTestCase.'); + if ($test->status()->isFailure() || $test->status()->isError()) { + $defectOccurred = true; + } } - - return $this->tests[0]::class . '::' . $this->tests[0]->nameWithDataSet(); } - public function valueObjectForEvents(): Event\Code\Phpt|Event\Code\TestMethod + public function name(): string { - return $this->tests[0]->valueObjectForEvents(); + return $this->tests[0]::class . '::' . $this->tests[0]->nameWithDataSet(); } /** @@ -84,10 +49,6 @@ public function valueObjectForEvents(): Event\Code\Phpt|Event\Code\TestMethod */ public function setData(int|string $dataName, array $data): void { - if ($this->isPhptTestCase()) { - throw new LogicException('Cannot call RepeatTestSuite::setData() on a PhptTestCase.'); - } - foreach ($this->tests as $test) { $test->setData($dataName, $data); } @@ -98,63 +59,8 @@ public function setData(int|string $dataName, array $data): void */ public function setDependencies(array $dependencies): void { - if ($this->isPhptTestCase()) { - throw new LogicException('Cannot call RepeatTestSuite::setDependencies() on a PhptTestCase.'); - } - foreach ($this->tests as $test) { $test->setDependencies($dependencies); } } - - /** - * @phpstan-assert-if-true non-empty-list $this->tests - * - * @phpstan-assert-if-false non-empty-list $this->tests - */ - public function isPhptTestCase(): bool - { - return $this->tests[0] instanceof PhptTestCase; - } - - private function runTestCase(): void - { - $defectOccurred = false; - - foreach ($this->tests as $test) { - if ($defectOccurred) { - $test->markSkippedForErrorInPreviousRepetition(); - - continue; - } - - $test->run(); - - if ($test->status()->isFailure() || $test->status()->isError()) { - $defectOccurred = true; - } - } - } - - private function runPhptTestCase(): void - { - $defectOccurred = false; - - foreach ($this->tests as $test) { - if ($defectOccurred) { - EventFacade::emitter()->testSkipped( - $this->valueObjectForEvents(), - 'Test repetition failure', - ); - - continue; - } - - $test->run(); - - if (!$test->passed()) { - $defectOccurred = true; - } - } - } } diff --git a/src/Framework/TestSuite.php b/src/Framework/TestSuite.php index 10fde3c207..b1f1a01247 100644 --- a/src/Framework/TestSuite.php +++ b/src/Framework/TestSuite.php @@ -39,6 +39,7 @@ use PHPUnit\Metadata\MetadataCollection; use PHPUnit\Runner\Exception as RunnerException; use PHPUnit\Runner\Filter\Factory; +use PHPUnit\Runner\Phpt\RepeatTestSuite as PhptRepeatTestSuite; use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; use PHPUnit\Runner\TestSuiteLoader; use PHPUnit\TestRunner\TestResult\Facade as TestResultFacade; @@ -239,7 +240,7 @@ public function addTestFile(string $filename, array $groups = [], int $repeatTim if ($repeatTimes === 1) { $test = new PhptTestCase($filename); } else { - $test = new RepeatTestSuite( + $test = new PhptRepeatTestSuite( array_map( static fn () => new PhptTestCase($filename), range(1, $repeatTimes), @@ -569,7 +570,7 @@ protected function addTestMethod(ReflectionClass $class, ReflectionMethod $metho return; } - if ($test instanceof TestCase || $test instanceof DataProviderTestSuite || $test instanceof RepeatTestSuite && !$test->isPhptTestCase()) { + if ($test instanceof TestCase || $test instanceof DataProviderTestSuite || $test instanceof RepeatTestSuite) { $test->setDependencies( Dependencies::dependencies($class->getName(), $methodName), ); diff --git a/src/Runner/Filter/NameFilterIterator.php b/src/Runner/Filter/NameFilterIterator.php index ce6696b4db..da5074f9e9 100644 --- a/src/Runner/Filter/NameFilterIterator.php +++ b/src/Runner/Filter/NameFilterIterator.php @@ -16,6 +16,7 @@ use PHPUnit\Framework\RepeatTestSuite; use PHPUnit\Framework\Test; use PHPUnit\Framework\TestSuite; +use PHPUnit\Runner\Phpt\RepeatTestSuite as PhptRepeatTestSuite; use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; use RecursiveFilterIterator; use RecursiveIterator; @@ -57,7 +58,7 @@ public function accept(): bool return true; } - if ($test instanceof PhptTestCase || ($test instanceof RepeatTestSuite && $test->isPhptTestCase())) { + if ($test instanceof PhptTestCase || $test instanceof PhptRepeatTestSuite) { return false; } diff --git a/src/Runner/Phpt/RepeatTestSuite.php b/src/Runner/Phpt/RepeatTestSuite.php new file mode 100644 index 0000000000..8aaa5155bf --- /dev/null +++ b/src/Runner/Phpt/RepeatTestSuite.php @@ -0,0 +1,46 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\Runner\Phpt; + +use PHPUnit\Event\Facade as EventFacade; +use PHPUnit\Framework\AbstractRepeatTestSuite; +use PHPUnit\Runner\Phpt\TestCase as PhptTestCase; + +/** + * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit + * + * @internal This class is not covered by the backward compatibility promise for PHPUnit + * + * @extends AbstractRepeatTestSuite + */ +final readonly class RepeatTestSuite extends AbstractRepeatTestSuite +{ + public function run(): void + { + $defectOccurred = false; + + foreach ($this->tests as $test) { + if ($defectOccurred) { + EventFacade::emitter()->testSkipped( + $this->valueObjectForEvents(), + 'Test repetition failure', + ); + + continue; + } + + $test->run(); + + if (!$test->passed()) { + $defectOccurred = true; + } + } + } +} From e9ff6a6827433c7f139c16234333c34e949ac6b3 Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Wed, 29 Oct 2025 20:59:22 +0100 Subject: [PATCH 19/21] Add $repeatAttemptNumber property in test event DTOs --- src/Event/Value/Test/Phpt.php | 24 +++++++- src/Event/Value/Test/TestMethod.php | 37 ++++++++++--- src/Event/Value/Test/TestMethodBuilder.php | 1 + src/Framework/TestCase.php | 55 ++++++++++++------- src/Runner/Phpt/TestCase.php | 13 ++++- ...orsTest.php => RepeatWithFailuresTest.php} | 2 +- .../repeat/dependent-test-with-failure.phpt | 2 +- .../repeat/error-skips-next-repetitions.phpt | 14 ++--- .../repeat/error-stop-on-failure-debug.phpt | 41 ++++++++++++++ .../repeat/error-stop-on-failure.phpt | 34 ++++++++++++ ...{simple-repeat.phpt => success-debug.phpt} | 0 tests/end-to-end/repeat/success.phpt | 23 ++++++++ 12 files changed, 203 insertions(+), 43 deletions(-) rename tests/end-to-end/repeat/_files/{RepeatWithErrorsTest.php => RepeatWithFailuresTest.php} (94%) create mode 100644 tests/end-to-end/repeat/error-stop-on-failure-debug.phpt create mode 100644 tests/end-to-end/repeat/error-stop-on-failure.phpt rename tests/end-to-end/repeat/{simple-repeat.phpt => success-debug.phpt} (100%) create mode 100644 tests/end-to-end/repeat/success.phpt diff --git a/src/Event/Value/Test/Phpt.php b/src/Event/Value/Test/Phpt.php index 65a3aec826..0652be42cd 100644 --- a/src/Event/Value/Test/Phpt.php +++ b/src/Event/Value/Test/Phpt.php @@ -16,6 +16,22 @@ */ final readonly class Phpt extends Test { + /** + * @var positive-int + */ + private int $repeatAttemptNumber; + + /** + * @param non-empty-string $file + * @param positive-int $repeatAttemptNumber + */ + public function __construct(string $file, int $repeatAttemptNumber = 1) + { + parent::__construct($file); + + $this->repeatAttemptNumber = $repeatAttemptNumber; + } + public function isPhpt(): true { return true; @@ -26,7 +42,7 @@ public function isPhpt(): true */ public function id(): string { - return $this->file(); + return $this->name(); } /** @@ -34,6 +50,10 @@ public function id(): string */ public function name(): string { - return $this->file(); + if ($this->repeatAttemptNumber === 1) { + return $this->file(); + } + + return $this->file() . " (repeat attempt #{$this->repeatAttemptNumber})"; } } diff --git a/src/Event/Value/Test/TestMethod.php b/src/Event/Value/Test/TestMethod.php index 4c97264543..c4c769b3d7 100644 --- a/src/Event/Value/Test/TestMethod.php +++ b/src/Event/Value/Test/TestMethod.php @@ -39,22 +39,29 @@ private MetadataCollection $metadata; private TestDataCollection $testData; + /** + * @var positive-int + */ + private int $repeatAttemptNumber; + /** * @param class-string $className * @param non-empty-string $methodName * @param non-empty-string $file * @param non-negative-int $line + * @param positive-int $repeatAttemptNumber */ - public function __construct(string $className, string $methodName, string $file, int $line, TestDox $testDox, MetadataCollection $metadata, TestDataCollection $testData) + public function __construct(string $className, string $methodName, string $file, int $line, TestDox $testDox, MetadataCollection $metadata, TestDataCollection $testData, int $repeatAttemptNumber = 1) { parent::__construct($file); - $this->className = $className; - $this->methodName = $methodName; - $this->line = $line; - $this->testDox = $testDox; - $this->metadata = $metadata; - $this->testData = $testData; + $this->className = $className; + $this->methodName = $methodName; + $this->line = $line; + $this->testDox = $testDox; + $this->metadata = $metadata; + $this->testData = $testData; + $this->repeatAttemptNumber = $repeatAttemptNumber; } /** @@ -129,7 +136,13 @@ public function nameWithClass(): string public function name(): string { if (!$this->testData->hasDataFromDataProvider()) { - return $this->methodName; + $name = $this->methodName; + + if ($this->repeatAttemptNumber !== 1) { + $name .= " (repeat attempt #{$this->repeatAttemptNumber})"; + } + + return $name; } $dataSetName = $this->testData->dataFromDataProvider()->dataSetName(); @@ -146,6 +159,12 @@ public function name(): string ); } - return $this->methodName . $dataSetName; + $name = $this->methodName . $dataSetName; + + if ($this->repeatAttemptNumber !== 1) { + $name .= " (repeat attempt #{$this->repeatAttemptNumber})"; + } + + return $name; } } diff --git a/src/Event/Value/Test/TestMethodBuilder.php b/src/Event/Value/Test/TestMethodBuilder.php index dc1a32ef8a..8218eddece 100644 --- a/src/Event/Value/Test/TestMethodBuilder.php +++ b/src/Event/Value/Test/TestMethodBuilder.php @@ -45,6 +45,7 @@ public static function fromTestCase(TestCase $testCase, bool $useTestCaseForTest $testDox, MetadataRegistry::parser()->forClassAndMethod($testCase::class, $methodName), self::dataFor($testCase), + $testCase->repeatAttemptNumber(), ); } diff --git a/src/Framework/TestCase.php b/src/Framework/TestCase.php index 8e74d2c19f..799c30730e 100644 --- a/src/Framework/TestCase.php +++ b/src/Framework/TestCase.php @@ -59,6 +59,7 @@ use AssertionError; use DeepCopy\DeepCopy; use PHPUnit\Event; +use PHPUnit\Event\InvalidArgumentException; use PHPUnit\Event\NoPreviousThrowableException; use PHPUnit\Framework\Constraint\Exception as ExceptionConstraint; use PHPUnit\Framework\Constraint\ExceptionCode; @@ -226,20 +227,35 @@ abstract class TestCase extends Assert implements Reorderable, SelfDescribing, T */ private mixed $errorLogCapture = false; private false|string $previousErrorLogTarget = false; + + /** + * @var positive-int + */ private readonly int $repeatTimes; - private int $currentRepeat; + + /** + * @var positive-int + */ + private readonly int $repeatAttemptNumber; /** * @param non-empty-string $name + * @param positive-int $repeatTimes + * @param positive-int $repeatAttemptNumber * * @internal This method is not covered by the backward compatibility promise for PHPUnit */ - final public function __construct(string $name, int $repeatTimes = 1, int $currentRepeat = 1) + final public function __construct(string $name, int $repeatTimes = 1, int $repeatAttemptNumber = 1) { - $this->methodName = $name; - $this->status = TestStatus::unknown(); - $this->repeatTimes = $repeatTimes; - $this->currentRepeat = $currentRepeat; + $this->methodName = $name; + $this->status = TestStatus::unknown(); + + if ($repeatTimes < $repeatAttemptNumber) { + throw new InvalidArgumentException("Given repeat attempt number \"{$repeatAttemptNumber}\" must be inferior or equal to repeat times \"{$repeatTimes}\""); + } + + $this->repeatTimes = $repeatTimes; + $this->repeatAttemptNumber = $repeatAttemptNumber; if (is_callable($this->sortId(), true)) { $this->providedTests = [new ExecutionOrderDependency($this->sortId())]; @@ -304,17 +320,6 @@ protected function tearDown(): void { } - /** - * @internal This method is not covered by the backward compatibility promise for PHPUnit - */ - public function newRepeatInstance(): static - { - $clone = clone $this; - $clone->currentRepeat++; - - return $clone; - } - /** * Returns a string representation of the test case. * @@ -658,7 +663,7 @@ final public function runBare(): void } } - if (!isset($e) && !isset($_e) && $this->currentRepeat === $this->repeatTimes) { + if (!isset($e) && !isset($_e) && $this->repeatAttemptNumber === $this->repeatTimes) { $emitter->testPassed( $this->valueObjectForEvents(), ); @@ -980,12 +985,22 @@ final public function wasPrepared(): bool return $this->wasPrepared; } + /** + * @return positive-int + * + * @internal This method is not covered by the backward compatibility promise for PHPUnit + */ + final public function repeatAttemptNumber(): int + { + return $this->repeatAttemptNumber; + } + /** * @internal This method is not covered by the backward compatibility promise for PHPUnit */ - public function markSkippedForErrorInPreviousRepetition(): void + final public function markSkippedForErrorInPreviousRepetition(): void { - $message = 'Test repetition failure'; + $message = "Test repetition #{$this->repeatAttemptNumber} failure"; Event\Facade::emitter()->testSkipped( $this->valueObjectForEvents(), diff --git a/src/Runner/Phpt/TestCase.php b/src/Runner/Phpt/TestCase.php index 564cc2c567..02b02609ef 100644 --- a/src/Runner/Phpt/TestCase.php +++ b/src/Runner/Phpt/TestCase.php @@ -78,12 +78,19 @@ final class TestCase implements Reorderable, SelfDescribing, Test private readonly string $filename; private bool $passed = false; + /** + * @var positive-int + */ + private int $repeatAttemptNumber; + /** * @param non-empty-string $filename + * @param positive-int $repeatAttemptNumber */ - public function __construct(string $filename) + public function __construct(string $filename, int $repeatAttemptNumber = 1) { - $this->filename = $filename; + $this->filename = $filename; + $this->repeatAttemptNumber = $repeatAttemptNumber; } public function count(): int @@ -297,7 +304,7 @@ public function requires(): array */ public function valueObjectForEvents(): Phpt { - return new Phpt($this->filename); + return new Phpt($this->filename, $this->repeatAttemptNumber); } /** diff --git a/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php b/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php similarity index 94% rename from tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php rename to tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php index 5ec8d246a7..6bfd931801 100644 --- a/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php +++ b/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php @@ -9,7 +9,7 @@ */ use PHPUnit\Framework\TestCase; -final class RepeatWithErrorsTest extends TestCase +final class RepeatWithFailuresTest extends TestCase { public function test1(): void { diff --git a/tests/end-to-end/repeat/dependent-test-with-failure.phpt b/tests/end-to-end/repeat/dependent-test-with-failure.phpt index 72bfd6eda4..0be2ded7a5 100644 --- a/tests/end-to-end/repeat/dependent-test-with-failure.phpt +++ b/tests/end-to-end/repeat/dependent-test-with-failure.phpt @@ -22,7 +22,7 @@ Time: %s, Memory: %s MB There was 1 failure: -1) DependentOfTestFailedInRepetitionTest::test1 +1) DependentOfTestFailedInRepetitionTest::test1 (repeat attempt #2) Failed asserting that true is false. %s/tests/end-to-end/repeat/_files/DependentOfTestFailedInRepetitionTest.php:%d diff --git a/tests/end-to-end/repeat/error-skips-next-repetitions.phpt b/tests/end-to-end/repeat/error-skips-next-repetitions.phpt index 8d236c54aa..67069463f2 100644 --- a/tests/end-to-end/repeat/error-skips-next-repetitions.phpt +++ b/tests/end-to-end/repeat/error-skips-next-repetitions.phpt @@ -6,7 +6,7 @@ $_SERVER['argv'][] = '--do-not-cache-result'; $_SERVER['argv'][] = '--no-configuration'; $_SERVER['argv'][] = '--repeat'; $_SERVER['argv'][] = '3'; -$_SERVER['argv'][] = __DIR__ . '/_files/RepeatWithErrorsTest.php'; +$_SERVER['argv'][] = __DIR__ . '/_files/RepeatWithFailuresTest.php'; require __DIR__ . '/../../bootstrap.php'; @@ -22,20 +22,20 @@ Time: %s, Memory: %s MB There were 3 failures: -1) RepeatWithErrorsTest::test1 +1) RepeatWithFailuresTest::test1 Failed asserting that true is false. -%s/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:%d +%s/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php:%d -2) RepeatWithErrorsTest::test2 +2) RepeatWithFailuresTest::test2 (repeat attempt #2) Failed asserting that true is false. -%s/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:%d +%s/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php:%d -3) RepeatWithErrorsTest::test3 +3) RepeatWithFailuresTest::test3 (repeat attempt #3) Failed asserting that true is false. -%s/tests/end-to-end/repeat/_files/RepeatWithErrorsTest.php:%d +%s/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php:%d FAILURES! Tests: 12, Assertions: 9, Failures: 3, Skipped: 3. diff --git a/tests/end-to-end/repeat/error-stop-on-failure-debug.phpt b/tests/end-to-end/repeat/error-stop-on-failure-debug.phpt new file mode 100644 index 0000000000..bba6848f8e --- /dev/null +++ b/tests/end-to-end/repeat/error-stop-on-failure-debug.phpt @@ -0,0 +1,41 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using PHP %s) +Test Runner Configured +Event Facade Sealed +Test Suite Loaded (%d tests) +Test Runner Started +Test Suite Sorted +Test Suite Filtered (3 tests) +Test Runner Execution Started (3 tests) +Test Suite Started (RepeatWithFailuresTest, 3 tests) +Test Preparation Started (RepeatWithFailuresTest::test2) +Test Prepared (RepeatWithFailuresTest::test2) +Test Finished (RepeatWithFailuresTest::test2) +Test Preparation Started (RepeatWithFailuresTest::test2) +Test Prepared (RepeatWithFailuresTest::test2) +Test Failed (RepeatWithFailuresTest::test2) +Failed asserting that true is false. +Test Finished (RepeatWithFailuresTest::test2) +Test Skipped (RepeatWithFailuresTest::test2) +Test repetition #3 failure +Test Suite Finished (RepeatWithFailuresTest, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 1) diff --git a/tests/end-to-end/repeat/error-stop-on-failure.phpt b/tests/end-to-end/repeat/error-stop-on-failure.phpt new file mode 100644 index 0000000000..6570ffa60b --- /dev/null +++ b/tests/end-to-end/repeat/error-stop-on-failure.phpt @@ -0,0 +1,34 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +.FS 3 / 3 (100%) + +Time: %s, Memory: %s MB + +There was 1 failure: + +1) RepeatWithFailuresTest::test2 (repeat attempt #2) +Failed asserting that true is false. + +%s/tests/end-to-end/repeat/_files/RepeatWithFailuresTest.php:%d + +FAILURES! +Tests: 3, Assertions: 2, Failures: 1, Skipped: 1. diff --git a/tests/end-to-end/repeat/simple-repeat.phpt b/tests/end-to-end/repeat/success-debug.phpt similarity index 100% rename from tests/end-to-end/repeat/simple-repeat.phpt rename to tests/end-to-end/repeat/success-debug.phpt diff --git a/tests/end-to-end/repeat/success.phpt b/tests/end-to-end/repeat/success.phpt new file mode 100644 index 0000000000..76f478290c --- /dev/null +++ b/tests/end-to-end/repeat/success.phpt @@ -0,0 +1,23 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: PHP 8.4.13 + +.... 4 / 4 (100%) + +Time: %s, Memory: %s MB + +OK (4 tests, 8 assertions) From 539fb970abcc5fb6fbec92064ea07ed485e1e623 Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Wed, 29 Oct 2025 21:39:11 +0100 Subject: [PATCH 20/21] add repeat test with dependency ordered by random --- .../repeat/dependent-test-random.phpt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/end-to-end/repeat/dependent-test-random.phpt diff --git a/tests/end-to-end/repeat/dependent-test-random.phpt b/tests/end-to-end/repeat/dependent-test-random.phpt new file mode 100644 index 0000000000..e1b6ed1509 --- /dev/null +++ b/tests/end-to-end/repeat/dependent-test-random.phpt @@ -0,0 +1,17 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTREGEX-- +(FSSS....)|(FS..SS..)|(FS....SS)|(..FSSS..)|(..FS..SS)|(....FSSS) From ae0c716313c6b44d9c9b1b5d8f7e09f5bc4883e2 Mon Sep 17 00:00:00 2001 From: Nicolas PHILIPPE Date: Mon, 3 Nov 2025 17:05:03 +0100 Subject: [PATCH 21/21] Cannot retry if depends on test which returns value --- src/Framework/RepeatTestSuite.php | 2 +- src/Framework/TestCase.php | 30 ++++++++++++++++++ ...pendentOfTestWhichReturnsSomethingTest.php | 27 ++++++++++++++++ ...ndent-on-test-which-returns-something.phpt | 31 +++++++++++++++++++ 4 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 tests/end-to-end/repeat/_files/DependentOfTestWhichReturnsSomethingTest.php create mode 100644 tests/end-to-end/repeat/dependent-on-test-which-returns-something.phpt diff --git a/src/Framework/RepeatTestSuite.php b/src/Framework/RepeatTestSuite.php index 572650720f..a94f1da3e4 100644 --- a/src/Framework/RepeatTestSuite.php +++ b/src/Framework/RepeatTestSuite.php @@ -33,7 +33,7 @@ public function run(): void $test->run(); - if ($test->status()->isFailure() || $test->status()->isError()) { + if ($test->status()->isFailure() || $test->status()->isError() || $test->status()->isSkipped()) { $defectOccurred = true; } } diff --git a/src/Framework/TestCase.php b/src/Framework/TestCase.php index 799c30730e..d8f31f795e 100644 --- a/src/Framework/TestCase.php +++ b/src/Framework/TestCase.php @@ -1531,6 +1531,12 @@ private function handleDependencies(): bool $returnValue = $passedTests->returnValue($dependencyTarget); + if ($this->repeatTimes > 1 && $returnValue !== null) { + $this->markSkippedForRepeatAndReturningDependency($dependency); + + return false; + } + if ($dependency->deepClone()) { $deepCopy = new DeepCopy; $deepCopy->skipUncloneable(false); @@ -1588,6 +1594,30 @@ private function markSkippedForMissingDependency(ExecutionOrderDependency $depen $this->status = TestStatus::skipped($message); } + /** + * @throws Exception + * @throws NoPreviousThrowableException + */ + private function markSkippedForRepeatAndReturningDependency(ExecutionOrderDependency $dependency): void + { + $message = sprintf( + 'This test depends on "%s" which returns a value. Such test cannot be run in repeat mode', + $dependency->targetIsClass() ? $dependency->getTargetClassName() : $dependency->getTarget(), + ); + + Event\Facade::emitter()->testTriggeredPhpunitWarning( + $this->valueObjectForEvents(), + $message, + ); + + Event\Facade::emitter()->testSkipped( + $this->valueObjectForEvents(), + $message, + ); + + $this->status = TestStatus::skipped($message); + } + private function startOutputBuffering(): void { ob_start(); diff --git a/tests/end-to-end/repeat/_files/DependentOfTestWhichReturnsSomethingTest.php b/tests/end-to-end/repeat/_files/DependentOfTestWhichReturnsSomethingTest.php new file mode 100644 index 0000000000..b24ac4c65f --- /dev/null +++ b/tests/end-to-end/repeat/_files/DependentOfTestWhichReturnsSomethingTest.php @@ -0,0 +1,27 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +use PHPUnit\Framework\Attributes\Depends; +use PHPUnit\Framework\TestCase; + +final class DependentOfTestWhichReturnsSomethingTest extends TestCase +{ + public function test1(): string + { + $this->assertTrue(true); + + return 'foo'; + } + + #[Depends('test1')] + public function test2(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/repeat/dependent-on-test-which-returns-something.phpt b/tests/end-to-end/repeat/dependent-on-test-which-returns-something.phpt new file mode 100644 index 0000000000..bc9f4406c9 --- /dev/null +++ b/tests/end-to-end/repeat/dependent-on-test-which-returns-something.phpt @@ -0,0 +1,31 @@ +--TEST-- +Repeat option +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit %s by Sebastian Bergmann and contributors. + +Runtime: %s + +..SS 4 / 4 (100%) + +Time: %s, Memory: %s MB + +1 test triggered 1 PHPUnit warning: + +1) DependentOfTestWhichReturnsSomethingTest::test2 +This test depends on "DependentOfTestWhichReturnsSomethingTest::test1" which returns a value. Such test cannot be run in repeat mode + +%s/tests/end-to-end/repeat/_files/DependentOfTestWhichReturnsSomethingTest.php:%d + +OK, but there were issues! +Tests: 4, Assertions: 2, PHPUnit Warnings: 1, Skipped: 2.