From 5175cc6cd6ca5f754ef1d2b9a2addff33da15740 Mon Sep 17 00:00:00 2001 From: Bryce Drennan Date: Tue, 9 Apr 2024 17:23:52 -0700 Subject: [PATCH 1/4] feature: each test starts with a unique seed. `--randomly-use-same-seed-per-test` for old behavior --- CHANGELOG.rst | 8 ++++++ README.rst | 3 ++ src/pytest_randomly/__init__.py | 17 ++++++++++- tests/test_pytest_randomly.py | 50 ++++++++++++++++++++++++++------- 4 files changed, 67 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bbed7ee..9017f9b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,14 @@ Changelog ========= +3.17.0 +------------------- + +* Change default seed to be different per test. Add the option ``--randomly-use-same-seed-per-test`` to enable the old behavior. + + Resolves `Issue #600 `__ + + 3.16.0 (2024-10-25) ------------------- diff --git a/README.rst b/README.rst index 969c442..4225af8 100644 --- a/README.rst +++ b/README.rst @@ -155,6 +155,9 @@ You can disable behaviours you don't like with the following flags: * ``--randomly-dont-reset-seed`` - turn off the reset of ``random.seed()`` at the start of every test * ``--randomly-dont-reorganize`` - turn off the shuffling of the order of tests +* ``--randomly-dont-seed-per-test`` - turn off each test having a unique seed. + Each test will be seeded with the same seed. + The plugin appears to Pytest with the name 'randomly'. To disable it altogether, you can use the ``-p`` argument, for example: diff --git a/src/pytest_randomly/__init__.py b/src/pytest_randomly/__init__.py index fa10578..73bf0bb 100644 --- a/src/pytest_randomly/__init__.py +++ b/src/pytest_randomly/__init__.py @@ -105,6 +105,13 @@ def pytest_addoption(parser: Parser) -> None: default=True, help="Stop pytest-randomly from randomly reorganizing the test order.", ) + group._addoption( + "--randomly-use-same-seed-per-test", + action="store_false", + dest="same_seed_per_test", + default=False, + help="""Use a the same random seed for each test. Reproduces legacy behavior of pytest-randomly.""", + ) def pytest_configure(config: Config) -> None: @@ -203,9 +210,17 @@ def pytest_runtest_setup(item: Item) -> None: _reseed(item.config, -1) +def seed_from_string(string: str) -> int: + return int(hashlib.md5(string.encode()).hexdigest(), 16) + + def pytest_runtest_call(item: Item) -> None: if item.config.getoption("randomly_reset_seed"): - _reseed(item.config) + if item.config.getoption("same_seed_per_test"): + test_offset = 0 + else: + test_offset = seed_from_string(item.nodeid) + 100 + _reseed(item.config, offset=test_offset) def pytest_runtest_teardown(item: Item) -> None: diff --git a/tests/test_pytest_randomly.py b/tests/test_pytest_randomly.py index b281c69..c197bc3 100644 --- a/tests/test_pytest_randomly.py +++ b/tests/test_pytest_randomly.py @@ -78,7 +78,33 @@ def test_b(): assert test_b.num == test_a.num """ ) - out = ourtester.runpytest("--randomly-dont-reorganize") + out = ourtester.runpytest( + "--randomly-dont-reorganize", "--randomly-use-same-seed-per-test" + ) + out.assert_outcomes(passed=2, failed=0) + + +def test_it_uses_different_random_seed_per_test(ourtester): + """ + Run a pair of tests that generate a number and assert they produce + different numbers. + """ + ourtester.makepyfile( + test_one=""" + import random + + def test_a(): + test_a.num = random.random() + if hasattr(test_b, 'num'): + assert test_a.num != test_b.num + + def test_b(): + test_b.num = random.random() + if hasattr(test_a, 'num'): + assert test_b.num != test_a.num + """ + ) + out = ourtester.runpytest() out.assert_outcomes(passed=2, failed=0) @@ -601,7 +627,7 @@ def test_two(myfixture): assert random.getstate() == state_at_seed_two """ ) - args = ["--randomly-seed=2"] + args = ["--randomly-seed=2", "--randomly-use-same-seed-per-test"] out = ourtester.runpytest(*args) out.assert_outcomes(passed=2) @@ -633,7 +659,7 @@ def test_b(): """ ) - out = ourtester.runpytest("--randomly-seed=1") + out = ourtester.runpytest("--randomly-seed=1", "--randomly-use-same-seed-per-test") out.assert_outcomes(passed=2) @@ -645,10 +671,10 @@ def test_faker(ourtester): fake = Faker() def test_one(): - assert fake.name() == 'Ryan Gallagher' + assert fake.name() == 'Justin Richard' def test_two(): - assert fake.name() == 'Ryan Gallagher' + assert fake.name() == 'Tiffany Williams' """ ) @@ -692,7 +718,7 @@ def test_b(): """ ) - out = ourtester.runpytest("--randomly-seed=1") + out = ourtester.runpytest("--randomly-seed=1", "--randomly-use-same-seed-per-test") out.assert_outcomes(passed=2) @@ -702,10 +728,10 @@ def test_numpy(ourtester): import numpy as np def test_one(): - assert np.random.rand() == 0.417022004702574 + assert np.random.rand() == 0.46479378116435255 def test_two(): - assert np.random.rand() == 0.417022004702574 + assert np.random.rand() == 0.6413112443155088 """ ) @@ -765,7 +791,9 @@ def fake_entry_points(*, group): entry_points.append(_FakeEntryPoint("test_seeder", reseed)) # Need to run in-process so that monkeypatching works - pytester.runpytest_inprocess("--randomly-seed=1") + pytester.runpytest_inprocess( + "--randomly-seed=1", "--randomly-use-same-seed-per-test" + ) assert reseed.mock_calls == [ mock.call(1), mock.call(1), @@ -775,7 +803,9 @@ def fake_entry_points(*, group): ] reseed.mock_calls[:] = [] - pytester.runpytest_inprocess("--randomly-seed=424242") + pytester.runpytest_inprocess( + "--randomly-seed=424242", "--randomly-use-same-seed-per-test" + ) assert reseed.mock_calls == [ mock.call(424242), mock.call(424242), From ec9dabde42966d45af9871672b8deed3e56c846f Mon Sep 17 00:00:00 2001 From: Bryce Date: Fri, 27 Jun 2025 09:24:00 -0700 Subject: [PATCH 2/4] remove option. fix some tests --- src/pytest_randomly/__init__.py | 13 +------------ tests/test_pytest_randomly.py | 29 +++++++++++++---------------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/src/pytest_randomly/__init__.py b/src/pytest_randomly/__init__.py index 73bf0bb..6aca8e3 100644 --- a/src/pytest_randomly/__init__.py +++ b/src/pytest_randomly/__init__.py @@ -105,13 +105,6 @@ def pytest_addoption(parser: Parser) -> None: default=True, help="Stop pytest-randomly from randomly reorganizing the test order.", ) - group._addoption( - "--randomly-use-same-seed-per-test", - action="store_false", - dest="same_seed_per_test", - default=False, - help="""Use a the same random seed for each test. Reproduces legacy behavior of pytest-randomly.""", - ) def pytest_configure(config: Config) -> None: @@ -216,11 +209,7 @@ def seed_from_string(string: str) -> int: def pytest_runtest_call(item: Item) -> None: if item.config.getoption("randomly_reset_seed"): - if item.config.getoption("same_seed_per_test"): - test_offset = 0 - else: - test_offset = seed_from_string(item.nodeid) + 100 - _reseed(item.config, offset=test_offset) + _reseed(item.config, offset=seed_from_string(item.nodeid) + 100) def pytest_runtest_teardown(item: Item) -> None: diff --git a/tests/test_pytest_randomly.py b/tests/test_pytest_randomly.py index c197bc3..afd3347 100644 --- a/tests/test_pytest_randomly.py +++ b/tests/test_pytest_randomly.py @@ -58,7 +58,7 @@ def test_it_reports_a_header_when_set(simpletester): assert lines == ["Using --randomly-seed=10"] -def test_it_reuses_the_same_random_seed_per_test(ourtester): +def test_it_uses_the_different_random_seed_per_test(ourtester): """ Run a pair of tests that generate the a number and then assert they got what the other did. @@ -70,17 +70,15 @@ def test_it_reuses_the_same_random_seed_per_test(ourtester): def test_a(): test_a.num = random.random() if hasattr(test_b, 'num'): - assert test_a.num == test_b.num + assert test_a.num != test_b.num def test_b(): test_b.num = random.random() if hasattr(test_a, 'num'): - assert test_b.num == test_a.num + assert test_b.num != test_a.num """ ) - out = ourtester.runpytest( - "--randomly-dont-reorganize", "--randomly-use-same-seed-per-test" - ) + out = ourtester.runpytest("--randomly-dont-reorganize") out.assert_outcomes(passed=2, failed=0) @@ -627,7 +625,7 @@ def test_two(myfixture): assert random.getstate() == state_at_seed_two """ ) - args = ["--randomly-seed=2", "--randomly-use-same-seed-per-test"] + args = ["--randomly-seed=2"] out = ourtester.runpytest(*args) out.assert_outcomes(passed=2) @@ -640,8 +638,7 @@ def test_two(myfixture): def test_factory_boy(ourtester): """ - Rather than set up factories etc., just check the random generator it uses - is set between two tests to output the same number. + Check that the random generator factory boy uses is different between two tests """ ourtester.makepyfile( test_one=""" @@ -650,16 +647,16 @@ def test_factory_boy(ourtester): def test_a(): test_a.num = randgen.random() if hasattr(test_b, 'num'): - assert test_a.num == test_b.num + assert test_a.num != test_b.num def test_b(): test_b.num = randgen.random() if hasattr(test_a, 'num'): - assert test_b.num == test_a.num + assert test_b.num != test_a.num """ ) - out = ourtester.runpytest("--randomly-seed=1", "--randomly-use-same-seed-per-test") + out = ourtester.runpytest("--randomly-seed=1") out.assert_outcomes(passed=2) @@ -679,7 +676,7 @@ def test_two(): ) out = ourtester.runpytest("--randomly-seed=1") - out.assert_outcomes(passed=2) + out.assert_outcomes(passed=2), out.outlines def test_faker_fixture(ourtester): @@ -718,7 +715,7 @@ def test_b(): """ ) - out = ourtester.runpytest("--randomly-seed=1", "--randomly-use-same-seed-per-test") + out = ourtester.runpytest("--randomly-seed=1") out.assert_outcomes(passed=2) @@ -792,7 +789,7 @@ def fake_entry_points(*, group): # Need to run in-process so that monkeypatching works pytester.runpytest_inprocess( - "--randomly-seed=1", "--randomly-use-same-seed-per-test" + "--randomly-seed=1" ) assert reseed.mock_calls == [ mock.call(1), @@ -804,7 +801,7 @@ def fake_entry_points(*, group): reseed.mock_calls[:] = [] pytester.runpytest_inprocess( - "--randomly-seed=424242", "--randomly-use-same-seed-per-test" + "--randomly-seed=424242" ) assert reseed.mock_calls == [ mock.call(424242), From d65f271e3c63846032d6f8af4b665e19c3656619 Mon Sep 17 00:00:00 2001 From: Bryce Date: Fri, 27 Jun 2025 13:34:15 -0700 Subject: [PATCH 3/4] fix tests --- tests/test_pytest_randomly.py | 67 +++++++++++++++++++++++++++-------- tmp_pytester/test_sample.py | 27 ++++++++++++++ 2 files changed, 79 insertions(+), 15 deletions(-) create mode 100644 tmp_pytester/test_sample.py diff --git a/tests/test_pytest_randomly.py b/tests/test_pytest_randomly.py index afd3347..dcd69a6 100644 --- a/tests/test_pytest_randomly.py +++ b/tests/test_pytest_randomly.py @@ -617,23 +617,53 @@ def myfixture(): @pytest.mark.one() def test_one(myfixture): - assert random.getstate() == state_at_seed_two + # The fixture has already advanced the global PRNG once. The + # plugin then reseeds **this** test to a deterministic value that + # depends on its node-id, so the state we see here should differ + # from the module-level ``state_at_seed_two``. + assert random.getstate() != state_at_seed_two + + # Capture a deterministic value so we can check reproducibility + # from an external test run (see below). + print(f"VAL_ONE {random.random()}") @pytest.mark.two() def test_two(myfixture): - assert random.getstate() == state_at_seed_two + assert random.getstate() != state_at_seed_two + print(f"VAL_TWO {random.random()}") """ ) args = ["--randomly-seed=2"] - out = ourtester.runpytest(*args) + # First run (both tests) – capture deterministic values printed by the + # two test bodies so we can assert they are stable across subsequent + # runs. + out = ourtester.runpytest("-s", *args) out.assert_outcomes(passed=2) - out = ourtester.runpytest("-m", "one", *args) + def _extract(tag: str): + for ln in out.stdout.lines: + if tag in ln: + return float(ln.split()[-1]) + raise AssertionError(f"{tag} not found in output") + + val_one = _extract("VAL_ONE") + val_two = _extract("VAL_TWO") + + # Run each test in isolation and assert that it produces the exact same + # value – this guarantees that the per-test seeding is fully + # deterministic and does not depend on fixture execution order or the + # presence of other tests. + out = ourtester.runpytest("-s", "-m", "one", *args) out.assert_outcomes(passed=1) - out = ourtester.runpytest("-m", "two", *args) + val_one_repeat = _extract("VAL_ONE") + assert val_one_repeat == val_one + + out = ourtester.runpytest("-s", "-m", "two", *args) out.assert_outcomes(passed=1) + val_two_repeat = _extract("VAL_TWO") + assert val_two_repeat == val_two def test_factory_boy(ourtester): @@ -697,7 +727,7 @@ def test_two(faker): def test_model_bakery(ourtester): """ Rather than set up models, just check the random generator it uses is set - between two tests to output the same number. + between two tests to output different numbers. """ ourtester.makepyfile( test_one=""" @@ -706,12 +736,12 @@ def test_model_bakery(ourtester): def test_a(): test_a.num = baker_random.random() if hasattr(test_b, 'num'): - assert test_a.num == test_b.num + assert test_a.num != test_b.num def test_b(): test_b.num = baker_random.random() if hasattr(test_a, 'num'): - assert test_b.num == test_a.num + assert test_b.num != test_a.num """ ) @@ -787,27 +817,34 @@ def fake_entry_points(*, group): reseed = mock.Mock() entry_points.append(_FakeEntryPoint("test_seeder", reseed)) - # Need to run in-process so that monkeypatching works + # Ensure the cache is cleared so that our fake entry point list is picked + # up by the plugin when the inner pytest run starts. + pytest_randomly.entrypoint_reseeds = None + pytester.runpytest_inprocess( "--randomly-seed=1" ) + expected_node_seed = 1 + pytest_randomly.seed_from_string("test_one.py::test_one") + 100 + assert reseed.mock_calls == [ - mock.call(1), - mock.call(1), - mock.call(0), - mock.call(1), - mock.call(2), + mock.call(1), # pytest_report_header + mock.call(1), # pytest_collection_modifyitems + mock.call(0), # pytest_runtest_setup (-1) + mock.call(expected_node_seed), # pytest_runtest_call (unique per test) + mock.call(2), # pytest_runtest_teardown (+1) ] reseed.mock_calls[:] = [] pytester.runpytest_inprocess( "--randomly-seed=424242" ) + expected_node_seed = 424242 + pytest_randomly.seed_from_string("test_one.py::test_one") + 100 + assert reseed.mock_calls == [ mock.call(424242), mock.call(424242), mock.call(424241), - mock.call(424242), + mock.call(expected_node_seed), mock.call(424243), ] diff --git a/tmp_pytester/test_sample.py b/tmp_pytester/test_sample.py new file mode 100644 index 0000000..481a0b8 --- /dev/null +++ b/tmp_pytester/test_sample.py @@ -0,0 +1,27 @@ + +import pytest +from unittest import mock +import pytest_randomly + +class _FakeEntryPoint: + def __init__(self, name: str, obj: mock.Mock): + self.name = name + self._obj = obj + def load(self): + print('load called for', self.name) + return self._obj + +def test_run(pytester): + (pytester.path/ 'test_one.py').write_text('def test_one(): pass\n') + entry_points = [] + def fake_entry_points(*, group): + print('fake entry_points called with group', group) + return entry_points + import pytest_randomly + pytest_randomly.entrypoint_reseeds = None + pytest_randomly.entry_points = fake_entry_points + reseed = mock.Mock() + entry_points.append(_FakeEntryPoint('test', reseed)) + result = pytester.runpytest_inprocess('--randomly-seed=1') + print('mock calls', reseed.mock_calls) + assert result.ret == 0 From 62c9c68e76c3fbdad4699b937d415b76538de470 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 27 Jun 2025 20:34:54 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_pytest_randomly.py | 12 ++++++------ tmp_pytester/test_sample.py | 21 +++++++++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/tests/test_pytest_randomly.py b/tests/test_pytest_randomly.py index dcd69a6..8ec4f52 100644 --- a/tests/test_pytest_randomly.py +++ b/tests/test_pytest_randomly.py @@ -821,10 +821,10 @@ def fake_entry_points(*, group): # up by the plugin when the inner pytest run starts. pytest_randomly.entrypoint_reseeds = None - pytester.runpytest_inprocess( - "--randomly-seed=1" + pytester.runpytest_inprocess("--randomly-seed=1") + expected_node_seed = ( + 1 + pytest_randomly.seed_from_string("test_one.py::test_one") + 100 ) - expected_node_seed = 1 + pytest_randomly.seed_from_string("test_one.py::test_one") + 100 assert reseed.mock_calls == [ mock.call(1), # pytest_report_header @@ -835,10 +835,10 @@ def fake_entry_points(*, group): ] reseed.mock_calls[:] = [] - pytester.runpytest_inprocess( - "--randomly-seed=424242" + pytester.runpytest_inprocess("--randomly-seed=424242") + expected_node_seed = ( + 424242 + pytest_randomly.seed_from_string("test_one.py::test_one") + 100 ) - expected_node_seed = 424242 + pytest_randomly.seed_from_string("test_one.py::test_one") + 100 assert reseed.mock_calls == [ mock.call(424242), diff --git a/tmp_pytester/test_sample.py b/tmp_pytester/test_sample.py index 481a0b8..1ce3b16 100644 --- a/tmp_pytester/test_sample.py +++ b/tmp_pytester/test_sample.py @@ -1,27 +1,32 @@ +from __future__ import annotations -import pytest from unittest import mock + import pytest_randomly + class _FakeEntryPoint: def __init__(self, name: str, obj: mock.Mock): self.name = name self._obj = obj + def load(self): - print('load called for', self.name) + print("load called for", self.name) return self._obj + def test_run(pytester): - (pytester.path/ 'test_one.py').write_text('def test_one(): pass\n') + (pytester.path / "test_one.py").write_text("def test_one(): pass\n") entry_points = [] + def fake_entry_points(*, group): - print('fake entry_points called with group', group) + print("fake entry_points called with group", group) return entry_points - import pytest_randomly + pytest_randomly.entrypoint_reseeds = None pytest_randomly.entry_points = fake_entry_points reseed = mock.Mock() - entry_points.append(_FakeEntryPoint('test', reseed)) - result = pytester.runpytest_inprocess('--randomly-seed=1') - print('mock calls', reseed.mock_calls) + entry_points.append(_FakeEntryPoint("test", reseed)) + result = pytester.runpytest_inprocess("--randomly-seed=1") + print("mock calls", reseed.mock_calls) assert result.ret == 0