From 5dd1eb4f4b6183f37faa29c0159c553c365f4ec1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Sep 2025 04:48:09 +0000 Subject: [PATCH 1/6] Initial plan From 857c41ea7e984c5bd80d19bc23470e43c4441959 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Sep 2025 05:05:08 +0000 Subject: [PATCH 2/6] Implement config randomization feature for runbms Co-authored-by: caizixian <2891235+caizixian@users.noreply.github.com> --- src/running/command/runbms.py | 17 ++++++++++++++++- tests/test_runbms.py | 18 +++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/running/command/runbms.py b/src/running/command/runbms.py index 35dbf03..126296a 100644 --- a/src/running/command/runbms.py +++ b/src/running/command/runbms.py @@ -32,6 +32,7 @@ import math import yaml from collections import defaultdict +import random if TYPE_CHECKING: from running.plugin.runbms import RunbmsPlugin @@ -43,6 +44,7 @@ skip_oom: Optional[int] skip_timeout: Optional[int] skip_log_compression: bool = False +randomize_configs: bool = False plugins: Dict[str, Any] resume: Optional[str] @@ -65,6 +67,10 @@ def setup_parser(subparsers): f.add_argument( "--skip-log-compression", action="store_true", help="Skip compressing log files" ) + f.add_argument( + "--randomize-configs", action="store_true", + help="Randomize the order of configs for each benchmark run" + ) def getid() -> str: @@ -279,7 +285,14 @@ def run_one_benchmark( for p in plugins.values(): p.start_invocation(hfac, size, bm, i) print(i, end="", flush=True) - for j, c in enumerate(configs): + + # Create order for configs - randomized if flag is set, otherwise sequential + config_indices = list(range(len(configs))) + if randomize_configs: + random.shuffle(config_indices) + + for j in config_indices: + c = configs[j] config_passed = False for p in plugins.values(): p.start_config(hfac, size, bm, i, c, j) @@ -412,6 +425,8 @@ def run(args): skip_timeout = args.get("skip_timeout") global skip_log_compression skip_log_compression = args.get("skip_log_compression") + global randomize_configs + randomize_configs = args.get("randomize_configs") # Load from configuration file global configuration configuration = Configuration.from_file(Path(os.getcwd()), args.get("CONFIG")) diff --git a/tests/test_runbms.py b/tests/test_runbms.py index 25057cb..7c41f47 100644 --- a/tests/test_runbms.py +++ b/tests/test_runbms.py @@ -1,5 +1,6 @@ -from running.command.runbms import spread +from running.command.runbms import spread, setup_parser import pytest +import argparse def test_spread_0(): @@ -18,3 +19,18 @@ def test_spread_1(): ) right = pytest.approx(1 + (i - 1) / 7) assert left == right + + +def test_randomize_configs_arg_parsing(): + """Test that --randomize-configs argument is parsed correctly""" + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + setup_parser(subparsers) + + # Test without the flag + args = parser.parse_args(['runbms', '/tmp/log', '/tmp/config.yml']) + assert args.randomize_configs == False + + # Test with the flag + args = parser.parse_args(['runbms', '--randomize-configs', '/tmp/log', '/tmp/config.yml']) + assert args.randomize_configs == True From 438a7ac9c059d1b610109e489bdfca17759c4cfe Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Sep 2025 05:12:30 +0000 Subject: [PATCH 3/6] Add tests for config randomization functionality Co-authored-by: caizixian <2891235+caizixian@users.noreply.github.com> --- tests/test_runbms.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_runbms.py b/tests/test_runbms.py index 7c41f47..f827020 100644 --- a/tests/test_runbms.py +++ b/tests/test_runbms.py @@ -1,6 +1,7 @@ from running.command.runbms import spread, setup_parser import pytest import argparse +import random def test_spread_0(): @@ -34,3 +35,30 @@ def test_randomize_configs_arg_parsing(): # Test with the flag args = parser.parse_args(['runbms', '--randomize-configs', '/tmp/log', '/tmp/config.yml']) assert args.randomize_configs == True + + +def test_config_randomization_logic(): + """Test that the config randomization logic works as expected""" + # Test the randomization logic independently + configs = ["config1", "config2", "config3", "config4", "config5"] + + # When randomize_configs is False, order should be preserved + config_indices = list(range(len(configs))) + # No shuffling should occur + original_order = config_indices.copy() + assert config_indices == original_order + + # When randomize_configs is True, shuffling should occur + # Test multiple times to make sure we get different orders (statistically) + config_indices = list(range(len(configs))) + different_orders = 0 + random.seed(42) # Set seed for reproducible test + + for _ in range(10): + test_indices = list(range(len(configs))) + random.shuffle(test_indices) + if test_indices != original_order: + different_orders += 1 + + # With 5 configs shuffled 10 times, we should get at least some different orders + assert different_orders > 0, "Shuffling should produce different orders" From c8d2bcc3a39e5a650532ab60a067146a2d654ae0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Sep 2025 05:21:53 +0000 Subject: [PATCH 4/6] Format code with black Co-authored-by: caizixian <2891235+caizixian@users.noreply.github.com> --- src/running/command/runbms.py | 9 +++++---- tests/test_runbms.py | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/running/command/runbms.py b/src/running/command/runbms.py index 126296a..d2d37ff 100644 --- a/src/running/command/runbms.py +++ b/src/running/command/runbms.py @@ -68,8 +68,9 @@ def setup_parser(subparsers): "--skip-log-compression", action="store_true", help="Skip compressing log files" ) f.add_argument( - "--randomize-configs", action="store_true", - help="Randomize the order of configs for each benchmark run" + "--randomize-configs", + action="store_true", + help="Randomize the order of configs for each benchmark run", ) @@ -285,12 +286,12 @@ def run_one_benchmark( for p in plugins.values(): p.start_invocation(hfac, size, bm, i) print(i, end="", flush=True) - + # Create order for configs - randomized if flag is set, otherwise sequential config_indices = list(range(len(configs))) if randomize_configs: random.shuffle(config_indices) - + for j in config_indices: c = configs[j] config_passed = False diff --git a/tests/test_runbms.py b/tests/test_runbms.py index f827020..10b583a 100644 --- a/tests/test_runbms.py +++ b/tests/test_runbms.py @@ -27,13 +27,15 @@ def test_randomize_configs_arg_parsing(): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers() setup_parser(subparsers) - + # Test without the flag - args = parser.parse_args(['runbms', '/tmp/log', '/tmp/config.yml']) + args = parser.parse_args(["runbms", "/tmp/log", "/tmp/config.yml"]) assert args.randomize_configs == False - + # Test with the flag - args = parser.parse_args(['runbms', '--randomize-configs', '/tmp/log', '/tmp/config.yml']) + args = parser.parse_args( + ["runbms", "--randomize-configs", "/tmp/log", "/tmp/config.yml"] + ) assert args.randomize_configs == True @@ -41,24 +43,24 @@ def test_config_randomization_logic(): """Test that the config randomization logic works as expected""" # Test the randomization logic independently configs = ["config1", "config2", "config3", "config4", "config5"] - + # When randomize_configs is False, order should be preserved config_indices = list(range(len(configs))) # No shuffling should occur original_order = config_indices.copy() assert config_indices == original_order - + # When randomize_configs is True, shuffling should occur # Test multiple times to make sure we get different orders (statistically) config_indices = list(range(len(configs))) different_orders = 0 random.seed(42) # Set seed for reproducible test - + for _ in range(10): test_indices = list(range(len(configs))) random.shuffle(test_indices) if test_indices != original_order: different_orders += 1 - + # With 5 configs shuffled 10 times, we should get at least some different orders assert different_orders > 0, "Shuffling should produce different orders" From 150c060eccfbaf576b18b104f8408c03e4aeec40 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Sep 2025 05:27:05 +0000 Subject: [PATCH 5/6] Update changelog and documentation for --randomize-configs flag Co-authored-by: caizixian <2891235+caizixian@users.noreply.github.com> --- docs/src/changelog.md | 2 ++ docs/src/commands/runbms.md | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/src/changelog.md b/docs/src/changelog.md index 2e7db56..1b16e2f 100644 --- a/docs/src/changelog.md +++ b/docs/src/changelog.md @@ -1,6 +1,8 @@ # Changelog ## Unreleased ### Added +#### Commands +- `runbms`: `--randomize-configs` (preview ⚠️) randomizes the order of configs for each invocation to help distinguish between system-related noise and configuration-specific issues. ### Changed diff --git a/docs/src/commands/runbms.md b/docs/src/commands/runbms.md index 21d5b2d..661176f 100644 --- a/docs/src/commands/runbms.md +++ b/docs/src/commands/runbms.md @@ -3,7 +3,7 @@ This subcommand runs benchmarks with different configs, possibly with varying he ## Usage ```console -runbms [-h|--help] [-i|--invocations INVOCATIONS] [-s|--slice SLICE] [-p|--id-prefix ID_PREFIX] [-m|--minheap-multiplier MINHEAP_MULTIPLIER] [--skip-oom SKIP_OOM] [--skip-timeout SKIP_TIMEOUT] [--resume RESUME] [--workdir WORKDIR] [--skip-log-compression] LOG_DIR CONFIG [N] [n ...] +runbms [-h|--help] [-i|--invocations INVOCATIONS] [-s|--slice SLICE] [-p|--id-prefix ID_PREFIX] [-m|--minheap-multiplier MINHEAP_MULTIPLIER] [--skip-oom SKIP_OOM] [--skip-timeout SKIP_TIMEOUT] [--resume RESUME] [--workdir WORKDIR] [--skip-log-compression] [--randomize-configs] LOG_DIR CONFIG [N] [n ...] ``` `-h`: print help message. @@ -35,6 +35,8 @@ If not specified, a temporary directory will be created under an OS-dependent lo `--skip-log-compression`: skip compressing log file as gzip. +`--randomize-configs` (preview ⚠️): randomize the order of configs for each invocation to help distinguish between system-related noise and configuration-specific issues. + `LOG_DIR`: where to store the results. This is required. From d9025f9a2643c5418ec687e888a415f474bdc89c Mon Sep 17 00:00:00 2001 From: Zixian Cai Date: Mon, 29 Sep 2025 15:55:27 +1000 Subject: [PATCH 6/6] Minor --- docs/src/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/changelog.md b/docs/src/changelog.md index 1b16e2f..df57e42 100644 --- a/docs/src/changelog.md +++ b/docs/src/changelog.md @@ -2,7 +2,7 @@ ## Unreleased ### Added #### Commands -- `runbms`: `--randomize-configs` (preview ⚠️) randomizes the order of configs for each invocation to help distinguish between system-related noise and configuration-specific issues. +- `runbms` gains an extra argument, `--randomize-configs`, to randomize the order of configs for each invocation to help distinguish between system-related noise and configuration-specific issues. ### Changed