Add --randomize-configs flag to runbms for noise/spike analysis

docs/src/changelog.md

@@ -1,6 +1,8 @@
 # Changelog
 ## Unreleased
 ### Added
+#### Commands
+- `runbms` gains an extra argument, `--randomize-configs`, to randomize the order of configs for each invocation to help distinguish between system-related noise and configuration-specific issues.
 
 ### Changed
 

docs/src/commands/runbms.md

@@ -3,7 +3,7 @@ This subcommand runs benchmarks with different configs, possibly with varying he
 
 ## Usage
 ```console
-runbms [-h|--help] [-i|--invocations INVOCATIONS] [-s|--slice SLICE] [-p|--id-prefix ID_PREFIX] [-m|--minheap-multiplier MINHEAP_MULTIPLIER] [--skip-oom SKIP_OOM] [--skip-timeout SKIP_TIMEOUT] [--resume RESUME] [--workdir WORKDIR] [--skip-log-compression] LOG_DIR CONFIG [N] [n ...]
+runbms [-h|--help] [-i|--invocations INVOCATIONS] [-s|--slice SLICE] [-p|--id-prefix ID_PREFIX] [-m|--minheap-multiplier MINHEAP_MULTIPLIER] [--skip-oom SKIP_OOM] [--skip-timeout SKIP_TIMEOUT] [--resume RESUME] [--workdir WORKDIR] [--skip-log-compression] [--randomize-configs] LOG_DIR CONFIG [N] [n ...]
 ```
 
 `-h`: print help message.
@@ -35,6 +35,8 @@ If not specified, a temporary directory will be created under an OS-dependent lo
 
 `--skip-log-compression`: skip compressing log file as gzip.
 
+`--randomize-configs` (preview ⚠️): randomize the order of configs for each invocation to help distinguish between system-related noise and configuration-specific issues.
+
 `LOG_DIR`: where to store the results.
 This is required.
 

src/running/command/runbms.py

@@ -32,6 +32,7 @@
 import math
 import yaml
 from collections import defaultdict
+import random
 
 if TYPE_CHECKING:
     from running.plugin.runbms import RunbmsPlugin
@@ -43,6 +44,7 @@
 skip_oom: Optional[int]
 skip_timeout: Optional[int]
 skip_log_compression: bool = False
+randomize_configs: bool = False
 plugins: Dict[str, Any]
 resume: Optional[str]
 
@@ -65,6 +67,11 @@ def setup_parser(subparsers):
     f.add_argument(
         "--skip-log-compression", action="store_true", help="Skip compressing log files"
     )
+    f.add_argument(
+        "--randomize-configs",
+        action="store_true",
+        help="Randomize the order of configs for each benchmark run",
+    )
 
 
 def getid() -> str:
@@ -279,7 +286,14 @@ def run_one_benchmark(
         for p in plugins.values():
             p.start_invocation(hfac, size, bm, i)
         print(i, end="", flush=True)
-        for j, c in enumerate(configs):
+
+        # Create order for configs - randomized if flag is set, otherwise sequential
+        config_indices = list(range(len(configs)))
+        if randomize_configs:
+            random.shuffle(config_indices)
+
+        for j in config_indices:
+            c = configs[j]
             config_passed = False
             for p in plugins.values():
                 p.start_config(hfac, size, bm, i, c, j)
@@ -412,6 +426,8 @@ def run(args):
         skip_timeout = args.get("skip_timeout")
         global skip_log_compression
         skip_log_compression = args.get("skip_log_compression")
+        global randomize_configs
+        randomize_configs = args.get("randomize_configs")
         # Load from configuration file
         global configuration
         configuration = Configuration.from_file(Path(os.getcwd()), args.get("CONFIG"))

tests/test_runbms.py

@@ -1,5 +1,7 @@
-from running.command.runbms import spread
+from running.command.runbms import spread, setup_parser
 import pytest
+import argparse
+import random
 
 
 def test_spread_0():
@@ -18,3 +20,47 @@ def test_spread_1():
         )
         right = pytest.approx(1 + (i - 1) / 7)
         assert left == right
+
+
+def test_randomize_configs_arg_parsing():
+    """Test that --randomize-configs argument is parsed correctly"""
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+    setup_parser(subparsers)
+
+    # Test without the flag
+    args = parser.parse_args(["runbms", "/tmp/log", "/tmp/config.yml"])
+    assert args.randomize_configs == False
+
+    # Test with the flag
+    args = parser.parse_args(
+        ["runbms", "--randomize-configs", "/tmp/log", "/tmp/config.yml"]
+    )
+    assert args.randomize_configs == True
+
+
+def test_config_randomization_logic():
+    """Test that the config randomization logic works as expected"""
+    # Test the randomization logic independently
+    configs = ["config1", "config2", "config3", "config4", "config5"]
+
+    # When randomize_configs is False, order should be preserved
+    config_indices = list(range(len(configs)))
+    # No shuffling should occur
+    original_order = config_indices.copy()
+    assert config_indices == original_order
+
+    # When randomize_configs is True, shuffling should occur
+    # Test multiple times to make sure we get different orders (statistically)
+    config_indices = list(range(len(configs)))
+    different_orders = 0
+    random.seed(42)  # Set seed for reproducible test
+
+    for _ in range(10):
+        test_indices = list(range(len(configs)))
+        random.shuffle(test_indices)
+        if test_indices != original_order:
+            different_orders += 1
+
+    # With 5 configs shuffled 10 times, we should get at least some different orders
+    assert different_orders > 0, "Shuffling should produce different orders"