From 9382133bc4dfdd59c037c490bf3f160411357ef6 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sun, 27 Jul 2025 03:17:59 +0000
Subject: [PATCH 1/9] Initial plan


From a6dfe49ed2bf37eea002e9687f8ce14811dd295a Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sun, 27 Jul 2025 03:30:31 +0000
Subject: [PATCH 2/9] Add --exit-on-failure flag to runbms for CI use

Co-authored-by: caizixian <2891235+caizixian@users.noreply.github.com>
---
 docs/src/commands/runbms.md   |  6 +++++-
 src/running/command/runbms.py | 23 +++++++++++++++++++++++
 tests/test_runbms.py          | 33 +++++++++++++++++++++++++++++++++
 3 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/docs/src/commands/runbms.md b/docs/src/commands/runbms.md
index 21d5b2d..fa6a5b9 100644
--- a/docs/src/commands/runbms.md
+++ b/docs/src/commands/runbms.md
@@ -3,7 +3,7 @@ This subcommand runs benchmarks with different configs, possibly with varying he
 
 ## Usage
 ```console
-runbms [-h|--help] [-i|--invocations INVOCATIONS] [-s|--slice SLICE] [-p|--id-prefix ID_PREFIX] [-m|--minheap-multiplier MINHEAP_MULTIPLIER] [--skip-oom SKIP_OOM] [--skip-timeout SKIP_TIMEOUT] [--resume RESUME] [--workdir WORKDIR] [--skip-log-compression] LOG_DIR CONFIG [N] [n ...]
+runbms [-h|--help] [-i|--invocations INVOCATIONS] [-s|--slice SLICE] [-p|--id-prefix ID_PREFIX] [-m|--minheap-multiplier MINHEAP_MULTIPLIER] [--skip-oom SKIP_OOM] [--skip-timeout SKIP_TIMEOUT] [--resume RESUME] [--workdir WORKDIR] [--skip-log-compression] [--exit-on-failure] LOG_DIR CONFIG [N] [n ...]
 ```
 
 `-h`: print help message.
@@ -35,6 +35,10 @@ If not specified, a temporary directory will be created under an OS-dependent lo
 
 `--skip-log-compression`: skip compressing log file as gzip.
 
+`--exit-on-failure`: exit with code 1 if any configuration fails.
+This is useful for CI environments where you need to detect failed runs without parsing the output.
+By default, `runbms` exits with code 0 even when some configurations fail.
+
 `LOG_DIR`: where to store the results.
 This is required.
 
diff --git a/src/running/command/runbms.py b/src/running/command/runbms.py
index 35dbf03..4f8d685 100644
--- a/src/running/command/runbms.py
+++ b/src/running/command/runbms.py
@@ -32,6 +32,7 @@
 import math
 import yaml
 from collections import defaultdict
+import sys
 
 if TYPE_CHECKING:
     from running.plugin.runbms import RunbmsPlugin
@@ -45,6 +46,8 @@
 skip_log_compression: bool = False
 plugins: Dict[str, Any]
 resume: Optional[str]
+exit_on_failure: bool = False
+any_config_failed: bool = False
 
 
 def setup_parser(subparsers):
@@ -65,6 +68,9 @@ def setup_parser(subparsers):
     f.add_argument(
         "--skip-log-compression", action="store_true", help="Skip compressing log files"
     )
+    f.add_argument(
+        "--exit-on-failure", action="store_true", help="Exit with code 1 if any configuration fails"
+    )
 
 
 def getid() -> str:
@@ -285,9 +291,12 @@ def run_one_benchmark(
                 p.start_config(hfac, size, bm, i, c, j)
             if skip_oom is not None and oomed_count[c] >= skip_oom:
                 print(".", end="", flush=True)
+                global any_config_failed
+                any_config_failed = True
                 continue
             if skip_timeout is not None and timeout_count[c] >= skip_timeout:
                 print(".", end="", flush=True)
+                any_config_failed = True
                 continue
             if resume:
                 log_filename_completed = get_filename_completed(bm, hfac, size, c)
@@ -314,16 +323,21 @@ def run_one_benchmark(
             if exit_status is SubprocessrExit.Timeout:
                 timeout_count[c] += 1
                 print(".", end="", flush=True)
+                any_config_failed = True
             elif exit_status is SubprocessrExit.Error:
                 print(".", end="", flush=True)
+                any_config_failed = True
             elif exit_status is SubprocessrExit.Normal:
                 if suite.is_passed(output):
                     config_passed = True
                     print(config_index_to_chr(j), end="", flush=True)
                 else:
                     print(".", end="", flush=True)
+                    any_config_failed = True
             elif exit_status is SubprocessrExit.Dryrun:
                 print(".", end="", flush=True)
+                # In dry-run mode, treat as failure for exit-on-failure purposes
+                any_config_failed = True
             else:
                 raise ValueError("Not a valid SubprocessrExit value")
             for p in plugins.values():
@@ -412,6 +426,8 @@ def run(args):
         skip_timeout = args.get("skip_timeout")
         global skip_log_compression
         skip_log_compression = args.get("skip_log_compression")
+        global exit_on_failure
+        exit_on_failure = args.get("exit_on_failure", False)
         # Load from configuration file
         global configuration
         configuration = Configuration.from_file(Path(os.getcwd()), args.get("CONFIG"))
@@ -474,6 +490,9 @@ def run(args):
                 Path(runbms_dir),
                 log_dir,
             )
+            # Check if we need to exit with failure code
+            if exit_on_failure and any_config_failed:
+                sys.exit(1)
             # early return
             return True
 
@@ -529,4 +548,8 @@ def run_N_ns(N, ns):
             else:
                 run_N_ns(N, ns)
 
+        # Check if we need to exit with failure code
+        if exit_on_failure and any_config_failed:
+            sys.exit(1)
+
         return True
diff --git a/tests/test_runbms.py b/tests/test_runbms.py
index 25057cb..6b05721 100644
--- a/tests/test_runbms.py
+++ b/tests/test_runbms.py
@@ -18,3 +18,36 @@ def test_spread_1():
         )
         right = pytest.approx(1 + (i - 1) / 7)
         assert left == right
+
+
+def test_exit_on_failure_flag_available():
+    """Test that the --exit-on-failure flag is available in the argument parser."""
+    from running.command.runbms import setup_parser
+    import argparse
+    
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+    setup_parser(subparsers)
+    
+    # Test that the flag is recognized and sets the correct default value
+    args = parser.parse_args(['runbms', '/tmp/logs', '/tmp/config.yml'])
+    assert hasattr(args, 'exit_on_failure')
+    assert args.exit_on_failure is False
+    
+    # Test that the flag can be set
+    args = parser.parse_args(['runbms', '--exit-on-failure', '/tmp/logs', '/tmp/config.yml'])
+    assert args.exit_on_failure is True
+
+
+def test_global_variables_initialization():
+    """Test that the new global variables are properly initialized."""
+    from running.command import runbms
+    
+    # Test that the new global variables exist
+    assert hasattr(runbms, 'exit_on_failure')
+    assert hasattr(runbms, 'any_config_failed')
+    
+    # Test default values (these are module-level globals)
+    # Note: These might be modified by other tests, so we just check they exist
+    assert isinstance(runbms.exit_on_failure, bool)
+    assert isinstance(runbms.any_config_failed, bool)

From 503704132a00cb233ca580646e38bb8b18c82831 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sun, 27 Jul 2025 03:40:52 +0000
Subject: [PATCH 3/9] Implement immediate exit on failure for --exit-on-failure
 flag

Co-authored-by: caizixian <2891235+caizixian@users.noreply.github.com>
---
 src/running/command/runbms.py | 12 ++++++++++++
 tests/test_runbms.py          | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/src/running/command/runbms.py b/src/running/command/runbms.py
index 4f8d685..a008bcf 100644
--- a/src/running/command/runbms.py
+++ b/src/running/command/runbms.py
@@ -293,10 +293,14 @@ def run_one_benchmark(
                 print(".", end="", flush=True)
                 global any_config_failed
                 any_config_failed = True
+                if exit_on_failure:
+                    sys.exit(1)
                 continue
             if skip_timeout is not None and timeout_count[c] >= skip_timeout:
                 print(".", end="", flush=True)
                 any_config_failed = True
+                if exit_on_failure:
+                    sys.exit(1)
                 continue
             if resume:
                 log_filename_completed = get_filename_completed(bm, hfac, size, c)
@@ -324,9 +328,13 @@ def run_one_benchmark(
                 timeout_count[c] += 1
                 print(".", end="", flush=True)
                 any_config_failed = True
+                if exit_on_failure:
+                    sys.exit(1)
             elif exit_status is SubprocessrExit.Error:
                 print(".", end="", flush=True)
                 any_config_failed = True
+                if exit_on_failure:
+                    sys.exit(1)
             elif exit_status is SubprocessrExit.Normal:
                 if suite.is_passed(output):
                     config_passed = True
@@ -334,10 +342,14 @@ def run_one_benchmark(
                 else:
                     print(".", end="", flush=True)
                     any_config_failed = True
+                    if exit_on_failure:
+                        sys.exit(1)
             elif exit_status is SubprocessrExit.Dryrun:
                 print(".", end="", flush=True)
                 # In dry-run mode, treat as failure for exit-on-failure purposes
                 any_config_failed = True
+                if exit_on_failure:
+                    sys.exit(1)
             else:
                 raise ValueError("Not a valid SubprocessrExit value")
             for p in plugins.values():
diff --git a/tests/test_runbms.py b/tests/test_runbms.py
index 6b05721..5e064ea 100644
--- a/tests/test_runbms.py
+++ b/tests/test_runbms.py
@@ -51,3 +51,35 @@ def test_global_variables_initialization():
     # Note: These might be modified by other tests, so we just check they exist
     assert isinstance(runbms.exit_on_failure, bool)
     assert isinstance(runbms.any_config_failed, bool)
+
+
+def test_exit_on_failure_immediate_exit():
+    """Test that sys.exit(1) is called immediately when exit_on_failure is True and a failure occurs."""
+    from running.command import runbms
+    import sys
+    from unittest.mock import patch
+    
+    # Save original values
+    original_exit_on_failure = runbms.exit_on_failure
+    original_any_config_failed = runbms.any_config_failed
+    
+    try:
+        # Set up the condition for immediate exit
+        runbms.exit_on_failure = True
+        runbms.any_config_failed = False
+        
+        # Mock sys.exit to verify it's called
+        with patch.object(sys, 'exit') as mock_exit:
+            # Simulate the code path where any_config_failed is set and immediate exit should occur
+            # This simulates what happens in run_one_benchmark when a failure is detected
+            runbms.any_config_failed = True
+            if runbms.exit_on_failure:
+                sys.exit(1)
+            
+            # Verify that sys.exit(1) was called
+            mock_exit.assert_called_once_with(1)
+    
+    finally:
+        # Restore original values
+        runbms.exit_on_failure = original_exit_on_failure
+        runbms.any_config_failed = original_any_config_failed

From daf0f96d46c46cfb6975e20045c4dcf3d8c7ad5d Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sun, 27 Jul 2025 03:50:24 +0000
Subject: [PATCH 4/9] Clean up any_config_failed global flag as it's no longer
 needed

Co-authored-by: caizixian <2891235+caizixian@users.noreply.github.com>
---
 src/running/command/runbms.py | 19 +++--------------
 tests/test_runbms.py          | 40 ++++++++++++++++-------------------
 2 files changed, 21 insertions(+), 38 deletions(-)

diff --git a/src/running/command/runbms.py b/src/running/command/runbms.py
index a008bcf..9733e7f 100644
--- a/src/running/command/runbms.py
+++ b/src/running/command/runbms.py
@@ -47,7 +47,6 @@
 plugins: Dict[str, Any]
 resume: Optional[str]
 exit_on_failure: bool = False
-any_config_failed: bool = False
 
 
 def setup_parser(subparsers):
@@ -69,7 +68,9 @@ def setup_parser(subparsers):
         "--skip-log-compression", action="store_true", help="Skip compressing log files"
     )
     f.add_argument(
-        "--exit-on-failure", action="store_true", help="Exit with code 1 if any configuration fails"
+        "--exit-on-failure",
+        action="store_true",
+        help="Exit with code 1 if any configuration fails",
     )
 
 
@@ -291,14 +292,11 @@ def run_one_benchmark(
                 p.start_config(hfac, size, bm, i, c, j)
             if skip_oom is not None and oomed_count[c] >= skip_oom:
                 print(".", end="", flush=True)
-                global any_config_failed
-                any_config_failed = True
                 if exit_on_failure:
                     sys.exit(1)
                 continue
             if skip_timeout is not None and timeout_count[c] >= skip_timeout:
                 print(".", end="", flush=True)
-                any_config_failed = True
                 if exit_on_failure:
                     sys.exit(1)
                 continue
@@ -327,12 +325,10 @@ def run_one_benchmark(
             if exit_status is SubprocessrExit.Timeout:
                 timeout_count[c] += 1
                 print(".", end="", flush=True)
-                any_config_failed = True
                 if exit_on_failure:
                     sys.exit(1)
             elif exit_status is SubprocessrExit.Error:
                 print(".", end="", flush=True)
-                any_config_failed = True
                 if exit_on_failure:
                     sys.exit(1)
             elif exit_status is SubprocessrExit.Normal:
@@ -341,13 +337,11 @@ def run_one_benchmark(
                     print(config_index_to_chr(j), end="", flush=True)
                 else:
                     print(".", end="", flush=True)
-                    any_config_failed = True
                     if exit_on_failure:
                         sys.exit(1)
             elif exit_status is SubprocessrExit.Dryrun:
                 print(".", end="", flush=True)
                 # In dry-run mode, treat as failure for exit-on-failure purposes
-                any_config_failed = True
                 if exit_on_failure:
                     sys.exit(1)
             else:
@@ -502,9 +496,6 @@ def run(args):
                 Path(runbms_dir),
                 log_dir,
             )
-            # Check if we need to exit with failure code
-            if exit_on_failure and any_config_failed:
-                sys.exit(1)
             # early return
             return True
 
@@ -560,8 +551,4 @@ def run_N_ns(N, ns):
             else:
                 run_N_ns(N, ns)
 
-        # Check if we need to exit with failure code
-        if exit_on_failure and any_config_failed:
-            sys.exit(1)
-
         return True
diff --git a/tests/test_runbms.py b/tests/test_runbms.py
index 5e064ea..7064ead 100644
--- a/tests/test_runbms.py
+++ b/tests/test_runbms.py
@@ -24,33 +24,33 @@ def test_exit_on_failure_flag_available():
     """Test that the --exit-on-failure flag is available in the argument parser."""
     from running.command.runbms import setup_parser
     import argparse
-    
+
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers()
     setup_parser(subparsers)
-    
+
     # Test that the flag is recognized and sets the correct default value
-    args = parser.parse_args(['runbms', '/tmp/logs', '/tmp/config.yml'])
-    assert hasattr(args, 'exit_on_failure')
+    args = parser.parse_args(["runbms", "/tmp/logs", "/tmp/config.yml"])
+    assert hasattr(args, "exit_on_failure")
     assert args.exit_on_failure is False
-    
+
     # Test that the flag can be set
-    args = parser.parse_args(['runbms', '--exit-on-failure', '/tmp/logs', '/tmp/config.yml'])
+    args = parser.parse_args(
+        ["runbms", "--exit-on-failure", "/tmp/logs", "/tmp/config.yml"]
+    )
     assert args.exit_on_failure is True
 
 
 def test_global_variables_initialization():
     """Test that the new global variables are properly initialized."""
     from running.command import runbms
-    
+
     # Test that the new global variables exist
-    assert hasattr(runbms, 'exit_on_failure')
-    assert hasattr(runbms, 'any_config_failed')
-    
+    assert hasattr(runbms, "exit_on_failure")
+
     # Test default values (these are module-level globals)
     # Note: These might be modified by other tests, so we just check they exist
     assert isinstance(runbms.exit_on_failure, bool)
-    assert isinstance(runbms.any_config_failed, bool)
 
 
 def test_exit_on_failure_immediate_exit():
@@ -58,28 +58,24 @@ def test_exit_on_failure_immediate_exit():
     from running.command import runbms
     import sys
     from unittest.mock import patch
-    
+
     # Save original values
     original_exit_on_failure = runbms.exit_on_failure
-    original_any_config_failed = runbms.any_config_failed
-    
+
     try:
         # Set up the condition for immediate exit
         runbms.exit_on_failure = True
-        runbms.any_config_failed = False
-        
+
         # Mock sys.exit to verify it's called
-        with patch.object(sys, 'exit') as mock_exit:
-            # Simulate the code path where any_config_failed is set and immediate exit should occur
+        with patch.object(sys, "exit") as mock_exit:
+            # Simulate the code path where immediate exit should occur
             # This simulates what happens in run_one_benchmark when a failure is detected
-            runbms.any_config_failed = True
             if runbms.exit_on_failure:
                 sys.exit(1)
-            
+
             # Verify that sys.exit(1) was called
             mock_exit.assert_called_once_with(1)
-    
+
     finally:
         # Restore original values
         runbms.exit_on_failure = original_exit_on_failure
-        runbms.any_config_failed = original_any_config_failed

From 0d89116f8fa11e7dc0df34bbdc8123a600586c90 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sun, 27 Jul 2025 04:00:49 +0000
Subject: [PATCH 5/9] Ignore --exit-on-failure flag in dry-run mode as
 requested

Co-authored-by: caizixian <2891235+caizixian@users.noreply.github.com>
---
 src/running/command/runbms.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/running/command/runbms.py b/src/running/command/runbms.py
index 9733e7f..3c10778 100644
--- a/src/running/command/runbms.py
+++ b/src/running/command/runbms.py
@@ -341,9 +341,6 @@ def run_one_benchmark(
                         sys.exit(1)
             elif exit_status is SubprocessrExit.Dryrun:
                 print(".", end="", flush=True)
-                # In dry-run mode, treat as failure for exit-on-failure purposes
-                if exit_on_failure:
-                    sys.exit(1)
             else:
                 raise ValueError("Not a valid SubprocessrExit value")
             for p in plugins.values():

From a03326088608b8f9666e8e2fd1908f61923294af Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sun, 27 Jul 2025 04:08:33 +0000
Subject: [PATCH 6/9] Remove useless test and update changelog for
 --exit-on-failure flag

Co-authored-by: caizixian <2891235+caizixian@users.noreply.github.com>
---
 docs/src/changelog.md |  2 ++
 tests/test_runbms.py  | 28 ----------------------------
 2 files changed, 2 insertions(+), 28 deletions(-)

diff --git a/docs/src/changelog.md b/docs/src/changelog.md
index 2e7db56..2cec3dd 100644
--- a/docs/src/changelog.md
+++ b/docs/src/changelog.md
@@ -1,6 +1,8 @@
 # Changelog
 ## Unreleased
 ### Added
+#### Commands
+- `runbms`: new `--exit-on-failure` flag to exit with code 1 when any benchmark configuration fails, making it suitable for CI environments.
 
 ### Changed
 
diff --git a/tests/test_runbms.py b/tests/test_runbms.py
index 7064ead..a709311 100644
--- a/tests/test_runbms.py
+++ b/tests/test_runbms.py
@@ -51,31 +51,3 @@ def test_global_variables_initialization():
     # Test default values (these are module-level globals)
     # Note: These might be modified by other tests, so we just check they exist
     assert isinstance(runbms.exit_on_failure, bool)
-
-
-def test_exit_on_failure_immediate_exit():
-    """Test that sys.exit(1) is called immediately when exit_on_failure is True and a failure occurs."""
-    from running.command import runbms
-    import sys
-    from unittest.mock import patch
-
-    # Save original values
-    original_exit_on_failure = runbms.exit_on_failure
-
-    try:
-        # Set up the condition for immediate exit
-        runbms.exit_on_failure = True
-
-        # Mock sys.exit to verify it's called
-        with patch.object(sys, "exit") as mock_exit:
-            # Simulate the code path where immediate exit should occur
-            # This simulates what happens in run_one_benchmark when a failure is detected
-            if runbms.exit_on_failure:
-                sys.exit(1)
-
-            # Verify that sys.exit(1) was called
-            mock_exit.assert_called_once_with(1)
-
-    finally:
-        # Restore original values
-        runbms.exit_on_failure = original_exit_on_failure

From 7dcb0dfd6fdce78e2145d287fb70f3c1ea324d9c Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Sun, 27 Jul 2025 04:27:57 +0000
Subject: [PATCH 7/9] Change --exit-on-failure flag to accept optional argument
 for custom exit code

Co-authored-by: caizixian <2891235+caizixian@users.noreply.github.com>
---
 docs/src/changelog.md         |  2 +-
 docs/src/commands/runbms.md   |  5 +++--
 src/running/command/runbms.py | 33 ++++++++++++++++++---------------
 tests/test_runbms.py          | 20 ++++++++++++++------
 4 files changed, 36 insertions(+), 24 deletions(-)

diff --git a/docs/src/changelog.md b/docs/src/changelog.md
index 2cec3dd..05ae688 100644
--- a/docs/src/changelog.md
+++ b/docs/src/changelog.md
@@ -2,7 +2,7 @@
 ## Unreleased
 ### Added
 #### Commands
-- `runbms`: new `--exit-on-failure` flag to exit with code 1 when any benchmark configuration fails, making it suitable for CI environments.
+- `runbms`: new `--exit-on-failure [CODE]` flag to exit with a specified code (default: 1) when any benchmark configuration fails, making it suitable for CI environments.
 
 ### Changed
 
diff --git a/docs/src/commands/runbms.md b/docs/src/commands/runbms.md
index fa6a5b9..65f12fa 100644
--- a/docs/src/commands/runbms.md
+++ b/docs/src/commands/runbms.md
@@ -3,7 +3,7 @@ This subcommand runs benchmarks with different configs, possibly with varying he
 
 ## Usage
 ```console
-runbms [-h|--help] [-i|--invocations INVOCATIONS] [-s|--slice SLICE] [-p|--id-prefix ID_PREFIX] [-m|--minheap-multiplier MINHEAP_MULTIPLIER] [--skip-oom SKIP_OOM] [--skip-timeout SKIP_TIMEOUT] [--resume RESUME] [--workdir WORKDIR] [--skip-log-compression] [--exit-on-failure] LOG_DIR CONFIG [N] [n ...]
+runbms [-h|--help] [-i|--invocations INVOCATIONS] [-s|--slice SLICE] [-p|--id-prefix ID_PREFIX] [-m|--minheap-multiplier MINHEAP_MULTIPLIER] [--skip-oom SKIP_OOM] [--skip-timeout SKIP_TIMEOUT] [--resume RESUME] [--workdir WORKDIR] [--skip-log-compression] [--exit-on-failure [CODE]] LOG_DIR CONFIG [N] [n ...]
 ```
 
 `-h`: print help message.
@@ -35,9 +35,10 @@ If not specified, a temporary directory will be created under an OS-dependent lo
 
 `--skip-log-compression`: skip compressing log file as gzip.
 
-`--exit-on-failure`: exit with code 1 if any configuration fails.
+`--exit-on-failure [CODE]`: exit with the specified code (default: 1) if any configuration fails.
 This is useful for CI environments where you need to detect failed runs without parsing the output.
 By default, `runbms` exits with code 0 even when some configurations fail.
+If the flag is provided without a code, it defaults to exit code 1.
 
 `LOG_DIR`: where to store the results.
 This is required.
diff --git a/src/running/command/runbms.py b/src/running/command/runbms.py
index 3c10778..67780ad 100644
--- a/src/running/command/runbms.py
+++ b/src/running/command/runbms.py
@@ -46,7 +46,7 @@
 skip_log_compression: bool = False
 plugins: Dict[str, Any]
 resume: Optional[str]
-exit_on_failure: bool = False
+exit_on_failure_code: Optional[int] = None
 
 
 def setup_parser(subparsers):
@@ -69,8 +69,11 @@ def setup_parser(subparsers):
     )
     f.add_argument(
         "--exit-on-failure",
-        action="store_true",
-        help="Exit with code 1 if any configuration fails",
+        nargs="?",
+        const=1,
+        type=int,
+        metavar="CODE",
+        help="Exit with specified code (default: 1) if any configuration fails",
     )
 
 
@@ -292,13 +295,13 @@ def run_one_benchmark(
                 p.start_config(hfac, size, bm, i, c, j)
             if skip_oom is not None and oomed_count[c] >= skip_oom:
                 print(".", end="", flush=True)
-                if exit_on_failure:
-                    sys.exit(1)
+                if exit_on_failure_code is not None:
+                    sys.exit(exit_on_failure_code)
                 continue
             if skip_timeout is not None and timeout_count[c] >= skip_timeout:
                 print(".", end="", flush=True)
-                if exit_on_failure:
-                    sys.exit(1)
+                if exit_on_failure_code is not None:
+                    sys.exit(exit_on_failure_code)
                 continue
             if resume:
                 log_filename_completed = get_filename_completed(bm, hfac, size, c)
@@ -325,20 +328,20 @@ def run_one_benchmark(
             if exit_status is SubprocessrExit.Timeout:
                 timeout_count[c] += 1
                 print(".", end="", flush=True)
-                if exit_on_failure:
-                    sys.exit(1)
+                if exit_on_failure_code is not None:
+                    sys.exit(exit_on_failure_code)
             elif exit_status is SubprocessrExit.Error:
                 print(".", end="", flush=True)
-                if exit_on_failure:
-                    sys.exit(1)
+                if exit_on_failure_code is not None:
+                    sys.exit(exit_on_failure_code)
             elif exit_status is SubprocessrExit.Normal:
                 if suite.is_passed(output):
                     config_passed = True
                     print(config_index_to_chr(j), end="", flush=True)
                 else:
                     print(".", end="", flush=True)
-                    if exit_on_failure:
-                        sys.exit(1)
+                    if exit_on_failure_code is not None:
+                        sys.exit(exit_on_failure_code)
             elif exit_status is SubprocessrExit.Dryrun:
                 print(".", end="", flush=True)
             else:
@@ -429,8 +432,8 @@ def run(args):
         skip_timeout = args.get("skip_timeout")
         global skip_log_compression
         skip_log_compression = args.get("skip_log_compression")
-        global exit_on_failure
-        exit_on_failure = args.get("exit_on_failure", False)
+        global exit_on_failure_code
+        exit_on_failure_code = args.get("exit_on_failure")
         # Load from configuration file
         global configuration
         configuration = Configuration.from_file(Path(os.getcwd()), args.get("CONFIG"))
diff --git a/tests/test_runbms.py b/tests/test_runbms.py
index a709311..0d15f34 100644
--- a/tests/test_runbms.py
+++ b/tests/test_runbms.py
@@ -32,13 +32,19 @@ def test_exit_on_failure_flag_available():
     # Test that the flag is recognized and sets the correct default value
     args = parser.parse_args(["runbms", "/tmp/logs", "/tmp/config.yml"])
     assert hasattr(args, "exit_on_failure")
-    assert args.exit_on_failure is False
+    assert args.exit_on_failure is None
 
-    # Test that the flag can be set
+    # Test that the flag can be set without argument (defaults to 1)
     args = parser.parse_args(
-        ["runbms", "--exit-on-failure", "/tmp/logs", "/tmp/config.yml"]
+        ["runbms", "/tmp/logs", "/tmp/config.yml", "--exit-on-failure"]
     )
-    assert args.exit_on_failure is True
+    assert args.exit_on_failure == 1
+
+    # Test that the flag can be set with custom argument
+    args = parser.parse_args(
+        ["runbms", "/tmp/logs", "/tmp/config.yml", "--exit-on-failure", "42"]
+    )
+    assert args.exit_on_failure == 42
 
 
 def test_global_variables_initialization():
@@ -46,8 +52,10 @@ def test_global_variables_initialization():
     from running.command import runbms
 
     # Test that the new global variables exist
-    assert hasattr(runbms, "exit_on_failure")
+    assert hasattr(runbms, "exit_on_failure_code")
 
     # Test default values (these are module-level globals)
     # Note: These might be modified by other tests, so we just check they exist
-    assert isinstance(runbms.exit_on_failure, bool)
+    assert runbms.exit_on_failure_code is None or isinstance(
+        runbms.exit_on_failure_code, int
+    )

From c666d4c83bb212d1a633dc3748b1759e4f1ac8cd Mon Sep 17 00:00:00 2001
From: Zixian Cai <u5937495@anu.edu.au>
Date: Sun, 27 Jul 2025 14:35:47 +1000
Subject: [PATCH 8/9] Update doc

---
 docs/src/commands/runbms.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/src/commands/runbms.md b/docs/src/commands/runbms.md
index 65f12fa..09d96be 100644
--- a/docs/src/commands/runbms.md
+++ b/docs/src/commands/runbms.md
@@ -3,7 +3,7 @@ This subcommand runs benchmarks with different configs, possibly with varying he
 
 ## Usage
 ```console
-runbms [-h|--help] [-i|--invocations INVOCATIONS] [-s|--slice SLICE] [-p|--id-prefix ID_PREFIX] [-m|--minheap-multiplier MINHEAP_MULTIPLIER] [--skip-oom SKIP_OOM] [--skip-timeout SKIP_TIMEOUT] [--resume RESUME] [--workdir WORKDIR] [--skip-log-compression] [--exit-on-failure [CODE]] LOG_DIR CONFIG [N] [n ...]
+runbms [-h|--help] [-i|--invocations INVOCATIONS] [-s|--slice SLICE] [-p|--id-prefix ID_PREFIX] [-m|--minheap-multiplier MINHEAP_MULTIPLIER] [--skip-oom SKIP_OOM] [--skip-timeout SKIP_TIMEOUT] [--resume RESUME] [--workdir WORKDIR] [--skip-log-compression] [--exit-on-failure CODE] LOG_DIR CONFIG [N] [n ...]
 ```
 
 `-h`: print help message.
@@ -14,7 +14,7 @@ Overrides `invocations` in the config file.
 `-s`: only use the specified heap sizes.
 This is a comma-separated string of integers or floating point numbers.
 For each slice `s` in `SLICE`, we run benchmarks at `s * minheap`.
-`N` and `n`s are ignored. 
+`N` and `n`s are ignored.
 
 `-p`: add a prefix to the folder names where the results are stored.
 By default, the folder that stores the result is named using the host name and the timestamp.
@@ -35,7 +35,7 @@ If not specified, a temporary directory will be created under an OS-dependent lo
 
 `--skip-log-compression`: skip compressing log file as gzip.
 
-`--exit-on-failure [CODE]`: exit with the specified code (default: 1) if any configuration fails.
+`--exit-on-failure` (preview ⚠️): exit with the specified code (default: 1) if any configuration fails.
 This is useful for CI environments where you need to detect failed runs without parsing the output.
 By default, `runbms` exits with code 0 even when some configurations fail.
 If the flag is provided without a code, it defaults to exit code 1.

From 6c33f4bc5469726fed7240db524f1ae17078111b Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 29 Sep 2025 06:30:05 +0000
Subject: [PATCH 9/9] Resolve merge conflicts between --exit-on-failure and
 --randomize-configs features

Co-authored-by: caizixian <2891235+caizixian@users.noreply.github.com>
---
 tests/test_runbms.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_runbms.py b/tests/test_runbms.py
index e53a41a..83599be 100644
--- a/tests/test_runbms.py
+++ b/tests/test_runbms.py
@@ -146,4 +146,3 @@ def test_config_randomization_logic():
 
     # With 5 configs shuffled 10 times, we should get at least some different orders
     assert different_orders > 0, "Shuffling should produce different orders"
->>>>>>> 939de1b865db4727bfe0f143131c223d8b520834