test bench fail 2

Author: lukaszstolarczuk

devops/scripts/benchmarks/main.py

@@ -108,28 +108,33 @@ def run_iterations(
     for iter in range(iters):
         log.info(f"running {benchmark.name()}, iteration {iter}... ")
         try:
-            bench_results = benchmark.run(
-                env_vars, run_trace=run_trace, force_trace=force_trace
-            )
-            if bench_results is None:
-                if options.exit_on_failure:
-                    raise RuntimeError(f"Benchmark produced no results!")
-                else:
-                    failures[benchmark.name()] = "benchmark produced no results!"
-                    return False
-
-            for bench_result in bench_results:
-                log.info(
-                    f"{benchmark.name()} complete ({bench_result.label}: {bench_result.value:.3f} {bench_result.unit})."
-                )
-                bench_result.name = bench_result.label
-                bench_result.lower_is_better = benchmark.lower_is_better()
-                bench_result.suite = benchmark.get_suite_name()
-
-                if bench_result.label not in results:
-                    results[bench_result.label] = []
-
-                results[bench_result.label].append(bench_result)
+            # XXX
+            failure_label = f"{benchmark.name()} iteration {iter}"
+            failures[failure_label] = (f"{failure_label}: TEST FAIL")
+            log.error(f"{failure_label}: TEST FAIL")
+            return False
+            # bench_results = benchmark.run(
+            #     env_vars, run_trace=run_trace, force_trace=force_trace
+            # )
+            # if bench_results is None:
+            #     if options.exit_on_failure:
+            #         raise RuntimeError(f"Benchmark produced no results!")
+            #     else:
+            #         failures[benchmark.name()] = "benchmark produced no results!"
+            #         return False
+
+            # for bench_result in bench_results:
+            #     log.info(
+            #         f"{benchmark.name()} complete ({bench_result.label}: {bench_result.value:.3f} {bench_result.unit})."
+            #     )
+            #     bench_result.name = bench_result.label
+            #     bench_result.lower_is_better = benchmark.lower_is_better()
+            #     bench_result.suite = benchmark.get_suite_name()
+
+            #     if bench_result.label not in results:
+            #         results[bench_result.label] = []
+
+            #     results[bench_result.label].append(bench_result)
         except Exception as e:
             failure_label = f"{benchmark.name()} iteration {iter}"
             if options.exit_on_failure: