From d25c73c77611cb345bdfb8b5ae979196b76e9e58 Mon Sep 17 00:00:00 2001 From: Kevin Truong Date: Tue, 30 Sep 2025 02:28:37 -0700 Subject: [PATCH 1/4] fixed benchmark report per language stats bug --- benchmark/benchmark.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index c3ad3daee3e..db13502b2e1 100755 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -704,8 +704,7 @@ def show(stat, red="red"): ) if verbose and len(lang_to_stats) > 0: - - def format_lang_stats(lang_stats): + def format_lang_stats(lang, lang_stats): # First, postprocess attributes for easier printing if lang_stats.completed_tests > 0: lang_stats.avg_duration_per_test = lang_stats.duration / float( @@ -716,7 +715,7 @@ def format_lang_stats(lang_stats): setattr(lang_stats, f"pass_num_{i}", num_passed) pass_rate = 100 * num_passed / float(lang_stats.completed_tests) setattr(lang_stats, f"pass_rate_{i}", pass_rate) - + # Then format attributes into ready-to-print strings for attr in lang_stats.__dict__: val = getattr(lang_stats, attr) @@ -742,7 +741,7 @@ def compute_lang_to_col_widths(lang_to_stats): print("======== Stats by language ========") print() - [format_lang_stats(lang_stats) for lang_stats in lang_to_stats.values()] + [format_lang_stats(lang, lang_stats) for lang, lang_stats in lang_to_stats.items()] lang_to_col_widths = compute_lang_to_col_widths(lang_to_stats) any_stats = list(lang_to_stats.values())[0] From 7a63e6117c8b7b8796ed9c3cd333c65c4d52c479 Mon Sep 17 00:00:00 2001 From: Kevin Truong Date: Tue, 30 Sep 2025 02:29:38 -0700 Subject: [PATCH 2/4] remove space --- benchmark/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index db13502b2e1..b81d3645bc7 100755 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -715,7 +715,7 @@ def format_lang_stats(lang, lang_stats): setattr(lang_stats, f"pass_num_{i}", num_passed) pass_rate = 100 * num_passed / float(lang_stats.completed_tests) setattr(lang_stats, f"pass_rate_{i}", pass_rate) - + # Then format attributes into ready-to-print strings for attr in lang_stats.__dict__: val = getattr(lang_stats, attr) From 8430ae22903ab839f98c3dfaeeb452336c3d7d0a Mon Sep 17 00:00:00 2001 From: Kevin Truong Date: Thu, 2 Oct 2025 16:50:56 -0700 Subject: [PATCH 3/4] changed to printing of pass_rate_i to pass_rate_i+1 for per language stats --- benchmark/benchmark.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index b81d3645bc7..d59a75ef7f9 100755 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -548,9 +548,9 @@ def add(attr_name, increment, global_stats, lang_stats): lang_stats.avg_duration_per_test = 0 lang_stats.cost = 0 for i in range(tries): - setattr(lang_stats, f"pass_rate_{i}", 0) + setattr(lang_stats, f"pass_rate_{i+1}", 0) for i in range(tries): - setattr(lang_stats, f"pass_num_{i}", 0) + setattr(lang_stats, f"pass_num_{i+1}", 0) lang_stats.error_outputs = 0 lang_stats.user_asks = 0 lang_stats.test_timeouts = 0 @@ -712,9 +712,9 @@ def format_lang_stats(lang, lang_stats): ) for i in range(tries): num_passed = lang_to_passed_tests[lang][i] - setattr(lang_stats, f"pass_num_{i}", num_passed) + setattr(lang_stats, f"pass_num_{i+1}", num_passed) pass_rate = 100 * num_passed / float(lang_stats.completed_tests) - setattr(lang_stats, f"pass_rate_{i}", pass_rate) + setattr(lang_stats, f"pass_rate_{i+1}", pass_rate) # Then format attributes into ready-to-print strings for attr in lang_stats.__dict__: From 3eac8bc8cd2a7e398b5c9f274ad46cd54ef2f5a0 Mon Sep 17 00:00:00 2001 From: Kevin Truong Date: Thu, 2 Oct 2025 16:55:31 -0700 Subject: [PATCH 4/4] fix whitespace --- benchmark/benchmark.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index d59a75ef7f9..02117242742 100755 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -548,9 +548,9 @@ def add(attr_name, increment, global_stats, lang_stats): lang_stats.avg_duration_per_test = 0 lang_stats.cost = 0 for i in range(tries): - setattr(lang_stats, f"pass_rate_{i+1}", 0) + setattr(lang_stats, f"pass_rate_{i + 1}", 0) for i in range(tries): - setattr(lang_stats, f"pass_num_{i+1}", 0) + setattr(lang_stats, f"pass_num_{i + 1}", 0) lang_stats.error_outputs = 0 lang_stats.user_asks = 0 lang_stats.test_timeouts = 0 @@ -636,7 +636,7 @@ def show(stat, red="red"): for i in range(tries): pass_rate = 100 * passed_tests[i] / res.completed_tests percents[i] = pass_rate - # console.print(f"{pass_rate:.1f}% correct after try {i+1}") + # console.print(f"{pass_rate:.1f}% correct after try {i + 1}") setattr(res, f"pass_rate_{i + 1}", f"{pass_rate:.1f}") setattr(res, f"pass_num_{i + 1}", passed_tests[i]) @@ -704,6 +704,7 @@ def show(stat, red="red"): ) if verbose and len(lang_to_stats) > 0: + def format_lang_stats(lang, lang_stats): # First, postprocess attributes for easier printing if lang_stats.completed_tests > 0: @@ -712,9 +713,9 @@ def format_lang_stats(lang, lang_stats): ) for i in range(tries): num_passed = lang_to_passed_tests[lang][i] - setattr(lang_stats, f"pass_num_{i+1}", num_passed) + setattr(lang_stats, f"pass_num_{i + 1}", num_passed) pass_rate = 100 * num_passed / float(lang_stats.completed_tests) - setattr(lang_stats, f"pass_rate_{i+1}", pass_rate) + setattr(lang_stats, f"pass_rate_{i + 1}", pass_rate) # Then format attributes into ready-to-print strings for attr in lang_stats.__dict__: