From 85db0f07d3ecae89eee0ec53ebefe0ce5766e16d Mon Sep 17 00:00:00 2001 From: muhmad Date: Tue, 10 Mar 2026 20:46:04 +0100 Subject: [PATCH 1/2] Fix inf rolling averages passed to compute_softmax Problem: When a miner has no valid scores in the moving average window, compute_smoothed_score assigns rolling_avg = float("inf") (line 178). These inf values are passed directly to compute_softmax (line 199). With a single inf, softmax works by accident: beta * inf = -inf, exp(-inf) = 0, so the inf miner gets 0 weight. But when ALL miners have inf (e.g. all are new with no scores yet), softmax breaks: scaled = -0.2 * [inf, inf, inf] = [-inf, -inf, -inf] shifted = [-inf] - max([-inf]) = [nan, nan, nan] (inf - inf = nan) exp([nan, nan, nan]) = [nan, nan, nan] weights = [nan, nan, nan] / nan = [nan, nan, nan] NaN weights crash downstream reward distribution. Even with a single inf, the behavior is fragile: the inf miner gets exactly 0 weight instead of the worst-but-nonzero weight that a miner with the worst real score would get. This silently excludes them from rewards rather than giving them the minimum. Fix: moving_average.py lines 184-193: Before passing to compute_softmax, replace inf rolling averages with the worst (highest) finite rolling average. This gives no-score miners the same weight as the worst real performer, and ensures compute_softmax always receives finite inputs. Co-Authored-By: Claude Opus 4.6 --- synth/validator/moving_average.py | 14 ++++++++++++++ tests/test_calculate_crps.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/synth/validator/moving_average.py b/synth/validator/moving_average.py index 168d3e96..989c88ed 100644 --- a/synth/validator/moving_average.py +++ b/synth/validator/moving_average.py @@ -167,6 +167,20 @@ def compute_smoothed_score( {"miner_id": miner_id, "rolling_avg": rolling_avg} ) + # Replace inf with the worst finite rolling average so that + # compute_softmax receives only finite values. Inf causes NaN + # weights when all miners have inf (inf - inf = nan in softmax). + finite_avgs = [ + r["rolling_avg"] + for r in rolling_avg_data + if np.isfinite(r["rolling_avg"]) + ] + if finite_avgs: + worst_finite = max(finite_avgs) + for r in rolling_avg_data: + if not np.isfinite(r["rolling_avg"]): + r["rolling_avg"] = worst_finite + # Add the miner UID to the results moving_averages_data = miner_data_handler.populate_miner_uid_in_miner_data( rolling_avg_data diff --git a/tests/test_calculate_crps.py b/tests/test_calculate_crps.py index 5376db06..64d3ee72 100644 --- a/tests/test_calculate_crps.py +++ b/tests/test_calculate_crps.py @@ -481,3 +481,31 @@ def test_high_freq_gap_intervals_produce_different_scores(self): self.assertEqual(len(gap_increments), 1) # gap: 1 evaluation self.assertEqual(len(reg_increments), 6) # regular: 6 evaluations self.assertNotEqual(score_gap, score_reg) + + +class TestSoftmaxWithInf(unittest.TestCase): + """Tests for compute_softmax handling of inf values.""" + + def test_softmax_single_inf_gives_zero_weight(self): + """A single inf score should get 0 weight.""" + scores = np.array([50, 60, 70, np.inf]) + weights = compute_softmax(scores, -0.2) + self.assertTrue(np.all(np.isfinite(weights))) + self.assertAlmostEqual(weights.sum(), 1.0, places=6) + self.assertEqual(weights[-1], 0.0) + + def test_softmax_all_inf_produces_nan(self): + """All-inf input produces NaN weights (the bug).""" + scores = np.array([np.inf, np.inf, np.inf]) + weights = compute_softmax(scores, -0.2) + # This proves the bug: all NaN weights + self.assertTrue(np.all(np.isnan(weights))) + + def test_softmax_all_finite_works(self): + """Normal finite scores produce valid weights.""" + scores = np.array([50, 60, 70, 80]) + weights = compute_softmax(scores, -0.2) + self.assertTrue(np.all(np.isfinite(weights))) + self.assertAlmostEqual(weights.sum(), 1.0, places=6) + # Lower score = better = higher weight (beta < 0) + self.assertGreater(weights[0], weights[-1]) From c3f51db4115ac2190a0a6af24e252ff4379afaea Mon Sep 17 00:00:00 2001 From: muhmad Date: Wed, 11 Mar 2026 10:17:11 +0100 Subject: [PATCH 2/2] Extract _replace_inf_rolling_avgs helper to reduce complexity Flake8 C901: compute_smoothed_score exceeded max-complexity=10. Extracted the inf-replacement logic into a helper function. Co-Authored-By: Claude Opus 4.6 --- synth/validator/moving_average.py | 32 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/synth/validator/moving_average.py b/synth/validator/moving_average.py index 989c88ed..5f27ec74 100644 --- a/synth/validator/moving_average.py +++ b/synth/validator/moving_average.py @@ -126,6 +126,24 @@ def apply_per_asset_coefficients( return df["prompt_score_v3"] +def _replace_inf_rolling_avgs(rolling_avg_data: list[dict]) -> None: + """Replace inf rolling averages with the worst finite value. + + Inf causes NaN weights when all miners have inf + (inf - inf = nan in softmax). + """ + finite_avgs = [ + r["rolling_avg"] + for r in rolling_avg_data + if np.isfinite(r["rolling_avg"]) + ] + if finite_avgs: + worst_finite = max(finite_avgs) + for r in rolling_avg_data: + if not np.isfinite(r["rolling_avg"]): + r["rolling_avg"] = worst_finite + + def compute_smoothed_score( miner_data_handler: MinerDataHandler, input_df: DataFrame, @@ -167,19 +185,7 @@ def compute_smoothed_score( {"miner_id": miner_id, "rolling_avg": rolling_avg} ) - # Replace inf with the worst finite rolling average so that - # compute_softmax receives only finite values. Inf causes NaN - # weights when all miners have inf (inf - inf = nan in softmax). - finite_avgs = [ - r["rolling_avg"] - for r in rolling_avg_data - if np.isfinite(r["rolling_avg"]) - ] - if finite_avgs: - worst_finite = max(finite_avgs) - for r in rolling_avg_data: - if not np.isfinite(r["rolling_avg"]): - r["rolling_avg"] = worst_finite + _replace_inf_rolling_avgs(rolling_avg_data) # Add the miner UID to the results moving_averages_data = miner_data_handler.populate_miner_uid_in_miner_data(