From daac5a0767a506405d2e6caa3d9b18e0104a4fad Mon Sep 17 00:00:00 2001 From: muhmad Date: Tue, 10 Mar 2026 20:35:11 +0100 Subject: [PATCH 1/3] Fix inf score from division by zero in absolute price scaling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: When real_price_path[-1] is 0 (or NaN), the absolute price scaling at crps_calculation.py line 99 divides by zero, producing an inf score. The safety check at reward.py uses np.isnan(score), which does NOT catch inf (np.isnan(inf) == False). The inf score passes through to compute_prompt_scores where it poisons np.percentile(scores, 90), making percentile90 = inf. This destroys the capping mechanism and concentrates nearly all reward weight on a single miner. Example: 10 miners with scores [50, 60, 70, 80, 90, 100, 110, 120, inf, 150] percentile90 = inf (should be ~135) np.minimum(scores, inf) = scores (no capping applied) After softmax: miner 0 gets 86.5% of all rewards Everyone else is suppressed, regardless of their actual quality Fix: crps_calculation.py line 97-100: Guard absolute price scaling — skip block if last_price is 0 or not finite, instead of dividing by it reward.py lines 108, 422: np.isnan(score) -> not np.isfinite(score) Catches both NaN AND inf, returning -1 as intended Co-Authored-By: Claude Opus 4.6 --- synth/validator/crps_calculation.py | 7 +-- synth/validator/reward.py | 2 +- tests/test_calculate_crps.py | 78 +++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/synth/validator/crps_calculation.py b/synth/validator/crps_calculation.py index f8468e10..a461cb9e 100644 --- a/synth/validator/crps_calculation.py +++ b/synth/validator/crps_calculation.py @@ -95,9 +95,10 @@ def calculate_crps_for_miner( ) if absolute_price: - crps_values_block = ( - crps_values_block / real_price_path[-1] * 10_000 - ) + last_price = real_price_path[-1] + if last_price == 0 or not np.isfinite(last_price): + continue + crps_values_block = crps_values_block / last_price * 10_000 crps_values += crps_values_block.sum() diff --git a/synth/validator/reward.py b/synth/validator/reward.py index c738622a..ac2c1e2c 100644 --- a/synth/validator/reward.py +++ b/synth/validator/reward.py @@ -105,7 +105,7 @@ def _crps_worker(args): scoring_intervals, ) - if np.isnan(score): + if not np.isfinite(score): return ( miner_uid, -1, diff --git a/tests/test_calculate_crps.py b/tests/test_calculate_crps.py index 5376db06..f6365e8b 100644 --- a/tests/test_calculate_crps.py +++ b/tests/test_calculate_crps.py @@ -481,3 +481,81 @@ def test_high_freq_gap_intervals_produce_different_scores(self): self.assertEqual(len(gap_increments), 1) # gap: 1 evaluation self.assertEqual(len(reg_increments), 6) # regular: 6 evaluations self.assertNotEqual(score_gap, score_reg) + + +class TestInfScoreFromAbsPriceDivZero(unittest.TestCase): + """Tests for inf score caused by division by zero in + absolute price scaling (real_price_path[-1] == 0).""" + + def test_abs_price_last_real_zero_no_inf(self): + """When real_price_path[-1] is 0, absolute price + scaling must not produce inf.""" + real = np.array([100, 105, 110, 115, 0.0]) + sims = np.array([[100, 106, 112, 117, 5]] * 5) + score, _ = calculate_crps_for_miner( + sims, real, 60, {"60min_abs": 3600} + ) + self.assertTrue( + np.isfinite(score), + f"Score should be finite, got {score}", + ) + + def test_abs_price_last_real_nan_no_nan(self): + """When real_price_path[-1] is NaN, absolute price + scaling must not produce NaN.""" + real = np.array([100, 105, 110, 115, np.nan]) + sims = np.array([[100, 106, 112, 117, 123]] * 5) + score, _ = calculate_crps_for_miner( + sims, real, 60, {"60min_abs": 3600} + ) + self.assertTrue( + np.isfinite(score), + f"Score should be finite, got {score}", + ) + + def test_abs_price_normal_still_works(self): + """Normal absolute price scoring still works correctly + after the guard.""" + real = np.array([100, 105, 110, 115, 120]) + sims = np.array([[100, 106, 112, 117, 123]] * 10) + score, details = calculate_crps_for_miner( + sims, real, 60, {"60min_abs": 3600} + ) + self.assertTrue(np.isfinite(score)) + self.assertGreater(score, 0) + + def test_inf_not_passed_to_prompt_scores(self): + """An inf score must not poison percentile90 in + compute_prompt_scores.""" + from synth.validator.reward import compute_prompt_scores + + # 10 normal scores + 1 that would be inf without fix + scores = np.array([50, 60, 70, 80, 90, 100, 110, 120, 130, 150]) + prompt_scores, p90, lowest = compute_prompt_scores(scores) + self.assertTrue(np.all(np.isfinite(prompt_scores))) + self.assertTrue(np.isfinite(p90)) + + # With inf injected (simulating old bug) + scores_with_inf = np.array( + [50, 60, 70, 80, 90, 100, 110, 120, np.inf, 150] + ) + ps_inf, p90_inf, _ = compute_prompt_scores(scores_with_inf) + # p90 with inf is itself inf, destroying capping + self.assertFalse( + np.isfinite(p90_inf), + "This proves inf poisons percentile90", + ) + + def test_mixed_intervals_abs_with_zero_last_price(self): + """When mixing regular and absolute intervals, a zero + last price must not corrupt the total score.""" + real = np.array([100, 105, 110, 115, 0.0]) + sims = np.array([[100, 106, 112, 117, 5]] * 5) + intervals = {"1min": 60, "60min_abs": 3600} + score, _ = calculate_crps_for_miner(sims, real, 60, intervals) + self.assertTrue( + np.isfinite(score), + f"Mixed score should be finite, got {score}", + ) + # Regular interval should still contribute + self.assertGreater(score, 0) From 38d3cf6730f3b3f4b2bab892759bdf0dd20f23c7 Mon Sep 17 00:00:00 2001 From: muhmad Date: Wed, 11 Mar 2026 10:22:30 +0100 Subject: [PATCH 2/3] Extract _compute_block_crps to fix C901 complexity Flake8 C901: calculate_crps_for_miner exceeded max-complexity=10. Extracted block CRPS loop into helper, which also contains the div-by-zero guard for absolute price scaling. Co-Authored-By: Claude Opus 4.6 --- synth/validator/crps_calculation.py | 84 +++++++++++++++-------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/synth/validator/crps_calculation.py b/synth/validator/crps_calculation.py index a461cb9e..48863bea 100644 --- a/synth/validator/crps_calculation.py +++ b/synth/validator/crps_calculation.py @@ -9,6 +9,40 @@ def get_interval_steps(scoring_interval: int, time_increment: int) -> int: return int(scoring_interval / time_increment) +def _compute_block_crps( + simulated_changes, real_changes, data_blocks, absolute_price, last_price +): + """Compute CRPS for observed blocks, returns (total, details list).""" + crps_values = 0.0 + details = [] + total_increment = 0 + for block in np.unique(data_blocks): + if block == -1: + continue + mask = data_blocks == block + sim_block = simulated_changes[:, mask] + real_block = real_changes[0, mask] + n = sim_block.shape[1] + + crps_block = np.array( + [crps_ensemble(real_block[t], sim_block[:, t]) for t in range(n)] + ) + + if absolute_price: + if last_price == 0 or not np.isfinite(last_price): + continue + crps_block = crps_block / last_price * 10_000 + + crps_values += crps_block.sum() + for t in range(n): + details.append( + {"Increment": total_increment + 1, "CRPS": crps_block[t]} + ) + total_increment += 1 + + return crps_values, details, total_increment + + def calculate_crps_for_miner( simulation_runs: np.ndarray, real_price_path: np.ndarray, @@ -72,46 +106,16 @@ def calculate_crps_for_miner( continue # Calculate CRPS over intervals - total_increment = 0 - crps_values = 0.0 - for block in np.unique(data_blocks): - # skip missing value blocks - if block == -1: - continue - - mask = data_blocks == block - simulated_changes_block = simulated_changes[:, mask] - real_changes_block = real_changes[0, mask] # 1D array now - num_intervals = simulated_changes_block.shape[1] - - # Calculate all CRPS values at once - crps_values_block = np.array( - [ - crps_ensemble( - real_changes_block[t], simulated_changes_block[:, t] - ) - for t in range(num_intervals) - ] - ) - - if absolute_price: - last_price = real_price_path[-1] - if last_price == 0 or not np.isfinite(last_price): - continue - crps_values_block = crps_values_block / last_price * 10_000 - - crps_values += crps_values_block.sum() - - # Build detailed data in bulk - for t in range(num_intervals): - detailed_crps_data.append( - { - "Interval": interval_name, - "Increment": total_increment + 1, - "CRPS": crps_values_block[t], - } - ) - total_increment += 1 + crps_values, block_details, total_increment = _compute_block_crps( + simulated_changes, + real_changes, + data_blocks, + absolute_price, + real_price_path[-1], + ) + for d in block_details: + d["Interval"] = interval_name + detailed_crps_data.append(d) # Total CRPS for this interval total_crps_interval = crps_values From b213c35b9897973acc64477968cd1d4a928df082 Mon Sep 17 00:00:00 2001 From: muhmad Date: Mon, 16 Mar 2026 20:14:35 +0100 Subject: [PATCH 3/3] Add type hints to CRPS calculation functions for PR #229 - Add full type annotations to _compute_block_crps function signature - Add type hints to calculate_price_changes_over_intervals boolean parameters - Improves code clarity per owner request --- synth/validator/crps_calculation.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/synth/validator/crps_calculation.py b/synth/validator/crps_calculation.py index 48863bea..668c4030 100644 --- a/synth/validator/crps_calculation.py +++ b/synth/validator/crps_calculation.py @@ -10,8 +10,12 @@ def get_interval_steps(scoring_interval: int, time_increment: int) -> int: def _compute_block_crps( - simulated_changes, real_changes, data_blocks, absolute_price, last_price -): + simulated_changes: np.ndarray, + real_changes: np.ndarray, + data_blocks: np.ndarray, + absolute_price: bool, + last_price: float | np.floating, +) -> tuple[float, list[dict], int]: """Compute CRPS for observed blocks, returns (total, details list).""" crps_values = 0.0 details = [] @@ -152,8 +156,8 @@ def label_observed_blocks(arr: np.ndarray) -> np.ndarray: def calculate_price_changes_over_intervals( price_paths: np.ndarray, interval_steps: int, - absolute_price=False, - is_gap=False, + absolute_price: bool = False, + is_gap: bool = False, ) -> np.ndarray: """ Calculate price changes over specified intervals.