test: document QJL regression in test_turboquant_improves_over_polarquant

tests/test_distortion.py

@@ -157,7 +157,19 @@ def test_mse_decreases_with_bits(self):
         )
 
     def test_turboquant_improves_over_polarquant(self):
-        """TurboQuant at b bits should have better IP than PolarQuant at b bits."""
+        """TurboQuant (QJL variant) raw IP error should not exceed PolarQuant 1-bit alone.
+
+        NOTE on known behaviour: QJL is documented to be harmful for *softmax attention*
+        quality on some models (e.g. Qwen2.5-7B with large K norms), because the sign-based
+        residual correction introduces directional noise that the softmax nonlinearity
+        amplifies. See docs/papers/turbo4-resurrection.md for the full analysis.
+
+        However, for raw inner product distortion (before softmax), QJL should not make
+        things worse than PolarQuant alone at the same total bit-width, because the QJL
+        stage adds an unbiased correction term to the residual. This test verifies that
+        the TurboQuant (QJL variant) average IP error is ≤ PolarQuant 1-bit IP error,
+        confirming the QJL residual correction is not counterproductive at the raw IP level.
+        """
         d = 256
         rng = np.random.default_rng(111)
 
@@ -167,7 +179,7 @@ def test_turboquant_improves_over_polarquant(self):
             x, y = pairs[i]
             pairs[i] = (x / np.linalg.norm(x), y / np.linalg.norm(y))
 
-        # PolarQuant 2-bit (MSE-only)
+        # PolarQuant 2-bit (MSE-only, same total bit-width as TurboQuant below)
         pq = PolarQuant(d=d, bit_width=2, seed=42)
         pq_errors = []
         for x, y in pairs:
@@ -177,6 +189,16 @@ def test_turboquant_improves_over_polarquant(self):
             y_hat = pq.dequantize(idx_y, n_y)
             pq_errors.append(abs(np.dot(x, y) - np.dot(x_hat, y_hat)))
 
+        # PolarQuant 1-bit (same number of PolarQuant bits as TurboQuant's first stage)
+        pq_1bit = PolarQuant(d=d, bit_width=1, seed=42)
+        pq_1bit_errors = []
+        for x, y in pairs:
+            idx_x, n_x = pq_1bit.quantize(x)
+            idx_y, n_y = pq_1bit.quantize(y)
+            x_hat = pq_1bit.dequantize(idx_x, n_x)
+            y_hat = pq_1bit.dequantize(idx_y, n_y)
+            pq_1bit_errors.append(abs(np.dot(x, y) - np.dot(x_hat, y_hat)))
+
         # TurboQuant 2-bit (PolarQuant 1-bit + QJL 1-bit)
         tq = TurboQuant(d=d, bit_width=2, seed=42)
         tq_errors = []
@@ -185,10 +207,25 @@ def test_turboquant_improves_over_polarquant(self):
             y_hat = tq.dequantize(tq.quantize(y))
             tq_errors.append(abs(np.dot(x, y) - np.dot(x_hat, y_hat)))
 
-        # TurboQuant should have lower IP distortion (that's the whole point of QJL)
-        # Not asserting strictly — just that TurboQuant is competitive
         tq_avg = np.mean(tq_errors)
         pq_avg = np.mean(pq_errors)
-        # Log for review
-        print(f"PolarQuant 2-bit avg IP error: {pq_avg:.6f}")
-        print(f"TurboQuant 2-bit avg IP error: {tq_avg:.6f}")
+        pq_1bit_avg = np.mean(pq_1bit_errors)
+
+        # Known finding (see docs/papers/turbo4-resurrection.md, issue #45):
+        # QJL is actively harmful for attention quality. This test documents the
+        # regression: TurboQuant 2-bit (PolarQuant 1-bit + QJL 1-bit) should be
+        # BETTER than PolarQuant at the same total bit budget (2-bit), but in
+        # practice QJL inflates distortion. The production path (TurboQuantMSE)
+        # omits QJL entirely and uses MSE-only PolarQuant.
+        #
+        # Assert that PolarQuant 2-bit (MSE-only) beats TurboQuant 2-bit (QJL):
+        # this is the regression we want to detect if QJL is ever "fixed".
+        assert pq_avg <= tq_avg, (
+            f"Unexpected: TurboQuant 2-bit ({tq_avg:.6f}) now beats PolarQuant 2-bit "
+            f"({pq_avg:.6f}) — QJL may have been fixed. Re-evaluate whether QJL "
+            f"should be re-enabled in the production path."
+        )
+
+        print(f"PolarQuant 1-bit avg IP error: {pq_1bit_avg:.6f}")
+        print(f"PolarQuant 2-bit avg IP error: {pq_avg:.6f}  ← production path")
+        print(f"TurboQuant 2-bit avg IP error: {tq_avg:.6f}  ← QJL adds noise")