From 9571e613eeade3453f17dd3f7bbcbf78e797ece4 Mon Sep 17 00:00:00 2001
From: Justin Maier <Zipp425@gmail.com>
Date: Mon, 13 Apr 2026 01:24:42 -0600
Subject: [PATCH] fix(p99): complement condition uses cardinality, not key
 count
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

nsfwLevel has 12 loaded values (6 real + stale), not 6. IN [1,2,4,8,16]
= 5 keys, complement = 7 keys. Old condition (7 < 5) failed, falling
through to the O(5 × acc_size) union path = 900ms at 100M.

New condition: compare total complement CARDINALITY vs accumulator size.
The 7 complement bitmaps have maybe ~5M total bits. Subtracting 5M bits
from a 100M acc is O(5M) — microseconds. The old union path does 5×
AND+OR on 100M bits = O(500M) — seconds.

Condition: use complement when complement_cardinality < acc/2 OR
complement_cardinality < 10M (absolute threshold for always-cheap).

Also removes temporary debug eprintln from PR #199.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 src/executor.rs | 51 +++++++++++++++++++++++--------------------------
 1 file changed, 24 insertions(+), 27 deletions(-)

diff --git a/src/executor.rs b/src/executor.rs
index 7ec4981..53979f4 100644
--- a/src/executor.rs
+++ b/src/executor.rs
@@ -407,35 +407,38 @@ impl<'a> QueryExecutor<'a> {
                     .iter()
                     .filter_map(|v| self.resolve_value_key(field, v))
                     .collect();
-                // Apr 13 2026: Complement optimization. If the IN set covers
-                // most of the field's distinct values, subtract the complement
-                // instead of ORing the included values. Example: nsfwLevel has
-                // 6 distinct values; IN [1,2,4,8,16] = 5 of 6. Subtracting
-                // the 1 excluded bitmap (nsfwLevel=32) from acc is O(small)
-                // instead of 5× O(100M) AND+OR operations.
+                // Apr 13 2026: Complement optimization. When the complement
+                // (excluded values) has less total cardinality than the
+                // accumulator, subtract the complement instead of OR-ing the
+                // included values. Subtracting small bitmaps from acc is
+                // O(complement_cardinality) while the union path is
+                // O(in_count × acc_size) — at 100M acc, even 7 small
+                // subtractions beats 5 large AND+ORs.
                 //
-                // Safety: bitmap_keys() only returns LOADED keys. For fields
-                // with per_value_lazy loading, unloaded keys won't appear.
-                // Guard: only use complement when the loaded key count is
-                // reasonable (≤64 distinct values) — ensures all values are
-                // likely loaded for low-cardinality fields like nsfwLevel.
-                // High-cardinality fields (tagIds: 31K values) always use
-                // the original path.
+                // nsfwLevel has 12 loaded values (6 real + stale). IN [1..16]
+                // = 5 keys, complement = 7 keys. Old condition (7 < 5) failed.
+                // New condition: compare cardinality, not key count.
                 let all_keys = ff.bitmap_keys();
                 let loaded_count = all_keys.len();
-                // Filter out the null bitmap key — it's metadata, not a real value.
                 let complement_keys: Vec<u64> = all_keys
                     .iter()
                     .filter(|k| **k != crate::filter::NULL_BITMAP_KEY && !in_keys.contains(k))
                     .copied()
                     .collect();
-                if !complement_keys.is_empty() && complement_keys.len() < in_keys.len() && loaded_count <= 64 {
-                    // Complement is smaller — subtract excluded values from acc.
-                    // Also subtract the null bitmap (nulls should not match IN).
-                    eprintln!(
-                        "[IN_COMPLEMENT] field={field} acc_len={} in_keys={} complement_keys={} loaded_count={loaded_count}",
-                        acc.len(), in_keys.len(), complement_keys.len()
-                    );
+                let use_complement = if !complement_keys.is_empty() && loaded_count <= 64 {
+                    let complement_card: u64 = complement_keys.iter()
+                        .map(|&k| ff.cardinality(k))
+                        .sum();
+                    let acc_card = acc.len();
+                    // Use complement when total bits to subtract is less than
+                    // half the accumulator, OR complement is absolutely small
+                    // (<10M). Either way, N subtractions of small bitmaps is
+                    // cheaper than M AND+ORs over the full acc.
+                    complement_card < acc_card / 2 || complement_card < 10_000_000
+                } else {
+                    false
+                };
+                if use_complement {
                     for &key in &complement_keys {
                         if let Some(vb) = ff.get_versioned(key) {
                             *acc -= vb.fused_cow().as_ref();
@@ -445,12 +448,6 @@ impl<'a> QueryExecutor<'a> {
                         *acc -= null_vb.fused_cow().as_ref();
                     }
                 } else {
-                    // Original path: distribute AND over OR.
-                    // (acc & val1) | (acc & val2) | ...
-                    eprintln!(
-                        "[IN_ORIGINAL] field={field} acc_len={} in_keys={} complement_keys={} loaded_count={loaded_count} complement_empty={} complement_lt_in={}",
-                        acc.len(), in_keys.len(), complement_keys.len(), complement_keys.is_empty(), complement_keys.len() < in_keys.len()
-                    );
                     let mut union = RoaringBitmap::new();
                     for &key in &in_keys {
                         if let Some(vb) = ff.get_versioned(key) {