XorBinaryFuse8 failed to construct on some data, at around 11511 entries #32

Author: thomasmueller

fastfilter/src/main/java/org/fastfilter/utils/Hash.java

@@ -35,4 +35,16 @@ public static int reduce(int hash, int n) {
         return (int) (((hash & 0xffffffffL) * n) >>> 32);
     }
 
+    /**
+     * Multiply two unsigned 64-bit values.
+     * See https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8188044
+     *
+     * @param a the first value
+     * @param b the second value
+     * @return the result
+     */
+    public static long multiplyHighUnsigned(long a, long b) {
+        return Math.multiplyHigh(a, b) + ((a >> 63) & b) + ((b >> 63) & a);
+    }
+
 }

fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java

@@ -42,7 +42,7 @@ public long getBitCount() {
     static int calculateSegmentLength(int arity, int size) {
         int segmentLength;
         if (arity == 3) {
-            segmentLength = 1 << (int) Math.floor(Math.log(size) / Math.log(3.33) + 2.25);
+            segmentLength = 1 << (int) Math.floor(Math.log(size) / Math.log(3.33) + 2.11);
         } else if (arity == 4) {
             segmentLength = 1 << (int) Math.floor(Math.log(size) / Math.log(2.91) - 0.5);
         } else {
@@ -200,17 +200,17 @@ private void addAll(long[] keys) {
             Arrays.fill(reverseOrder, 0);
 
             // TODO
-            System.out.println("WARNING: hashIndex " + hashIndex + "\n");
-            if (hashIndex >= 0) {
-                System.out.println(size + " keys; arrayLength " + arrayLength + " reverseOrderPos " + reverseOrderPos);
-            }
+            // if (hashIndex > 10) {
+            //     System.out.println("WARNING: hashIndex " + hashIndex + "\n");
+            //     System.out.println(size + " keys; arrayLength " + arrayLength + " reverseOrderPos " + reverseOrderPos + " segmentLength " + segmentLength + " segmentCount " + segmentCount);
+            // }
             if (hashIndex > 100) {
                 // if construction doesn't succeed eventually,
                 // then there is likely a problem with the hash function
-                break;
+                throw new IllegalArgumentException("Construction failed after " + hashIndex + " retries for size " + size);
             }
             // use a new random numbers
-            seed++;
+            seed = Hash.randomSeed();
         }
         alone = null;
         t2count = null;
@@ -246,8 +246,14 @@ public boolean mayContain(long key) {
         return (f & 0xff) == 0;
     }
 
+    @Override
+    public String toString() {
+        return "segmentLength " + segmentLength + " segmentCount " + segmentCount;
+    }
+
     int getHashFromHash(long hash, int index) {
         long h = Hash.reduce((int) (hash >>> 32), segmentCountLength);
+        // long h = Hash.multiplyHighUnsigned(hash, segmentCountLength);
         h += index * segmentLength;
         // keep the lower 36 bits
         long hh = hash & ((1L << 36) - 1);

fastfilter/src/test/java/org/fastfilter/xor/SmallSetTest.java

@@ -16,4 +16,37 @@ public void small() {
         XorSimple.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL});
         XorSimple2.construct(new long[]{0xef9bddc5166c081cL, 0x33bf87adaa46dcfcL});
     }
+    
+    @Test
+    public void verySmallSizes() {
+        int n = 1;
+        for (; n < 2000; n++) {
+            testWithSize(n);
+        }
+        for (; n < 20000; n += 7) {
+            testWithSize(n);
+        }
+    }
+        
+    @Test
+    public void smallSizes() {
+        long lastTime = System.currentTimeMillis();
+        for (int n = 1; n < 1_500_000; n = (int) ((n * 1.01) + 7)) {
+            XorBinaryFuse8 f = testWithSize(n);
+            long now = System.currentTimeMillis();
+            if (now - lastTime > 5000) {
+                lastTime = now;
+                System.out.println("n=" + n + " " + f.toString());
+            }
+        }
+    }
+    
+    private static XorBinaryFuse8 testWithSize(int n) {
+        long[] keys = new long[n];
+        for (int i = 0; i < n; i++) {
+            keys[i] = i;
+        }
+        return XorBinaryFuse8.construct(keys);
+    }
+
 }