QGram and NGram

tdebatty · tdebatty · commit 886a4b895ec0 · 2015-01-29T17:11:40.000+01:00
+ Shingling
diff --git a/README.md b/README.md
@@ -76,5 +76,48 @@ public class MyApp {
 }
 ```
 
+## Q-Gram
+
+Q-Gram edit distance, not to confuse with N-Gram distance defined by Kondrak (below), is the relative number of n-grams both strings have in common.
+
+```java
+import info.debatty.java.stringsimilarity.*;
+
+public class MyApp {
+    
+    public static void main(String[] args) {
+        QGram dig = new QGram(2);
+        
+        // Should be 2: CD and CE
+        System.out.println(dig.absoluteDistance("ABCD", "ABCE"));
+        
+        // Should be 0.5 (2 / 4)
+        System.out.println(dig.distance("ABCD", "ABCE"));
+    }
+}
+```
+
+## N-Gram similarity (Kondrak)
+
+N-Gram Similarity as defined by Kondrak, "N-Gram Similarity and Distance", String Processing and Information Retrieval, Lecture Notes in Computer Science Volume 3772, 2005, pp 115-126.
+
+http://webdocs.cs.ualberta.ca/~kondrak/papers/spire05.pdf
+
+The algorithm uses affixing with special character '\n' two increase the weight of first characters. The normalization is achieved by dividing the total similarity score the original length of the longer word.
+
+```java
+import info.debatty.java.stringsimilarity.*;
+
+public class MyApp {
+
+    public static void main(String[] args) {
+        NGram twogram = new NGram(2);
+
+        // Should be 0.41666
+        System.out.println(twogram.distance("ABCD", "ABTUIO"));
+    }
+}
+```
+
 
 
diff --git a/src/info/debatty/java/stringsimilarity/KShingling.java b/src/info/debatty/java/stringsimilarity/KShingling.java
@@ -0,0 +1,104 @@
+package info.debatty.java.stringsimilarity;
+
+import java.io.Serializable;
+import java.security.InvalidParameterException;
+import java.util.HashSet;
+
+/**
+ * A k-shingling is a set of unique k-grams, used to measure the similarity of 
+ * two documents.
+ * 
+ * Generally speaking, a k-gram is any sequence of k tokens. We use here the 
+ * definition from  Leskovec, Rajaraman & Ullman (2014),
+ * "Mining of Massive Datasets", Cambridge University Press:
+ * Multiple subsequent spaces are replaced by a single space, and a k-gram is a 
+ * sequence of k characters.
+ * 
+ * @author Thibault Debatty http://www.debatty.info
+ */
+public class KShingling extends HashSet<String> implements Serializable {
+    
+
+    public static void main(String[] args) {
+        String s1 = "my string,  \n  my song";
+        String s2 = "another string, from a song";
+        KShingling ks = new KShingling(4);
+        ks.parse(s1);
+        ks.parse(s2);
+        System.out.println(ks.toString());
+        
+        for (boolean b : ks.booleanVectorOf(s1)) {
+            System.out.print(b ? "1" : "0");
+        }
+        System.out.print("\n");
+        
+        for (boolean b : ks.booleanVectorOf(s2)) {
+            System.out.print(b ? "1" : "0");
+        }
+        System.out.print("\n");
+        
+        ks.add("Another shingle?");
+    }
+    
+    protected int k = 5;
+    
+    public KShingling() {
+        super();
+    }
+    
+    public KShingling(int k) {
+        super();
+        this.setK(k);
+    }
+    
+    public int getK() {
+        return k;
+    }
+    
+    /**
+     * Set the size of k-grams.
+     * Default value is 5 (recommended for emails).
+     * A good rule of thumb is to imagine that there are only 20 characters 
+     * and estimate the number of k-shingles as 20^k. For large documents, 
+     * such as research articles, choice k = 9 is considered safe.
+     * @param k 
+     */
+    public final void setK(int k) {
+        if (k <= 0) {
+            throw new InvalidParameterException("k should be positive!");
+        }
+        
+        this.k = k;
+    }
+    
+    public boolean parse(String s) {
+        s = s.replaceAll("\\s+", " ");
+        for (int i = 0; i < (s.length() - k + 1); i++) {
+            this.add(s.substring(i, i+k));
+        }
+        return true;
+    }
+    
+    @Override
+    public boolean add(String s) {
+        if (s.length() != k) {
+            throw  new InvalidParameterException("This size of this String (" +
+                    s.length() + ") is different from k (" + k + ")");
+        }
+        
+        return super.add(s);
+    }
+    
+    public boolean[] booleanVectorOf(String s) {
+        boolean[] r = new boolean[this.size()];
+        
+        int i = 0;
+        for (String shingle : this) {
+            r[i] = s.contains(shingle);            
+            i++;
+        }
+        
+        return r;
+    }
+    
+}
diff --git a/src/info/debatty/java/stringsimilarity/NGram.java b/src/info/debatty/java/stringsimilarity/NGram.java
@@ -1,17 +1,42 @@
 package info.debatty.java.stringsimilarity;
 
 /**
+ * N-Gram Similarity as defined by Kondrak, "N-Gram Similarity and Distance",
+ * String Processing and Information Retrieval, Lecture Notes in Computer 
+ * Science Volume 3772, 2005, pp 115-126.
+ * 
+ * The algorithm uses affixing with special character '\n' two increase the 
+ * weight of first characters. The normalization is achieved by dividing the 
+ * total similarity score the original length of the longer word.
  * 
  * http://webdocs.cs.ualberta.ca/~kondrak/papers/spire05.pdf
- * @author tibo
  */
-public class NGram {
+public class NGram implements StringSimilarityInterface {
+    
+    public static void main(String[] args) {
+        NGram twogram = new NGram(2);
+        
+        System.out.println(twogram.distance("ABCD", "ABTUIO"));
+    }
+
+    private final int n;
+    
+    public NGram(int n) {
+        this.n = n;
+    }
+
+    public NGram() {
+        this.n = 2;
+    }
 
-    public static double Distance(String s0, String s1) {
-        return Distance(s0, s1, 2);
+    @Override
+    public double similarity(String s1, String s2) {
+        return distance(s1, s2);
     }
 
-    private static double Distance(String s0, String s1, int n) {
+    @Override
+    public double distance(String s0, String s1) {
+        final char special = '\n';
         final int sl = s0.length();
         final int tl = s1.length();
 
@@ -41,7 +66,7 @@ private static double Distance(String s0, String s1, int n) {
         //construct sa with prefix
         for (int i = 0; i < sa.length; i++) {
             if (i < n - 1) {
-                sa[i] = 0; //add prefix
+                sa[i] = special; //add prefix
             } else {
                 sa[i] = s0.charAt(i - n + 1);
             }
@@ -63,7 +88,7 @@ private static double Distance(String s0, String s1, int n) {
             //construct t_j n-gram 
             if (j < n) {
                 for (int ti = 0; ti < n - j; ti++) {
-                    t_j[ti] = 0; //add prefix
+                    t_j[ti] = special; //add prefix
                 }
                 for (int ti = n - j; ti < n; ti++) {
                     t_j[ti] = s1.charAt(ti - (n - j));
@@ -79,7 +104,7 @@ private static double Distance(String s0, String s1, int n) {
                 for (int ni = 0; ni < n; ni++) {
                     if (sa[i - 1 + ni] != t_j[ni]) {
                         cost++;
-                    } else if (sa[i - 1 + ni] == 0) { //discount matches on prefix
+                    } else if (sa[i - 1 + ni] == special) { //discount matches on prefix
                         tn--;
                     }
                 }
diff --git a/src/info/debatty/java/stringsimilarity/QGram.java b/src/info/debatty/java/stringsimilarity/QGram.java
@@ -0,0 +1,73 @@
+package info.debatty.java.stringsimilarity;
+
+/**
+ * QGram similarity is the relative number of n-grams both strings have in 
+ * common.
+ * 
+ * @author Thibault Debatty
+ */
+public class QGram implements StringSimilarityInterface {
+    
+    public static void main(String[] args) {
+        QGram dig = new QGram(2);
+        
+        // Should be 2: CD and CE
+        System.out.println(dig.absoluteDistance("ABCD", "ABCE"));
+        
+        // Should be 0.5 (2 / 4)
+        System.out.println(dig.distance("ABCD", "ABCE"));
+    }
+    
+    private int n;
+
+    public QGram(int n) {
+        this.n = n;
+    }
+    
+    public QGram() {
+        this.n = 3;
+    }
+    
+    @Override
+    public double similarity(String s1, String s2) {
+        return 1.0 - distance(s1, s2);
+    }
+
+    @Override
+    public double distance(String s1, String s2) {
+        KShingling sh = new KShingling(n);
+        sh.parse(s1);
+        sh.parse(s2);
+        
+        boolean[] b1 = sh.booleanVectorOf(s1);
+        boolean[] b2 = sh.booleanVectorOf(s2);
+        
+        int d = 0;
+        for (int i = 0; i < b1.length; i++) {
+            if (b1[i] != b2[i]) {
+                d++;
+            }
+        }
+        
+        return ((double) d) / sh.size();
+    }
+    
+    public int absoluteDistance(String s1, String s2) {
+        KShingling sh = new KShingling(n);
+        sh.parse(s1);
+        sh.parse(s2);
+        
+        boolean[] b1 = sh.booleanVectorOf(s1);
+        boolean[] b2 = sh.booleanVectorOf(s2);
+        
+        int d = 0;
+        for (int i = 0; i < b1.length; i++) {
+            if (b1[i] != b2[i]) {
+                d++;
+            }
+        }
+        
+        return d;
+    }
+    
+}
