Speed optimization

Author: tdebatty

src/main/java/info/debatty/java/stringsimilarity/Cosine.java

@@ -24,18 +24,10 @@
 
 package info.debatty.java.stringsimilarity;
 
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Set;
-
 /**
- * Implements Cosine Similarity.
- * The strings are first transformed in vectors of occurences of k-shingles 
- * (sequences of k characters). In this n-dimensional space, the similarity
- * between the two strings is the cosine of their respective vectors.
  * @author Thibault Debatty
  */
-public class Cosine implements StringSimilarityInterface {
+public class Cosine extends SetBasedStringSimilarity {
 
     /**
      * @param args the command line arguments
@@ -59,75 +51,54 @@ public static void main(String[] args) {
         System.out.println(cos.similarity("ABAB", "BAB"));
     }
 
-    private int k;
-    
+    /**
+     * Implements Cosine Similarity.
+     * The strings are first transformed in vectors of occurrences of k-shingles 
+     * (sequences of k characters). In this n-dimensional space, the similarity
+     * between the two strings is the cosine of their respective vectors.
+     * 
+     * @param k 
+     */
     public Cosine(int k) {
-        this.k = k;
+        super(k);
     }
 
     public Cosine() {
-        this.k = 3;
+        super(3);
     }
 
-    /**
-     * Computes the cosine similarity of s1 and s2.
-     * The strings are first converted to vectors in the space of k-shingles. 
-     * The cosine similarity is computed as V1 . V2 / (|V1| * |V2|)
-     * @param s1
-     * @param s2
-     * @return Cosine similarity
-     */
-    public double similarity(String s1, String s2) {
-        if (s1.equals(s2)) {
-            return 1.0;
-        }
-        
-        
-        if (s1.equals("") || s2.equals("")) {
-            return 0.0;
-        }
-        
-        KShingling ks = new KShingling(this.k);
-        HashMap<String, Integer> profile1 = ks.getProfile(s1);
-        HashMap<String, Integer> profile2 = ks.getProfile(s2);
+
+    public double similarity(int[] profile1, int[] profile2) {
 
-        return dotProduct(profile1, profile2) / (norm(profile1) * norm(profile2));   
+        return dotProduct(profile1, profile2) / (norm(profile1) * norm(profile2));
     }
 
-    public double distance(String s1, String s2) {
-        return 1.0 - similarity(s1, s2);
-    }
+
 
     /**
      * Compute the norm L2 : sqrt(Sum_i( v_i^2))
      * @param profile
      * @return L2 norm
      */
-    protected static double norm(HashMap<String, Integer> profile) {
+    protected static double norm(int[] profile) {
         double agg = 0;
 
-        for (int v : profile.values()) {
+        for (int v : profile) {
             agg += v * v;
         }
 
         return Math.sqrt(agg);
     }
 
-    protected static double dotProduct(HashMap<String, Integer> profile1,
-            HashMap<String, Integer> profile2) {
+    protected static double dotProduct(int[] profile1, int[] profile2) {
+        int length = Math.max(profile1.length, profile2.length);
+        profile1 = java.util.Arrays.copyOf(profile1, length);
+        profile2 = java.util.Arrays.copyOf(profile2, length);
 
         double agg = 0;
-        Set<String> union = new HashSet<String>();
-        union.addAll(profile1.keySet());
-        union.addAll(profile2.keySet());
-        
-        for (String key : union) {
-            int v1 = profile1.containsKey(key) ? profile1.get(key) : 0;
-            int v2 = profile2.containsKey(key) ? profile2.get(key) : 0;
-            agg += v1 * v2;
+        for (int i = 0; i < length; i++) {
+            agg += profile1[i] * profile2[i];
         }
-        
         return agg;
     }
-    
 }

src/main/java/info/debatty/java/stringsimilarity/Jaccard.java

@@ -24,15 +24,11 @@
 
 package info.debatty.java.stringsimilarity;
 
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Set;
-
 /**
  * 
  * @author Thibault Debatty
  */
-public class Jaccard implements StringSimilarityInterface {
+public class Jaccard extends SetBasedStringSimilarity {
 
     /**
      * @param args the command line arguments
@@ -47,7 +43,7 @@ public static void main(String[] args) {
         System.out.println(j2.similarity("ABCDE", "ABCDF"));
     }
 
-    private final int k;
+    
 
     /**
      * The strings are first transformed into sets of k-shingles (sequences of k
@@ -57,17 +53,13 @@ public static void main(String[] args) {
      * @param k 
      */
     public Jaccard(int k) {
-        this.k = k;
+        super(k);
     }
 
     public Jaccard() {
-        this.k = 3;
+        super(3);
     }
 
-    public double similarity(String s1, String s2) {
-        KShingling ks = new KShingling(this.k);
-        return similarity(ks.getProfile(s1), ks.getProfile(s2));
-    }
 
     /**
      * Compute and return the Jaccard index similarity between two string profiles.
@@ -82,23 +74,24 @@ public double similarity(String s1, String s2) {
      * @param profile2
      * @return 
      */
-    public double similarity(HashMap<String,Integer> profile1,
-        HashMap<String,Integer> profile2) {
-        Set<String> set1 = profile1.keySet();
-        Set<String> set2 = profile2.keySet();
-        
-        Set union = new HashSet();
-        union.addAll(set1);
-        union.addAll(set2);
+    public double similarity(int[] profile1, int[] profile2) {
+        int length = Math.max(profile1.length, profile2.length);
+        profile1 = java.util.Arrays.copyOf(profile1, length);
+        profile2 = java.util.Arrays.copyOf(profile2, length);
 
-        Set inter = new HashSet(set1);
-        inter.retainAll(set2);
+        int inter = 0;
+        int union = 0;
 
-        return (double) inter.size() / union.size();
-    }
-
-    public double distance(String s1, String s2) {
-        return 1.0 - similarity(s1, s2);
-    }
+        for (int i = 0; i < length; i++) {
+            if (profile1[i] > 0 || profile2[i] > 0) {
+                union++;
+                
+                if (profile1[i] > 0 && profile2[i] > 0) {
+                    inter++;
+                }
+            }
+        }
 
+        return (double) inter / union;
+    }
 }

src/main/java/info/debatty/java/stringsimilarity/KShingling.java

@@ -1,7 +1,10 @@
 package info.debatty.java.stringsimilarity;
 
 import java.security.InvalidParameterException;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
 import java.util.regex.Pattern;
 
 /**
@@ -30,7 +33,8 @@ public static void main(String[] args) {
         System.out.println(ks.getProfile("ABCAB"));
     }
 
-    protected int k = 5;
+    protected int k;
+    private HashMap<String, Integer> shingles = new HashMap<String, Integer>();
 
     /**
      * k-shingling is the operation of transforming a string (or text document) into
@@ -42,35 +46,28 @@ public static void main(String[] args) {
      * "Mining of Massive Datasets", Cambridge University Press:
      * Multiple subsequent spaces are replaced by a single space, and a k-gram is a 
      * sequence of k characters. 
+     * 
+     * Default value of k is 5 (recommended for emails).
+     * A good rule of thumb is to imagine that there are only 20 characters 
+     * and estimate the number of k-shingles as 20^k. For large documents, 
+     * such as research articles, k = 9 is considered a safe choice.
      */
     public KShingling() {
-        
+        k = 5;
     }
 
     public KShingling(int k) {
-        this.setK(k);
-    }
-    
-    public int getK() {
-        return k;
-    }
-    
-    /**
-     * Set the size of k-grams.
-     * Default value is 5 (recommended for emails).
-     * A good rule of thumb is to imagine that there are only 20 characters 
-     * and estimate the number of k-shingles as 20^k. For large documents, 
-     * such as research articles, choice k = 9 is considered safe.
-     * @param k 
-     */
-    public final void setK(int k) {
         if (k <= 0) {
             throw new InvalidParameterException("k should be positive!");
         }
 
         this.k = k;
     }
 
+    public int getK() {
+        return k;
+    }
+    
     /**
      * Pattern for finding multiple following spaces
      */
@@ -91,7 +88,32 @@ public final void setK(int k) {
      * @param s
      * @return 
      */
-    public HashMap<String, Integer> getProfile(String s) {
+    public int[] getProfile(String s) {
+        ArrayList<Integer> r = new ArrayList<Integer>(shingles.size());
+        for (int i = 0; i < shingles.size(); i++) {
+            r.add(0);
+        }
+        
+        s = spaceReg.matcher(s).replaceAll(" ");
+        String shingle;
+        for (int i = 0; i < (s.length() - k + 1); i++) {
+            shingle = s.substring(i, i+k);
+            int position;
+            
+            if (shingles.containsKey(shingle)) {
+                position = shingles.get(shingle);
+                r.set(position, r.get(position) + 1);
+                
+            } else {
+                shingles.put(shingle, shingles.size());
+                r.add(1);
+            }
+            
+        }
+        
+        return convertIntegers(r);
+        
+        /*
         HashMap<String, Integer> r = new HashMap<String, Integer>(s.length() / 2);
         s = spaceReg.matcher(s).replaceAll(" ");
         String kgram;
@@ -107,6 +129,15 @@ public HashMap<String, Integer> getProfile(String s) {
                 r.put(kgram, 1);
             }
         }
-        return r;
+        return r;*/
+    }
+ 
+    public static int[] convertIntegers(List<Integer> integers) {
+        int[] ret = new int[integers.size()];
+        Iterator<Integer> iterator = integers.iterator();
+        for (int i = 0; i < ret.length; i++) {
+            ret[i] = iterator.next().intValue();
+        }
+        return ret;
     }
 }