bug fixes and more optimizations

Author: yooper

src/Analysis/Keywords/Rake.php

@@ -113,7 +113,7 @@ public function getKeywordScores()
             $tokens = explode(" ", $phrase);            
             foreach($tokens as $keyword)
             {                
-                $degrees[$keyword] += substr_count($phrase, " ")+1;                                    
+                $degrees[$keyword] += count($tokens);                                    
             }  
         }      
 

src/Filters/WhitespaceFilter.php

@@ -12,7 +12,7 @@ class WhitespaceFilter implements ITokenTransformation
 {
     public function transform($word) 
     {
-        return preg_replace("/(^\s+)|(\s+$)/us", "", preg_replace( '/\s+/', ' ', $word ));
+        return preg_replace("/\s[[:space:]]+/", " ", str_replace(["\r", "\n"], ' ', $word));
     }
 
 }

src/Tokenizers/PennTreeBankTokenizer.php

@@ -9,7 +9,7 @@
  * Based on http://www.cis.upenn.edu/~treebank/tokenizer.sed
  * @author yooper
  */
-class PennTreeBankTokenizer extends WhitespaceTokenizer
+class PennTreeBankTokenizer
 {
     /**
      *
@@ -24,11 +24,11 @@ public function __construct()
 
     /**
      * Calls internal functions to handle data processing
-     * @param type $string
+     * @param string $str
      */
     public function tokenize($str)
     {
-        return parent::tokenize($this->execute($str));
+        return preg_split('/[\pZ\pC]+/u', $this->execute($str), null, PREG_SPLIT_NO_EMPTY);
     }
     /**
      * Handles the data processing

src/Tokenizers/WhitespaceTokenizer.php

@@ -9,7 +9,7 @@ class WhitespaceTokenizer extends TokenizerAbstract
 {
     public function tokenize($string)
     {
-        return preg_split('/[\pZ\pC]+/u', $string, null, PREG_SPLIT_NO_EMPTY);
+        return mb_split('\s+', $string);
     }
 }