move nktl downloader to ptadownloader. Added in morph stemmer, it is dependent on having wordnet installed

Author: yooper

composer.json

@@ -25,7 +25,7 @@
     },
     "require" : {
         "php": ">=5.5",
-        "yooper/stop-words": "dev-master",
+        "yooper/stop-words": "^1.0",
         "symfony/console": ">=2.7"
     },
     "require-dev": {

src/Console/Commands/NltkPackageInstallAllCommand.php

@@ -11,14 +11,14 @@
 
 
 /**
- * Install all the nltk data packages
+ * Install all the pta data packages
  * @author yooper
  */
 class NltkPackageInstallAllCommand extends Command
 {
     protected function configure()
     {
-        $this->setName('nltk:install:all')
+        $this->setName('pta:install:all')
             ->setDescription('Install all packages from pta data');              
     }
 

src/Console/Commands/NltkPackageInstallCommand.php

@@ -20,12 +20,12 @@ class NltkPackageInstallCommand extends Command
 {
     protected function configure()
     {
-        $this->setName('nltk:install:package')
+        $this->setName('pta:install:package')
             ->setDescription('Install the selected corpus')
             ->addArgument(
                 'package',
                 InputArgument::REQUIRED,
-                'You must selected a valid package id, use nltk:list to explore the available options.'
+                'You must selected a valid package id, use pta:list to explore the available options.'
             );               
     }
 
@@ -47,7 +47,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
         }
 
         if(!$packageFound) { 
-            $output->writeln("Package {$packageId} was not found, try textconsole nltk:list, to see the available packages");
+            $output->writeln("Package {$packageId} was not found, try textconsole pta:list, to see the available packages");
         } else {
             $output->writeln("Package {$package->getId()} - {$package->getName()} was installed into {$download->getInstallDir()}");
         }        

src/Console/Commands/NltkPackageListCommand.php

@@ -17,12 +17,12 @@ class NltkPackageListCommand extends Command
 {
     protected function configure()
     {
-        $this->setName('nltk:list')
-            ->setDescription('List Corpora available in the nltk data repo.')
+        $this->setName('pta:list')
+            ->setDescription('List Corpora available in the pta data repo.')
             ->addArgument(
                 'url',
                 InputArgument::OPTIONAL,
-                'Use a different url to download the nltk package list.'
+                'Use a different url to download the pta/nltk package list.'
             );               
     }
 

src/Downloaders/DownloadPackageFactory.php

@@ -52,11 +52,14 @@ static public function download(Package $package)
     }
 
     /**
-     * Verify the packages checksum against the downloaded file
+     * Verify the packages checksum against the downloaded file, if it exists
      * @return boolean
      */
     public function verifyChecksum()
     {
+        if(empty($this->getPackage()->getChecksum())) {
+            return true;
+        }
         return $this->getPackage()->getChecksum() === md5($this->getDownloadFullPath());
     }
 
@@ -169,7 +172,7 @@ protected function downloadRemoteFile()
      */
     public function getDownloadFullPath()
     {
-       return  sys_get_temp_dir().DIRECTORY_SEPARATOR.'nltk-downloads'
+       return  sys_get_temp_dir().DIRECTORY_SEPARATOR.'pta-downloads'
                .DIRECTORY_SEPARATOR.$this->getPackage()->getSubdir()
                .DIRECTORY_SEPARATOR.basename($this->getPackage()->getUrl());
     }

src/Downloaders/NltkCorporaIndexDownloader.php

@@ -89,7 +89,7 @@ public function getXmlContent()
      */
     protected function getCacheFileName()
     {
-        return 'nltk-list.xml';
+        return 'pta-list.xml';
     }
 
 

src/Stemmers/MorphStemmer.php

@@ -0,0 +1,66 @@
+<?php
+
+namespace TextAnalysis\Stemmers;
+
+use TextAnalysis\Indexes\WordnetIndex;
+use TextAnalysis\Corpus\WordnetCorpus;
+use TextAnalysis\Interfaces\IStemmer;
+
+/**
+ * 
+ * Use a morph stemmer to stem to the base word
+ * @author dcardin
+ */
+class MorphStemmer implements IStemmer
+{
+    /**
+     *
+     * @var array
+     */
+    protected $cache = [];
+   
+    /**
+     *
+     * @var WordnetIndex
+     */
+    protected $wordnetIndex = null;
+    
+    public function __construct() 
+    {
+        $this->wordnetIndex = new WordnetIndex(new WordnetCorpus(get_storage_path('corpora/wordnet')));
+    }
+    
+    /**
+     * 
+     * @return WordnetIndex
+     */
+    public function getWordnetIndex()
+    {
+        return $this->wordnetIndex;
+    }
+    
+    /**
+     * 
+     * @param string $token
+     * @return string
+     */
+    public function stem($token) 
+    {
+        if(!isset($this->cache[$token])) {
+            if(mb_strlen($token) < 3){ 
+                $this->cache[$token] = $token;
+            } else {
+                $this->cache[$token] = $this->getWordnetIndex()->getMorph($token);
+            }            
+        }
+        return $this->cache[$token];        
+    }
+    
+    public function __destruct() 
+    {
+        unset($this->cache);
+        unset($this->wordnetIndex);
+    }
+   
+
+}

tests/TextAnalysis/Stemmers/MorphStemmerTest.php

@@ -0,0 +1,23 @@
+<?php
+
+namespace Tests\TextAnalysis\Stemmers;
+
+use TextAnalysis\Stemmers\MorphStemmer;
+
+/**
+ * Description of MorphStemmerTest
+ *
+ * @author yooper
+ */
+class MorphStemmerTest extends \PHPUnit_Framework_TestCase
+{
+    public function testMorphStemmer()
+    {
+        if( getenv('SKIP_TEST')) {
+            return;
+        }        
+        $stemmer = new MorphStemmer();
+        $this->assertEquals('university', $stemmer->stem('universities'));
+            
+    }
+}