add support for download progress bar for pta:install:package. Made API for Stanford taggers easier

Author: yooper

README.md

@@ -16,25 +16,36 @@ composer require yooper/php-text-analysis
 Documentation for the library resides in the wiki. 
 https://github.com/yooper/php-text-analysis/wiki
 
+
+
+
 Dictionary Installation
 =============
 
 Not required unless you use the dictionary stemmers
 
-*For Ubuntu*
+*For Ubuntu < 16*
 ```
 sudo apt-get install libpspell-dev 
 sudo apt-get install php5-pspell
 sudo apt-get install aspell-en
 sudo apt-get install php5-enchant
 ```
+*For Ubuntu >= 16*
+```
+sudo apt-get install libpspell-dev php7.0-pspell aspell-en php7.0-enchant
+```
+
+
 *For Centos* 
 ```
 sudo yum install php5-pspell
 sudo yum install aspell-en
 sudo yum install php5-enchant
 ```
 
+*PHP Pecl Stem* is not currently available in php 7.0. 
+
 
 Tokenize
 =============

src/Console/Commands/NltkPackageInstallCommand.php

@@ -6,18 +6,30 @@
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Helper\ProgressBar;
 
 use TextAnalysis\Downloaders\DownloadPackageFactory as DPF;
 use TextAnalysis\Downloaders\NltkCorporaIndexDownloader;
 
 
+
 /**
  * Installs the selected nltk corpus package 
  *
  * @author yooper
  */
 class NltkPackageInstallCommand extends Command
 {
+    /**
+    * @var ProgressBar
+    */
+    protected $progressBar = null;
+    
+    /**
+     * @var \Symfony\Component\Console\Output\OutputInterface
+     */
+    private $output;
+
     protected function configure()
     {
         $this->setName('pta:install:package')
@@ -31,25 +43,88 @@ protected function configure()
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {        
+        $this->output = $output;
         $packageId = $input->getArgument('package');
 
         $listPackages = (new NltkCorporaIndexDownloader())->getPackages();
 
-        $packageFound = false;
+        $packageFound = null;
 
         foreach($listPackages as $package) 
         { 
             if($packageId == $package->getId()) {
-                $packageFound = true;
-                $download = DPF::download($package);                
+                $packageFound = $package;
                 break;
             }
         }
 
         if(!$packageFound) { 
             $output->writeln("Package {$packageId} was not found, try textconsole pta:list, to see the available packages");
         } else {
+            
+            $download = DPF::download($package);  
+            // Create stream context.
+            $context = stream_context_create([], ['notification' => [$this, 'progress']]);
+
+            // Pipe file.
+            $resource = fopen($packageFound->getUrl(), 'r', null, $context);
+            $stream = fopen($download->getDownloadFullPath(), 'w+');
+            if (!$stream) {
+                $output->writeln("Package {$packageFound->getId()} - {$packageFound->getName()} install failed, permission denied to create file into {$download->getDownloadFullPath()}");
+            }
+
+            stream_copy_to_stream($resource, $stream);
+            
+            if (!fclose($stream)) {
+                $output->writeln("Could not save file {$download->getDownloadFullPath()}");
+            }
+
+            // End output.
+            $this->progressBar->finish();
+
+            if(!$download->verifyChecksum()) {
+                $output->writeln("Bad checksum for the downloaded package {$packageFound->getId()}");
+                exit;        
+            }
+            $download->unpackPackage();
+            $output->writeln(PHP_EOL);
             $output->writeln("Package {$package->getId()} - {$package->getName()} was installed into {$download->getInstallDir()}");
         }        
     }
+
+    /**
+     * @param int $notificationCode
+     * @param int $severity
+     * @param string $message
+     * @param int $messageCode
+     * @param int $bytesTransferred
+     * @param int $bytesMax
+     */
+    public function progress($notificationCode, $severity, $message, $messageCode, $bytesTransferred, $bytesMax)
+    {
+        if (STREAM_NOTIFY_REDIRECTED === $notificationCode) {
+            $this->progressBar->clear();
+            $this->progressBar = null;
+            return;
+        }
+
+        if (STREAM_NOTIFY_FILE_SIZE_IS === $notificationCode) {
+            if ($this->progressBar) {
+                $this->progressBar->clear();
+            }
+            $this->progressBar = new ProgressBar($this->output, $bytesMax);
+        }
+
+        if (STREAM_NOTIFY_PROGRESS === $notificationCode) {
+            if (is_null($this->progressBar)) {
+                $this->progressBar = new ProgressBar($this->output);
+            }
+            $this->progressBar->setProgress($bytesTransferred);
+        }
+
+        if (STREAM_NOTIFY_COMPLETED === $notificationCode) {
+            $this->finish($bytesTransferred);
+        }
+    }
+
 }

src/Downloaders/DownloadPackageFactory.php

@@ -31,14 +31,7 @@ protected function __construct(Package $package)
         if(file_exists($this->getDownloadFullPath()) && $this->verifyChecksum()) {
             return;
         }     
-        
-        $this->downloadRemoteFile();
-        
-        if($this->verifyChecksum()) {
-            throw new Exception("Bad checksum for the downloaded package {$this->getPackage()->getId()}");            
-        }
-        
-        $this->unpackPackage();        
+               
     }
 
     /**
@@ -67,7 +60,7 @@ public function verifyChecksum()
      * de-compress the downloaded corpus into the install directory, or
      * copy the files into the install directory
      */
-    protected function unpackPackage()
+    public function unpackPackage()
     {
         // it is zipped, we must unzip it
         if($this->getPackage()->getUnzip()) {
@@ -150,21 +143,6 @@ public function initialize()
 
     }
 
-    /**
-     * @todo improve downloader code, make it more robust
-     */
-    protected function downloadRemoteFile()
-    {
-        $handle = fopen($this->getPackage()->getUrl(), "rb");
-        $fp = fopen($this->getDownloadFullPath(), 'w');
-        $content = '';
-        while (!feof($handle)) {
-          $content = fread($handle, 8192);
-          fwrite($fp, $content);
-        }
-        fclose($handle);
-        fclose($fp);
-    }
 
     /**
      * Has the full path to where the download should go

src/Taggers/StanfordAbstract.php

@@ -160,7 +160,7 @@ public function getPathToJava()
         if(getenv('JAVA_HOME')) {
             return getenv('JAVA_HOME');
         } else {
-            return 'java';
+            throw new RuntimeException('env JAVA_HOME must be set.');
         }
     }
 

src/Taggers/StanfordNerTagger.php

@@ -12,7 +12,17 @@
  */
 class StanfordNerTagger extends StanfordAbstract
 {
-    public function __construct($jarPath, $classifierPath, $javaOptions = array(), $separator = '/') {
+    public function __construct($jarPath = null, $classifierPath = null, $javaOptions = array(), $separator = '/') {
+        $nerPath = 'taggers/stanford-ner-2015-12-09';
+        
+        if(!$jarPath) {
+            $jarPath = get_storage_path($nerPath).'stanford-ner.jar';
+        }
+
+        if(!$classifierPath) {
+            $classifierPath = get_storage_path($nerPath.DIRECTORY_SEPARATOR."classifiers")."english.all.3class.distsim.crf.ser.gz";
+        }
+
         parent::__construct($jarPath, $classifierPath, $javaOptions, $separator);
         // created the temp file
         $this->tmpFilePath = tempnam(sys_get_temp_dir(), "stanford_ner_");

src/Taggers/StanfordPosTagger.php

@@ -12,8 +12,18 @@
  */
 class StanfordPosTagger extends StanfordAbstract
 {
-    public function __construct($jarPath, $classifierPath, $javaOptions = array(), $separator = '/') {
-        parent::__construct($jarPath, $classifierPath, $javaOptions, $separator);
+    public function __construct($jarPath = null, $modelPath = null, $javaOptions = array(), $separator = '/') 
+    {
+        $posPath = 'taggers/stanford-postagger-2015-12-09'; 
+        if(!$jarPath) {
+            $jarPath = get_storage_path($posPath).'stanford-postagger-3.6.0.jar';
+        }
+
+        if(!$modelPath) {
+            $modelPath = get_storage_path($posPath.DIRECTORY_SEPARATOR."models")."english-left3words-distsim.tagger";
+        }
+
+        parent::__construct($jarPath, $modelPath, $javaOptions, $separator);
         // created the temp file
         $this->tmpFilePath = tempnam(sys_get_temp_dir(), "stanford_pos_");
     }   

tests/TextAnalysis/Adapters/EnchantAdapterTest.php

@@ -17,7 +17,7 @@ public function testSpelling()
         }        
         $adapter = new EnchantAdapter();
         $this->assertEquals('run', $adapter->suggest("runn")[0]);                
-        $this->assertEquals('cooper', $adapter->suggest("yooper")[0]); 
+        $this->assertEquals('looper', $adapter->suggest("yooper")[0]); 
         $this->assertEquals('flute', $adapter->suggest("flute")[0]);         
     }
 }

tests/TextAnalysis/Stemmers/DictionaryStemmerTest.php

@@ -14,7 +14,7 @@ class DictionaryStemmerTest extends \PHPUnit_Framework_TestCase
 {
     public function testPspell()
     {       
-        if( getenv('SKIP_TEST')) {
+        if( getenv('SKIP_TEST') || !extension_loaded('stem')) {
             return;
         }
 

tests/TextAnalysis/Stemmers/SnowballStemmerTest.php

@@ -12,7 +12,7 @@ class SnowballStemmerTest extends \PHPUnit_Framework_TestCase
 {
     public function testDefaultEnglish()
     {       
-        if( getenv('SKIP_TEST')) {
+        if( getenv('SKIP_TEST') || !extension_loaded('stem')) {
             return;
         }
         $stemmer = new SnowballStemmer('english', $exceptions = ['universities' => 'university']);
@@ -27,7 +27,7 @@ public function testDefaultEnglish()
 
     public function testSwedish()
     {       
-        if( getenv('SKIP_TEST')) {
+        if( getenv('SKIP_TEST') || !extension_loaded('stem')) {
             return;
         }
         $stemmer = new SnowballStemmer('swedish');
@@ -36,7 +36,7 @@ public function testSwedish()
 
     public function testException()
     {
-        if( getenv('SKIP_TEST')) {
+        if( getenv('SKIP_TEST') || !extension_loaded('stem')) {
             return;
         }      
         $this->setExpectedException('Exception');

tests/TextAnalysis/Taggers/StanfordNerTaggerTest.php

@@ -14,6 +14,8 @@
  */
 class StanfordNerTaggerTest extends \PHPUnit_Framework_TestCase
 {
+    protected $nerPath = 'taggers/stanford-ner-2015-12-09';
+
     protected $text = "Marquette County is a county located in the Upper Peninsula of the US state of Michigan. As of the 2010 census, the population was 67,077.";
 
     public function testJarNotFound()
@@ -29,7 +31,7 @@ public function testClassiferNotFound()
             return;
         }           
 
-        $tagger = new StanfordNerTagger(get_storage_path('ner').'stanford-ner.jar', "classifier.gz");
+        $tagger = new StanfordNerTagger(get_storage_path($this->nerPath).'stanford-ner.jar', "classifier.gz");
         $this->setExpectedException('RuntimeException', 'Classifier not found classifier.gz');
         $tagger->tag([]);        
     }
@@ -51,20 +53,16 @@ public function testTempCreatedFile()
 
     public function testStanfordNer()
     {
-        if( getenv('SKIP_TEST') || !getenv('JAVA_HOME')) {
+        if( getenv('SKIP_TEST')) {
             return;
         }        
-        
+    
         $document = new TokensDocument((new WhitespaceTokenizer())->tokenize($this->text));
-        
-        $jarPath = get_storage_path('ner').'stanford-ner.jar';
-        $classiferPath = get_storage_path('ner'.DIRECTORY_SEPARATOR."classifiers")."english.all.3class.distsim.crf.ser.gz";
-        
-        $tagger = new StanfordNerTagger($jarPath, $classiferPath);
+        $tagger = new StanfordNerTagger();
         $output = $tagger->tag($document->getDocumentData());
 
         $this->assertFileExists($tagger->getTmpFilePath());        
-        $this->assertEquals(138, filesize($tagger->getTmpFilePath()));        
+        $this->assertEquals(138, filesize($tagger->getTmpFilePath())); 
         $this->assertEquals(['Michigan','LOCATION'], $output[15], "Did you set JAVA_HOME env variable?");        
     }