Merge pull request #421 from PyThaiNLP/corpus-version

Update check corpus version

Author: bact

pythainlp/cli/data.py

@@ -90,7 +90,7 @@ def catalog(self, argv):
         corpus_names = sorted(corpus_db.keys())
         print("Dataset/corpus available for download:")
         for name in corpus_names:
-            print(f"- {name} {corpus_db[name]['version']}", end="")
+            print(f"- {name} {corpus_db[name]['latest_version']}", end="")
             corpus_info = corpus.get_corpus_db_detail(name)
             if corpus_info:
                 print(f"  (Local: {corpus_info['version']})")

pythainlp/corpus/__init__.py

@@ -40,7 +40,7 @@
 _CORPUS_DB_URL = (
     "https://raw.githubusercontent.com/"
     "PyThaiNLP/pythainlp-corpus/"
-    "2.2/db.json"
+    "fix-thainer/db.json"  # 2.2
 )
 
 # local corpus catalog filename

pythainlp/corpus/core.py

@@ -189,7 +189,7 @@ def _check_hash(dst: str, md5: str) -> None:
                 raise Exception("Hash does not match expected.")
 
 
-def download(name: str, force: bool = False, url: str = None) -> bool:
+def download(name: str, force: bool = False, url: str = None, version: str = None) -> bool:
     """
     Download corpus.
 
@@ -199,6 +199,7 @@ def download(name: str, force: bool = False, url: str = None) -> bool:
     :param str name: corpus name
     :param bool force: force download
     :param str url: URL of the corpus catalog
+    :param str version: Version of the corpus
     :return: **True** if the corpus is found and succesfully downloaded.
              Otherwise, it returns **False**.
     :rtype: bool
@@ -233,39 +234,49 @@ def download(name: str, force: bool = False, url: str = None) -> bool:
         local_db = TinyDB(corpus_db_path())
         query = Query()
 
-        corpus = corpus_db[name]
+        corpus = corpus_db[name.lower()]
         print("Corpus:", name)
-        found = local_db.search(query.name == name)
+        if version is None:
+            version = corpus['latest_version']
+        corpus_versions = corpus["versions"][version]
+        file_name = corpus_versions["filename"]
+        found = local_db.search((query.name == name) & (query.version == version))
 
         # If not found in local, download
         if force or not found:
-            print(f"- Downloading: {name} {corpus['version']}")
-            _download(corpus["download"], corpus["file_name"])
-            _check_hash(corpus["file_name"], corpus["md5"])
+            print(f"- Downloading: {name} {version}")
+            _download(
+                corpus_versions["download_url"],
+                file_name,
+            )
+            _check_hash(
+                file_name,
+                corpus_versions["md5"],
+            )
 
             if found:
                 local_db.update(
-                    {"version": corpus["version"]}, query.name == name
+                    {"version": version}, query.name == name
                 )
             else:
                 local_db.insert(
                     {
                         "name": name,
-                        "version": corpus["version"],
-                        "file_name": corpus["file_name"],
+                        "version": version,
+                        "file_name": file_name,
                     }
                 )
         else:
             if local_db.search(
-                query.name == name and query.version == corpus["version"]
+                query.name == name and query.version == version
             ):
                 # Already has the same version
                 print("- Already up to date.")
             else:
                 # Has the corpus but different version
                 current_ver = local_db.search(query.name == name)[0]["version"]
                 print(f"- Existing version: {current_ver}")
-                print(f"- New version available: {corpus['version']}")
+                print(f"- New version available: {version}")
                 print("- Use download(data_name, force=True) to update")
 
         local_db.close()

pythainlp/tag/named_entity.py

@@ -13,7 +13,7 @@
 from pythainlp.tokenize import word_tokenize
 from pythainlp.util import isthai
 
-_CORPUS_NAME = "thainer-1-4"
+_CORPUS_NAME = "thainer"
 _TOKENIZER_ENGINE = "newmm"  # should be the same as one used in training data
 
 

tests/test_corpus.py

@@ -52,6 +52,8 @@ def test_corpus(self):
         self.assertIsNotNone(get_corpus_db_detail("test"))  # corpus exists
         self.assertTrue(remove("test"))  # remove existing
         self.assertFalse(remove("test"))  # remove non-existing
+        self.assertTrue(download(name="test", version="0.1"))
+        self.assertTrue(remove("test"))
 
     def test_tnc(self):
         self.assertIsNotNone(tnc.word_freqs())

tests/test_util.py

@@ -361,10 +361,11 @@ def test_thaiword_to_date(self):
             now + timedelta(days=-2), thaiword_to_date("วานซืน", now)
         )
 
-        self.assertEqual(
-            thaiword_to_date("วันนี้").day + 1,
-            thaiword_to_date("พรุ่งนี้").day,
-        )
+        # it's error if "พรุ่งนี้" is 1 not 32.
+        #self.assertEqual(
+        #    thaiword_to_date("วันนี้").day + 1,
+        #    thaiword_to_date("พรุ่งนี้").day,
+        #)
         self.assertIsNone(thaiword_to_date("วันไหน"))
 
     # ### pythainlp.util.trie