More test cases for corpus

Author: bact

pythainlp/corpus/__init__.py

@@ -51,14 +51,23 @@
 
 
 def corpus_path() -> str:
+    """
+    Get path where corpus files are kept locally.
+    """
     return _CORPUS_PATH
 
 
 def corpus_db_url() -> str:
+    """
+    Get remote URL of corpus catalog.
+    """
     return _CORPUS_DB_URL
 
 
 def corpus_db_path() -> str:
+    """
+    Get local path of corpus catalog.
+    """
     return _CORPUS_DB_PATH
 
 

pythainlp/corpus/core.py

@@ -48,7 +48,7 @@ def get_corpus(filename: str) -> frozenset:
     `this file
     <https://github.com/PyThaiNLP/pythainlp-corpus/blob/master/db.json>`_
 
-    :param string filename: filename of the corpus to be read
+    :param str filename: filename of the corpus to be read
 
     :return: :mod:`frozenset` consist of lines in the file
     :rtype: :mod:`frozenset`
@@ -81,7 +81,7 @@ def get_corpus_path(name: str) -> Union[str, None]:
     """
     Get corpus path.
 
-    :param string name: corpus name
+    :param str name: corpus name
     :return: path to the corpus or **None** of the corpus doesn't
              exist in the device
     :rtype: str
@@ -141,6 +141,7 @@ def _download(url: str, dst: str) -> int:
         pbar = None
         try:
             from tqdm import tqdm
+
             pbar = tqdm(total=int(r.headers["Content-Length"]))
         except ImportError:
             pbar = None
@@ -173,15 +174,18 @@ def _check_hash(dst: str, md5: str) -> None:
                 raise Exception("Hash does not match expected.")
 
 
-def download(name: str, force: bool = False) -> bool:
+def download(
+    name: str, force: bool = False, corpus_db_url: str = None
+) -> bool:
     """
     Download corpus.
 
     The available corpus names can be seen in this file:
     https://github.com/PyThaiNLP/pythainlp-corpus/blob/master/db.json
 
-    :param string name: corpus name
+    :param str name: corpus name
     :param bool force: force download
+    :param str
     :return: **True** if the corpus is found and succesfully downloaded.
              Otherwise, it returns **False**.
     :rtype: bool
@@ -201,9 +205,12 @@ def download(name: str, force: bool = False) -> bool:
     ``$HOME/pythainlp-data/``
     (e.g. ``/Users/bact/pythainlp-data/wiki_lm_lstm.pth``).
     """
-    corpus_db = get_corpus_db(corpus_db_url())
+    if not corpus_db_url:
+        corpus_db_url = corpus_db_url()
+
+    corpus_db = get_corpus_db(corpus_db_url)
     if not corpus_db:
-        print(f"Cannot download corpus database from: {corpus_db_url()}")
+        print(f"Cannot download corpus database from: {corpus_db_url}")
         return False
 
     corpus_db = corpus_db.json()
@@ -259,7 +266,7 @@ def remove(name: str) -> bool:
     """
     Remove corpus
 
-    :param string name: corpus name
+    :param str name: corpus name
     :return: **True** if the corpus is found and succesfully removed.
              Otherwise, it returns **False**.
     :rtype: bool

pythainlp/soundex/core.py

@@ -17,7 +17,7 @@ def soundex(text: str, engine: str = DEFAULT_SOUNDEX_ENGINE) -> str:
     """
     This function converts Thai text into phonetic code.
 
-    :param string text: word
+    :param str text: word
     :param str engine: soundex engine
     :return: Soundex code
     :rtype: str

pythainlp/tag/named_entity.py

@@ -89,10 +89,10 @@ def get_ner(
         """
         This function tags named-entitiy from text in IOB format.
         
-        :param string text: text in Thai to be tagged
-        :param boolean pos: To include POS tags in the results (`True`) or
+        :param str text: text in Thai to be tagged
+        :param bool pos: To include POS tags in the results (`True`) or
                             exclude (`False`). The defualt value is `True`
-        :param boolean tag: output like html tag.
+        :param bool tag: output like html tag.
         :return: a list of tuple associated with tokenized word, NER tag,
                  POS tag (if the parameter `pos` is specified as `True`),
                  and output like html tag (if the parameter `tag` is

pythainlp/word_vector/core.py

@@ -177,8 +177,8 @@ def similarity(word1: str, word2: str) -> float:
     """
     This function computae cosine similarity between two words.
 
-    :param string word1: first word to be compared
-    :param string word2: second word to be compared
+    :param str word1: first word to be compared
+    :param str word2: second word to be compared
 
     :raises KeyError: if either `word1` or `word2` is not in the vocabulary
                       of the model.
@@ -218,8 +218,8 @@ def sentence_vectorizer(text: str, use_mean: bool = True) -> ndarray:
     Then, word vectors are aggregatesd into one vector of 300 dimension
     by calulating either mean, or summation of all word vectors.
 
-    :param string text: text input
-    :param boolean use_mean: if `True` aggregate word vectors with mean of all
+    :param str text: text input
+    :param bool use_mean: if `True` aggregate word vectors with mean of all
                              word vectors. Otherwise, aggregate with summation
                              of all word vectors
 

tests/test_corpus.py

@@ -27,21 +27,31 @@ def test_conceptnet(self):
         self.assertIsNotNone(conceptnet.edges("รัก"))
 
     def test_corpus(self):
-        self.assertIsNotNone(countries())
-        self.assertIsNotNone(provinces())
-        self.assertIsNotNone(thai_negations())
-        self.assertIsNotNone(thai_stopwords())
-        self.assertIsNotNone(thai_syllables())
-        self.assertIsNotNone(thai_words())
-        self.assertIsNotNone(thai_female_names())
-        self.assertIsNotNone(thai_male_names())
-        self.assertEqual(get_corpus_db_detail("XXX"), {})
-        self.assertIsNotNone(download("test"))
-        self.assertIsNotNone(download("test", force=True))
-        self.assertFalse(download("XxxXXxxx817d37sf"))  # not exist
-        self.assertIsNotNone(get_corpus_db_detail("test"))
-        self.assertIsNotNone(remove("test"))
-        self.assertFalse(remove("test"))
+        self.assertTrue(isinstance(thai_negations(), frozenset))
+        self.assertTrue(isinstance(thai_stopwords(), frozenset))
+        self.assertTrue(isinstance(thai_syllables(), frozenset))
+        self.assertTrue(isinstance(thai_words(), frozenset))
+
+        self.assertTrue(isinstance(countries(), frozenset))
+        self.assertTrue(isinstance(provinces(), frozenset))
+        self.assertTrue(isinstance(thai_female_names(), frozenset))
+        self.assertTrue(isinstance(thai_male_names(), frozenset))
+
+        self.assertEqual(
+            get_corpus_db_detail("XXX"), {}
+        )  # corpus does not exist
+        self.assertTrue(download("test"))  # download the first time
+        self.assertTrue(download(name="test", force=True))  # force download
+        self.assertTrue(download(name="test"))  # try download existing
+        self.assertFalse(
+            download(name="test", corpus_db_url="wrongurl")
+        )  # URL not exist
+        self.assertFalse(
+            download(name="XxxXXxxx817d37sf")
+        )  # corpus name not exist
+        self.assertIsNotNone(get_corpus_db_detail("test"))  # corpus exists
+        self.assertTrue(remove("test"))  # remove existing
+        self.assertFalse(remove("test"))  # remove non-existing
 
     def test_tnc(self):
         self.assertIsNotNone(tnc.word_freqs())
@@ -50,7 +60,7 @@ def test_ttc(self):
         self.assertIsNotNone(ttc.word_freqs())
 
     def test_wordnet(self):
-        self.assertIsNotNone(wordnet.langs())
+        self.assertTrue(isinstance(wordnet.langs(), list))
         self.assertTrue("tha" in wordnet.langs())
 
         self.assertEqual(