Merge pull request #214 from bact/dev

Reorder if checks for dict_trie and more test cases

Author: bact

pythainlp/corpus/tnc.py

@@ -18,10 +18,14 @@
 
 def word_freq(word: str, domain: str = "all") -> int:
     """
-    Get word frequency of a word.
+    **Not officially supported.**
+    Get word frequency of a word by domain.
     This function will make a query to the server of Thai National Corpus.
     Internet connection is required.
 
+    **IMPORTANT:** Currently (as of 29 April 2019) always return 0,
+    as the service URL has been changed and the code is not updated yet.
+
     :param string word: word
     :param string domain: domain
     """
@@ -39,6 +43,7 @@ def word_freq(word: str, domain: str = "all") -> int:
         "others": "0",
     }
     url = "http://www.arts.chula.ac.th/~ling/TNCII/corp.php"
+    # New URL is http://www.arts.chula.ac.th/~ling/tnc3/
     data = {"genre[]": "", "domain[]": listdomain[domain], "sortby": "perc", "p": word}
 
     r = requests.post(url, data=data)

pythainlp/tokenize/__init__.py

@@ -186,16 +186,17 @@ def dict_trie(dict_source: Union[str, Iterable[str], Trie]) -> Trie:
     """
     trie = None
 
-    if isinstance(dict_source, str):
+    if isinstance(dict_source, Trie):
+        trie = dict_source
+    elif isinstance(dict_source, str):
         # Receive a file path of the dict to read
         with open(dict_source, "r", encoding="utf8") as f:
             _vocabs = f.read().splitlines()
             trie = Trie(_vocabs)
     elif isinstance(dict_source, Iterable):
+        # Note: Trie and str are both Iterable, Iterable check should be here
         # Received a sequence type object of vocabs
         trie = Trie(dict_source)
-    elif isinstance(dict_source, Trie):
-        trie = dict_source
     else:
         raise TypeError(
             "Type of dict_source must be marisa_trie.Trie, or Iterable[str], or str (path to source file)"

tests/__init__.py

@@ -3,14 +3,17 @@
 Unit test
 """
 import datetime
+import os
 import unittest
 from collections import Counter
 
 from nltk.corpus import wordnet as wn
 from pythainlp import word_vector
 from pythainlp.corpus import (
+    _CORPUS_PATH,
     conceptnet,
     countries,
+    download,
     provinces,
     remove,
     thai_negations,
@@ -20,44 +23,44 @@
     tnc,
     ttc,
     wordnet,
-    download,
 )
+from pythainlp.corpus.common import _THAI_WORDS_FILENAME
 from pythainlp.soundex import lk82, metasound, soundex, udom83
-from pythainlp.spell import correct, spell, NorvigSpellChecker
+from pythainlp.spell import NorvigSpellChecker, correct, spell
 from pythainlp.summarize import summarize
 from pythainlp.tag import perceptron, pos_tag, pos_tag_sents, unigram
 from pythainlp.tag.locations import tag_provinces
 from pythainlp.tag.named_entity import ThaiNameTagger
+from pythainlp.tokenize import DEFAULT_DICT_TRIE, FROZEN_DICT_TRIE, Tokenizer
+from pythainlp.tokenize import deepcut as tokenize_deepcut
 from pythainlp.tokenize import (
-    DEFAULT_DICT_TRIE,
-    FROZEN_DICT_TRIE,
-    Tokenizer,
     dict_trie,
     dict_word_tokenize,
     etcc,
     longest,
     multi_cut,
     newmm,
+)
+from pythainlp.tokenize import pyicu as tokenize_pyicu
+from pythainlp.tokenize import (
     sent_tokenize,
     subword_tokenize,
     syllable_tokenize,
     tcc,
     word_tokenize,
 )
-from pythainlp.tokenize import deepcut as tokenize_deepcut
-from pythainlp.tokenize import pyicu as tokenize_pyicu
 from pythainlp.transliterate import romanize, transliterate
 from pythainlp.transliterate.ipa import trans_list, xsampa_list
 from pythainlp.transliterate.royin import romanize as romanize_royin
 from pythainlp.util import (
     arabic_digit_to_thai_digit,
     bahttext,
     collate,
+    countthai,
     deletetone,
     digit_to_text,
     eng_to_thai,
     find_keyword,
-    countthai,
     isthai,
     isthaichar,
     normalize,
@@ -70,8 +73,8 @@
     thai_digit_to_arabic_digit,
     thai_strftime,
     thai_to_eng,
-    thaiword_to_num,
     thaicheck,
+    thaiword_to_num,
 )
 
 
@@ -139,17 +142,25 @@ def test_soundex(self):
         self.assertIsNotNone(soundex("a", engine="metasound"))
         self.assertIsNotNone(soundex("a", engine="XXX"))
 
+        self.assertEqual(lk82(None), "")
+        self.assertEqual(lk82(""), "")
+        self.assertEqual(lk82("เหตุ"), lk82("เหด"))
         self.assertEqual(lk82("รถ"), "ร3000")
         self.assertIsNotNone(lk82("เกาะ"))
         self.assertIsNotNone(lk82("อุยกูร์"))
         self.assertIsNotNone(lk82("หยากไย่"))
         self.assertIsNotNone(lk82("หอ"))
-        self.assertEqual(lk82(""), "")
         self.assertEqual(lk82("น์"), "")
 
-        self.assertEqual(udom83("รถ"), "ร800000")
         self.assertEqual(udom83(None), "")
+        self.assertEqual(udom83(""), "")
+        self.assertEqual(udom83("เหตุ"), udom83("เหด"))
+        self.assertEqual(udom83("รถ"), "ร800000")
 
+        self.assertEqual(metasound(None), "")
+        self.assertEqual(metasound(""), "")
+        self.assertEqual(metasound("เหตุ"), metasound("เหด"))
+        self.assertEqual(metasound("รักษ์"), metasound("รัก"))
         self.assertEqual(metasound("บูรณะ"), "บ550")
         self.assertEqual(metasound("คน"), "ค500")
         self.assertEqual(metasound("คนA"), "ค500")
@@ -161,8 +172,11 @@ def test_soundex(self):
         self.assertIsNotNone(metasound("มา"))
         self.assertIsNotNone(metasound("ยา"))
         self.assertIsNotNone(metasound("วา"))
-        self.assertEqual(metasound("รักษ์"), metasound("รัก"))
-        self.assertEqual(metasound(""), "")
+        self.assertIsNotNone(metasound("บูชา"))
+        self.assertIsNotNone(metasound("กมลา"))
+        self.assertIsNotNone(metasound("กาโวกาโว"))
+        self.assertIsNotNone(metasound("สุวรรณา"))
+        self.assertIsNotNone(metasound("ดอยบอย"))
 
     # ### pythainlp.spell
 
@@ -324,6 +338,9 @@ def test_word_tokenize(self):
         self.assertIsNotNone(dict_trie(["ทดสอบ", "สร้าง", "Trie"]))
         self.assertIsNotNone(dict_trie(thai_words()))
         self.assertIsNotNone(dict_trie(FROZEN_DICT_TRIE))
+        self.assertIsNotNone(
+            dict_trie(os.path.join(_CORPUS_PATH, _THAI_WORDS_FILENAME))
+        )
 
         self.assertIsNotNone(word_tokenize("รถไฟฟ้าBTS", custom_dict=DEFAULT_DICT_TRIE))
         self.assertIsNotNone(
@@ -339,6 +356,9 @@ def test_Tokenizer(self):
         t_test.set_tokenize_engine("longest")
         self.assertEqual(t_test.word_tokenize(None), [])
 
+        t_test = Tokenizer()
+        self.assertEqual(t_test.word_tokenize("ก"), ["ก"])
+
     def test_word_tokenize_icu(self):
         self.assertEqual(tokenize_pyicu.segment(None), [])
         self.assertEqual(tokenize_pyicu.segment(""), [])