Merge pull request #703 from PyThaiNLP/dev

Update add-word_detokenize from dev

Author: wannaphong

.github/workflows/windows-test.yml

@@ -0,0 +1,59 @@
+name: Windows Unit test and code coverage
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+      - 'docs/**'
+  pull_request:
+    branches:
+      - dev
+    paths-ignore:
+      - '**.md'
+      - 'docs/**'
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest]
+        python-version: [3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+        auto-activate-base: true
+        auto-update-conda: true
+    - shell: powershell
+      run: |
+        conda info
+        conda list
+    - name: Install PyTorch
+      shell: powershell
+      run: |
+        pip install torch==1.8.1
+    - name: Install dependencies
+      shell: powershell
+      run: |
+        python -m pip install --disable-pip-version-check --user --upgrade pip setuptools
+        python -m pip --version
+        python -m pip install pytest coverage coveralls
+        conda install -y -c conda-forge fairseq
+        python -m pip install https://www.dropbox.com/s/o6p2sj5z50iim1e/PyICU-2.3.1-cp38-cp38-win_amd64.whl?dl=1
+        python -m pip install -r docker_requirements.txt
+        python -m pip install .[full]
+        python -m nltk.downloader omw-1.4
+        python -m pip install spacy deepcut
+    - name: Test
+      shell: powershell
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        COVERALLS_SERVICE_NAME: github
+      run: |
+        coverage run -m unittest discover
+        coveralls

docs/api/soundex.rst

@@ -11,6 +11,7 @@ Modules
 .. autofunction:: lk82
 .. autofunction:: udom83
 .. autofunction:: metasound
+.. autofunction:: prayut_and_somchaip
 
 References
 ----------
@@ -23,3 +24,5 @@ References
         Master Thesis. Chulalongkorn University, Thailand.
 
 .. [#lk82] วิชิต หล่อจีระชุณห์กุล และ เจริญ คุวินทร์พันธุ์. `โปรแกรมการสืบค้นคำไทยตามเสียงอ่าน (Thai Soundex) <http://guru.sanook.com/1520/>`_.
+
+.. [#prayut_and_somchaip] Prayut Suwanvisat, Somchai Prasitjutrakul. Thai-English Cross-Language Transliterated Word Retrieval using Soundex Technique. In 1998 [cited 2022 Sep 8]. Available from: https://www.cp.eng.chula.ac.th/~somchai/spj/papers/ThaiText/ncsec98-clir.pdf

pythainlp/corpus/core.py

@@ -47,7 +47,7 @@ def get_corpus_db_detail(name: str, version: str = None) -> dict:
     :return: details about a corpus
     :rtype: dict
     """
-    with open(corpus_db_path(), "r", encoding="utf-8") as f:
+    with open(corpus_db_path(), "r", encoding="utf-8-sig") as f:
         local_db = json.load(f)
 
     if version is None:
@@ -378,7 +378,7 @@ def download(
 
     # check if corpus is available
     if name in corpus_db:
-        with open(corpus_db_path(), "r", encoding="utf-8") as f:
+        with open(corpus_db_path(), "r", encoding="utf-8-sig") as f:
             local_db = json.load(f)
 
         corpus = corpus_db[name]
@@ -509,7 +509,7 @@ def remove(name: str) -> bool:
     if _CHECK_MODE == "1":
         print("PyThaiNLP is read-only mode. It can't remove corpus.")
         return False
-    with open(corpus_db_path(), "r", encoding="utf-8") as f:
+    with open(corpus_db_path(), "r", encoding="utf-8-sig") as f:
         db = json.load(f)
     data = [
         corpus for corpus in db["_default"].values() if corpus["name"] == name

pythainlp/soundex/__init__.py

@@ -10,11 +10,13 @@
     "lk82",
     "metasound",
     "udom83",
+    "prayut_and_somchaip",
 ]
 
 from pythainlp.soundex.lk82 import lk82
 from pythainlp.soundex.metasound import metasound
 from pythainlp.soundex.udom83 import udom83
+from pythainlp.soundex.prayut_and_somchaip import prayut_and_somchaip
 
 DEFAULT_SOUNDEX_ENGINE = "udom83"
 

pythainlp/soundex/core.py

@@ -7,18 +7,21 @@
 from pythainlp.soundex.lk82 import lk82
 from pythainlp.soundex.metasound import metasound
 from pythainlp.soundex.udom83 import udom83
+from pythainlp.soundex.prayut_and_somchaip import prayut_and_somchaip
 from pythainlp.soundex import DEFAULT_SOUNDEX_ENGINE
 
 # Other Thai soundex systems (not implemented yet): Arun91, KSS97
 # [KSS97] https://linux.thai.net/~thep/soundex/soundex.html
 
 
-def soundex(text: str, engine: str = DEFAULT_SOUNDEX_ENGINE) -> str:
+def soundex(text: str, engine: str = DEFAULT_SOUNDEX_ENGINE, length: int = 4) -> str:
     """
     This function converts Thai text into phonetic code.
 
     :param str text: word
     :param str engine: soundex engine
+    :param int length: preferred length of the Soundex code (default is 4)\
+        for metasound and prayut_and_somchaip only
     :return: Soundex code
     :rtype: str
 
@@ -29,6 +32,8 @@ def soundex(text: str, engine: str = DEFAULT_SOUNDEX_ENGINE) -> str:
           Wannee Udompanich [#lk82]_
         * *metasound* - Thai soundex algorithm based on a combination
           of Metaphone and Soundex proposed by Snae & Brückner [#metasound]_
+        * *prayut_and_somchaip* - Thai-English Cross-Language Transliterated
+          Word Retrieval using Soundex Technique  [#prayut_and_somchaip]_
 
     :Example:
     ::
@@ -54,12 +59,18 @@ def soundex(text: str, engine: str = DEFAULT_SOUNDEX_ENGINE) -> str:
         soundex("ปัจจุบัน"), soundex("ปัจจุบัน", engine='lk82'), \\
             soundex("ปัจจุบัน", engine='metasound')
         # output: ('ป775300', 'ป3E54', 'ป223')
+
+        soundex("vp", engine="prayut_and_somchaip")
+        # output: '11'
+        soundex("วีพี", engine="prayut_and_somchaip")
+        # output: '11'
     """
     if engine == "lk82":
-        _soundex = lk82
+        _soundex = lk82(text)
+    elif engine == "prayut_and_somchaip":
+        _soundex = prayut_and_somchaip(text, length=length)
     elif engine == "metasound":
-        _soundex = metasound
+        _soundex = metasound(text, length=length)
     else:  # default, use "udom83"
-        _soundex = udom83
-
-    return _soundex(text)
+        _soundex = udom83(text)
+    return _soundex

pythainlp/soundex/metasound.py

@@ -37,7 +37,7 @@ def metasound(text: str, length: int = 4) -> str:
     :Example:
     ::
 
-        from pythainlp.metasound import metasound
+        from pythainlp.soundex.metasound import metasound
 
         metasound("ลัก")
         # output: 'ล100'

pythainlp/soundex/prayut_and_somchaip.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""
+Thai-English Cross-Language Transliterated Word Retrieval
+using Soundex Technique
+
+References:
+Prayut Suwanvisat, Somchai Prasitjutrakul.Thai-English Cross-Language Transliterated Word Retrieval using Soundex Technique. In 1998 [cited 2022 Sep 8]. Available from: https://www.cp.eng.chula.ac.th/~somchai/spj/papers/ThaiText/ncsec98-clir.pdf
+"""
+from pythainlp import thai_characters
+_C0 = "AEIOUHWYอ"
+_C1 = "BFPVบฝฟปผพภว"
+_C2 = "CGJKQSXZขฃคฅฆฉขฌกจซศษส"
+_C3 = "DTฎดฏตฐฑฒถทธ"
+_C4 = "Lลฬ"
+_C5 = "MNมณน"
+_C6 = "Rร"
+_C7 = "AEIOUอ"
+_C8 = "Hหฮ"
+_C1_1 = "Wว"
+_C9 = "Yยญ"
+_C52 = "ง"
+
+
+def prayut_and_somchaip(text: str, length: int = 4) -> str:
+    """
+    This function converts English-Thai Cross-Language Transliterated Word into
+    phonetic code with the mactching technique called **Soundex** [#prayut_and_somchaip]_.
+
+    :param str text: English-Thai Cross-Language Transliterated Word
+    :param int length: preferred length of the Soundex code (default is 4)
+
+    :return: Soundex for the given text
+    :rtype: str
+
+    :Example:
+    ::
+
+        from pythainlp.soundex.prayut_and_somchaip import prayut_and_somchaip
+
+        prayut_and_somchaip("king", 2)
+        # output: '52'
+
+        prayut_and_somchaip("คิง", 2)
+        # output: '52'
+    """
+    if not text or not isinstance(text, str):
+        return ""
+    text = text.upper()
+    # keep only consonants (English-Thai)
+    chars = []
+    for ch in text:
+        if ch in thai_characters+"ABCDEFGHIJKLMNOPQRSTUVWXYZ":
+            chars.append(ch)
+
+    i = 0
+    while i < len(chars):
+        if i == 0 and chars[i] in _C0:
+            chars[i] = "0"
+        elif chars[i] in _C1:
+            chars[i] = "1"
+        elif chars[i] in _C2:
+            chars[i] = "2"
+        elif chars[i] in _C3:
+            chars[i] = "3"
+        elif chars[i] in _C4:
+            chars[i] = "4"
+        elif chars[i] in _C5:
+            chars[i] = "5"
+        elif chars[i] in _C6:
+            chars[i] = "6"
+        elif chars[i] in _C52:
+            chars[i] = "52"
+        elif chars[i] in _C7 and i != 0:
+            chars[i] = "7"
+        elif chars[i] in _C8 and i != 0:
+            chars[i] = "8"
+        elif chars[i] in _C1_1 and i != 0:
+            chars[i] = "1"
+        elif chars[i] in _C9 and i != 0:
+            chars[i] = "9"
+        else:
+            chars[i] = None
+        i += 1
+    chars = list(
+        ''.join([i for i in chars if i is not None])
+    )
+    return "".join(chars[-length:])

pythainlp/tag/_tag_perceptron.py

@@ -202,7 +202,7 @@ def load(self, loc: str) -> None:
         :param str loc: model path
         """
         try:
-            with open(loc, "r", encoding='utf-8') as f:
+            with open(loc, "r", encoding='utf-8-sig') as f:
                 w_td_c = json.load(f)
         except IOError:
             msg = "Missing trontagger.json file."

pythainlp/tokenize/crfcut.py

@@ -189,7 +189,10 @@ def segment(text: str) -> List[str]:
     :param str text: text to be tokenized to sentences
     :return: list of words, tokenized from the text
     """
-    toks = word_tokenize(text)
+    if isinstance(text, str):
+        toks = word_tokenize(text)
+    else:
+        toks = text
     feat = extract_features(toks)
     labs = _tagger.tag(feat)
     labs[-1] = "E"  # make sure it cuts the last sentence

tests/test_soundex.py

@@ -2,14 +2,18 @@
 
 import unittest
 
-from pythainlp.soundex import lk82, metasound, soundex, udom83
+from pythainlp.soundex import lk82, metasound, soundex, udom83, prayut_and_somchaip
 
 
 class TestSoundexPackage(unittest.TestCase):
     def test_soundex(self):
         self.assertIsNotNone(soundex("a", engine="lk82"))
         self.assertIsNotNone(soundex("a", engine="udom83"))
         self.assertIsNotNone(soundex("a", engine="metasound"))
+        self.assertEqual(
+            soundex("vp", engine="prayut_and_somchaip"),
+            soundex("วีพี", engine="prayut_and_somchaip")
+        )
         self.assertIsNotNone(soundex("a", engine="XXX"))
 
         self.assertEqual(lk82(None), "")
@@ -54,3 +58,18 @@ def test_soundex(self):
         self.assertIsNotNone(metasound("กาโวกาโว"))
         self.assertIsNotNone(metasound("สุวรรณา"))
         self.assertIsNotNone(metasound("ดอยบอย"))
+
+        self.assertEqual(prayut_and_somchaip(None), "")
+        self.assertEqual(prayut_and_somchaip(""), "")
+        self.assertEqual(prayut_and_somchaip("vp"), "11")
+        self.assertIsNotNone(prayut_and_somchaip("บา"))
+        self.assertIsNotNone(prayut_and_somchaip("go"))
+        self.assertIsNotNone(prayut_and_somchaip("อด"))
+        self.assertIsNotNone(prayut_and_somchaip("ลน"))
+        self.assertIsNotNone(prayut_and_somchaip("มอ"))
+        self.assertIsNotNone(prayut_and_somchaip("รอ"))
+        self.assertIsNotNone(prayut_and_somchaip("ขอ"))
+        self.assertIsNotNone(prayut_and_somchaip("บน"))
+        self.assertIsNotNone(prayut_and_somchaip("ณาญ"))
+        self.assertIsNotNone(prayut_and_somchaip("กาง"))
+        self.assertIsNotNone(prayut_and_somchaip("ว้าว"))