Refactor and revise variable names

Author: bact

docs/api/tokenize.rst

@@ -10,9 +10,9 @@ Modules
 
 .. autofunction:: clause_tokenize
 .. autofunction:: sent_tokenize
-.. autofunction:: word_tokenize
-.. autofunction:: syllable_tokenize
 .. autofunction:: subword_tokenize
+.. autofunction:: syllable_tokenize
+.. autofunction:: word_tokenize
 .. autoclass:: Tokenizer
    :members:
 

pythainlp/tokenize/__init__.py

@@ -7,6 +7,7 @@
     "THAI2FIT_TOKENIZER",
     "Tokenizer",
     "Trie",
+    "clause_tokenize",
     "sent_tokenize",
     "subword_tokenize",
     "syllable_tokenize",

pythainlp/tokenize/core.py

@@ -8,10 +8,10 @@
 from pythainlp.tokenize import (
     DEFAULT_SENT_TOKENIZE_ENGINE,
     DEFAULT_SUBWORD_TOKENIZE_ENGINE,
+    DEFAULT_SYLLABLE_DICT_TRIE,
     DEFAULT_SYLLABLE_TOKENIZE_ENGINE,
-    DEFAULT_WORD_TOKENIZE_ENGINE,
     DEFAULT_WORD_DICT_TRIE,
-    DEFAULT_SYLLABLE_DICT_TRIE,
+    DEFAULT_WORD_TOKENIZE_ENGINE,
 )
 from pythainlp.util.trie import Trie, dict_trie
 
@@ -20,7 +20,7 @@ def clause_tokenize(doc: List[str]) -> List[List[str]]:
     """
     Clause tokenizer. (or Clause segmentation)
 
-    Tokenizes running word list into list of claues (list of strings).
+    Tokenizes running word list into list of clauses (list of strings).
     split by CRF trained on LST20 Corpus.
 
     :param str doc: word list to be clause
@@ -36,8 +36,9 @@ def clause_tokenize(doc: List[str]) -> List[List[str]]:
         ['และ', 'คุณ', 'เล่น', 'มือถือ'],
         ['ส่วน', 'น้อง', 'เขียน', 'โปรแกรม']]
     """
-    from .lst20 import clause_tokenize as cla
-    return cla(doc)
+    from .crfcls import segment
+
+    return segment(doc)
 
 
 def word_tokenize(

pythainlp/tokenize/crfcls.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+"""
+Clause segmenter
+"""
+from typing import List
+
+import pycrfsuite
+from pythainlp.corpus import get_corpus_path
+from pythainlp.tag import pos_tag
+
+
+def _doc2features(doc, i):
+    # features from current word
+    curr_word = doc[i][0]
+    curr_pos = doc[i][1]
+    features = {
+        "word.word": curr_word,
+        "word.isspace": curr_word.isspace(),
+        "word.isdigit()": curr_word.isdigit(),
+        "postag": curr_pos,
+    }
+
+    # features from previous word
+    if i > 0:
+        prev_word = doc[i - 1][0]
+        prev_pos = doc[i - 1][1]
+        features["word.prevword"] = prev_word
+        features["word.previsspace"] = prev_word.isspace()
+        features["word.prevwordisdigit"] = prev_word.isdigit()
+        features["word.prepostag"] = prev_pos
+    else:
+        features["BOS"] = True  # Beginning of Sequence
+
+    # features from next word
+    if i < len(doc) - 1:
+        next_word = doc[i + 1][0]
+        next_pos = doc[i + 1][1]
+        features["word.nextword"] = next_word
+        features["word.nextisspace"] = next_word.isspace()
+        features["word.nextwordisdigit"] = next_word.isdigit()
+        features["word.nextpostag"] = next_pos
+    else:
+        features["EOS"] = True  # End of Sequence
+
+    return features
+
+
+def _extract_features(doc):
+    return [_doc2features(doc, i) for i in range(len(doc))]
+
+
+_CORPUS_NAME = "lst20-cls"
+tagger = pycrfsuite.Tagger()
+tagger.open(get_corpus_path(_CORPUS_NAME))
+
+
+def segment(doc: List[str]) -> List[List[str]]:
+    word_tags = pos_tag(doc, corpus="lst20")
+    features = _extract_features(word_tags)
+    word_markers = list(zip(doc, tagger.tag(features)))
+
+    clauses = []
+    temp = []
+    len_doc = len(doc) - 1
+    for i, word_marker in enumerate(word_markers):
+        word, marker = word_marker
+        if marker == "E_CLS" or i == len_doc:
+            temp.append(word)
+            clauses.append(temp)
+            temp = []
+        else:
+            temp.append(word)
+
+    return clauses

pythainlp/tokenize/crfcut.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 """
-CRFCut - Thai sentence segmentor.
+CRFCut - Thai sentence segmenter.
 
 Thai sentence segmentation using conditional random field,
 default model trained on TED dataset

pythainlp/tokenize/lst20.py

@@ -2,7 +2,12 @@
 
 import unittest
 
-from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE, Tokenizer, attacut
+from pythainlp.tokenize import (
+    DEFAULT_WORD_DICT_TRIE,
+    Tokenizer,
+    attacut,
+    clause_tokenize,
+)
 from pythainlp.tokenize import deepcut as tokenize_deepcut
 from pythainlp.tokenize import etcc, longest, multi_cut, newmm
 from pythainlp.tokenize import pyicu as tokenize_pyicu
@@ -15,7 +20,6 @@
 )
 from pythainlp.tokenize.ssg import segment as ssg_segment
 from pythainlp.util import dict_trie
-from pythainlp.tokenize import clause_tokenize
 
 
 class TestTokenizePackage(unittest.TestCase):
@@ -187,10 +191,7 @@ def setUp(self):
 
     def test_clause_tokenize(self):
         self.assertIsNotNone(clause_tokenize(["ฉัน", "ทดสอบ"]))
-        self.assertIsInstance(
-            clause_tokenize(["ฉัน", "ทดสอบ"]),
-            list
-        )
+        self.assertIsInstance(clause_tokenize(["ฉัน", "ทดสอบ"]), list)
 
     def test_Tokenizer(self):
         t_test = Tokenizer(DEFAULT_WORD_DICT_TRIE)