Merge pull request #407 from PyThaiNLP/thainer-newcrf

Port Thai NER from sklearn-crfsuite to python-crfsuite

Author: wannaphong

README.md

@@ -83,7 +83,6 @@ where `extras` can be
   - `icu` (for ICU, International Components for Unicode, support in transliteration and tokenization)
   - `ipa` (for IPA, International Phonetic Alphabet, support in transliteration)
   - `ml` (to support ULMFiT models for classification)
-  - `ner` (for named-entity recognizer)
   - `thai2fit` (for Thai word vector)
   - `thai2rom` (for machine-learnt romanization)
   - `wordnet` (for Thai WordNet API)
@@ -205,7 +204,6 @@ pip install pythainlp[extra1,extra2,...]
   - `icu` (สำหรับการถอดตัวสะกดเป็นสัทอักษรและการตัดคำด้วย ICU)
   - `ipa` (สำหรับการถอดตัวสะกดเป็นสัทอักษรสากล (IPA))
   - `ml` (สำหรับการรองรับโมเดล ULMFiT)
-  - `ner` (สำหรับการติดป้ายชื่อเฉพาะ (named-entity))
   - `thai2fit` (สำหรับ word vector)
   - `thai2rom` (สำหรับการถอดตัวสะกดเป็นอักษรละติน)
   - `wordnet` (สำหรับ API WordNet ภาษาไทย)

docs/notes/installation.rst

@@ -18,7 +18,6 @@ where ``extras`` can be
   - ``icu`` (for ICU, International Components for Unicode, support in transliteration and tokenization)
   - ``ipa`` (for IPA, International Phonetic Alphabet, support in transliteration)
   - ``ml`` (to support ULMFiT models for classification)
-  - ``ner`` (for named-entity recognizer)
   - ``thai2fit`` (for Thai word vector)
   - ``thai2rom`` (for machine-learnt romanization)
   - ``full`` (install everything)

pythainlp/corpus/__init__.py

@@ -39,7 +39,7 @@
 _CORPUS_DB_URL = (
     "https://raw.githubusercontent.com/"
     + "PyThaiNLP/pythainlp-corpus/"
-    + "2.1/db.json"
+    + "2.2/db.json"
 )
 
 _CORPUS_DB_FILENAME = "db.json"

pythainlp/tag/named_entity.py

@@ -7,7 +7,7 @@
 
 from typing import List, Tuple, Union
 
-import sklearn_crfsuite
+import pycrfsuite
 from pythainlp.corpus import download, get_corpus_path, thai_stopwords
 from pythainlp.tag import pos_tag
 from pythainlp.tokenize import word_tokenize
@@ -76,18 +76,12 @@ def __init__(self):
         """
         Thai named-entity recognizer
         """
-        self.__data_path = get_corpus_path("thainer-1-3")
+        self.__data_path = get_corpus_path("thainer-1-4")
         if not self.__data_path:
-            download("thainer-1-3")
-            self.__data_path = get_corpus_path("thainer-1-3")
-        self.crf = sklearn_crfsuite.CRF(
-            algorithm="lbfgs",
-            c1=0.1,
-            c2=0.1,
-            max_iterations=500,
-            all_possible_transitions=True,
-            model_filename=self.__data_path,
-        )
+            download("thainer-1-4")
+            self.__data_path = get_corpus_path("thainer-1-4")
+        self.crf = pycrfsuite.Tagger()
+        self.crf.open(self.__data_path)
 
     def get_ner(
         self, text: str, pos: bool = True, tag: bool = False
@@ -148,7 +142,7 @@ def get_ner(
             self.__tokens, engine="perceptron", corpus="orchid_ud"
         )
         self.__x_test = self.__extract_features(self.__pos_tags)
-        self.__y = self.crf.predict_single(self.__x_test)
+        self.__y = self.crf.tag(self.__x_test)
 
         self.sent_ner = [
             (self.__pos_tags[i][0], data) for i, data in enumerate(self.__y)

setup.py

@@ -51,8 +51,7 @@
     "benchmarks": ["numpy>=1.16.1", "pandas>=0.24"],
     "icu": ["pyicu>=2.3"],
     "ipa": ["epitran>=1.1"],
-    "ml": ["numpy>=1.16.1", "torch>=1.0.0"],
-    "ner": ["sklearn-crfsuite>=0.3.6"],
+    "ml": ["numpy>=1.16", "torch>=1.0.0"],
     "ssg": ["ssg>=0.0.6"],
     "thai2fit": ["emoji>=0.5.1", "gensim>=3.2.0", "numpy>=1.16.1"],
     "thai2rom": ["torch>=1.0.0", "numpy>=1.16.1"],
@@ -66,7 +65,6 @@
         "numpy>=1.16.1",
         "pandas>=0.24",
         "pyicu>=2.3",
-        "sklearn-crfsuite>=0.3.6",
         "ssg>=0.0.6",
         "torch>=1.0.0",
     ],