Merge pull request #170 from PyThaiNLP/1.7

PyThaiNLP 1.7.2

Author: wannaphong

docs/_build/doctrees/api/ulmfit.doctree

@@ -171,6 +171,7 @@ <h1>All modules for which code is available</h1>
 <li><a href="pythainlp/summarize.html">pythainlp.summarize</a></li>
 <li><a href="pythainlp/tag.html">pythainlp.tag</a></li>
 <li><a href="pythainlp/tokenize.html">pythainlp.tokenize</a></li>
+<li><a href="pythainlp/ulmfit/utils.html">pythainlp.ulmfit.utils</a></li>
 <li><a href="pythainlp/word_vector/thai2vec.html">pythainlp.word_vector.thai2vec</a></li>
 </ul>
 

docs/_build/doctrees/environment.pickle

@@ -183,8 +183,12 @@ <h1 id="index">Index</h1>
 <h2 id="A">A</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
-      <li><a href="api/word_vector.html#pythainlp.word_vector.thai2vec.about">about() (in module pythainlp.word_vector.thai2vec)</a>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.about">about() (in module pythainlp.ulmfit.utils)</a>
+
+      <ul>
+        <li><a href="api/word_vector.html#pythainlp.word_vector.thai2vec.about">(in module pythainlp.word_vector.thai2vec)</a>
 </li>
+      </ul></li>
   </ul></td>
 </tr></table>
 
@@ -207,6 +211,8 @@ <h2 id="D">D</h2>
 </li>
   </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.document_vector">document_vector() (in module pythainlp.ulmfit.utils)</a>
+</li>
       <li><a href="api/word_vector.html#pythainlp.word_vector.thai2vec.doesnt_match">doesnt_match() (in module pythainlp.word_vector.thai2vec)</a>
 </li>
   </ul></td>
@@ -215,11 +221,15 @@ <h2 id="D">D</h2>
 <h2 id="G">G</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.get_all">get_all() (in module pythainlp.ulmfit.utils)</a>
+</li>
       <li><a href="api/word_vector.html#pythainlp.word_vector.thai2vec.get_model">get_model() (in module pythainlp.word_vector.thai2vec)</a>
 </li>
   </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="api/ner.html#pythainlp.ner.thainer.get_ner">get_ner() (pythainlp.ner.thainer method)</a>
+</li>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.get_texts">get_texts() (in module pythainlp.ulmfit.utils)</a>
 </li>
   </ul></td>
 </tr></table>
@@ -242,6 +252,10 @@ <h2 id="L">L</h2>
 
 <h2 id="M">M</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
+  <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.merge_wgts">merge_wgts() (in module pythainlp.ulmfit.utils)</a>
+</li>
+  </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="api/word_vector.html#pythainlp.word_vector.thai2vec.most_similar_cosmul">most_similar_cosmul() (in module pythainlp.word_vector.thai2vec)</a>
 </li>
@@ -258,6 +272,8 @@ <h2 id="N">N</h2>
   </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="api/number.html#pythainlp.number.num_to_thai_num">num_to_thai_num() (in module pythainlp.number)</a>
+</li>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.numericalizer">numericalizer() (in module pythainlp.ulmfit.utils)</a>
 </li>
       <li><a href="api/number.html#pythainlp.number.numtowords">numtowords() (in module pythainlp.number)</a>
 </li>
@@ -268,12 +284,24 @@ <h2 id="P">P</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="api/tag.html#pythainlp.tag.pos_tag">pos_tag() (in module pythainlp.tag)</a>
+</li>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.ThaiTokenizer.proc_all">proc_all() (pythainlp.ulmfit.utils.ThaiTokenizer static method)</a>
+</li>
+  </ul></td>
+  <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.ThaiTokenizer.proc_all_mp">proc_all_mp() (pythainlp.ulmfit.utils.ThaiTokenizer static method)</a>
+</li>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.ThaiTokenizer.proc_text">proc_text() (pythainlp.ulmfit.utils.ThaiTokenizer method)</a>
 </li>
   </ul></td>
 </tr></table>
 
 <h2 id="R">R</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
+  <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.ThaiTokenizer.replace_rep">replace_rep() (pythainlp.ulmfit.utils.ThaiTokenizer static method)</a>
+</li>
+  </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="api/romanization.html#pythainlp.romanization.romanization">romanization() (in module pythainlp.romanization)</a>
 
@@ -293,11 +321,13 @@ <h2 id="S">S</h2>
 </li>
       <li><a href="api/sentiment.html#pythainlp.sentiment.sentiment">sentiment() (in module pythainlp.sentiment)</a>
 </li>
-  </ul></td>
-  <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="api/word_vector.html#pythainlp.word_vector.thai2vec.similarity">similarity() (in module pythainlp.word_vector.thai2vec)</a>
 </li>
+  </ul></td>
+  <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="api/spell.html#pythainlp.spell.spell">spell() (in module pythainlp.spell)</a>
+</li>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.ThaiTokenizer.sub_br">sub_br() (pythainlp.ulmfit.utils.ThaiTokenizer method)</a>
 </li>
       <li><a href="api/tokenizer.html#pythainlp.tokenize.subword_tokenize">subword_tokenize() (in module pythainlp.tokenize)</a>
 </li>
@@ -315,15 +345,19 @@ <h2 id="T">T</h2>
 </li>
       <li><a href="api/change.html#pythainlp.change.texttothai">texttothai() (in module pythainlp.change)</a>
 </li>
-  </ul></td>
-  <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="api/romanization.html#pythainlp.romanization.thai2rom.thai2rom">thai2rom (class in pythainlp.romanization.thai2rom)</a>
 </li>
+  </ul></td>
+  <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="api/number.html#pythainlp.number.thai_num_to_num">thai_num_to_num() (in module pythainlp.number)</a>
 </li>
       <li><a href="api/number.html#pythainlp.number.thai_num_to_text">thai_num_to_text() (in module pythainlp.number)</a>
 </li>
       <li><a href="api/ner.html#pythainlp.ner.thainer">thainer (class in pythainlp.ner)</a>
+</li>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.ThaiTokenizer">ThaiTokenizer (class in pythainlp.ulmfit.utils)</a>
+</li>
+      <li><a href="api/ulmfit.html#pythainlp.ulmfit.utils.ThaiTokenizer.tokenize">tokenize() (pythainlp.ulmfit.utils.ThaiTokenizer method)</a>
 </li>
   </ul></td>
 </tr></table>

docs/_build/html/_modules/index.html

@@ -11,7 +11,7 @@
 from tqdm import tqdm
 
 CORPUS_DB_URL = (
-    "https://raw.githubusercontent.com/PyThaiNLP/pythainlp-corpus/master/db.json"
+    "https://github.com/PyThaiNLP/pythainlp-corpus/raw/1.7/db.json"
 )
 
 # __all__ = ["thaipos", "thaiword","alphabet","tone","country","wordnet"]

docs/_build/html/_modules/pythainlp/ulmfit/utils.html

@@ -4,7 +4,6 @@
 import codecs
 import re
 
-import nltk
 from pythainlp.corpus.thaisyllable import get_data as syllable_dict
 from pythainlp.corpus.thaiword import get_data as word_dict
 from six.moves import zip
@@ -104,9 +103,9 @@ def sent_tokenize(text, engine="whitespace+newline"):
     :return: a list of text, split by whitespace or new line.
     """
     if engine == "whitespace":
-        sentences = nltk.tokenize.WhitespaceTokenizer().tokenize(text)
+        sentences = re.split(r' +', text, re.U)
     else:
-        sentences = re.sub(r"\n+|\s+", "|", text, re.U).split("|")
+        sentences = text.split()
 
     return sentences