Updated crfcut.py

Crfcut creating issues for split using terminal punctuation commonly '.' (full stop) which should be treated as end of the sentence, Modified the function such that it should split using terminal punctuations and avoid any kind of empty strings.

Author: varunkatiyar819

pythainlp/tokenize/crfcut.py

@@ -198,12 +198,22 @@ def segment(text: str) -> List[str]:
     feat = extract_features(toks)
     labs = _tagger.tag(feat)
     labs[-1] = "E"  # make sure it cuts the last sentence
+  
+    #To ensure splitting of sentences using Terminal Punctuation
+    for idx, _ in enumerate(toks):
+      if(toks[idx].strip().endswith(('!', '.', '?'))):
+          labs[idx] = "E"
+      
+      #Spaces or empty strings would no longer be treated as end of the sentence.
+      elif(toks[idx].strip() == ""):
+          labs[idx] = "I"
 
     sentences = []
     sentence = ""
     for i, w in enumerate(toks):
         sentence = sentence + w
-        if labs[i] == "E":
+        #Constraining empty strings to get added, to avoid any sort of unusual behaviour due to empty strings.
+        if labs[i] == "E" and sentence != '':
             sentences.append(sentence)
             sentence = ""