Merge pull request #912 from S2P2/fix-join-broken-num

Fix empty string ('') added (in some cases) when using word_tokenize with join_broken_num=True

Author: bact

pythainlp/tokenize/_utils.py

@@ -61,8 +61,8 @@ def rejoin_formatted_num(segments: List[str]) -> List[str]:
                 connected_token += segments[segment_idx]
                 pos += len(segments[segment_idx])
                 segment_idx += 1
-
-            tokens_joined.append(connected_token)
+            if connected_token:
+                tokens_joined.append(connected_token)
             match = next(matching_results, None)
         else:
             tokens_joined.append(token)