Merge pull request #418 from PyThaiNLP/unittest-main

Add unit test to CLI functions

Author: bact

pythainlp/__main__.py

@@ -5,33 +5,35 @@
 from pythainlp import cli
 
 
-def main(args=None):
+def main(argv=None):
     """ThaiNLP command line."""
-    if args is None:
-        args = sys.argv[1:]
+    if not argv:
+        argv = sys.argv
 
     parser = argparse.ArgumentParser(
-        "thainlp", usage="thainlp <command> [options]"
+        prog="thainlp",
+        description="Thai natural language processing.",
+        usage=(
+            "thainlp <command> [options]\n\n"
+            "Example:\n\n"
+            "thainlp data catalog\n\n"
+            "--"
+        ),
     )
-
     parser.add_argument(
         "command",
         type=str,
         choices=cli.COMMANDS,
         help="text processing action",
     )
 
-    args = parser.parse_args(sys.argv[1:2])
-
+    args = parser.parse_args(argv[1:2])
     cli.exit_if_empty(args.command, parser)
 
     if hasattr(cli, args.command):
         command = getattr(cli, args.command)
-        command.App(sys.argv)
-    else:
-        print(f"Command not available: {args.command}")
-        print("Please run with --help for alternatives")
+        command.App(argv)
 
 
 if __name__ == "__main__":
-    main()
+    main(argv=sys.argv)

pythainlp/cli/__init__.py

@@ -2,7 +2,7 @@
 import sys
 from argparse import ArgumentParser
 
-from . import data, soundex, tag, tokenize
+from pythainlp.cli import data, soundex, tag, tokenize
 
 # a command should be a verb when possible
 COMMANDS = sorted(["data", "soundex", "tag", "tokenize"])

pythainlp/cli/data.py

@@ -10,7 +10,26 @@
 class App:
     def __init__(self, argv):
         parser = argparse.ArgumentParser(
-            prog="data", usage="thainlp data <subcommand>",
+            prog="data",
+            description="Manage dataset/corpus.",
+            usage=(
+                "thainlp data <subcommand>\n\n"
+                "subcommands:\n\n"
+                "catalog                show list of available datasets\n"
+                "info <dataset_name>    show information about the dataset\n"
+                "get <dataset_name>     download the dataset\n"
+                "rm <dataset_name>      remove the dataset\n"
+                "path                   show full path to data directory\n\n"
+                "Example:\n\n"
+                "thainlp data get thai2fit_wv\n\n"
+                "Current data path:\n\n"
+                f"{get_pythainlp_data_path()}\n\n"
+                "To change PyThaiNLP data path, set the operating system's\n"
+                "PYTHAINLP_DATA_DIR environment variable.\n\n"
+                "For more information about corpora that PyThaiNLP use, see:\n"
+                "https://github.com/PyThaiNLP/pythainlp-corpus/\n\n"
+                "--"
+            ),
         )
         parser.add_argument(
             "subcommand",
@@ -78,8 +97,10 @@ def catalog(self, argv):
             else:
                 print()
 
-        print("\nUse subcommand 'get' to download dataset.")
-        print("Example: thainlp data get crfcut")
+        print(
+            "\nUse subcommand 'get' to download a dataset.\n\n"
+            "Example: thainlp data get crfcut\n"
+        )
 
     def path(self, argv):
         """Print path for local dataset."""

pythainlp/cli/soundex.py

@@ -14,6 +14,18 @@ def __init__(self, argv):
         parser = argparse.ArgumentParser(
             prog="soundex",
             description="Convert a text to its sound-based index.",
+            usage=(
+                "thainlp soundex [-a algorithm] <text>\n\n"
+                "algorithms:\n\n"
+                "udom83\n"
+                "lk82\n"
+                "metasound\n\n"
+                f"Default soundex algorithm is {DEFAULT_SOUNDEX_ENGINE}.\n\n"
+                "<text> should be inside double quotes.\n\n"
+                "Example:\n\n"
+                'thainlp soundex -a lk82 "มอเตอร์ไซค์"\n\n'
+                "--"
+            ),
         )
         parser.add_argument(
             "-a",

pythainlp/cli/tag.py

@@ -9,11 +9,10 @@
 
 class SubAppBase:
     def __init__(self, name, argv):
-        parser = argparse.ArgumentParser(name)
+        parser = argparse.ArgumentParser(**cli.make_usage("tag " + name))
         parser.add_argument(
             "text", type=str, help="input text",
         )
-
         parser.add_argument(
             "-s",
             "--sep",
@@ -43,17 +42,30 @@ def __init__(self, *args, **kwargs):
 
 class App:
     def __init__(self, argv):
-        parser = argparse.ArgumentParser(**cli.make_usage("tag"))
-        parser.add_argument("subcommand", type=str, help="[pos]")
+        parser = argparse.ArgumentParser(
+            prog="tag",
+            description="Annotate a text with linguistic information",
+            usage=(
+                'thainlp tag <tag_type> [--sep "<separator>"] "<text>"\n\n'
+                "tag_type:\n\n"
+                "pos                part-of-speech\n\n"
+                "<separator> and <text> should be inside double quotes.\n"
+                "<text> should be a tokenized text, "
+                "with tokens separated by <separator>.\n\n"
+                "Example:\n\n"
+                'thainlp tag pos -s " " "แรงดึงดูด เก็บ หัว คุณ ลง"\n\n'
+                "--"
+            ),
+        )
+        parser.add_argument("tag_type", type=str, help="[pos]")
 
         args = parser.parse_args(argv[2:3])
-
-        cli.exit_if_empty(args.subcommand, parser)
-        subcommand = str.lower(args.subcommand)
+        cli.exit_if_empty(args.tag_type, parser)
+        tag_type = str.lower(args.tag_type)
 
         argv = argv[3:]
 
-        if subcommand == "pos":
+        if tag_type == "pos":
             POSTaggingApp("Part-of-Speech tagging", argv)
         else:
-            print(f"Tag type not available: {subcommand}")
+            print(f"Tag type not available: {tag_type}")

pythainlp/cli/tokenize.py

@@ -108,25 +108,45 @@ def __init__(self, *args, **kwargs):
 
 class App:
     def __init__(self, argv):
-        parser = argparse.ArgumentParser(**cli.make_usage("tokenize"))
+        parser = argparse.ArgumentParser(
+            prog="tokenize",
+            description="Break a text into small units (tokens).",
+            usage=(
+                'thainlp tokenize <token_type> [options] "<text>"\n\n'
+                "token_type:\n\n"
+                "subword            subword (may not be a linguistic unit)\n"
+                "syllable           syllable\n"
+                "word               word\n"
+                "sent               sentence\n\n"
+                "options:\n\n"
+                "--sep or -s <separator>    specify custom separator\n"
+                "                           (default is a space)\n"
+                "--algo or -a <algorithm>   tokenization algorithm\n"
+                "                           (see API doc for more info)\n"
+                "--keep-whitespace or -w    keep whitespaces in output\n"
+                "                           (default)\n\n"
+                "<separator> and <text> should be inside double quotes.\n\n"
+                "Example:\n\n"
+                'thainlp tokenize word -s "|" "ใต้แสงนีออนเปลี่ยวเหงา"\n\n'
+                "--"
+            ),
+        )
         parser.add_argument(
-            "subcommand", type=str, help="[subword|syllable|word|sent]",
+            "token_type", type=str, help="[subword|syllable|word|sent]",
         )
 
         args = parser.parse_args(argv[2:3])
-
-        cli.exit_if_empty(args.subcommand, parser)
-        subcommand = str.lower(args.subcommand)
+        cli.exit_if_empty(args.token_type, parser)
+        token_type = str.lower(args.token_type)
 
         argv = argv[3:]
-
-        if subcommand.startswith("w"):
+        if token_type.startswith("w"):
             WordTokenizationApp("word", argv)
-        elif subcommand.startswith("sy"):
+        elif token_type.startswith("sy"):
             SyllableTokenizationApp("syllable", argv)
-        elif subcommand.startswith("su"):
+        elif token_type.startswith("su"):
             SubwordTokenizationApp("subword", argv)
-        elif subcommand.startswith("se"):
+        elif token_type.startswith("se"):
             SentenceTokenizationApp("sent", argv)
         else:
-            print(f"Subcommand not available: {subcommand}")
+            print(f"Token type not available: {token_type}")

pythainlp/corpus/corpus_license.md

@@ -1,5 +1,8 @@
 # Corpus License
 
+For more information about corpora that PyThaiNLP use,
+see [https://github.com/PyThaiNLP/pythainlp-corpus/](https://github.com/PyThaiNLP/pythainlp-corpus/).
+
 ## tha-wn.db
 
 ```

tests/__init__.py

@@ -3,7 +3,6 @@
 Unit test.
 
 Each file in tests/ is for each main package.
-#TODO Test for CLI
 """
 import sys
 import unittest

tests/test_cli.py

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+from argparse import ArgumentError
+from types import ModuleType
+
+from pythainlp import __main__, cli
+
+
+class TestMainPackage(unittest.TestCase):
+    def test_cli_main(self):
+        # call with no argument, should exit with 2
+        with self.assertRaises(SystemExit) as ex:
+            __main__.main()
+        self.assertEqual(ex.exception.code, 2)
+
+        with self.assertRaises((ArgumentError, SystemExit)):
+            self.assertIsNone(__main__.main(["thainlp"]))
+
+        with self.assertRaises((ArgumentError, SystemExit)):
+            self.assertIsNone(
+                __main__.main(["thainlp", "NOT_EXIST", "command"])
+            )
+
+        self.assertIsNone(__main__.main(["thainlp", "data", "path"]))
+
+    def test_cli_data(self):
+        self.assertIsInstance(getattr(cli, "data"), ModuleType)
+
+        with self.assertRaises(SystemExit) as ex:
+            cli.data.App(["thainlp", "data"])
+        self.assertEqual(ex.exception.code, 2)
+
+        self.assertIsNotNone(cli.data.App(["thainlp", "data", "catalog"]))
+        self.assertIsNotNone(cli.data.App(["thainlp", "data", "path"]))
+        self.assertIsNotNone(cli.data.App(["thainlp", "data", "get", "test"]))
+        self.assertIsNotNone(cli.data.App(["thainlp", "data", "info", "test"]))
+        self.assertIsNotNone(cli.data.App(["thainlp", "data", "rm", "test"]))
+        self.assertIsNotNone(
+            cli.data.App(["thainlp", "data", "get", "NOT_EXIST"])
+        )
+        self.assertIsNotNone(
+            cli.data.App(["thainlp", "data", "info", "NOT_EXIST"])
+        )
+        self.assertIsNotNone(
+            cli.data.App(["thainlp", "data", "rm", "NOT_EXIST"])
+        )
+
+    def test_cli_soundex(self):
+        self.assertIsInstance(getattr(cli, "soundex"), ModuleType)
+
+        with self.assertRaises(SystemExit) as ex:
+            cli.data.App(["thainlp", "soundex"])
+        self.assertEqual(ex.exception.code, 2)
+
+        self.assertIsNotNone(cli.soundex.App(["thainlp", "soundex", "ทดสอบ"]))
+
+    def test_cli_tag(self):
+        self.assertIsInstance(getattr(cli, "tag"), ModuleType)
+
+        with self.assertRaises(SystemExit) as ex:
+            cli.data.App(["thainlp", "tag"])
+        self.assertEqual(ex.exception.code, 2)
+
+        self.assertIsNotNone(
+            cli.tag.App(
+                [
+                    "thainlp",
+                    "tag",
+                    "pos",
+                    "-s",
+                    " ",
+                    "มอเตอร์ไซค์ ความว่างเปล่า",
+                ]
+            )
+        )
+        self.assertIsNotNone(
+            cli.tag.App(
+                [
+                    "thainlp",
+                    "tag",
+                    "role",
+                    "-s",
+                    " ",
+                    "มอเตอร์ไซค์ ความว่างเปล่า",
+                ]
+            )
+        )
+
+    def test_cli_tokenize(self):
+        self.assertIsInstance(getattr(cli, "tokenize"), ModuleType)
+
+        with self.assertRaises(SystemExit) as ex:
+            cli.data.App(["thainlp", "tokenize"])
+        self.assertEqual(ex.exception.code, 2)
+
+        self.assertIsNotNone(
+            cli.tokenize.App(
+                ["thainlp", "tokenize", "NOT_EXIST", "ไม่มีอยู่ จริง"]
+            )
+        )
+        self.assertIsNotNone(
+            cli.tokenize.App(
+                [
+                    "thainlp",
+                    "tokenize",
+                    "subword",
+                    "-s",
+                    "|",
+                    "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้",
+                ]
+            )
+        )
+        self.assertIsNotNone(
+            cli.tokenize.App(
+                [
+                    "thainlp",
+                    "tokenize",
+                    "syllable",
+                    "-s",
+                    "|",
+                    "-w",
+                    "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้",
+                ]
+            )
+        )
+        self.assertIsNotNone(
+            cli.tokenize.App(
+                [
+                    "thainlp",
+                    "tokenize",
+                    "word",
+                    "-nw",
+                    "-a",
+                    "newmm",
+                    "-s",
+                    "|",
+                    "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้",
+                ]
+            )
+        )
+        self.assertIsNotNone(
+            cli.tokenize.App(
+                [
+                    "thainlp",
+                    "tokenize",
+                    "sent",
+                    "-s",
+                    "|",
+                    (
+                        "ถ้าฉันยิงกระต่ายได้ ฉันก็ยิงฟาสซิสต์ได้"
+                        "กระสุนสำหรับสมองของคุณวันนี้"
+                        "แต่คุณก็จะลืมมันไปทั้งหมดอีกครั้ง"
+                    ),
+                ]
+            )
+        )