Merge pull request #61 from MarcCote/command_parsing

Command parsing

Author: mgermain

scripts/smart_dispatch.py

@@ -35,10 +35,10 @@ def main():
             jobname = args.commandsFile.name
             commands = smartdispatch.get_commands_from_file(args.commandsFile)
         else:
-            # Commands that needs to be parsed and unfolded.
-            arguments = map(smartdispatch.unfold_argument, args.commandAndOptions)
-            jobname = smartdispatch.generate_name_from_arguments(arguments, max_length=235)
-            commands = smartdispatch.get_commands_from_arguments(arguments)
+            # Command that needs to be parsed and unfolded.
+            command = " ".join(args.commandAndOptions)
+            jobname = smartdispatch.generate_name_from_command(command, max_length=235)
+            commands = smartdispatch.unfold_command(command)
 
         commands = smartdispatch.replace_uid_tag(commands)
 

smartdispatch/argument_template.py

@@ -0,0 +1,41 @@
+import re
+from collections import OrderedDict
+
+
+def build_argument_templates_dictionnary():
+    # Order matter, if some regex is more greedy than another, the it should go after
+    argument_templates = OrderedDict()
+    argument_templates[RangeArgumentTemplate.__name__] = RangeArgumentTemplate()
+    argument_templates[ListArgumentTemplate.__name__] = ListArgumentTemplate()
+    return argument_templates
+
+
+class ArgumentTemplate(object):
+    def __init__(self):
+        self.regex = ""
+
+    def unfold(self, match):
+        raise NotImplementedError("Subclass must implement method `unfold(self, match)`!")
+
+
+class ListArgumentTemplate(ArgumentTemplate):
+    def __init__(self):
+        self.regex = "\[[^]]*\]"
+
+    def unfold(self, match):
+        return match[1:-1].split(' ')
+
+
+class RangeArgumentTemplate(ArgumentTemplate):
+    def __init__(self):
+        self.regex = "\[(\d+):(\d+)(?::(\d+))?\]"
+
+    def unfold(self, match):
+        groups = re.search(self.regex, match).groups()
+        start = int(groups[0])
+        end = int(groups[1])
+        step = 1 if groups[2] is None else int(groups[2])
+        return map(str, range(start, end, step))
+
+
+argument_templates = build_argument_templates_dictionnary()

smartdispatch/smartdispatch.py

@@ -1,11 +1,13 @@
 from __future__ import absolute_import
 
 import os
+import re
 import itertools
 from datetime import datetime
 
 import smartdispatch
 from smartdispatch import utils
+from smartdispatch.argument_template import argument_templates
 
 UID_TAG = "{UID}"
 
@@ -98,45 +100,47 @@ def get_commands_from_file(fileobj):
     return fileobj.read().strip().split('\n')
 
 
-def get_commands_from_arguments(arguments):
-    ''' Obtains commands from the product of every unfolded arguments.
+def unfold_command(command):
+    ''' Unfolds a command into a list of unfolded commands.
+
+    Unfolding is performed for every folded arguments (see *Arguments templates*)
+    found in `command`. Then, resulting commands are generated using the product
+    of every unfolded arguments.
 
     Parameters
     ----------
-    arguments : list of list of str
-        list of unfolded arguments
+    command : list of str
+        command to unfold
 
     Returns
     -------
     commands : list of str
-        commands resulting from the product of every unfolded arguments
-    '''
-    return [" ".join(argvalues) for argvalues in itertools.product(*arguments)]
-
-
-def unfold_argument(argument):
-    ''' Unfolds a folded argument into a list of unfolded arguments.
+        commands obtained after unfolding `command`
 
-    An argument can be folded e.g. a list of unfolded arguments separated by spaces.
-    An unfolded argument unfolds to itself.
+    Arguments template
+    ------------------
+    *list*: "[item1 item2 ... itemN]"
+    *range*: "[start:end]" or "[start:end:step]"
+    '''
+    text = utils.encode_escaped_characters(command)
 
-    Parameters
-    ----------
-    argument : str
-        argument to unfold
+    # Build the master regex with all argument's regex
+    regex = "(" + "|".join(["(?P<{0}>{1})".format(name, arg.regex) for name, arg in argument_templates.items()]) + ")"
 
-    Returns
-    -------
-    unfolded_arguments : list of str
-        result of the unfolding
+    pos = 0
+    arguments = []
+    for match in re.finditer(regex, text):
+        # Add already unfolded argument
+        arguments.append([text[pos:match.start()]])
 
-    Complex arguments
-    -----------------
-    *list (space)*: "item1 item2 ... itemN"
-    '''
+        # Unfold argument
+        argument_template_name, matched_text = next((k, v) for k, v in match.groupdict().items() if v is not None)
+        arguments.append(argument_templates[argument_template_name].unfold(matched_text))
+        pos = match.end()
 
-    # Suppose `argument`is a space separated list
-    return argument.split(" ")
+    arguments.append([text[pos:]])  # Add remaining unfolded arguments
+    arguments = [map(utils.decode_escaped_characters, argvalues) for argvalues in arguments]
+    return ["".join(argvalues) for argvalues in itertools.product(*arguments)]
 
 
 def replace_uid_tag(commands):

smartdispatch/tests/test_argument_template.py

@@ -0,0 +1,35 @@
+import re
+from nose.tools import assert_true
+from numpy.testing import assert_array_equal
+
+from smartdispatch.argument_template import ListArgumentTemplate, RangeArgumentTemplate
+
+
+def test_list_argument_template():
+    # Test valid enumeration folded arguments
+    arg = ListArgumentTemplate()
+    folded_arguments = [("[]", [""]),
+                        ("[1]", ["1"]),
+                        ("[1 ]", ["1", ""]),
+                        ("[1 2 3]", ["1", "2", "3"]),
+                        ]
+
+    for folded_argument, unfolded_arguments in folded_arguments:
+        match = re.match(arg.regex, folded_argument)
+        assert_true(match is not None)
+        assert_array_equal(arg.unfold(match.group()), unfolded_arguments)
+
+    # Test invalid enumeration folded arguments
+    assert_true(re.match(arg.regex, "[1 2 3") is None)
+
+
+def test_range_argument_template():
+    arg = RangeArgumentTemplate()
+    folded_arguments = [("[1:4]", ["1", "2", "3"]),
+                        ("[1:4:2]", ["1", "3"]),
+                        ]
+
+    for folded_argument, unfolded_arguments in folded_arguments:
+        match = re.match(arg.regex, folded_argument)
+        assert_true(match is not None)
+        assert_array_equal(arg.unfold(match.group()), unfolded_arguments)

smartdispatch/tests/test_smartdispatch.py

@@ -77,32 +77,71 @@ def test_get_commands_from_file():
     assert_array_equal(smartdispatch.get_commands_from_file(fileobj), commands)
 
 
-def test_get_commands_from_arguments():
-    # Test single unfolded arguments
-    args = [["arg"]]
-    assert_equal(smartdispatch.get_commands_from_arguments(args), ["arg"])
-
-    args = [["args1_a", "args1_b"]]
-    assert_equal(smartdispatch.get_commands_from_arguments(args), ["args1_a", "args1_b"])
-
-    # Test multiple unfolded arguments
-    args = [["args1"], ["args2"]]
-    assert_equal(smartdispatch.get_commands_from_arguments(args), ["args1 args2"])
-
-    args = [["args1_a", "args1_b", "args1_c"], ["args2_a", "args2_b"]]
-    assert_equal(smartdispatch.get_commands_from_arguments(args), ["args1_a args2_a", "args1_a args2_b",
-                                                                   "args1_b args2_a", "args1_b args2_b",
-                                                                   "args1_c args2_a", "args1_c args2_b"])
-
-
-def test_unfold_argument():
-    # Test simple argument
-    for arg in ["arg1", "[arg1"]:
-        assert_array_equal(smartdispatch.unfold_argument(arg), [arg])
-
-    # Test list (space)
-    for arg in ["arg1 arg2", "arg1 ", " arg1"]:
-        assert_array_equal(smartdispatch.unfold_argument(arg), arg.split(" "))
+def test_unfold_command():
+    # Test with one argument
+    cmd = "ls"
+    assert_equal(smartdispatch.unfold_command(cmd), ["ls"])
+
+    cmd = "echo 1"
+    assert_equal(smartdispatch.unfold_command(cmd), ["echo 1"])
+
+    # Test two arguments
+    cmd = "echo [1 2]"
+    assert_equal(smartdispatch.unfold_command(cmd), ["echo 1", "echo 2"])
+
+    cmd = "echo test [1 2] yay"
+    assert_equal(smartdispatch.unfold_command(cmd), ["echo test 1 yay", "echo test 2 yay"])
+
+    cmd = "echo test[1 2]"
+    assert_equal(smartdispatch.unfold_command(cmd), ["echo test1", "echo test2"])
+
+    cmd = "echo test[1 2]yay"
+    assert_equal(smartdispatch.unfold_command(cmd), ["echo test1yay", "echo test2yay"])
+
+    # Test multiple folded arguments
+    cmd = "python my_command.py [0.01 0.000001 0.00000000001] -1 [omicron mu]"
+    assert_equal(smartdispatch.unfold_command(cmd), ["python my_command.py 0.01 -1 omicron",
+                                                     "python my_command.py 0.01 -1 mu",
+                                                     "python my_command.py 0.000001 -1 omicron",
+                                                     "python my_command.py 0.000001 -1 mu",
+                                                     "python my_command.py 0.00000000001 -1 omicron",
+                                                     "python my_command.py 0.00000000001 -1 mu"])
+
+    # Test multiple folded arguments and not unfoldable brackets
+    cmd = "python my_command.py [0.01 0.000001 0.00000000001] -1 \[[42 133,666]\] slow [omicron mu]"
+    assert_equal(smartdispatch.unfold_command(cmd), ["python my_command.py 0.01 -1 [42] slow omicron",
+                                                     "python my_command.py 0.01 -1 [42] slow mu",
+                                                     "python my_command.py 0.01 -1 [133,666] slow omicron",
+                                                     "python my_command.py 0.01 -1 [133,666] slow mu",
+                                                     "python my_command.py 0.000001 -1 [42] slow omicron",
+                                                     "python my_command.py 0.000001 -1 [42] slow mu",
+                                                     "python my_command.py 0.000001 -1 [133,666] slow omicron",
+                                                     "python my_command.py 0.000001 -1 [133,666] slow mu",
+                                                     "python my_command.py 0.00000000001 -1 [42] slow omicron",
+                                                     "python my_command.py 0.00000000001 -1 [42] slow mu",
+                                                     "python my_command.py 0.00000000001 -1 [133,666] slow omicron",
+                                                     "python my_command.py 0.00000000001 -1 [133,666] slow mu"])
+
+    # Test multiple different folded arguments
+    cmd = "python my_command.py [0.01 0.001] -[1:5] slow"
+    assert_equal(smartdispatch.unfold_command(cmd), ["python my_command.py 0.01 -1 slow",
+                                                     "python my_command.py 0.01 -2 slow",
+                                                     "python my_command.py 0.01 -3 slow",
+                                                     "python my_command.py 0.01 -4 slow",
+                                                     "python my_command.py 0.001 -1 slow",
+                                                     "python my_command.py 0.001 -2 slow",
+                                                     "python my_command.py 0.001 -3 slow",
+                                                     "python my_command.py 0.001 -4 slow"])
+
+    cmd = "python my_command.py -[1:5] slow [0.01 0.001]"
+    assert_equal(smartdispatch.unfold_command(cmd), ["python my_command.py -1 slow 0.01",
+                                                     "python my_command.py -1 slow 0.001",
+                                                     "python my_command.py -2 slow 0.01",
+                                                     "python my_command.py -2 slow 0.001",
+                                                     "python my_command.py -3 slow 0.01",
+                                                     "python my_command.py -3 slow 0.001",
+                                                     "python my_command.py -4 slow 0.01",
+                                                     "python my_command.py -4 slow 0.001"])
 
 
 def test_replace_uid_tag():

smartdispatch/utils.py

@@ -35,6 +35,24 @@ def slugify(value):
     return str(re.sub('[-\s]+', '_', value))
 
 
+def encode_escaped_characters(text, escaping_character="\\"):
+    """ Escape the escaped character using its hex representation """
+    def hexify(match):
+        return "\\x{0}".format(match.group()[-1].encode("hex"))
+
+    return re.sub(r"\\.", hexify, text)
+
+
+def decode_escaped_characters(text):
+    """ Convert hex representation to the character it represents """
+    if len(text) == 0:
+        return ''
+
+    def unhexify(match):
+        return match.group()[2:].decode("hex")
+
+    return re.sub(r"\\x..", unhexify, text)
+
 @contextmanager
 def open_with_lock(*args, **kwargs):
     """ Context manager for opening file with an exclusive lock. """