Merge pull request #496 from python-cmd2/embedded_newlines

Fix #495 by allowing embedded newlines in unclosed quote marks when entering multiline commands

Author: tleonhardt

cmd2/cmd2.py

@@ -1858,8 +1858,30 @@ def _complete_statement(self, line: str) -> Statement:
         pipe runs out. We can't refactor it because we need to retain
         backwards compatibility with the standard library version of cmd.
         """
-        statement = self.statement_parser.parse(self.preparse(line))
-        while statement.multiline_command and not statement.terminator:
+        # preparse() is deprecated, use self.register_postparsing_hook() instead
+        line = self.preparse(line)
+
+        while True:
+            try:
+                statement = self.statement_parser.parse(line)
+                if statement.multiline_command and statement.terminator:
+                    # we have a completed multiline command, we are done
+                    break
+                if not statement.multiline_command:
+                    # it's not a multiline command, but we parsed it ok
+                    # so we are done
+                    break
+            except ValueError:
+                # we have unclosed quotation marks, lets parse only the command
+                # and see if it's a multiline
+                statement = self.statement_parser.parse_command_only(line)
+                if not statement.multiline_command:
+                    # not a multiline command, so raise the exception
+                    raise
+
+            # if we get here we must have:
+            #   - a multiline command with no terminator
+            #   - a multiline command with unclosed quotation marks
             if not self.quit_on_sigint:
                 try:
                     newline = self.pseudo_raw_input(self.continuation_prompt)
@@ -1885,7 +1907,6 @@ def _complete_statement(self, line: str) -> Statement:
                     newline = '\n'
                     self.poutput(newline)
                 line = '{}\n{}'.format(statement.raw, newline)
-            statement = self.statement_parser.parse(line)
 
         if not statement.command:
             raise EmptyStatement()

cmd2/parsing.py

@@ -407,8 +407,8 @@ def parse_command_only(self, rawinput: str) -> Statement:
         """Partially parse input into a Statement object.
 
         The command is identified, and shortcuts and aliases are expanded.
-        Terminators, multiline commands, and output redirection are not
-        parsed.
+        Multiline commands are identified, but terminators and output
+        redirection are not parsed.
 
         This method is used by tab completion code and therefore must not
         generate an exception if there are unclosed quotes.
@@ -420,8 +420,8 @@ def parse_command_only(self, rawinput: str) -> Statement:
           - args
 
         Different from parse(), this method does not remove redundant whitespace
-        within statement.args. It does however, ensure args does not have leading
-        or trailing whitespace.
+        within statement.args. It does however, ensure args does not have
+        leading or trailing whitespace.
         """
         # expand shortcuts and aliases
         line = self._expand(rawinput)
@@ -447,13 +447,20 @@ def parse_command_only(self, rawinput: str) -> Statement:
             if not command or not args:
                 args = None
 
+        # set multiline
+        if command in self.multiline_commands:
+            multiline_command = command
+        else:
+            multiline_command = None
+
         # build the statement
         # string representation of args must be an empty string instead of
         # None for compatibility with standard library cmd
         statement = Statement('' if args is None else args,
                               raw=rawinput,
                               command=command,
                               args=args,
+                              multiline_command=multiline_command,
                               )
         return statement
 

tests/test_cmd2.py

@@ -1471,7 +1471,8 @@ def test_multiline_complete_empty_statement_raises_exception(multiline_app):
         multiline_app._complete_statement('')
 
 def test_multiline_complete_statement_without_terminator(multiline_app):
-    # Mock out the input call so we don't actually wait for a user's response on stdin when it looks for more input
+    # Mock out the input call so we don't actually wait for a user's response
+    # on stdin when it looks for more input
     m = mock.MagicMock(name='input', return_value='\n')
     builtins.input = m
 
@@ -1481,6 +1482,20 @@ def test_multiline_complete_statement_without_terminator(multiline_app):
     statement = multiline_app._complete_statement(line)
     assert statement == args
     assert statement.command == command
+    assert statement.multiline_command == command
+
+def test_multiline_complete_statement_with_unclosed_quotes(multiline_app):
+    # Mock out the input call so we don't actually wait for a user's response
+    # on stdin when it looks for more input
+    m = mock.MagicMock(name='input', side_effect=['quotes', '" now closed;'])
+    builtins.input = m
+
+    line = 'orate hi "partially open'
+    statement = multiline_app._complete_statement(line)
+    assert statement == 'hi "partially open\nquotes\n" now closed'
+    assert statement.command == 'orate'
+    assert statement.multiline_command == 'orate'
+    assert statement.terminator == ';'
 
 
 def test_clipboard_failure(base_app, capsys):

tests/test_parsing.py

@@ -376,7 +376,7 @@ def test_parse_multiline_with_complete_comment(parser):
     assert statement.argv == ['multiline', 'command', 'is', 'done']
     assert statement.terminator == ';'
 
-def test_parse_multiline_termninated_by_empty_line(parser):
+def test_parse_multiline_terminated_by_empty_line(parser):
     line = 'multiline command ends\n\n'
     statement = parser.parse(line)
     assert statement.multiline_command == 'multiline'
@@ -386,6 +386,23 @@ def test_parse_multiline_termninated_by_empty_line(parser):
     assert statement.argv == ['multiline', 'command', 'ends']
     assert statement.terminator == '\n'
 
+@pytest.mark.parametrize('line,terminator',[
+    ('multiline command "with\nembedded newline";', ';'),
+    ('multiline command "with\nembedded newline";;;', ';'),
+    ('multiline command "with\nembedded newline";; ;;', ';'),
+    ('multiline command "with\nembedded newline" &', '&'),
+    ('multiline command "with\nembedded newline" & &', '&'),
+    ('multiline command "with\nembedded newline"\n\n', '\n'),
+])
+def test_parse_multiline_with_embedded_newline(parser, line, terminator):
+    statement = parser.parse(line)
+    assert statement.multiline_command == 'multiline'
+    assert statement.command == 'multiline'
+    assert statement.args == 'command "with\nembedded newline"'
+    assert statement == statement.args
+    assert statement.argv == ['multiline', 'command', 'with\nembedded newline']
+    assert statement.terminator == terminator
+
 def test_parse_multiline_ignores_terminators_in_comments(parser):
     line = 'multiline command "with term; ends" now\n\n'
     statement = parser.parse(line)
@@ -584,6 +601,16 @@ def test_parse_command_only_none(parser, line):
     assert statement.args is None
     assert statement == ''
 
+def test_parse_command_only_multiline(parser):
+    line = 'multiline with partially "open quotes and no terminator'
+    statement = parser.parse_command_only(line)
+    assert statement.command == 'multiline'
+    assert statement.multiline_command == 'multiline'
+    assert statement.args == 'with partially "open quotes and no terminator'
+    assert statement == statement.args
+    assert statement.command_and_args == line
+
+
 def test_statement_initialization(parser):
     string = 'alias'
     statement = cmd2.Statement(string)