Fix literal block multiline parsing

Fixes #8394

4 files changed, 109 insertions, 53 deletions

diff --git a/lib/ansible/module_utils/splitter.py b/lib/ansible/module_utils/splitter.py
index be7555847d..d633c2e289 100644
--- a/lib/ansible/module_utils/splitter.py
+++ b/lib/ansible/module_utils/splitter.py
@@ -76,7 +76,7 @@ def split_args(args):
         do_decode = True
     except UnicodeDecodeError:
         do_decode = False
-    tokens = args.split(' ')
+    items = args.split(' ')
 
     # iterate over the tokens, and reassemble any that may have been
     # split on a space inside a jinja2 block.
@@ -92,56 +92,70 @@ def split_args(args):
     block_depth   = 0 # used to count nested jinja2 {% %} blocks
     comment_depth = 0 # used to count nested jinja2 {# #} blocks
 
-    # now we loop over each split token, coalescing tokens if the white space
+    # now we loop over each split chunk, coalescing tokens if the white space
     # split occurred within quotes or a jinja2 block of some kind
-    for token in tokens:
-
-        # store the previous quoting state for checking later
-        was_inside_quotes = inside_quotes
-        quote_char = _get_quote_state(token, quote_char)
-        inside_quotes = quote_char is not None
-
-        # multiple conditions may append a token to the list of params,
-        # so we keep track with this flag to make sure it only happens once
-        # append means add to the end of the list, don't append means concatenate
-        # it to the end of the last token
-        appended = False
-
-        # if we're inside quotes now, but weren't before, append the token 
-        # to the end of the list, since we'll tack on more to it later
-        # otherwise, if we're inside any jinja2 block, inside quotes, or we were
-        # inside quotes (but aren't now) concat this token to the last param
-        if inside_quotes and not was_inside_quotes:
-            params.append(token)
-            appended = True
-        elif print_depth or block_depth or comment_depth or inside_quotes or was_inside_quotes:
-            params[-1] = "%s %s" % (params[-1], token)
-            appended = True
-
-        # if the number of paired block tags is not the same, the depth has changed, so we calculate that here
-        # and may append the current token to the params (if we haven't previously done so)
-        prev_print_depth = print_depth
-        print_depth = _count_jinja2_blocks(token, print_depth, "{{", "}}")
-        if print_depth != prev_print_depth and not appended:
-            params.append(token)
-            appended = True
-
-        prev_block_depth = block_depth
-        block_depth = _count_jinja2_blocks(token, block_depth, "{%", "%}")
-        if block_depth != prev_block_depth and not appended:
-            params.append(token)
-            appended = True
-
-        prev_comment_depth = comment_depth
-        comment_depth = _count_jinja2_blocks(token, comment_depth, "{#", "#}")
-        if comment_depth != prev_comment_depth and not appended:
-            params.append(token)
-            appended = True
-
-        # finally, if we're at zero depth for all blocks and not inside quotes, and have not
-        # yet appended anything to the list of params, we do so now
-        if not (print_depth or block_depth or comment_depth) and not inside_quotes and not appended and token != '':
-            params.append(token)
+    for item in items:
+
+        # we split on spaces and newlines separately, so that we
+        # can tell which character we split on for reassembly
+        # inside quotation characters
+        tokens = item.split('\n')
+
+        for idx,token in enumerate(tokens):
+
+            # if we're at the end of the enumeration, the character separator
+            # used when reassembling quoted bits should be a space, otherwise
+            # it will be a newline character
+            spacer = ' '
+            if idx > 0:
+                spacer = '\n'
+
+            # store the previous quoting state for checking later
+            was_inside_quotes = inside_quotes
+            quote_char = _get_quote_state(token, quote_char)
+            inside_quotes = quote_char is not None
+
+            # multiple conditions may append a token to the list of params,
+            # so we keep track with this flag to make sure it only happens once
+            # append means add to the end of the list, don't append means concatenate
+            # it to the end of the last token
+            appended = False
+
+            # if we're inside quotes now, but weren't before, append the token
+            # to the end of the list, since we'll tack on more to it later
+            # otherwise, if we're inside any jinja2 block, inside quotes, or we were
+            # inside quotes (but aren't now) concat this token to the last param
+            if inside_quotes and not was_inside_quotes:
+                params.append(token)
+                appended = True
+            elif print_depth or block_depth or comment_depth or inside_quotes or was_inside_quotes:
+                params[-1] = "%s%s%s" % (params[-1], spacer, token)
+                appended = True
+
+            # if the number of paired block tags is not the same, the depth has changed, so we calculate that here
+            # and may append the current token to the params (if we haven't previously done so)
+            prev_print_depth = print_depth
+            print_depth = _count_jinja2_blocks(token, print_depth, "{{", "}}")
+            if print_depth != prev_print_depth and not appended:
+                params.append(token)
+                appended = True
+
+            prev_block_depth = block_depth
+            block_depth = _count_jinja2_blocks(token, block_depth, "{%", "%}")
+            if block_depth != prev_block_depth and not appended:
+                params.append(token)
+                appended = True
+
+            prev_comment_depth = comment_depth
+            comment_depth = _count_jinja2_blocks(token, comment_depth, "{#", "#}")
+            if comment_depth != prev_comment_depth and not appended:
+                params.append(token)
+                appended = True
+
+            # finally, if we're at zero depth for all blocks and not inside quotes, and have not
+            # yet appended anything to the list of params, we do so now
+            if not (print_depth or block_depth or comment_depth) and not inside_quotes and not appended and token != '':
+                params.append(token)
 
     # If we're done and things are not at zero depth or we're still inside quotes,
     # raise an error to indicate that the args were unbalanced
diff --git a/test/integration/roles/test_command_shell/tasks/main.yml b/test/integration/roles/test_command_shell/tasks/main.yml
index a7eae09545..db6f14084e 100644
--- a/test/integration/roles/test_command_shell/tasks/main.yml
+++ b/test/integration/roles/test_command_shell/tasks/main.yml
@@ -168,3 +168,23 @@
   assert:
     that:
       - "shell_result4.changed == False"
+
+- name: execute a shell command using a literal multiline block
+  shell: |
+    echo this is a
+    "multiline echo"
+    "with a new line
+    in quotes"
+    | md5sum
+    | tr -s ' '
+    | cut -f1 -d ' '
+  register: shell_result5
+
+- debug: var=shell_result5
+
+- name: assert the multiline shell command ran as expected
+  assert:
+    that:
+      - "shell_result5.changed"
+      - "shell_result5.stdout == '32f3cc201b69ed8afa3902b80f554ca8'"
+
diff --git a/test/integration/roles/test_copy/tasks/main.yml b/test/integration/roles/test_copy/tasks/main.yml
index b1d53a748a..e168bead9b 100644
--- a/test/integration/roles/test_copy/tasks/main.yml
+++ b/test/integration/roles/test_copy/tasks/main.yml
@@ -158,4 +158,25 @@
 - name: assert that the directory was not changed
   assert:
     that:
-      - "not copy_result5|changed"
\ No newline at end of file
+      - "not copy_result5|changed"
+
+# issue 8394
+- name: create a file with content and a literal multiline block
+  copy: |
+    content='this is the first line
+    this is the second line
+
+    this line is after an empty line
+    this line is the last line
+    '
+    dest={{output_dir}}/multiline.txt
+  register: copy_result6
+
+- debug: var=copy_result6
+
+- name: assert the multiline file was created correctly
+  assert:
+    that:
+      - "copy_result6.changed"
+      - "copy_result6.dest == '{{output_dir|expanduser}}/multiline.txt'"
+      - "copy_result6.md5sum == '1627d51e7e607c92cf1a502bf0c6cce3'"
diff --git a/test/units/TestUtils.py b/test/units/TestUtils.py
index 935da0ff8f..b091603626 100644
--- a/test/units/TestUtils.py
+++ b/test/units/TestUtils.py
@@ -705,8 +705,9 @@ class TestUtils(unittest.TestCase):
         # jinja2 loop blocks with lots of complexity
         _test_combo(
             # in memory of neighbors cat
-            'a {% if x %} y {%else %} {{meow}} {% endif %} cookiechip\ndone',
-            ['a', '{% if x %}', 'y', '{%else %}', '{{meow}}', '{% endif %}', 'cookiechip\ndone']
+            # we only preserve newlines inside of quotes
+            'a {% if x %} y {%else %} {{meow}} {% endif %} "cookie\nchip"\ndone',
+            ['a', '{% if x %}', 'y', '{%else %}', '{{meow}}', '{% endif %}', '"cookie\nchip"', 'done']
         )
 
         # test space preservation within quotes