Fix multiple multiline matching issues in pcre2grep.

git-svn-id: svn://vcs.exim.org/pcre2/code/trunk@878 6239d852-aaf2-0410-a92c-79f79f948069

5 files changed, 177 insertions, 42 deletions

diff --git a/ChangeLog b/ChangeLog
index a0ad0b2..6881e62 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -45,6 +45,12 @@ misbehaviour for subsequent references to groups that started outside the
 recursion. ACCEPT in an assertion now closes only those groups that were 
 started within that assertion. Fixes oss-fuzz issues 3852 and 3891.
 
+13. Multiline matching in pcre2grep was misbehaving if the pattern matched 
+within a line, and then matched again at the end of the line and over into 
+subsequent lines. Behaviour was different with and without colouring, and 
+sometimes context lines were incorrectly printed and/or line endings were lost. 
+All these issues should now be fixed.
+
 
 Version 10.30 14-August-2017
 ----------------------------
diff --git a/RunGrepTest b/RunGrepTest
index cf88c78..a7496cb 100755
--- a/RunGrepTest
+++ b/RunGrepTest
@@ -248,7 +248,7 @@ echo "---------------------------- Test 35 -----------------------------" >>test
 echo "RC=$?" >>testtrygrep
 
 echo "---------------------------- Test 36 -----------------------------" >>testtrygrep
-(cd $srcdir; $valgrind $vjs $pcre2grep -L -r --include=grepinput --exclude 'grepinput$' --exclude=grepinput8 --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep -L -r --include=grepinput --exclude 'grepinput$' --exclude=grepinput8 --exclude=grepinputM --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep
 echo "RC=$?" >>testtrygrep
 
 echo "---------------------------- Test 37 -----------------------------" >>testtrygrep
@@ -391,6 +391,12 @@ echo "RC=$?" >>testtrygrep
 echo "---------------------------- Test 70 -----------------------------" >>testtrygrep
 (cd $srcdir; $valgrind $vjs $pcre2grep --color=always -M "triple:\t.*\n\n" ./testdata/grepinput3) >>testtrygrep
 echo "RC=$?" >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep --color=always -M -n "triple:\t.*\n\n" ./testdata/grepinput3) >>testtrygrep
+echo "RC=$?" >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep -M "triple:\t.*\n\n" ./testdata/grepinput3) >>testtrygrep
+echo "RC=$?" >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep -M -n "triple:\t.*\n\n" ./testdata/grepinput3) >>testtrygrep
+echo "RC=$?" >>testtrygrep
 
 echo "---------------------------- Test 71 -----------------------------" >>testtrygrep
 (cd $srcdir; $valgrind $vjs $pcre2grep -o "^01|^02|^03" ./testdata/grepinput) >>testtrygrep
@@ -494,25 +500,25 @@ echo "---------------------------- Test 95 -----------------------------" >>test
 echo "RC=$?" >>testtrygrep
 
 echo "---------------------------- Test 96 -----------------------------" >>testtrygrep
-(cd $srcdir; $valgrind $vjs $pcre2grep -L -r --include-dir=testdata --exclude '^(?!grepinput)' 'fox' ./test* | sort) >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep -L -r --include-dir=testdata --exclude '^(?!grepinput)' --exclude=grepinputM 'fox' ./test* | sort) >>testtrygrep
 echo "RC=$?" >>testtrygrep
 
 echo "---------------------------- Test 97 -----------------------------" >>testtrygrep
 echo "grepinput$" >testtemp1grep
 echo "grepinput8" >>testtemp1grep
-(cd $srcdir; $valgrind $vjs $pcre2grep -L -r --include=grepinput --exclude-from $builddir/testtemp1grep --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep -L -r --include=grepinput --exclude=grepinputM --exclude-from $builddir/testtemp1grep --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep
 echo "RC=$?" >>testtrygrep
 
 echo "---------------------------- Test 98 -----------------------------" >>testtrygrep
 echo "grepinput$" >testtemp1grep
 echo "grepinput8" >>testtemp1grep
-(cd $srcdir; $valgrind $vjs $pcre2grep -L -r --exclude=grepinput3 --include=grepinput --exclude-from $builddir/testtemp1grep --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep -L -r --exclude=grepinput3 --exclude=grepinputM --include=grepinput --exclude-from $builddir/testtemp1grep --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep
 echo "RC=$?" >>testtrygrep
 
 echo "---------------------------- Test 99 -----------------------------" >>testtrygrep
 echo "grepinput$" >testtemp1grep
 echo "grepinput8" >testtemp2grep
-(cd $srcdir; $valgrind $vjs $pcre2grep -L -r --include grepinput --exclude-from $builddir/testtemp1grep --exclude-from=$builddir/testtemp2grep --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep -L -r --include grepinput --exclude=grepinputM --exclude-from $builddir/testtemp1grep --exclude-from=$builddir/testtemp2grep --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep
 echo "RC=$?" >>testtrygrep
 
 echo "---------------------------- Test 100 ------------------------------" >>testtrygrep
@@ -582,7 +588,7 @@ echo "---------------------------- Test 115 -----------------------------" >>tes
 echo "RC=$?" >>testtrygrep
 
 echo "---------------------------- Test 116 -----------------------------" >>testtrygrep
-(cd $srcdir; $valgrind $vjs $pcre2grep -th 'the' testdata/grepinput*) >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep --exclude=grepinputM -th 'the' testdata/grepinput*) >>testtrygrep
 echo "RC=$?" >>testtrygrep
 
 echo "---------------------------- Test 117 -----------------------------" >>testtrygrep
@@ -610,10 +616,20 @@ echo "---------------------------- Test 122 -----------------------------" >>tes
 (cd $srcdir; $valgrind $vjs $pcre2grep -w 'cat|dog' testdata/grepinputv) >>testtrygrep
 echo "RC=$?" >>testtrygrep
 
-echo "---------------------------- Test 122 -----------------------------" >>testtrygrep
+echo "---------------------------- Test 123 -----------------------------" >>testtrygrep
 (cd $srcdir; $valgrind $vjs $pcre2grep -w 'dog|cat' testdata/grepinputv) >>testtrygrep
 echo "RC=$?" >>testtrygrep
 
+echo "---------------------------- Test 124 -----------------------------" >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep -Mn --colour=always 'start[\s]+end' testdata/grepinputM) >>testtrygrep
+echo "RC=$?" >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep -Mn --colour=always -A2 'start[\s]+end' testdata/grepinputM) >>testtrygrep
+echo "RC=$?" >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep -Mn 'start[\s]+end' testdata/grepinputM) >>testtrygrep
+echo "RC=$?" >>testtrygrep
+(cd $srcdir; $valgrind $vjs $pcre2grep -Mn -A2 'start[\s]+end' testdata/grepinputM) >>testtrygrep
+echo "RC=$?" >>testtrygrep
+
 
 # Now compare the results.
 
diff --git a/src/pcre2grep.c b/src/pcre2grep.c
index 1649d5a..bec07e1 100644
--- a/src/pcre2grep.c
+++ b/src/pcre2grep.c
@@ -2505,7 +2505,10 @@ while (ptr < endptr)
   match = match_patterns(ptr, length, options, startoffset, &mrc);
   options = PCRE2_NOTEMPTY;
 
-  /* If it's a match or a not-match (as required), do what's wanted. */
+  /* If it's a match or a not-match (as required), do what's wanted. NOTE: Use
+  only FWRITE_IGNORE() - which is just a packaged fwrite() that ignores its
+  return code - to output data lines, so that binary zeroes are treated as just
+  another data character. */
 
   if (match != invert)
     {
@@ -2734,27 +2737,6 @@ while (ptr < endptr)
       if (printname != NULL) fprintf(stdout, "%s:", printname);
       if (number) fprintf(stdout, "%d:", linenumber);
 
-      /* In multiline mode, we want to print to the end of the line in which
-      the end of the matched string is found, so we adjust linelength and the
-      line number appropriately, but only when there actually was a match
-      (invert not set). Because the PCRE2_FIRSTLINE option is set, the start of
-      the match will always be before the first newline sequence. */
-
-      if (multiline & !invert)
-        {
-        char *endmatch = ptr + offsets[1];
-        t = ptr;
-        while (t <= endmatch)
-          {
-          t = end_of_line(t, endptr, &endlinelength);
-          if (t < endmatch) linenumber++; else break;
-          }
-        linelength = t - ptr - endlinelength;
-        }
-
-      /*** NOTE: Use only fwrite() to output the data line, so that binary
-      zeroes are treated as just another data character. */
-
       /* This extra option, for Jeffrey Friedl's debugging requirements,
       replaces the matched string, or a specific captured string if it exists,
       with X. When this happens, colouring is ignored. */
@@ -2771,20 +2753,48 @@ while (ptr < endptr)
       else
 #endif
 
-      /* We have to split the line(s) up if colouring, and search for further
-      matches, but not of course if the line is a non-match. */
+      /* In multiline mode, or if colouring, we have to split the line(s) up
+      and search for further matches, but not of course if the line is a
+      non-match. In multiline mode this is necessary in case there is another
+      match that spans the end of the current line. When colouring we want to
+      colour all matches. */
 
-      if (do_colour && !invert)
+      if ((multiline || do_colour) && !invert)
         {
         int plength;
         FWRITE_IGNORE(ptr, 1, offsets[0], stdout);
         print_match(ptr + offsets[0], offsets[1] - offsets[0]);
         for (;;)
           {
-          startoffset = offsets[1];
-          if (startoffset >= linelength + endlinelength ||
-              !match_patterns(ptr, length, options, startoffset, &mrc))
-            break;
+          startoffset = offsets[1];  /* Advance after previous match. */
+
+          /* If the current match ended past the end of the line (only possible
+          in multiline mode), we must move on to the line in which it did end
+          before searching for more matches. Because the PCRE2_FIRSTLINE option
+          is set, the start of the match will always be before the first
+          newline sequence. */
+
+          while (startoffset > linelength + endlinelength)
+            {
+            ptr += linelength + endlinelength;
+            filepos += (int)(linelength + endlinelength);
+            linenumber++;
+            startoffset -= (int)(linelength + endlinelength);
+            t = end_of_line(ptr, endptr, &endlinelength);
+            linelength = t - ptr - endlinelength;
+            length = (size_t)(endptr - ptr);
+            }
+
+          /* If startoffset is at the exact end of the line it means this
+          complete line was the final part of the match, so there is nothing
+          more to do. */
+
+          if (startoffset == linelength + endlinelength) break;
+
+          /* Otherwise, run a match from within the final line, and if found,
+          loop for any that may follow. */
+
+          if (!match_patterns(ptr, length, options, startoffset, &mrc)) break;
           FWRITE_IGNORE(ptr + startoffset, 1, offsets[0] - startoffset, stdout);
           print_match(ptr + offsets[0], offsets[1] - offsets[0]);
           }
@@ -2797,7 +2807,7 @@ while (ptr < endptr)
         if (plength > 0) FWRITE_IGNORE(ptr + startoffset, 1, plength, stdout);
         }
 
-      /* Not colouring; no need to search for further matches */
+      /* Not colouring or multiline; no need to search for further matches. */
 
       else FWRITE_IGNORE(ptr, 1, linelength + endlinelength, stdout);
       }
diff --git a/testdata/grepinputM b/testdata/grepinputM
new file mode 100644
index 0000000..9119e3d
--- /dev/null
+++ b/testdata/grepinputM
@@ -0,0 +1,17 @@
+Data file for multiline tests of multiple matches.
+
+start end in between start
+end and following
+Other stuff
+
+start end in between start
+end and following start
+end other stuff
+
+start end in between start
+
+end
+
+** These two lines must be last.
+start end in between start
+end
diff --git a/testdata/grepoutput b/testdata/grepoutput
index 52e0d17..7e963fb 100644
--- a/testdata/grepoutput
+++ b/testdata/grepoutput
@@ -487,6 +487,7 @@ RC=0
 ./testdata/grepinput:456
 ./testdata/grepinput3:0
 ./testdata/grepinput8:0
+./testdata/grepinputM:0
 ./testdata/grepinputv:1
 ./testdata/grepinputx:0
 RC=0
@@ -600,6 +601,33 @@ RC=0
 triple:	t6_txt	s2_tag	s_txt	p_tag	p_txt	o_tag	o_txt
 
 RC=0
+1:triple:	t1_txt	s1_tag	s_txt	p_tag	p_txt	o_tag	o_txt
+
+6:triple:	t3_txt	s2_tag	s_txt	p_tag	p_txt	o_tag	o_txt
+
+8:triple:	t4_txt	s1_tag	s_txt	p_tag	p_txt	o_tag	o_txt
+
+13:triple:	t6_txt	s2_tag	s_txt	p_tag	p_txt	o_tag	o_txt
+
+RC=0
+triple:	t1_txt	s1_tag	s_txt	p_tag	p_txt	o_tag	o_txt
+
+triple:	t3_txt	s2_tag	s_txt	p_tag	p_txt	o_tag	o_txt
+
+triple:	t4_txt	s1_tag	s_txt	p_tag	p_txt	o_tag	o_txt
+
+triple:	t6_txt	s2_tag	s_txt	p_tag	p_txt	o_tag	o_txt
+
+RC=0
+1:triple:	t1_txt	s1_tag	s_txt	p_tag	p_txt	o_tag	o_txt
+
+6:triple:	t3_txt	s2_tag	s_txt	p_tag	p_txt	o_tag	o_txt
+
+8:triple:	t4_txt	s1_tag	s_txt	p_tag	p_txt	o_tag	o_txt
+
+13:triple:	t6_txt	s2_tag	s_txt	p_tag	p_txt	o_tag	o_txt
+
+RC=0
 ---------------------------- Test 71 -----------------------------
 01
 RC=0
@@ -793,21 +821,23 @@ RC=0
 37216,12
 RC=0
 ---------------------------- Test 113 -----------------------------
-478
+480
 RC=0
 ---------------------------- Test 114 -----------------------------
 testdata/grepinput:469
 testdata/grepinput3:0
 testdata/grepinput8:0
+testdata/grepinputM:2
 testdata/grepinputv:3
 testdata/grepinputx:6
-TOTAL:478
+TOTAL:480
 RC=0
 ---------------------------- Test 115 -----------------------------
 testdata/grepinput:469
+testdata/grepinputM:2
 testdata/grepinputv:3
 testdata/grepinputx:6
-TOTAL:478
+TOTAL:480
 RC=0
 ---------------------------- Test 116 -----------------------------
 478
@@ -816,9 +846,10 @@ RC=0
 469
 0
 0
+2
 3
 6
-478
+480
 RC=0
 ---------------------------- Test 118 -----------------------------
 testdata/grepinput3
@@ -846,7 +877,62 @@ RC=0
 over the lazy dog.
 The word is cat in this line
 RC=0
----------------------------- Test 122 -----------------------------
+---------------------------- Test 123 -----------------------------
 over the lazy dog.
 The word is cat in this line
 RC=0
+---------------------------- Test 124 -----------------------------
+3:start end in between start
+end and following
+7:start end in between start
+end and following start
+end other stuff
+11:start end in between start
+
+end
+16:start end in between start
+end
+RC=0
+3:start end in between start
+end and following
+5-Other stuff
+6-
+7:start end in between start
+end and following start
+end other stuff
+10-
+11:start end in between start
+
+end
+14-
+15-** These two lines must be last.
+16:start end in between start
+end
+RC=0
+3:start end in between start
+end and following
+7:start end in between start
+end and following start
+end other stuff
+11:start end in between start
+
+end
+16:start end in between start
+end
+RC=0
+3:start end in between start
+end and following
+5-Other stuff
+6-
+7:start end in between start
+end and following start
+end other stuff
+10-
+11:start end in between start
+
+end
+14-
+15-** These two lines must be last.
+16:start end in between start
+end
+RC=0