Stage 29b: Process quotes and comments by buffer, not bytes.

* ltdl/m4/gnulib-cache.m4: Import memchr2 module.
* m4/input.c (m4__next_token): Add buffer reads to quote and
comment parsing.
* NEWS: Document this.

Signed-off-by: Eric Blake <ebb9@byu.net>

4 files changed, 119 insertions, 9 deletions

diff --git a/ChangeLog b/ChangeLog
index d61afa14..ad5e8a45 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2009-02-17  Eric Blake  <ebb9@byu.net>
+
+	Stage 29b: Process quotes and comments by buffer, not bytes.
+	Search for quote and comment delimiters by buffer when possible.
+	Memory impact: none.
+	Speed impact: noticeable improvement, from fewer function calls.
+	* ltdl/m4/gnulib-cache.m4: Import memchr2 module.
+	* m4/input.c (m4__next_token): Add buffer reads to quote and
+	comment parsing.
+	* NEWS: Document this.
+
 2009-02-16  Eric Blake  <ebb9@byu.net>
 
 	Stage 29a: Process dnl and macro names by buffer, not bytes.
diff --git a/NEWS b/NEWS
index 772216dc..1f25484e 100644
--- a/NEWS
+++ b/NEWS
@@ -42,11 +42,6 @@ promoted to 2.0.
 *** The `-L'/`--nesting-limit' command-line option now performs argument
     validation and accepts an optional multiplier suffix.
 
-*** The `-o'/`--error-output' command-line options, which were replaced by
-    `--debugfile' in M4 1.4.7, now issue a deprecation warning.  This
-    warning interferes with all versions of Autoconf prior to 2.61, so plan
-    on installing an updated Autoconf when installing M4 2.0.
-
 *** New `-p'/`--pushdef' and `--popdef' command-line options allow more
     control over macro definitions from the command line between input
     files.
@@ -217,6 +212,14 @@ promoted to 2.0.
 ** Remove the undocumented command-line option '-N', as no one complained
    about the assertion failure regression that it introduced in 1.4.7.
 
+** The `-o'/`--error-output' command-line options, which were replaced by
+   `--debugfile' in 1.4.7, now issue a deprecation warning.  This warning
+   harmlessly triggers with versions of Autoconf 2.60 and earlier, but can
+   be silenced by applying this patch:
+     http://git.sv.gnu.org/gitweb/?p=autoconf.git;a=commitdiff;h=714eeee87
+
+** Improve the speed of the input engine.
+
 ** Fix the `m4wrap' builtin to accumulate wrapped text in FIFO order, as
    required by POSIX.  The manual mentions a way to restore the LIFO order
    present in earlier GNU M4 versions.  NOTE: this change exposes a bug
diff --git a/ltdl/m4/gnulib-cache.m4 b/ltdl/m4/gnulib-cache.m4
index 1cda6d49..f8436dce 100644
--- a/ltdl/m4/gnulib-cache.m4
+++ b/ltdl/m4/gnulib-cache.m4
@@ -15,7 +15,7 @@
 
 
 # Specification in the form of a command-line invocation:
-#   gnulib-tool --import --dir=. --local-dir=local --lib=libgnu --source-base=gnu --m4-base=ltdl/m4 --doc-base=doc --tests-base=tests/gnu --aux-dir=build-aux --with-tests --libtool --macro-prefix=M4 assert autobuild avltree-oset binary-io clean-temp cloexec close-stream closein config-h configmake dirname error exit fdl-1.3 fflush filenamecat flexmember fopen fopen-safer freadptr freadseek fseeko gendocs gettext git-version-gen gnumakefile gnupload gpl-3.0 intprops memmem mkstemp obstack obstack-printf-posix progname propername quote regex regexprops-generic sprintf-posix stdbool stdlib-safer strnlen strtod strtol tempname unlocked-io vasnprintf-posix verify verror xalloc xalloc-die xmemdup0 xprintf-posix xstrndup xvasprintf-posix
+#   gnulib-tool --import --dir=. --local-dir=local --lib=libgnu --source-base=gnu --m4-base=ltdl/m4 --doc-base=doc --tests-base=tests/gnu --aux-dir=build-aux --with-tests --libtool --macro-prefix=M4 assert autobuild avltree-oset binary-io clean-temp cloexec close-stream closein config-h configmake dirname error exit fdl-1.3 fflush filenamecat flexmember fopen fopen-safer freadptr freadseek fseeko gendocs gettext git-version-gen gnumakefile gnupload gpl-3.0 intprops memchr2 memmem mkstemp obstack obstack-printf-posix progname propername quote regex regexprops-generic sprintf-posix stdbool stdlib-safer strnlen strtod strtol tempname unlocked-io vasnprintf-posix verify verror xalloc xalloc-die xmemdup0 xprintf-posix xstrndup xvasprintf-posix
 
 # Specification in the form of a few gnulib-tool.m4 macro invocations:
 gl_LOCAL_DIR([local])
@@ -49,6 +49,7 @@ gl_MODULES([
   gnupload
   gpl-3.0
   intprops
+  memchr2
   memmem
   mkstemp
   obstack
diff --git a/m4/input.c b/m4/input.c
index 36a1481e..0fb41015 100644
--- a/m4/input.c
+++ b/m4/input.c
@@ -26,6 +26,7 @@
 
 #include "freadptr.h"
 #include "freadseek.h"
+#include "memchr2.h"
 
 /* Define this to see runtime debug info.  Implied by DEBUG.  */
 /*#define DEBUG_INPUT */
@@ -1857,8 +1858,64 @@ m4__next_token (m4 *context, m4_symbol_value *token, int *line,
 	type = M4_TOKEN_STRING;
 	while (1)
 	  {
-	    ch = next_char (context, obs && m4__quote_age (M4SYNTAX), false,
-			    false);
+	    /* Start with buffer search for either potential delimiter.  */
+	    size_t len;
+	    const char *buffer = next_buffer (context, &len,
+					      obs && m4__quote_age (M4SYNTAX));
+	    if (buffer)
+	      {
+		const char *p = buffer;
+		if (m4_is_syntax_single_quotes (M4SYNTAX))
+		  do
+		    {
+		      p = (char *) memchr2 (p, *context->syntax->quote.str1,
+					    *context->syntax->quote.str2,
+					    buffer + len - p);
+		    }
+		  while (p && m4__quote_age (M4SYNTAX)
+			 && (*p++ == *context->syntax->quote.str2
+			     ? --quote_level : ++quote_level));
+		else
+		  {
+		    size_t remaining = len;
+		    assert (context->syntax->quote.len1 == 1
+			    && context->syntax->quote.len2 == 1);
+		    while (remaining && !m4_has_syntax (M4SYNTAX, *p,
+							(M4_SYNTAX_LQUOTE
+							 | M4_SYNTAX_RQUOTE)))
+		      {
+			p++;
+			remaining--;
+		      }
+		    if (!remaining)
+		      p = NULL;
+		  }
+		if (p)
+		  {
+		    if (m4__quote_age (M4SYNTAX))
+		      {
+			assert (!quote_level
+				&& context->syntax->quote.len1 == 1
+				&& context->syntax->quote.len2 == 1);
+			obstack_grow (obs_safe, buffer, p - buffer - 1);
+			consume_buffer (context, p - buffer);
+			break;
+		      }
+		    obstack_grow (obs_safe, buffer, p - buffer);
+		    ch = to_uchar (*p);
+		    consume_buffer (context, p - buffer + 1);
+		  }
+		else
+		  {
+		    obstack_grow (obs_safe, buffer, len);
+		    consume_buffer (context, len);
+		    continue;
+		  }
+	      }
+	    /* Fall back to byte-wise search.  */
+	    else
+	      ch = next_char (context, obs && m4__quote_age (M4SYNTAX), false,
+			      false);
 	    if (ch == CHAR_EOF)
 	      {
 		if (!caller)
@@ -1914,7 +1971,45 @@ m4__next_token (m4 *context, m4_symbol_value *token, int *line,
 	  obstack_1grow (obs_safe, ch);
 	while (1)
 	  {
-	    ch = next_char (context, false, false, false);
+	    /* Start with buffer search for potential end delimiter.  */
+	    size_t len;
+	    const char *buffer = next_buffer (context, &len, false);
+	    if (buffer)
+	      {
+		const char *p;
+		if (m4_is_syntax_single_comments (M4SYNTAX))
+		  p = (char *) memchr (buffer, *context->syntax->comm.str2,
+				       len);
+		else
+		  {
+		    size_t remaining = len;
+		    assert (context->syntax->comm.len2 == 1);
+		    p = buffer;
+		    while (remaining
+			   && !m4_has_syntax (M4SYNTAX, *p, M4_SYNTAX_ECOMM))
+		      {
+			p++;
+			remaining--;
+		      }
+		    if (!remaining)
+		      p = NULL;
+		  }
+		if (p)
+		  {
+		    obstack_grow (obs_safe, buffer, p - buffer);
+		    ch = to_uchar (*p);
+		    consume_buffer (context, p - buffer + 1);
+		  }
+		else
+		  {
+		    obstack_grow (obs_safe, buffer, len);
+		    consume_buffer (context, len);
+		    continue;
+		  }
+	      }
+	    /* Fall back to byte-wise search.  */
+	    else
+	      ch = next_char (context, false, false, false);
 	    if (ch == CHAR_EOF)
 	      {
 		if (!caller)