Introduce testmaker.m4 and testmaker.sh

The existing method of generating multiple tests from l4 files is not
flexible enough to handle multilanguage - reliance on the C
preprocessor prevents that. This commit creates replacement machinery
that's more flexible and will greatly reduce the complexity of adding
new back ends.

The idea is to generate test .l files by combining a
backend-independent ruleset with backend-dependent boilerplate for
test mains and preambles.

This commit creates the rules files, the m4 to template them into
tests, and a shell wrapper to drive the m4 expansion.  The autoconf
integration works OK, but requires additional rules at the
end of tests/Makefile.am that probably ought to be autogenerated.

There was one other change forced by this.  In order for the new
testmaker to work, it needs to be possible to analyze test file names
into stems and option modifiers.  Having underscores in the stem
would make the analysis slower and bug-prone.

I wanted to change them to "-" rather than deleting them, but this
gives autoconf fits - it seems to not like filenames with embedded
dashes in certain contexts.

At this commit, only one test - array_nr.l - is generated using the new
system.

13 files changed, 401 insertions, 34 deletions

diff --git a/tests/.gitignore b/tests/.gitignore
index 30a881e..ad3aef0 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -6,6 +6,7 @@ alloc_extra_cpp
 alloc_extra_cpp.c
 alloc_extra_c99
 alloc_extra_c99.c
+array_nr.l
 array_nr
 array_nr.c
 array_r
@@ -109,10 +110,10 @@ rescan_nr.direct
 rescan_nr.direct.c
 rescan_r.direct
 rescan_r.direct.c
-quote_in_comment_cpp
-quote_in_comment_cpp.c
-quote_in_comment_c99
-quote_in_comment_c99.c
+quoteincomment_cpp
+quoteincomment_cpp.c
+quoteincomment_c99
+quoteincomment_c99.c
 quotes_cpp
 quotes_cpp.c
 quotes_c99
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 22d6eba..a1e1607 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -98,8 +98,8 @@ simple_tests = \
 	prefix_nr \
 	prefix_r \
 	prefix_c99 \
-	quote_in_comment_cpp \
-	quote_in_comment_c99 \
+	quoteincomment_cpp \
+	quoteincomment_c99 \
 	quotes_cpp \
 	quotes_c99 \
 	string_nr \
@@ -136,8 +136,8 @@ ONE_TESTS = \
 	lineno_c99.one \
 	lineno_trailing.one
 
-quote_in_comment_cpp_SOURCES = quote_in_comment_cpp.l
-quote_in_comment_c99_SOURCES = quote_in_comment_c99.l
+quoteincomment_cpp_SOURCES = quoteincomment_cpp.l
+quoteincomment_c99_SOURCES = quoteincomment_c99.l
 alloc_extra_cpp_SOURCES = alloc_extra_cpp.l
 alloc_extra_c99_SOURCES = alloc_extra_c99.l
 array_nr_SOURCES = array_nr.l
@@ -225,6 +225,7 @@ yyextra_c99_SOURCES = yyextra_c99.l
 CLEANFILES = \
 	alloc_extra_cpp.c \
 	alloc_extra_c99.c \
+	array_nr.l \
 	array_nr.c \
 	array_r.c \
 	array_c99.c \
@@ -286,8 +287,8 @@ CLEANFILES = \
 	prefix_r.c \
 	prefix_c99.c \
 	pthread.c \
-	quote_in_comment_cpp.c \
-	quote_in_comment_c99.c \
+	quoteincomment_cpp.c \
+	quoteincomment_c99.c \
 	quotes_cpp.c \
 	reject_nr.reject.c \
 	reject_r.reject.c \
@@ -355,7 +356,7 @@ EXTRA_DIST = \
 	reject.txt \
 	rescan_nr.direct.txt \
 	rescan_r.direct.txt \
-	quote_in_comment.txt \
+	quoteincomment.txt \
 	quotes.txt \
 	top.txt \
 	yyextra.txt \
@@ -550,3 +551,8 @@ tableopts_ver_r%.c: tableopts.l4 $(FLEX)
 
 tableopts_ver_r%.$(OBJEXT):  tableopts_ver_r%.c  
 	$(AM_V_CC)$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -DTEST_IS_REENTRANT -c -o $@ $<
+
+# Rules for tests generated from rulesets start here
+
+array_nr.l: array.rules testmaker.m4
+	$(SHELL) testmaker.sh array_nr.l
diff --git a/tests/array_nr.l b/tests/array.rules
index f4f0f1b..cd6fe91 100644
--- a/tests/array_nr.l
+++ b/tests/array.rules
@@ -1,5 +1,4 @@
-/*
- * This file is part of flex.
+/* This file is part of flex.
  * 
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -21,16 +20,10 @@
  * PURPOSE.
  */
 
-%{
-/* A template scanner file to build "scanner.c". */
-#include <stdio.h>
-#include <stdlib.h>
-#include "config.h"
-/*#include "parser.h" */
+M4_TEST_PREAMBLE`'dnl
 
-%}
-
-%option 8bit prefix="test"
+M4_TEST_OPTIONS`'dnl
+%option 8bit
 %option nounput nomain noyywrap noinput
 %option warn array
 
@@ -41,15 +34,4 @@
 
 
 %%
-
-int main (void);
-
-int
-main (void)
-{
-    yyin = stdin;
-    yyout = stdout;
-    yylex();
-    printf("TEST RETURNING OK.\n");
-    return 0;
-}
+M4_TEST_POSTAMBLE`'dnl
diff --git a/tests/basic.rules b/tests/basic.rules
new file mode 100644
index 0000000..fcddf82
--- /dev/null
+++ b/tests/basic.rules
@@ -0,0 +1,50 @@
+/*
+ * This file is part of flex.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE.
+ */
+
+/* TEST scanner.
+
+   Sample Input:
+       # this is a comment
+       foo = true
+       bar = "string value"
+       integer = 43
+*/
+
+M4_TEST_PREAMBLE`'dnl
+
+M4_TEST_OPTIONS`'dnl
+%option nounput noyywrap noyylineno warn nodefault noinput
+
+IDENT [[:alnum:]_-]
+WS    [[:blank:]]
+%%
+
+^{IDENT}+{WS}*={WS}*(true|false){WS}*\r?\n    { return 100;}
+^{IDENT}+{WS}*={WS}*\"[^\"\n\r]*\"{WS}*\r?\n  { return 101;}
+^{IDENT}+{WS}*={WS}*[[:digit:]]+{WS}*\r?\n    { return 102;}
+^{WS}*#.*\r?\n     { }
+^{WS}*\r?\n        { }
+.|\n  {M4_TEST_FAILMESSAGE}
+
+%%
+M4_TEST_POSTAMBLE
diff --git a/tests/extended.rules b/tests/extended.rules
new file mode 100644
index 0000000..0dc601d
--- /dev/null
+++ b/tests/extended.rules
@@ -0,0 +1,52 @@
+/*
+ * This file is part of flex.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE.
+ */
+/* Output should match the input. */
+/* FIXME: Assumes ";" is a legal statememt terminator */
+
+M4_TEST_PREAMBLE`'dnl
+
+M4_TEST_OPTIONS`'dnl
+%option 8bit nounput nomain noyywrap noinput warn
+%%
+
+abc(?# Single Line Comment )def    yyecho();
+ghi(?#
+    multi-line
+    comment
+    )jkl                           yyecho();
+
+mno(?#
+    multi-line //
+    comment with ##
+    ~~!@#$ %^&*(@-_+=\|,.<>/ ?: ;
+    punctuation
+    )pqr                           yyecho();
+(?# Start of a rule.)stu           yyecho();
+vwxyz(?#End of a rule.)            yyecho();
+A(?x: B
+    /* comment */
+    C D)                           yyecho();
+
+\n                                 yyecho();
+%%
+M4_TEST_POSTAMBLE
diff --git a/tests/quoteincomment.rules b/tests/quoteincomment.rules
new file mode 100644
index 0000000..e28ee34
--- /dev/null
+++ b/tests/quoteincomment.rules
@@ -0,0 +1,32 @@
+/*
+ * This file is part of flex.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE.
+ */
+M4_TEST_PREAMBLE`'dnl
+
+M4_TEST_OPTIONS`'dnl
+%option 8bit noyywrap
+%%
+.|\n	{ yyecho();
+   //' "
+  }
+%%
+M4_TEST_POSTAMBLE
diff --git a/tests/quote_in_comment.txt b/tests/quoteincomment.txt
index 9daeafb..9daeafb 100644
--- a/tests/quote_in_comment.txt
+++ b/tests/quoteincomment.txt
diff --git a/tests/quote_in_comment_c99.l b/tests/quoteincomment_c99.l
index 2d3db98..2d3db98 100644
--- a/tests/quote_in_comment_c99.l
+++ b/tests/quoteincomment_c99.l
diff --git a/tests/quote_in_comment_cpp.l b/tests/quoteincomment_cpp.l
index cb17e2f..cb17e2f 100644
--- a/tests/quote_in_comment_cpp.l
+++ b/tests/quoteincomment_cpp.l
diff --git a/tests/reject.rules b/tests/reject.rules
new file mode 100644
index 0000000..d44ade9
--- /dev/null
+++ b/tests/reject.rules
@@ -0,0 +1,35 @@
+/*
+ * This file is part of flex.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE.
+ */
+
+M4_TEST_PREAMBLE`'dnl
+
+M4_TEST_OPTIONS`'dnl
+%option 8bit
+%option nounput nomain noyywrap noinput
+%option warn reject
+%%
+
+.              { ECHO; }
+.|\n           ;
+
+%%
diff --git a/tests/tableopts.rules b/tests/tableopts.rules
new file mode 100644
index 0000000..65c5b7e
--- /dev/null
+++ b/tests/tableopts.rules
@@ -0,0 +1,39 @@
+/*
+ * This file is part of flex.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE.
+ */
+The simple grammar used as a test ofb table-compression options
+
+
+M4_TEST_PREAMBLE`'dnl
+
+M4_TEST_OPTIONS`'dnl
+%option 8bit
+%option nounput nomain noyywrap noinput
+%option warn yylineno
+%%
+
+foo|bar         ;
+[[:digit:]]+    ;
+[[:blank:]]+    ;
+.|\n            ;
+%%
+M4_TEST_POSTAMBLE
diff --git a/tests/testmaker.m4 b/tests/testmaker.m4
new file mode 100644
index 0000000..088dd3c
--- /dev/null
+++ b/tests/testmaker.m4
@@ -0,0 +1,106 @@
+dnl Multi-backend test template
+dnl
+dnl Supply per-backend boilerplate for Flex tests.
+dnl See testmaker.sh for how it's used.
+dnl
+dnl M4_TEST_BACKEND must be set to a recognized back end name.
+dnl For purposes of this file, "nr" and "r" are treated as
+dnl separate back ends - non-reentrant and reentrant C through
+dnl the default cpp skeleton.  At time of writing this file,
+dnl c99 is the only other; it is expected this will change.
+dnl
+dnl The purpose of this file is to properly set three macros:
+dnl
+dnl M4_TEST_PREAMBLE = the contyents of s top secton, if required, and
+dnl the emit option required to srt the back end.
+dnl
+dnl M4_TEST_COMPRESSION = compression option.
+dnl
+dnl M4_TEST_FAILMESSAGE = A line of code required to issue dnl a
+dnl failure notification to standard error and exit with a failure status.
+dnl
+dnl M4_TEST_POSTAMBLE = the test main.
+dnl
+dnl These macros are to be expanded by files with a .rules extension
+dnl that contain pure flex rulesets and no backend-specific code.
+dnl
+define(`M4_TEST_FAILMESSAGE', `INVALID BACK END')dnl
+dnl
+ifelse(M4_TEST_BACKEND, `nr', `dnl
+define(`M4_TEST_PREAMBLE', `dnl
+%{
+#include "config.h"
+#include <stdio.h>
+%}
+')dnl close preamble
+define(`M4_TEST_FAILMESSAGE', `dnl
+fprintf(stderr,"Invalid line.\n"); exit(-1);
+')dnl close failmessage
+define(`M4_TEST_POSTAMBLE', `dnl
+int main (void)
+{
+    yyin = stdin;
+    yyout = stdout;
+    while( yylex() )
+    {
+    }
+    printf("TEST RETURNING OK.\n");
+    return 0;
+}
+')dnl close postamble
+')dnl close nr
+dnl
+ifelse(M4_TEST_BACKEND, `r', `dnl
+define(`M4_TEST_PREAMBLE', `dnl
+%{
+#include "config.h"
+#include <stdio.h>
+%}
+')dnl close preamble
+define(`M4_TEST_FAILMESSAGE', `fprintf(stderr,"Invalid line.\n"); exit(-1);')
+define(`M4_TEST_POSTAMBLE', `dnl
+int main (void)
+{
+    yyscan_t  lexer;
+    yylex_init( &lexer );
+    yyset_out ( stdout,lexer);
+    yyset_in  ( stdin, lexer);
+    while( yylex(lexer) )
+    {
+    }
+    yylex_destroy( lexer );
+    printf("TEST RETURNING OK.\n");
+    return 0;
+}
+')dnl close postamble
+')dnl close r
+dnl
+dnl A hypothetical example
+ifelse(M4_TEST_BACKEND, `go', `dnl
+define(`M4_TEST_PREAMBLE', `dnl
+%top{
+package main
+
+import (
+    "fmt"
+    "log"
+    "os"
+)
+%}
+%option emit="go"
+')dnl close preamble
+define(`M4_TEST_FAILMESSAGE', `log.Fatal("Invalid line"); os.Exit(-1);')
+define(`M4_TEST_POSTAMBLE', `dnl
+func main(void) {
+	lexer := new(FlexLexer)
+	lexer.yysetOut(os.Stdout)
+	lexer.yysetIn(os.Stdin)
+	for lexer.yylex() {
+	}
+	fmt.Printf("TEST RETURNING OK.\n")
+	os.Exit(0)
+}
+')dnl clpse postamble
+')dnl close go
+dnl
+dnl Additional backends go here
diff --git a/tests/testmaker.sh b/tests/testmaker.sh
new file mode 100644
index 0000000..70c0501
--- /dev/null
+++ b/tests/testmaker.sh
@@ -0,0 +1,64 @@
+#!/bin/sh
+#
+# testmaker.sh - asssemble tests from backend-ndependent rulesets and
+# backend-dependent boilerplate.
+#
+# The single argument is a testfile name to be generated.
+# With the -d option, dump to stdourather than crating the file.
+#
+# A typical test load name: tableopts_opt_nr_Ca_opt, this breaks
+# down as tableopts_{tag}_{backend}_{compression}_{tag}.
+# The backend field can be nr, r, c99 and will eventually have more values.
+# The compression options are the flags that woulld nprmally be passed to
+# Flex; the possibilities are Ca Ce Cf C_F Cm Cem Cae Caef Cae_F Cam Caem.
+#
+if [ "$1" = -d ] ; then
+    shift
+    outdev=/dev/stdout
+else
+    outdev="$1"
+fi
+
+testfile=$1
+
+trap "rm -f /tmp/testmaker$$" EXIT INT QUIT
+
+set `echo $testfile | tr '.' ' '`
+if [ "$2" != "l" ]
+then
+    echo "$0: Don't know how to make anything but a .l file: $parts" >&2
+    exit 1
+fi
+
+set -- `echo $1 | tr '_' ' '`
+stem=$1
+options=""
+for part in $*; do
+    # This is the only pace in this dcript that you need to modify
+    # to add a new back end - just add a line on the pattern of
+    # the c99 one. Of course testmaker.m4 will require the
+    # right boilerplate code for this to work.
+    #
+    # Yes, cpp is an alias for nr.
+    case ${part} in
+	cpp|nr) backend=nr; ;;
+	r) backend=r; options="${options} reentrant";;
+	c99) backend=r; options="${options} reentrant emit=\"c99\"" ;;
+    esac
+done
+
+if [ -z "$backend" ]
+then
+    echo "Can't identify a back end part in ${testfile}" >&2;
+    exit 1;
+fi
+
+(
+    printf "define(\`M4_TEST_BACKEND', \`${backend}')dnl\n"
+    if [ -z "${options}" ] ; then
+        printf "define(\`M4_TEST_OPTIONS', \`')dnl\n"
+    else
+        printf "define(\`M4_TEST_OPTIONS', \`%%option${options}\n')dnl\n"
+    fi
+    cat testmaker.m4 ${stem}.rules
+) | m4 >${outdev}