More flattening.

18 files changed, 197 insertions, 158 deletions

diff --git a/test/Makefile.am b/test/Makefile.am
index 1697c3ff..2c89a76c 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -1,4 +1,4 @@
-DIST_SUBDIRS =  python http cxx ruby
+DIST_SUBDIRS =  python cxx ruby
 
 TESTS = runtests.mk
 
@@ -6,7 +6,7 @@ TEST_CASES = \
 	backtrack1.lm \
 	backtrack2.lm \
 	backtrack3.lm \
-	dns.lm \
+	binary1.lm \
 	accum1.lm \
 	accum2.lm \
 	accum3.lm \
@@ -60,7 +60,9 @@ TEST_CASES = \
 	sprintf.lm \
 	div.lm \
 	scope1.lm \
-	export1.lm
+	export1.lm \
+	multiregion1.lm \
+	multiregion2.lm
 
 INPUT = \
 	accum1.in \
@@ -78,7 +80,7 @@ INPUT = \
 	counting2.in \
 	counting3.in \
 	counting4.in \
-	dns.in \
+	binary1.in \
 	func.in \
 	heredoc.in \
 	island.in \
@@ -107,7 +109,9 @@ INPUT = \
 	undofrag1.in \
 	undofrag2.in \
 	undofrag3.in \
-	export1.in
+	export1.in \
+	multiregion1.in \
+	multiregion2.in
 
 EXPECTED_OUTPUT = \
 	accum1.exp \
@@ -134,7 +138,7 @@ EXPECTED_OUTPUT = \
 	counting3.exp \
 	counting4.exp \
 	div.exp \
-	dns.exp \
+	binary1.exp \
 	exit1.exp \
 	exit2.exp \
 	func.exp \
@@ -167,6 +171,8 @@ EXPECTED_OUTPUT = \
 	undofrag1.exp \
 	undofrag2.exp \
 	undofrag3.exp \
-	export1.exp
+	export1.exp \
+	multiregion1.exp \
+	multiregion2.exp
 
 EXTRA_DIST = genmf TESTS runtests.mk $(TEST_CASES) $(INPUT) $(EXPECTED_OUTPUT)
diff --git a/test/TESTS b/test/TESTS
index d9c8fe95..eaf93902 100644
--- a/test/TESTS
+++ b/test/TESTS
@@ -3,7 +3,6 @@
 
 SUBDIRS="
 	python
-	http
 	cxx
 	ruby
 "
@@ -12,7 +11,7 @@ TESTS="
 	backtrack1.lm
 	backtrack2.lm
 	backtrack3.lm
-	dns.lm
+	binary1.lm
 	accum1.lm
 	accum2.lm
 	accum3.lm
@@ -59,6 +58,8 @@ TESTS="
 	til.lm
 	matchex.lm
 	maxlen.lm
+	multiregion1.lm
+	multiregion2.lm
 	superid.lm
 	tags1.lm
 	tags2.lm
diff --git a/test/dns.exp b/test/binary1.exp
index b0d76eac..b0d76eac 100644
--- a/test/dns.exp
+++ b/test/binary1.exp
diff --git a/test/dns.in b/test/binary1.in
index 8da7d3dc..8da7d3dc 100644
--- a/test/dns.in
+++ b/test/binary1.in
diff --git a/test/dns.lm b/test/binary1.lm
index 87cfaab1..87cfaab1 100644
--- a/test/dns.lm
+++ b/test/binary1.lm
diff --git a/test/diff/Makefile b/test/diff/Makefile
deleted file mode 100644
index d647261a..00000000
--- a/test/diff/Makefile
+++ /dev/null
@@ -1,34 +0,0 @@
-#
-#   Copyright 2002-2006 Adrian Thurston <thurston@complang.org>
-#
-
-#   This file is part of Colm.
-#
-#   Colm is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published by
-#   the Free Software Foundation; either version 2 of the License, or
-#   (at your option) any later version.
-#
-#   Colm is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#   GNU General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with Colm; if not, write to the Free Software
-#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
-
-
-SRC = $(wildcard *.lm)
-BIN = $(SRC:%.lm=%.bin)
-COLM = ../../colm/colm
-
-all: $(BIN)
-
-$(BIN): $(COLM)
-
-$(BIN): %.bin: %.lm
-	$(COLM) $<
-
-clean:
-	rm -f *.cpp *.bin
diff --git a/test/dnstools/Makefile b/test/dnstools/Makefile
deleted file mode 100644
index 79f30cff..00000000
--- a/test/dnstools/Makefile
+++ /dev/null
@@ -1,11 +0,0 @@
-#
-#   Copyright 2007 Adrian Thurston <thurston@complang.org>
-#
-
-all: extract
-
-extract: extract.c
-	gcc -Wall -o extract extract.c -lpcap
-
-clean:
-	rm -f extract
diff --git a/test/dnstools/dumpdns b/test/dnstools/dumpdns
deleted file mode 100644
index 3409afee..00000000
--- a/test/dnstools/dumpdns
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-#
-
-# Use this script to capture dns packets to a dump file. Then use extract to
-# break up the packets and strip the headers, leaving just dns packets.
-#
-# usage: bash dumpdns <dumpfile>
-
-[ -z "$1" ] && exit
-
-tcpdump -s 0 -w $1 udp port 53
diff --git a/test/dnstools/extract.c b/test/dnstools/extract.c
deleted file mode 100644
index 8af026a3..00000000
--- a/test/dnstools/extract.c
+++ /dev/null
@@ -1,48 +0,0 @@
-#include <pcap.h>
-
-/*
- * Break up a dump file and strip headers, leaving just 
- * the dns portion of packets.
- */
-
-char outname[1024];
-char errbuf[PCAP_ERRBUF_SIZE];
-
-int main( int argc, char **argv )
-{
-	int packet;
-	pcap_t *p;
-	if ( argc != 3 ) {
-		fprintf( stderr, "usage: get <dumpfile> <rootname>\n" );
-		return 1;
-	}
-
-	p = pcap_open_offline( argv[1], errbuf );
-
-	for ( packet = 0; ; packet++ ) {
-		FILE *file;
-		unsigned long len;
-		struct pcap_pkthdr h;
-		const u_char *data;
-		
-		data = pcap_next( p, &h );
-		if ( data == 0 )
-			break;
-
-		if ( h.caplen < h.len )
-			fprintf( stderr, "warning: packet number %02d is short\n", packet );
-
-		/* The magic number is the size of the headers we want to strip. */
-		data += 42;
-		len = h.caplen - 42;
-
-		sprintf( outname, "%s-%04d", argv[2], packet );
-		file = fopen( outname, "wb" );
-		fwrite( data, 1, len, file );
-		fclose( file );
-	}
-
-	pcap_close( p );
-
-	return 0;
-}
diff --git a/test/http/http1.exp b/test/http/http1.exp
deleted file mode 100644
index c16250bf..00000000
--- a/test/http/http1.exp
+++ /dev/null
@@ -1,7 +0,0 @@
-HTTP/1.0 200 OK

-Date: Fri, 31 Dec 1999 23:59:59 GMT

-Content-Type: text/plain

-

-<request><request_line><method>GET</method><_literal_0002> </_literal_0002><request_uri>/hi/there/</request_uri><_literal_0002> </_literal_0002><http_version><_literal_0001>HTTP/</_literal_0001><http_number>1.1</http_number></http_version><CRLF>

-</CRLF></request_line><_repeat_header></_repeat_header><CRLF>

-</CRLF></request>
\ No newline at end of file
diff --git a/test/http/http1.in b/test/http/http1.in
deleted file mode 100644
index c1416f84..00000000
--- a/test/http/http1.in
+++ /dev/null
@@ -1,2 +0,0 @@
-GET /hi/there/ HTTP/1.1

-

diff --git a/test/multiregion1.exp b/test/multiregion1.exp
new file mode 100644
index 00000000..35413bc8
--- /dev/null
+++ b/test/multiregion1.exp
@@ -0,0 +1,86 @@
+Index: fsmrun.cpp
+===================================================================
+--- newfilename.cpp (revision 4555)
++++ fsmrun.cpp	(working copy)
+@@ -150,7 +150,7 @@
+ 	peof = 0;
+ 	if ( parser != 0 ) {
+ 		region = parser->getNextRegion();
+-		cs = getStateFromNextRegion();
++		cs = tables->entryByRegion[region];
+ 	}
+ 	else {
+ 		region = 0;
+@@ -189,7 +189,7 @@
+ 
+ 	tokstart = 0;
+ 	region = parser->getNextRegion();
+-	cs = getStateFromNextRegion();
++	cs = tables->entryByRegion[region];
+ }
+ 
+ void FsmRun::sendToken( int id )
+@@ -222,7 +222,7 @@
+ 		parser = newParser;
+ 
+ 		region = parser->getNextRegion();
+-		cs = getStateFromNextRegion();
++		cs = tables->entryByRegion[region];
+ 	}
+ 	else {
+ 		#ifdef LOG_ACTIONS
+@@ -355,7 +355,7 @@
+ 
+ 			/* Set the current state from the next region. */
+ 			region = parser->getNextRegion();
+-			cs = getStateFromNextRegion();
++			cs = tables->entryByRegion[region];
+ 		}
+ 	}
+ 
+@@ -452,7 +452,7 @@
+ 		/* First thing check for error. */
+ 		if ( cs == tables->errorState ) {
+ 			if ( parser != 0 ) {
+-				if ( getStateFromNextRegion( 1 ) != 0 ) {
++				if ( parser->getNextRegion( 1 ) != 0 ) {
+ 					#ifdef LOG_BACKTRACK
+ 					cerr << "scanner failed, trying next region" << endl;
+ 					#endif
+@@ -462,7 +462,7 @@
+ 
+ 					parser->nextRegionInd += 1;
+ 					region = parser->getNextRegion();
+-					cs = getStateFromNextRegion();
++					cs = tables->entryByRegion[region];
+ 					cerr << "new token region: " << 
+ 							parser->tables->gbl->regionInfo[region].name << endl;
+ 					continue;
+@@ -495,7 +495,7 @@
+ 					}
+ 					else {
+ 						region = parser->getNextRegion();
+-						cs = getStateFromNextRegion();
++						cs = tables->entryByRegion[region];
+ 						cerr << "new token region: " << 
+ 								parser->tables->gbl->regionInfo[region].name << endl;
+ 						continue;
+Index: junk.cpp
+===================================================================
+--- 
++++ junk.cpp	(working copy)
+Index: fsmrun.h
+===================================================================
+--- fsmrun.h	(revision 4557)
++++ fsmrun.h	(working copy)
+@@ -197,10 +197,6 @@
+ 	void runOnInputStream( PdaRun *parser, InputStream &in );
+ 	void execute();
+ 
+-	/* Offset can be used to look at the next nextRegionInd. */
+-	int getStateFromNextRegion( int offset = 0 )
+-		{ return tables->entryByRegion[parser->getNextRegion(offset)]; }
+-
+ 	FsmTables *tables;
+ 	PdaRun *parser;
+ 	InputStream *inputStream;
diff --git a/test/diff/input1.diff b/test/multiregion1.in
index b0021f67..b0021f67 100644
--- a/test/diff/input1.diff
+++ b/test/multiregion1.in
diff --git a/test/diff/diff.lm b/test/multiregion1.lm
index 37232025..7a75fd9b 100644
--- a/test/diff/diff.lm
+++ b/test/multiregion1.lm
@@ -48,11 +48,9 @@ def file_diff
 def start 
 	[file_diff*]
 
+parse S: start( stdin )
 
-start S = parse start( stdin )
-
-for OF:old_file in S {
-	print( 'old file: ', OF )
+for OF: old_file in S {
 	# Get the first word and check if it is 
 	# the file we are interested in.
 	if match OF [
@@ -62,23 +60,9 @@ for OF:old_file in S {
 		]
 	{
 		OF = construct old_file
-			["--- newfilename.cpp" Rest "\n"]
-		print_xml( OF )
+			["--- newfilename.cpp " Rest "\n"]
 	}
 }
 
 print( S )
 
-#		for Header: file_header in lhs {
-#			old_file OF = old_file in Header
-#			if match OF 
-#				[old_file_start "lmparse.kl" word_list newline]
-#			{
-#				Header = construct file_header
-#					~Index: rewritten
-#					~===================================================================
-#					~--- this is the file (asldkfj)
-#					~+++ this is the file (ewir)
-#			}
-#		}
-
diff --git a/test/multiregion2.exp b/test/multiregion2.exp
new file mode 100644
index 00000000..3e3a984c
--- /dev/null
+++ b/test/multiregion2.exp
@@ -0,0 +1,17 @@
+GET /hi/there/ HTTP/1.1

+

+GET /hithere/ HTTP/1.1

+Host: localhost:3535

+User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.12) Gecko/20080207 Ubuntu/7.10 (gutsy) Firefox/2.0.0.12

+Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5

+Accept-Language: en-us,en;q=0.5

+Accept-Encoding: gzip,deflate

+Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7

+Keep-Alive: 300

+Connection: keep-alive

+Cache-Control: max-age=0

+

+GET foo HTTP/1.1

+hello: foo

+hi: there my friend

+

diff --git a/test/multiregion2.in b/test/multiregion2.in
new file mode 100644
index 00000000..25a07c41
--- /dev/null
+++ b/test/multiregion2.in
@@ -0,0 +1,19 @@
+GET /hi/there/ HTTP/1.1

+

+GET /hithere/ HTTP/1.1

+Host: localhost:3535

+User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.12) Gecko/20080207 Ubuntu/7.10 (gutsy) Firefox/2.0.0.12

+Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5

+Accept-Language: en-us,en;q=0.5

+Accept-Encoding: gzip,deflate

+Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7

+Keep-Alive: 300

+Connection: keep-alive

+Cache-Control: max-age=0

+

+GET foo HTTP/1.1

+hello: foo

+hi: there

+	my

+ friend

+

diff --git a/test/http/http.lm b/test/multiregion2.lm
index 39c56206..f86d931c 100644
--- a/test/http/http.lm
+++ b/test/multiregion2.lm
@@ -45,8 +45,19 @@ def request_line
 
 token field_name /token_char+/
 
-token field_value 
-	/(^(CR|LF) | CR LF (SP|HT))* CR LF/
+lex field_value
+{
+	token fv_plain /(^(CR|LF))*/
+	token fv_ext /CR LF (SP|HT)/
+	token fv_term /CR LF/
+}
+
+def fv 
+	[fv_plain]
+|	[fv_ext]
+
+def field_value 
+	[fv* fv_term]
 
 def header 
 	[field_name ':' field_value]
@@ -58,11 +69,19 @@ def header
 def request
 	[request_line header* CRLF]
 
-R: request = parse_stop request( stdin )
 
-print( 'HTTP/1.0 200 OK\r\n' )
-print( 'Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n' )
-print( 'Content-Type: text/plain\r\n' )
-print( '\r\n' )
-print_xml( R )
+parse R: request*( stdin )
+
+if !R {
+	print( error() )
+	exit( 1 )
+}
+
+for FV: fv in R 
+{
+	if match FV [fv_ext]
+		FV = cons fv " "
+}
+
+print( R )
 
diff --git a/test/runtests.mk b/test/runtests.mk
index 742a730f..f7c4979d 100755
--- a/test/runtests.mk
+++ b/test/runtests.mk
@@ -1,12 +1,12 @@
 #!/usr/bin/make -f 
 
-SUBDIRS =  python http cxx ruby
+SUBDIRS =  python cxx ruby
 
 TESTS = \
 	backtrack1.lm \
 	backtrack2.lm \
 	backtrack3.lm \
-	dns.lm \
+	binary1.lm \
 	accum1.lm \
 	accum2.lm \
 	accum3.lm \
@@ -53,6 +53,8 @@ TESTS = \
 	til.lm \
 	matchex.lm \
 	maxlen.lm \
+	multiregion1.lm \
+	multiregion2.lm \
 	superid.lm \
 	tags1.lm \
 	tags2.lm \
@@ -68,7 +70,7 @@ DIFFS = \
 	backtrack1.diff \
 	backtrack2.diff \
 	backtrack3.diff \
-	dns.diff \
+	binary1.diff \
 	accum1.diff \
 	accum2.diff \
 	accum3.diff \
@@ -115,6 +117,8 @@ DIFFS = \
 	til.diff \
 	matchex.diff \
 	maxlen.diff \
+	multiregion1.diff \
+	multiregion2.diff \
 	superid.diff \
 	tags1.diff \
 	tags2.diff \
@@ -165,14 +169,14 @@ backtrack3.out: backtrack3.bin
 
 backtrack3.bin: backtrack3.lm ./../colm/colm
 	./../colm/colm backtrack3.lm
-dns.diff: dns.out dns.exp
-	@diff -u dns.exp dns.out > dns.diff || ( cat dns.diff; rm dns.diff )
+binary1.diff: binary1.out binary1.exp
+	@diff -u binary1.exp binary1.out > binary1.diff || ( cat binary1.diff; rm binary1.diff )
 
-dns.out: dns.bin
-	./dns.bin  < dns.in > dns.out
+binary1.out: binary1.bin
+	./binary1.bin  < binary1.in > binary1.out
 
-dns.bin: dns.lm ./../colm/colm
-	./../colm/colm dns.lm
+binary1.bin: binary1.lm ./../colm/colm
+	./../colm/colm binary1.lm
 accum1.diff: accum1.out accum1.exp
 	@diff -u accum1.exp accum1.out > accum1.diff || ( cat accum1.diff; rm accum1.diff )
 
@@ -541,6 +545,22 @@ maxlen.out: maxlen.bin
 
 maxlen.bin: maxlen.lm ./../colm/colm
 	./../colm/colm maxlen.lm
+multiregion1.diff: multiregion1.out multiregion1.exp
+	@diff -u multiregion1.exp multiregion1.out > multiregion1.diff || ( cat multiregion1.diff; rm multiregion1.diff )
+
+multiregion1.out: multiregion1.bin
+	./multiregion1.bin  < multiregion1.in > multiregion1.out
+
+multiregion1.bin: multiregion1.lm ./../colm/colm
+	./../colm/colm multiregion1.lm
+multiregion2.diff: multiregion2.out multiregion2.exp
+	@diff -u multiregion2.exp multiregion2.out > multiregion2.diff || ( cat multiregion2.diff; rm multiregion2.diff )
+
+multiregion2.out: multiregion2.bin
+	./multiregion2.bin  < multiregion2.in > multiregion2.out
+
+multiregion2.bin: multiregion2.lm ./../colm/colm
+	./../colm/colm multiregion2.lm
 superid.diff: superid.out superid.exp
 	@diff -u superid.exp superid.out > superid.diff || ( cat superid.diff; rm superid.diff )