prints, alias, <<

35 files changed, 372 insertions, 317 deletions

diff --git a/colm/colm.lm b/colm/colm.lm
index 1bebaeac..991bd733 100644
--- a/colm/colm.lm
+++ b/colm/colm.lm
@@ -7,6 +7,7 @@ lex
 	token RL / 'rl' /
 	token IGNORE / 'ignore' /
 	token PRINT / 'print' /
+	token PRINTS / 'prints' /
 	token PRINT_XML / 'print_xml' /
 	token PRINT_XML_AC / 'print_xml_ac' /
 	token PARSE / 'parse' /
@@ -23,7 +24,7 @@ lex
 	token ELSIF / 'elsif' /
 	token ELSE / 'else' /
 	token IN / 'in' /
-	token PARSER / 'parser' /
+	token PARSER / 'parser' | 'accum' /
 	token LIST / 'list' /
 	token VECTOR / 'vector' /
 	token MAP / 'map' /
@@ -35,6 +36,7 @@ lex
 	token BREAK / 'break' /
 	token REJECT / 'reject' /
 	token REDUCEFIRST / 'reducefirst' /
+	token ALIAS / 'alias' /
 
 	token MAKE_TOKEN / 'make_token' /
 	token MAKE_TREE / 'make_tree' /
@@ -89,6 +91,7 @@ lex
 	token PERCENT /'%'/
 	token PLUS /'+'/
 	token MINUS /'-'/
+	token LTLT /'<<'/
 
 	ignore / ( '\n' | '\t' | ' ' )+ /
 	ignore / '#' . ( ^'\n' )* . '\n' /
@@ -181,7 +184,10 @@ def root_item
 |	[ExportDef: export_def]
 #|	[pre_eof]
 #|	[precedence]
-#|	[typedef]
+|	[AliasDef: alias_def]
+
+def alias_def
+	[ALIAS Id: id TypeRef: type_ref]
 
 def context_item
 	[ContextVarDef: context_var_def]
@@ -353,6 +359,7 @@ def var_def
 
 def print_stmt
 	[Tree: PRINT POPEN CodeExprList: code_expr* PCLOSE]
+|	[PrintStream: PRINTS POPEN CodeExprList: code_expr* PCLOSE]
 |	[Xml: PRINT_XML POPEN CodeExprList: code_expr* PCLOSE]
 |	[XmlAc: PRINT_XML_AC POPEN CodeExprList: code_expr* PCLOSE]
 
@@ -396,6 +403,7 @@ def code_factor
 |	[False: FALSE]
 |	[POPEN ParenCodeExpr: code_expr PCLOSE]
 |	[Send: SEND ToVarRef: var_ref Accumulate: accumulate]
+|	[ToVarRef: var_ref Send: LTLT Accumulate: accumulate]
 |	[Parse: PARSE OptCapture: opt_capture TypeRef: type_ref
 		OptFieldInit: opt_field_init Accumulate: accumulate]
 |	[ParseStop: PARSE_STOP OptCapture: opt_capture TypeRef: type_ref
diff --git a/colm/load.cc b/colm/load.cc
index fa0dd5fc..f3d71931 100644
--- a/colm/load.cc
+++ b/colm/load.cc
@@ -589,6 +589,13 @@ struct LoadSource
 		return exportStmt( objField, LangStmt::AssignType, expr );
 	}
 
+	void walkAliasDef( alias_def aliasDef )
+	{
+		String id = aliasDef.Id().text().c_str();
+		TypeRef *typeRef = walkTypeRef( aliasDef.TypeRef() );
+		alias( internal, id, typeRef );
+	}
+
 	CodeBlock *walkOptTranslate( opt_translate optTranslate )
 	{
 		CodeBlock *block = 0;
@@ -916,6 +923,8 @@ LangStmt *LoadSource::walkPrintStmt( print_stmt &printStmt )
 	LangStmt::Type type;
 	if ( printStmt.Tree() != 0 )
 		type = LangStmt::PrintType;
+	else if ( printStmt.PrintStream() != 0 )
+		type = LangStmt::PrintStreamType;
 	else if ( printStmt.Xml() != 0 )
 		type = LangStmt::PrintXMLType;
 	else if ( printStmt.XmlAc() != 0 )
@@ -1706,6 +1715,9 @@ void LoadSource::walkRootItem( root_item &rootItem, StmtList *stmtList )
 	else if ( rootItem.ExportDef() != 0 ) {
 		walkExportDef( rootItem.ExportDef() );
 	}
+	else if ( rootItem.AliasDef() != 0 ) {
+		walkAliasDef( rootItem.AliasDef() );
+	}
 }
 
 bool walkOptNoIgnore( opt_no_ignore OptNoIngore )
diff --git a/test/combine.sh b/test/combine.sh
index bcdb0cd3..60157613 100755
--- a/test/combine.sh
+++ b/test/combine.sh
@@ -29,4 +29,5 @@ for fn in $@; do
 	> ${fn%.lm}.tst;
 	cat ${fn%.lm}.tst > $fn;
 	rm ${fn%.lm}.tst;
+	git add $fn
 done
diff --git a/test/mailbox.exp b/test/mailbox.exp
deleted file mode 100644
index 97c2e223..00000000
--- a/test/mailbox.exp
+++ /dev/null
@@ -1,30 +0,0 @@
-<start><_repeat_message><message><from_line>From thurston  Tue Jan  2 21:16:50 2007
-</from_line><_repeat_header><header><header_name>Return-Path</header_name><colon>: </colon><header_content>&lt;unknown&gt;
-</header_content></header><header><header_name>X-Spam-Level</header_name><colon>: </colon><header_content>*
-</header_content></header><header><header_name>Received</header_name><colon>: </colon><header_content>from [109.111.71.111] (helo=twfmtr)
-	by zifreax with smtp (Exim 4.43)
-	id 1H1vfs-0005LN-HW; Tue, 2 Jan 2007 21:16:16 -0500
-</header_content></header><header><header_name>Message-ID</header_name><colon>: </colon><header_content>&lt;459B113F.8050903@immoarthabitatge.com&gt;
-</header_content></header><header><header_name>X-Keywords</header_name><colon>:                   </colon><header_content>
-</header_content></header><header><header_name>X-UID</header_name><colon>: </colon><header_content>1
-</header_content></header></_repeat_header><blank_line>
-</blank_line><_repeat_simple_line><simple_line>Content-Type: text/html; charset=ISO-8859-1
-</simple_line><simple_line>&lt;/body&gt;
-</simple_line><simple_line>&lt;/html&gt;
-</simple_line><simple_line>
-</simple_line></_repeat_simple_line></message><message><from_line>From thurston  Wed Jan  3 02:35:48 2007
-</from_line><_repeat_header><header><header_name>Return-Path</header_name><colon>: </colon><header_content>&lt;unknown&gt;
-</header_content></header><header><header_name>X-Spam-Checker-Version</header_name><colon>: </colon><header_content>SpamAssassin 3.1.1 (2006-03-10) on mambo.cs.queensu.ca
-</header_content></header><header><header_name>X-Spam-Level</header_name><colon>: </colon><header_content>**
-</header_content></header><header><header_name>X-Spam-Status</header_name><colon>: </colon><header_content>No, score=2.9 required=5.0 tests=BAYES_20,EXTRA_MPART_TYPE,
-	HTML_40_50,HTML_IMAGE_ONLY_16,HTML_MESSAGE,RCVD_IN_BL_SPAMCOP_NET 
-	autolearn=no version=3.1.1
-</header_content></header><header><header_name>X-Bogosity</header_name><colon>: </colon><header_content>Unsure, tests=bogofilter, spamicity=0.971708, version=1.0.2
-</header_content></header><header><header_name>Status</header_name><colon>: </colon><header_content>RO
-</header_content></header><header><header_name>X-UID</header_name><colon>: </colon><header_content>2
-</header_content></header></_repeat_header><blank_line>
-</blank_line><_repeat_simple_line><simple_line>------=_NextPart_000_0010_01C72F11.F137BD60
-</simple_line><simple_line>	charset="windows-1252"
-</simple_line><simple_line>Content-Transfer-Encoding: quoted-printable
-</simple_line><simple_line>
-</simple_line></_repeat_simple_line></message></_repeat_message></start>
\ No newline at end of file
diff --git a/test/mailbox.in b/test/mailbox.in
deleted file mode 100644
index 412f8bed..00000000
--- a/test/mailbox.in
+++ /dev/null
@@ -1,29 +0,0 @@
-From thurston  Tue Jan  2 21:16:50 2007
-Return-Path: <unknown>
-X-Spam-Level: *
-Received: from [109.111.71.111] (helo=twfmtr)
-	by zifreax with smtp (Exim 4.43)
-	id 1H1vfs-0005LN-HW; Tue, 2 Jan 2007 21:16:16 -0500
-Message-ID: <459B113F.8050903@immoarthabitatge.com>
-X-Keywords:                   
-X-UID: 1
-
-Content-Type: text/html; charset=ISO-8859-1
-</body>
-</html>
-
-From thurston  Wed Jan  3 02:35:48 2007
-Return-Path: <unknown>
-X-Spam-Checker-Version: SpamAssassin 3.1.1 (2006-03-10) on mambo.cs.queensu.ca
-X-Spam-Level: **
-X-Spam-Status: No, score=2.9 required=5.0 tests=BAYES_20,EXTRA_MPART_TYPE,
-	HTML_40_50,HTML_IMAGE_ONLY_16,HTML_MESSAGE,RCVD_IN_BL_SPAMCOP_NET 
-	autolearn=no version=3.1.1
-X-Bogosity: Unsure, tests=bogofilter, spamicity=0.971708, version=1.0.2
-Status: RO
-X-UID: 2
-
-------=_NextPart_000_0010_01C72F11.F137BD60
-	charset="windows-1252"
-Content-Transfer-Encoding: quoted-printable
-
diff --git a/test/mailbox.lm b/test/mailbox.lm
index df45a65a..4cf5fb11 100644
--- a/test/mailbox.lm
+++ b/test/mailbox.lm
@@ -1,3 +1,4 @@
+##### LM #####
 
 # lines, and fromlines 
 lex
@@ -41,3 +42,65 @@ def start
 
 parse S: start[ stdin ]
 print_xml( S.tree )
+print( '\n' )
+##### IN #####
+From thurston  Tue Jan  2 21:16:50 2007
+Return-Path: <unknown>
+X-Spam-Level: *
+Received: from [109.111.71.111] (helo=twfmtr)
+	by zifreax with smtp (Exim 4.43)
+	id 1H1vfs-0005LN-HW; Tue, 2 Jan 2007 21:16:16 -0500
+Message-ID: <459B113F.8050903@immoarthabitatge.com>
+X-Keywords:                   
+X-UID: 1
+
+Content-Type: text/html; charset=ISO-8859-1
+</body>
+</html>
+
+From thurston  Wed Jan  3 02:35:48 2007
+Return-Path: <unknown>
+X-Spam-Checker-Version: SpamAssassin 3.1.1 (2006-03-10) on mambo.cs.queensu.ca
+X-Spam-Level: **
+X-Spam-Status: No, score=2.9 required=5.0 tests=BAYES_20,EXTRA_MPART_TYPE,
+	HTML_40_50,HTML_IMAGE_ONLY_16,HTML_MESSAGE,RCVD_IN_BL_SPAMCOP_NET 
+	autolearn=no version=3.1.1
+X-Bogosity: Unsure, tests=bogofilter, spamicity=0.971708, version=1.0.2
+Status: RO
+X-UID: 2
+
+------=_NextPart_000_0010_01C72F11.F137BD60
+	charset="windows-1252"
+Content-Transfer-Encoding: quoted-printable
+
+##### EXP #####
+<start><_repeat_message><message><from_line>From thurston  Tue Jan  2 21:16:50 2007
+</from_line><_repeat_header><header><header_name>Return-Path</header_name><colon>: </colon><header_content>&lt;unknown&gt;
+</header_content></header><header><header_name>X-Spam-Level</header_name><colon>: </colon><header_content>*
+</header_content></header><header><header_name>Received</header_name><colon>: </colon><header_content>from [109.111.71.111] (helo=twfmtr)
+	by zifreax with smtp (Exim 4.43)
+	id 1H1vfs-0005LN-HW; Tue, 2 Jan 2007 21:16:16 -0500
+</header_content></header><header><header_name>Message-ID</header_name><colon>: </colon><header_content>&lt;459B113F.8050903@immoarthabitatge.com&gt;
+</header_content></header><header><header_name>X-Keywords</header_name><colon>:                   </colon><header_content>
+</header_content></header><header><header_name>X-UID</header_name><colon>: </colon><header_content>1
+</header_content></header></_repeat_header><blank_line>
+</blank_line><_repeat_simple_line><simple_line>Content-Type: text/html; charset=ISO-8859-1
+</simple_line><simple_line>&lt;/body&gt;
+</simple_line><simple_line>&lt;/html&gt;
+</simple_line><simple_line>
+</simple_line></_repeat_simple_line></message><message><from_line>From thurston  Wed Jan  3 02:35:48 2007
+</from_line><_repeat_header><header><header_name>Return-Path</header_name><colon>: </colon><header_content>&lt;unknown&gt;
+</header_content></header><header><header_name>X-Spam-Checker-Version</header_name><colon>: </colon><header_content>SpamAssassin 3.1.1 (2006-03-10) on mambo.cs.queensu.ca
+</header_content></header><header><header_name>X-Spam-Level</header_name><colon>: </colon><header_content>**
+</header_content></header><header><header_name>X-Spam-Status</header_name><colon>: </colon><header_content>No, score=2.9 required=5.0 tests=BAYES_20,EXTRA_MPART_TYPE,
+	HTML_40_50,HTML_IMAGE_ONLY_16,HTML_MESSAGE,RCVD_IN_BL_SPAMCOP_NET 
+	autolearn=no version=3.1.1
+</header_content></header><header><header_name>X-Bogosity</header_name><colon>: </colon><header_content>Unsure, tests=bogofilter, spamicity=0.971708, version=1.0.2
+</header_content></header><header><header_name>Status</header_name><colon>: </colon><header_content>RO
+</header_content></header><header><header_name>X-UID</header_name><colon>: </colon><header_content>2
+</header_content></header></_repeat_header><blank_line>
+</blank_line><_repeat_simple_line><simple_line>------=_NextPart_000_0010_01C72F11.F137BD60
+</simple_line><simple_line>	charset="windows-1252"
+</simple_line><simple_line>Content-Transfer-Encoding: quoted-printable
+</simple_line><simple_line>
+</simple_line></_repeat_simple_line></message></_repeat_message></start>
diff --git a/test/maxlen.exp b/test/maxlen.exp
deleted file mode 100644
index c8ceeabc..00000000
--- a/test/maxlen.exp
+++ /dev/null
@@ -1 +0,0 @@
-<maxlen::start><maxlen::restricted_list><maxlen::open></maxlen::open><maxlen::_repeat_item><maxlen::item><maxlen::id>a</maxlen::id></maxlen::item><maxlen::item><maxlen::id>b</maxlen::id></maxlen::item><maxlen::item><maxlen::id>c</maxlen::id></maxlen::item></maxlen::_repeat_item></maxlen::restricted_list><maxlen::_repeat_id><maxlen::id>d</maxlen::id><maxlen::id>e</maxlen::id><maxlen::id>f</maxlen::id><maxlen::id>g</maxlen::id></maxlen::_repeat_id></maxlen::start>
\ No newline at end of file
diff --git a/test/maxlen.in b/test/maxlen.in
deleted file mode 100644
index a0fa0cfd..00000000
--- a/test/maxlen.in
+++ /dev/null
@@ -1 +0,0 @@
-a b c d e f g
diff --git a/test/maxlen.lm b/test/maxlen.lm
index e4a619dc..481a3fac 100644
--- a/test/maxlen.lm
+++ b/test/maxlen.lm
@@ -1,3 +1,4 @@
+##### LM #####
 
 context maxlen
 
@@ -49,3 +50,8 @@ MaxLen.allow = 3
 
 parse S: maxlen::start(MaxLen)[stdin]
 print_xml( S.tree )
+print('\n')
+##### IN #####
+a b c d e f g
+##### EXP #####
+<maxlen::start><maxlen::restricted_list><maxlen::open></maxlen::open><maxlen::_repeat_item><maxlen::item><maxlen::id>a</maxlen::id></maxlen::item><maxlen::item><maxlen::id>b</maxlen::id></maxlen::item><maxlen::item><maxlen::id>c</maxlen::id></maxlen::item></maxlen::_repeat_item></maxlen::restricted_list><maxlen::_repeat_id><maxlen::id>d</maxlen::id><maxlen::id>e</maxlen::id><maxlen::id>f</maxlen::id><maxlen::id>g</maxlen::id></maxlen::_repeat_id></maxlen::start>
diff --git a/test/multiregion1.exp b/test/multiregion1.exp
deleted file mode 100644
index 35413bc8..00000000
--- a/test/multiregion1.exp
+++ /dev/null
@@ -1,86 +0,0 @@
-Index: fsmrun.cpp
-===================================================================
---- newfilename.cpp (revision 4555)
-+++ fsmrun.cpp	(working copy)
-@@ -150,7 +150,7 @@
- 	peof = 0;
- 	if ( parser != 0 ) {
- 		region = parser->getNextRegion();
--		cs = getStateFromNextRegion();
-+		cs = tables->entryByRegion[region];
- 	}
- 	else {
- 		region = 0;
-@@ -189,7 +189,7 @@
- 
- 	tokstart = 0;
- 	region = parser->getNextRegion();
--	cs = getStateFromNextRegion();
-+	cs = tables->entryByRegion[region];
- }
- 
- void FsmRun::sendToken( int id )
-@@ -222,7 +222,7 @@
- 		parser = newParser;
- 
- 		region = parser->getNextRegion();
--		cs = getStateFromNextRegion();
-+		cs = tables->entryByRegion[region];
- 	}
- 	else {
- 		#ifdef LOG_ACTIONS
-@@ -355,7 +355,7 @@
- 
- 			/* Set the current state from the next region. */
- 			region = parser->getNextRegion();
--			cs = getStateFromNextRegion();
-+			cs = tables->entryByRegion[region];
- 		}
- 	}
- 
-@@ -452,7 +452,7 @@
- 		/* First thing check for error. */
- 		if ( cs == tables->errorState ) {
- 			if ( parser != 0 ) {
--				if ( getStateFromNextRegion( 1 ) != 0 ) {
-+				if ( parser->getNextRegion( 1 ) != 0 ) {
- 					#ifdef LOG_BACKTRACK
- 					cerr << "scanner failed, trying next region" << endl;
- 					#endif
-@@ -462,7 +462,7 @@
- 
- 					parser->nextRegionInd += 1;
- 					region = parser->getNextRegion();
--					cs = getStateFromNextRegion();
-+					cs = tables->entryByRegion[region];
- 					cerr << "new token region: " << 
- 							parser->tables->gbl->regionInfo[region].name << endl;
- 					continue;
-@@ -495,7 +495,7 @@
- 					}
- 					else {
- 						region = parser->getNextRegion();
--						cs = getStateFromNextRegion();
-+						cs = tables->entryByRegion[region];
- 						cerr << "new token region: " << 
- 								parser->tables->gbl->regionInfo[region].name << endl;
- 						continue;
-Index: junk.cpp
-===================================================================
---- 
-+++ junk.cpp	(working copy)
-Index: fsmrun.h
-===================================================================
---- fsmrun.h	(revision 4557)
-+++ fsmrun.h	(working copy)
-@@ -197,10 +197,6 @@
- 	void runOnInputStream( PdaRun *parser, InputStream &in );
- 	void execute();
- 
--	/* Offset can be used to look at the next nextRegionInd. */
--	int getStateFromNextRegion( int offset = 0 )
--		{ return tables->entryByRegion[parser->getNextRegion(offset)]; }
--
- 	FsmTables *tables;
- 	PdaRun *parser;
- 	InputStream *inputStream;
diff --git a/test/multiregion1.in b/test/multiregion1.in
deleted file mode 100644
index b0021f67..00000000
--- a/test/multiregion1.in
+++ /dev/null
@@ -1,86 +0,0 @@
-Index: fsmrun.cpp
-===================================================================
---- fsmrun.cpp	(revision 4555)
-+++ fsmrun.cpp	(working copy)
-@@ -150,7 +150,7 @@
- 	peof = 0;
- 	if ( parser != 0 ) {
- 		region = parser->getNextRegion();
--		cs = getStateFromNextRegion();
-+		cs = tables->entryByRegion[region];
- 	}
- 	else {
- 		region = 0;
-@@ -189,7 +189,7 @@
- 
- 	tokstart = 0;
- 	region = parser->getNextRegion();
--	cs = getStateFromNextRegion();
-+	cs = tables->entryByRegion[region];
- }
- 
- void FsmRun::sendToken( int id )
-@@ -222,7 +222,7 @@
- 		parser = newParser;
- 
- 		region = parser->getNextRegion();
--		cs = getStateFromNextRegion();
-+		cs = tables->entryByRegion[region];
- 	}
- 	else {
- 		#ifdef LOG_ACTIONS
-@@ -355,7 +355,7 @@
- 
- 			/* Set the current state from the next region. */
- 			region = parser->getNextRegion();
--			cs = getStateFromNextRegion();
-+			cs = tables->entryByRegion[region];
- 		}
- 	}
- 
-@@ -452,7 +452,7 @@
- 		/* First thing check for error. */
- 		if ( cs == tables->errorState ) {
- 			if ( parser != 0 ) {
--				if ( getStateFromNextRegion( 1 ) != 0 ) {
-+				if ( parser->getNextRegion( 1 ) != 0 ) {
- 					#ifdef LOG_BACKTRACK
- 					cerr << "scanner failed, trying next region" << endl;
- 					#endif
-@@ -462,7 +462,7 @@
- 
- 					parser->nextRegionInd += 1;
- 					region = parser->getNextRegion();
--					cs = getStateFromNextRegion();
-+					cs = tables->entryByRegion[region];
- 					cerr << "new token region: " << 
- 							parser->tables->gbl->regionInfo[region].name << endl;
- 					continue;
-@@ -495,7 +495,7 @@
- 					}
- 					else {
- 						region = parser->getNextRegion();
--						cs = getStateFromNextRegion();
-+						cs = tables->entryByRegion[region];
- 						cerr << "new token region: " << 
- 								parser->tables->gbl->regionInfo[region].name << endl;
- 						continue;
-Index: junk.cpp
-===================================================================
---- 
-+++ junk.cpp	(working copy)
-Index: fsmrun.h
-===================================================================
---- fsmrun.h	(revision 4557)
-+++ fsmrun.h	(working copy)
-@@ -197,10 +197,6 @@
- 	void runOnInputStream( PdaRun *parser, InputStream &in );
- 	void execute();
- 
--	/* Offset can be used to look at the next nextRegionInd. */
--	int getStateFromNextRegion( int offset = 0 )
--		{ return tables->entryByRegion[parser->getNextRegion(offset)]; }
--
- 	FsmTables *tables;
- 	PdaRun *parser;
- 	InputStream *inputStream;
diff --git a/test/multiregion1.lm b/test/multiregion1.lm
index 3eb05714..c80f612c 100644
--- a/test/multiregion1.lm
+++ b/test/multiregion1.lm
@@ -1,3 +1,4 @@
+##### LM #####
 
 
 token newline / '\n' /
@@ -66,3 +67,177 @@ for OF: old_file in S {
 
 print( S )
 
+##### IN #####
+Index: fsmrun.cpp
+===================================================================
+--- fsmrun.cpp	(revision 4555)
++++ fsmrun.cpp	(working copy)
+@@ -150,7 +150,7 @@
+ 	peof = 0;
+ 	if ( parser != 0 ) {
+ 		region = parser->getNextRegion();
+-		cs = getStateFromNextRegion();
++		cs = tables->entryByRegion[region];
+ 	}
+ 	else {
+ 		region = 0;
+@@ -189,7 +189,7 @@
+ 
+ 	tokstart = 0;
+ 	region = parser->getNextRegion();
+-	cs = getStateFromNextRegion();
++	cs = tables->entryByRegion[region];
+ }
+ 
+ void FsmRun::sendToken( int id )
+@@ -222,7 +222,7 @@
+ 		parser = newParser;
+ 
+ 		region = parser->getNextRegion();
+-		cs = getStateFromNextRegion();
++		cs = tables->entryByRegion[region];
+ 	}
+ 	else {
+ 		#ifdef LOG_ACTIONS
+@@ -355,7 +355,7 @@
+ 
+ 			/* Set the current state from the next region. */
+ 			region = parser->getNextRegion();
+-			cs = getStateFromNextRegion();
++			cs = tables->entryByRegion[region];
+ 		}
+ 	}
+ 
+@@ -452,7 +452,7 @@
+ 		/* First thing check for error. */
+ 		if ( cs == tables->errorState ) {
+ 			if ( parser != 0 ) {
+-				if ( getStateFromNextRegion( 1 ) != 0 ) {
++				if ( parser->getNextRegion( 1 ) != 0 ) {
+ 					#ifdef LOG_BACKTRACK
+ 					cerr << "scanner failed, trying next region" << endl;
+ 					#endif
+@@ -462,7 +462,7 @@
+ 
+ 					parser->nextRegionInd += 1;
+ 					region = parser->getNextRegion();
+-					cs = getStateFromNextRegion();
++					cs = tables->entryByRegion[region];
+ 					cerr << "new token region: " << 
+ 							parser->tables->gbl->regionInfo[region].name << endl;
+ 					continue;
+@@ -495,7 +495,7 @@
+ 					}
+ 					else {
+ 						region = parser->getNextRegion();
+-						cs = getStateFromNextRegion();
++						cs = tables->entryByRegion[region];
+ 						cerr << "new token region: " << 
+ 								parser->tables->gbl->regionInfo[region].name << endl;
+ 						continue;
+Index: junk.cpp
+===================================================================
+--- 
++++ junk.cpp	(working copy)
+Index: fsmrun.h
+===================================================================
+--- fsmrun.h	(revision 4557)
++++ fsmrun.h	(working copy)
+@@ -197,10 +197,6 @@
+ 	void runOnInputStream( PdaRun *parser, InputStream &in );
+ 	void execute();
+ 
+-	/* Offset can be used to look at the next nextRegionInd. */
+-	int getStateFromNextRegion( int offset = 0 )
+-		{ return tables->entryByRegion[parser->getNextRegion(offset)]; }
+-
+ 	FsmTables *tables;
+ 	PdaRun *parser;
+ 	InputStream *inputStream;
+##### EXP #####
+Index: fsmrun.cpp
+===================================================================
+--- newfilename.cpp (revision 4555)
++++ fsmrun.cpp	(working copy)
+@@ -150,7 +150,7 @@
+ 	peof = 0;
+ 	if ( parser != 0 ) {
+ 		region = parser->getNextRegion();
+-		cs = getStateFromNextRegion();
++		cs = tables->entryByRegion[region];
+ 	}
+ 	else {
+ 		region = 0;
+@@ -189,7 +189,7 @@
+ 
+ 	tokstart = 0;
+ 	region = parser->getNextRegion();
+-	cs = getStateFromNextRegion();
++	cs = tables->entryByRegion[region];
+ }
+ 
+ void FsmRun::sendToken( int id )
+@@ -222,7 +222,7 @@
+ 		parser = newParser;
+ 
+ 		region = parser->getNextRegion();
+-		cs = getStateFromNextRegion();
++		cs = tables->entryByRegion[region];
+ 	}
+ 	else {
+ 		#ifdef LOG_ACTIONS
+@@ -355,7 +355,7 @@
+ 
+ 			/* Set the current state from the next region. */
+ 			region = parser->getNextRegion();
+-			cs = getStateFromNextRegion();
++			cs = tables->entryByRegion[region];
+ 		}
+ 	}
+ 
+@@ -452,7 +452,7 @@
+ 		/* First thing check for error. */
+ 		if ( cs == tables->errorState ) {
+ 			if ( parser != 0 ) {
+-				if ( getStateFromNextRegion( 1 ) != 0 ) {
++				if ( parser->getNextRegion( 1 ) != 0 ) {
+ 					#ifdef LOG_BACKTRACK
+ 					cerr << "scanner failed, trying next region" << endl;
+ 					#endif
+@@ -462,7 +462,7 @@
+ 
+ 					parser->nextRegionInd += 1;
+ 					region = parser->getNextRegion();
+-					cs = getStateFromNextRegion();
++					cs = tables->entryByRegion[region];
+ 					cerr << "new token region: " << 
+ 							parser->tables->gbl->regionInfo[region].name << endl;
+ 					continue;
+@@ -495,7 +495,7 @@
+ 					}
+ 					else {
+ 						region = parser->getNextRegion();
+-						cs = getStateFromNextRegion();
++						cs = tables->entryByRegion[region];
+ 						cerr << "new token region: " << 
+ 								parser->tables->gbl->regionInfo[region].name << endl;
+ 						continue;
+Index: junk.cpp
+===================================================================
+--- 
++++ junk.cpp	(working copy)
+Index: fsmrun.h
+===================================================================
+--- fsmrun.h	(revision 4557)
++++ fsmrun.h	(working copy)
+@@ -197,10 +197,6 @@
+ 	void runOnInputStream( PdaRun *parser, InputStream &in );
+ 	void execute();
+ 
+-	/* Offset can be used to look at the next nextRegionInd. */
+-	int getStateFromNextRegion( int offset = 0 )
+-		{ return tables->entryByRegion[parser->getNextRegion(offset)]; }
+-
+ 	FsmTables *tables;
+ 	PdaRun *parser;
+ 	InputStream *inputStream;
diff --git a/test/multiregion2.exp b/test/multiregion2.exp
deleted file mode 100644
index 3e3a984c..00000000
--- a/test/multiregion2.exp
+++ /dev/null
@@ -1,17 +0,0 @@
-GET /hi/there/ HTTP/1.1

-

-GET /hithere/ HTTP/1.1

-Host: localhost:3535

-User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.12) Gecko/20080207 Ubuntu/7.10 (gutsy) Firefox/2.0.0.12

-Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5

-Accept-Language: en-us,en;q=0.5

-Accept-Encoding: gzip,deflate

-Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7

-Keep-Alive: 300

-Connection: keep-alive

-Cache-Control: max-age=0

-

-GET foo HTTP/1.1

-hello: foo

-hi: there my friend

-

diff --git a/test/multiregion2.in b/test/multiregion2.in
deleted file mode 100644
index 25a07c41..00000000
--- a/test/multiregion2.in
+++ /dev/null
@@ -1,19 +0,0 @@
-GET /hi/there/ HTTP/1.1

-

-GET /hithere/ HTTP/1.1

-Host: localhost:3535

-User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.12) Gecko/20080207 Ubuntu/7.10 (gutsy) Firefox/2.0.0.12

-Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5

-Accept-Language: en-us,en;q=0.5

-Accept-Encoding: gzip,deflate

-Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7

-Keep-Alive: 300

-Connection: keep-alive

-Cache-Control: max-age=0

-

-GET foo HTTP/1.1

-hello: foo

-hi: there

-	my

- friend

-

diff --git a/test/multiregion2.lm b/test/multiregion2.lm
index 0b19b15b..61455ba8 100644
--- a/test/multiregion2.lm
+++ b/test/multiregion2.lm
@@ -1,3 +1,4 @@
+##### LM #####
 #
 # Character classes
 #
@@ -83,3 +84,41 @@ for FV: fv in R {
 
 print( R )
 
+##### IN #####
+GET /hi/there/ HTTP/1.1

+

+GET /hithere/ HTTP/1.1

+Host: localhost:3535

+User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.12) Gecko/20080207 Ubuntu/7.10 (gutsy) Firefox/2.0.0.12

+Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5

+Accept-Language: en-us,en;q=0.5

+Accept-Encoding: gzip,deflate

+Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7

+Keep-Alive: 300

+Connection: keep-alive

+Cache-Control: max-age=0

+

+GET foo HTTP/1.1

+hello: foo

+hi: there

+	my

+ friend

+

+##### EXP #####
+GET /hi/there/ HTTP/1.1

+

+GET /hithere/ HTTP/1.1

+Host: localhost:3535

+User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.12) Gecko/20080207 Ubuntu/7.10 (gutsy) Firefox/2.0.0.12

+Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5

+Accept-Language: en-us,en;q=0.5

+Accept-Encoding: gzip,deflate

+Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7

+Keep-Alive: 300

+Connection: keep-alive

+Cache-Control: max-age=0

+

+GET foo HTTP/1.1

+hello: foo

+hi: there my friend

+

diff --git a/test/mutualrec.exp b/test/mutualrec.exp
deleted file mode 100644
index 6884b2cf..00000000
--- a/test/mutualrec.exp
+++ /dev/null
@@ -1 +0,0 @@
-f1() = 22
diff --git a/test/mutualrec.lm b/test/mutualrec.lm
index bfaf0d24..2eafd289 100644
--- a/test/mutualrec.lm
+++ b/test/mutualrec.lm
@@ -1,3 +1,4 @@
+##### LM #####
 
 int f1( i: int )
 {
@@ -13,3 +14,5 @@ int f2( i: int )
 }
 
 print( "f1() = [$f1(0)]\n" )
+##### EXP #####
+f1() = 22
diff --git a/test/nestedcomm.exp b/test/nestedcomm.exp
deleted file mode 100644
index f460488f..00000000
--- a/test/nestedcomm.exp
+++ /dev/null
@@ -1,5 +0,0 @@
-( (this is a nested comment /*sdf;asd_++_stuff) )
-hello there ( (this is a nested comment /*sdf;asd_++_stuff) ) and this is not
-<nested><_repeat_id><id>hello</id><id>there</id><id>and</id><id>this</id><id>is</id><id>not</id></_repeat_id></nested>
-<nested><_repeat_id><id>hello</id><_ignore_0001> </_ignore_0001><id>there</id><_ignore_0001> </_ignore_0001><nested_comment><_literal_0007>(</_literal_0007><_repeat_nc_item><nc_item><nc_data> </nc_data></nc_item><nc_item><nested_comment><_literal_0007>(</_literal_0007><_repeat_nc_item><nc_item><nc_data>this is a nested comment /*sdf;asd_++_stuff</nc_data></nc_item></_repeat_nc_item><_literal_0009>)</_literal_0009></nested_comment></nc_item><nc_item><nc_data> </nc_data></nc_item></_repeat_nc_item><_literal_0009>)</_literal_0009></nested_comment><_ignore_0001> </_ignore_0001><id>and</id><_ignore_0001> </_ignore_0001><id>this</id><_ignore_0001> </_ignore_0001><id>is</id><_ignore_0001> </_ignore_0001><id>not</id></_repeat_id></nested>
-hello there ( (this is a nested comment /*sdf;asd_++_stuff) ) and this is not
diff --git a/test/nestedcomm.in b/test/nestedcomm.in
deleted file mode 100644
index 11789576..00000000
--- a/test/nestedcomm.in
+++ /dev/null
@@ -1 +0,0 @@
-hello there ( (this is a nested comment /*sdf;asd_++_stuff) ) and this is not
diff --git a/test/nestedcomm.lm b/test/nestedcomm.lm
index 5adc8902..381c5c22 100644
--- a/test/nestedcomm.lm
+++ b/test/nestedcomm.lm
@@ -1,3 +1,4 @@
+##### LM #####
 #
 # Tokens
 #
@@ -45,3 +46,11 @@ print( '\n' )
 print_xml_ac( ^P )
 print( '\n' )
 print( ^P '\n' )
+##### IN #####
+hello there ( (this is a nested comment /*sdf;asd_++_stuff) ) and this is not
+##### EXP #####
+( (this is a nested comment /*sdf;asd_++_stuff) )
+hello there ( (this is a nested comment /*sdf;asd_++_stuff) ) and this is not
+<nested><_repeat_id><id>hello</id><id>there</id><id>and</id><id>this</id><id>is</id><id>not</id></_repeat_id></nested>
+<nested><_repeat_id><id>hello</id><_ignore_0001> </_ignore_0001><id>there</id><_ignore_0001> </_ignore_0001><nested_comment><_literal_0007>(</_literal_0007><_repeat_nc_item><nc_item><nc_data> </nc_data></nc_item><nc_item><nested_comment><_literal_0007>(</_literal_0007><_repeat_nc_item><nc_item><nc_data>this is a nested comment /*sdf;asd_++_stuff</nc_data></nc_item></_repeat_nc_item><_literal_0009>)</_literal_0009></nested_comment></nc_item><nc_item><nc_data> </nc_data></nc_item></_repeat_nc_item><_literal_0009>)</_literal_0009></nested_comment><_ignore_0001> </_ignore_0001><id>and</id><_ignore_0001> </_ignore_0001><id>this</id><_ignore_0001> </_ignore_0001><id>is</id><_ignore_0001> </_ignore_0001><id>not</id></_repeat_id></nested>
+hello there ( (this is a nested comment /*sdf;asd_++_stuff) ) and this is not
diff --git a/test/order1.exp b/test/order1.exp
deleted file mode 100644
index fc6cc2ec..00000000
--- a/test/order1.exp
+++ /dev/null
@@ -1,4 +0,0 @@
-#include "some_header.h"
-
-#include <assert.h>
-
diff --git a/test/order1.in b/test/order1.in
deleted file mode 100644
index 8f7d9635..00000000
--- a/test/order1.in
+++ /dev/null
@@ -1,12 +0,0 @@
-#include "some_header.h"
-
-%%
-
-tcp protocol FOO 
-{
-	port 99
-
-	client c
-	server s
-}
-
diff --git a/test/order1.lm b/test/order1.lm
index e7260118..f4a6998f 100644
--- a/test/order1.lm
+++ b/test/order1.lm
@@ -1,3 +1,4 @@
+##### LM #####
 
 lex
 	token c_single_lit /( 'L'? "'" ( [^'\\\n] | '\\' any )* "'" )/
@@ -94,3 +95,21 @@ Output <<
 	"
 
 print( Output.finish() )
+##### IN #####
+#include "some_header.h"
+
+%%
+
+tcp protocol FOO 
+{
+	port 99
+
+	client c
+	server s
+}
+
+##### EXP #####
+#include "some_header.h"
+
+#include <assert.h>
+
diff --git a/test/order2.exp b/test/order2.exp
deleted file mode 100644
index fc6cc2ec..00000000
--- a/test/order2.exp
+++ /dev/null
@@ -1,4 +0,0 @@
-#include "some_header.h"
-
-#include <assert.h>
-
diff --git a/test/order2.in b/test/order2.in
deleted file mode 100644
index 8f7d9635..00000000
--- a/test/order2.in
+++ /dev/null
@@ -1,12 +0,0 @@
-#include "some_header.h"
-
-%%
-
-tcp protocol FOO 
-{
-	port 99
-
-	client c
-	server s
-}
-
diff --git a/test/order2.lm b/test/order2.lm
index f19168a8..e057bbca 100644
--- a/test/order2.lm
+++ b/test/order2.lm
@@ -1,3 +1,4 @@
+##### LM #####
 
 lex
 	token c_single_lit /( 'L'? "'" ( [^'\\\n] | '\\' any )* "'" )/
@@ -94,3 +95,21 @@ Output <<
 	"#include <assert.h>
 	"
 print( Output.finish() )
+##### IN #####
+#include "some_header.h"
+
+%%
+
+tcp protocol FOO 
+{
+	port 99
+
+	client c
+	server s
+}
+
+##### EXP #####
+#include "some_header.h"
+
+#include <assert.h>
+
diff --git a/test/prints.exp b/test/prints.exp
deleted file mode 100644
index c849c57d..00000000
--- a/test/prints.exp
+++ /dev/null
@@ -1 +0,0 @@
-fd stdout: a b c
diff --git a/test/prints.in b/test/prints.in
deleted file mode 100644
index 3774da60..00000000
--- a/test/prints.in
+++ /dev/null
@@ -1 +0,0 @@
-a b c
diff --git a/test/prints.lm b/test/prints.lm
index cb8e37e5..1a72d216 100644
--- a/test/prints.lm
+++ b/test/prints.lm
@@ -1,3 +1,4 @@
+##### LM #####
 lex
 	token word /[a-z]+/
 	ignore /[\t\n ]+/
@@ -10,3 +11,7 @@ parse Start: start[stdin]
 
 prints( stderr 'fd stderr: ' Start.tree '\n' )
 prints( stdout 'fd stdout: ' Start.tree '\n' )
+##### IN #####
+a b c
+##### EXP #####
+fd stdout: a b c
diff --git a/test/pull1.exp b/test/pull1.exp
deleted file mode 100644
index aa3a0fe0..00000000
--- a/test/pull1.exp
+++ /dev/null
@@ -1 +0,0 @@
-this is in
diff --git a/test/pull1.in b/test/pull1.in
deleted file mode 100644
index f4d2e4a0..00000000
--- a/test/pull1.in
+++ /dev/null
@@ -1 +0,0 @@
-this is input for a non-parse pull
diff --git a/test/pull1.lm b/test/pull1.lm
index bc559671..f86bd6c9 100644
--- a/test/pull1.lm
+++ b/test/pull1.lm
@@ -1,2 +1,7 @@
+##### LM #####
 String: str = stdin.pull( 10 )
 print( String '\n' )
+##### IN #####
+this is input for a non-parse pull
+##### EXP #####
+this is in
diff --git a/test/pull2.exp b/test/pull2.exp
deleted file mode 100644
index aa3a0fe0..00000000
--- a/test/pull2.exp
+++ /dev/null
@@ -1 +0,0 @@
-this is in
diff --git a/test/pull2.in b/test/pull2.in
deleted file mode 100644
index f4d2e4a0..00000000
--- a/test/pull2.in
+++ /dev/null
@@ -1 +0,0 @@
-this is input for a non-parse pull
diff --git a/test/pull2.lm b/test/pull2.lm
index 1a18b829..9adb42f2 100644
--- a/test/pull2.lm
+++ b/test/pull2.lm
@@ -1,3 +1,8 @@
-Stream: stream = open( 'pull2.in' 'r' )
+##### LM #####
+Stream: stream = open( 'working/pull2.in' 'r' )
 String: str = Stream.pull( 10 )
 print( String '\n' )
+##### IN #####
+this is input for a non-parse pull
+##### EXP #####
+this is in