Merge branch 'master' of ssh://thurston@complang.org/var/git/colm

7 files changed, 129 insertions, 39 deletions

diff --git a/colm/colm.lm b/colm/colm.lm
index 8f36e237..be147c31 100644
--- a/colm/colm.lm
+++ b/colm/colm.lm
@@ -183,23 +183,23 @@ def start
 	[RootItemList: root_item*]
 
 def root_item
-	[RlDef: rl_def]
-|	[LiteralDef: literal_def]
-|	[TokenDef: token_def]
-|	[IgnoreDef: ignore_def]
-|	[CflDef: cfl_def]
-|	[RegionDef: region_def]
-|	[ContextDef: context_def]
-|	[NamespaceDef: namespace_def]
-|	[FunctionDef: function_def]
-|	[IterDef: iter_def]
-|	[Statement: statement]
-|	[GlobalDef: global_def]
-|	[ExportDef: export_def]
-|	[PreEof: pre_eof]
-|	[PrecedenceDef: precedence_def]
-|	[AliasDef: alias_def]
-|	[Include: include]
+	[RlDef: rl_def] commit
+|	[LiteralDef: literal_def] commit
+|	[TokenDef: token_def] commit
+|	[IgnoreDef: ignore_def] commit
+|	[CflDef: cfl_def] commit
+|	[RegionDef: region_def] commit
+|	[ContextDef: context_def] commit
+|	[NamespaceDef: namespace_def] commit
+|	[FunctionDef: function_def] commit
+|	[IterDef: iter_def] commit
+|	[Statement: statement] commit
+|	[GlobalDef: global_def] commit
+|	[ExportDef: export_def] commit
+|	[PreEof: pre_eof] commit
+|	[PrecedenceDef: precedence_def] commit
+|	[AliasDef: alias_def] commit
+|	[Include: include] commit
 
 def include
 	[INCLUDE File: lit]
@@ -227,19 +227,19 @@ def alias_def
 	[ALIAS Id: id TypeRef: type_ref]
 
 def context_item
-	[ContextVarDef: context_var_def]
-|	[LiteralDef: literal_def]
-|	[RlDef: rl_def]
-|	[TokenDef: token_def]
-|	[IgnoreDef: ignore_def]
-|	[CflDef: cfl_def]
-|	[RegionDef: region_def]
-|	[ContextDef: context_def]
-|	[FunctionDef: function_def]
-|	[IterDef: iter_def]
-|	[ExportDef: export_def]
-|	[PreEof: pre_eof]
-|	[PrecedenceDef: precedence_def]
+	[ContextVarDef: context_var_def] commit
+|	[LiteralDef: literal_def] commit
+|	[RlDef: rl_def] commit
+|	[TokenDef: token_def] commit
+|	[IgnoreDef: ignore_def] commit
+|	[CflDef: cfl_def] commit
+|	[RegionDef: region_def] commit
+|	[ContextDef: context_def] commit
+|	[FunctionDef: function_def] commit
+|	[IterDef: iter_def] commit
+|	[ExportDef: export_def] commit
+|	[PreEof: pre_eof] commit
+|	[PrecedenceDef: precedence_def] commit
 
 def export_def
 	[EXPORT VarDef: var_def OptDefInit: opt_def_init]
@@ -310,7 +310,7 @@ def opt_lex_expr
 
 def token_def
 	[TOKEN Id: id VarDefList: var_def* 
-		LEX_FSLASH OptExpr: opt_lex_expr LEX_FSLASH
+		NiLeft: opt_no_ignore LEX_FSLASH OptExpr: opt_lex_expr LEX_FSLASH NiRight: opt_no_ignore
 		OptTranslate: opt_translate]
 
 def opt_translate
diff --git a/colm/conscolm.cc b/colm/conscolm.cc
index e359af88..adaabb6d 100644
--- a/colm/conscolm.cc
+++ b/colm/conscolm.cc
@@ -78,7 +78,9 @@ void LoadColm::walkProdList( LelDefList *outProdList, prod_list &prodList )
 	prod_el_list prodElList = prodList.Prod().ProdElList();
 	walkProdElList( outElList, prodElList );
 
-	Production *prod = BaseParser::production( internal, outElList, false, 0, 0 );
+	bool commit = prodList.Prod().OptCommit().Commit() != 0;
+
+	Production *prod = BaseParser::production( internal, outElList, commit, 0, 0 );
 	prodAppend( outProdList, prod );
 }
 
@@ -300,7 +302,7 @@ void LoadColm::consParseStmt( StmtList *stmtList )
 	stmtList->append( parseStmt );
 }
 
-void LoadColm::consExportStmt( StmtList *stmtList )
+void LoadColm::consExportTree( StmtList *stmtList )
 {
 	QualItemVect *qual = new QualItemVect;
 	qual->append( QualItem( internal, String( "P" ), QualItem::Dot ) );
@@ -314,6 +316,20 @@ void LoadColm::consExportStmt( StmtList *stmtList )
 	stmtList->append( programExport );
 }
 
+void LoadColm::consExportError( StmtList *stmtList )
+{
+	QualItemVect *qual = new QualItemVect;
+	qual->append( QualItem( internal, String( "P" ), QualItem::Dot ) );
+	LangVarRef *varRef = LangVarRef::cons( internal, qual, String("error") );
+	LangExpr *expr = LangExpr::cons( LangTerm::cons( internal, LangTerm::VarRefType, varRef ) );
+
+	NamespaceQual *nspaceQual = NamespaceQual::cons( namespaceStack.top() );
+	TypeRef *typeRef = TypeRef::cons( internal, nspaceQual, String("str"), RepeatNone );
+	ObjectField *program = ObjectField::cons( internal, typeRef, String("ColmError") );
+	LangStmt *programExport = exportStmt( program, LangStmt::AssignType, expr );
+	stmtList->append( programExport );
+}
+
 void LoadColm::go()
 {
 	LoadColm::init();
@@ -352,7 +368,8 @@ void LoadColm::go()
 	colmDeleteProgram( program );
 
 	consParseStmt( stmtList );
-	consExportStmt( stmtList );
+	consExportTree( stmtList );
+	consExportError( stmtList );
 
 	pd->rootCodeBlock = CodeBlock::cons( stmtList, 0 );
 }
diff --git a/colm/conscolm.h b/colm/conscolm.h
index 0b641a24..2494900b 100644
--- a/colm/conscolm.h
+++ b/colm/conscolm.h
@@ -61,7 +61,8 @@ struct LoadColm
 
 	/* Constructing statements needed to parse and export the input. */
 	void consParseStmt( StmtList *stmtList );
-	void consExportStmt( StmtList *stmtList );
+	void consExportTree( StmtList *stmtList );
+	void consExportError( StmtList *stmtList );
 
 	void go();
 };
diff --git a/colm/consinit.cc b/colm/consinit.cc
index cb76f38b..03179982 100644
--- a/colm/consinit.cc
+++ b/colm/consinit.cc
@@ -664,12 +664,23 @@ void ConsInit::prodElList()
 	definition( "prod_el_list",  prod1, prod2 );
 }
 
+void ConsInit::optCommit()
+{
+	ProdEl *prodEl1 = prodRefName( "Commit", "COMMIT" );
+	Production *prod1 = production( prodEl1 );
+
+	Production *prod2 = production();
+
+	definition( "opt_commit",  prod1, prod2 );
+}
+
 void ConsInit::prod()
 {
 	ProdEl *prodEl1 = prodRefLit( "'['" );
 	ProdEl *prodEl2 = prodRefName( "ProdElList", "prod_el_list" );
 	ProdEl *prodEl3 = prodRefLit( "']'" );
-	Production *prod1 = production( prodEl1, prodEl2, prodEl3 );
+	ProdEl *prodEl4 = prodRefName( "OptCommit", "opt_commit" );
+	Production *prod1 = production( prodEl1, prodEl2, prodEl3, prodEl4 );
 
 	definition( "prod",  prod1 );
 }
@@ -784,6 +795,7 @@ void ConsInit::go()
 	keyword( "'token'" );
 	keyword( "'ignore'" );
 	keyword( "NI", "'ni'" );
+	keyword( "COMMIT", "'commit'" );
 
 	idToken();
 	literalToken();
@@ -814,6 +826,7 @@ void ConsInit::go()
 	optProdName();
 	prodEl();
 	prodElList();
+	optCommit();
 	prod();
 	prodList();
 	ignore();
diff --git a/colm/consinit.h b/colm/consinit.h
index 3490c2c5..b92115a5 100644
--- a/colm/consinit.h
+++ b/colm/consinit.h
@@ -91,6 +91,7 @@ struct ConsInit
 	void prodList();
 	void prod();
 	void startProd();
+	void optCommit();
 
 	void parseInput( StmtList *stmtList );
 	void exportTree( StmtList *stmtList );
diff --git a/colm/load.cc b/colm/load.cc
index fc477cbe..e199a8fe 100644
--- a/colm/load.cc
+++ b/colm/load.cc
@@ -298,6 +298,9 @@ struct LoadSource
 	{
 		String name = TokenDef.Id().text().c_str();
 
+		bool niLeft = walkOptNoIgnore( TokenDef.NiLeft() );
+		bool niRight = walkOptNoIgnore( TokenDef.NiRight() );
+
 		ObjectDef *objectDef = walkVarDefList( TokenDef.VarDefList() );
 		objectDef->name = name;
 
@@ -309,7 +312,7 @@ struct LoadSource
 
 		CodeBlock *translate = walkOptTranslate( TokenDef.OptTranslate() );
 
-		defineToken( internal, name, join, objectDef, translate, false, false, false );
+		defineToken( internal, name, join, objectDef, translate, false, niLeft, niRight );
 	}
 
 	String walkOptId( opt_id optId )
@@ -617,10 +620,11 @@ struct LoadSource
 
 		/* Extract the parse tree. */
 		start Start = ColmTree( program );
+		str Error = ColmError( program );
 
 		if ( Start == 0 ) {
 			gblErrorCount += 1;
-			std::cerr << inputFileName << ": include parse error" << std::endl;
+			std::cerr << inputFileName << ": parse error: " << Error.text() << std::endl;
 			return 0;
 		}
 
@@ -1859,10 +1863,11 @@ void LoadSource::go()
 
 	/* Extract the parse tree. */
 	start Start = ColmTree( program );
+	str Error = ColmError( program );
 
 	if ( Start == 0 ) {
 		gblErrorCount += 1;
-		std::cerr << inputFileName << ": parse error" << std::endl;
+		std::cerr << inputFileName << ": parse error: " << Error.text() << std::endl;
 		return;
 	}
 
diff --git a/test/ignore5.lm b/test/ignore5.lm
new file mode 100644
index 00000000..d49464d4
--- /dev/null
+++ b/test/ignore5.lm
@@ -0,0 +1,53 @@
+##### LM #####
+lex
+	ignore /space+/
+	literal '*', '!', ';'
+	token POPEN /'('/ ni
+	token PCLOSE ni /')'/
+	token id /[a-zA-Z_0-9]+/ 
+end
+
+lex
+	ignore /space+/
+	token inner_t /[a-zA-Z_0-9]+/ 
+
+	literal ''
+end
+
+def inner
+	['' inner_t*]
+
+def item 
+	[id]
+|	[POPEN inner PCLOSE]
+
+def start 
+	[item* ';']
+
+parse StartP: start[ stdin ]
+Start: start = StartP.tree
+
+if ( ! Start ) {
+	print( 'parse error\n' )
+	exit( 0 )
+}
+
+for I: item in Start {
+	print( 'item: .' %I '.\n' )
+	if match I [ O: POPEN Inner: inner C: PCLOSE ]
+		print( 'innr: .' %O '.' %Inner '.' %C '.\n' )
+}
+
+##### IN #####
+a b c (   d   ) e (  ) f g;
+##### EXP #####
+item: .a .
+item: .b .
+item: .c .
+item: .(   d   ) .
+innr: .(.   d   .) .
+item: .e .
+item: .(  ) .
+innr: .(.  .) .
+item: .f .
+item: .g.