Need to collect tokens and nonterminals at parse time so we can declare the

language elements in the declarating pass.

7 files changed, 56 insertions, 16 deletions

diff --git a/colm/fsmcodegen.cc b/colm/fsmcodegen.cc
index 59f35ce0..d098b0c3 100644
--- a/colm/fsmcodegen.cc
+++ b/colm/fsmcodegen.cc
@@ -221,7 +221,7 @@ void FsmCodeGen::LM_SWITCH( ostream &ret, InlineItem *item,
 				"; goto st" << redFsm->errState->id << ";\n";
 	}
 
-	for ( TokenDefList::Iter lmi = item->tokenRegion->tokenDefList; lmi.lte(); lmi++ ) {
+	for ( TokenDefListReg::Iter lmi = item->tokenRegion->tokenDefList; lmi.lte(); lmi++ ) {
 		if ( lmi->inLmSelect ) {
 			assert( lmi->token != 0 );
 			ret << "	case " << lmi->longestMatchId << ":\n";
diff --git a/colm/fsmexec.cc b/colm/fsmexec.cc
index d6c28f2f..80e0cdc0 100644
--- a/colm/fsmexec.cc
+++ b/colm/fsmexec.cc
@@ -62,7 +62,7 @@ void execAction( FsmRun *fsmRun, GenAction *genAction )
 				fsmRun->cs = fsmRun->tables->errorState;
 			}
 			else {
-				for ( TokenDefList::Iter lmi = item->tokenRegion->tokenDefList; 
+				for ( TokenDefListReg::Iter lmi = item->tokenRegion->tokenDefList; 
 						lmi.lte(); lmi++ )
 				{
 					if ( lmi->inLmSelect && fsmRun->act == lmi->longestMatchId )
diff --git a/colm/lmparse.kl b/colm/lmparse.kl
index 6295bfae..7ac3671a 100644
--- a/colm/lmparse.kl
+++ b/colm/lmparse.kl
@@ -353,6 +353,10 @@ cfl_def: cfl_def_head obj_var_list properties_list cfl_prod_list
 
 		if ( $3->property & PROPERTY_REDUCE_FIRST )
 			langEl->reduceFirst = true;
+
+		Namespace *nspace = namespaceStack.top();
+		NtDef *ntDef = new NtDef( $1->langEl->name, nspace );
+		nspace->ntDefList.append( ntDef );
 	};
 
 nonterm cfl_def_head
@@ -970,6 +974,8 @@ literal_item: TK_Literal
 			region->tokenDefList.append( tokenDef );
 			ldel = nspace->literalDict.insert( interp, tokenDef );
 			pd->nextTokenId += 1;
+
+			nspace->tokenDefList.append( tokenDef );
 		}
 
 		if ( !insideRegion ) {
@@ -1032,6 +1038,7 @@ token_def:
 		TokenDef *tokenDef = new TokenDef( name, String(), join, tokEl, $1->loc, 
 				pd->nextTokenId++, nspace, region );
 		region->tokenDefList.append( tokenDef );
+		nspace->tokenDefList.append( tokenDef );
 		tokEl->tokenDef = tokenDef;
 
 		tokenDef->reCaptureVect = reCaptureVect;
diff --git a/colm/parsedata.cc b/colm/parsedata.cc
index a1984ec0..a3959e07 100644
--- a/colm/parsedata.cc
+++ b/colm/parsedata.cc
@@ -994,7 +994,7 @@ void ParseData::analyzeAction( Action *action, InlineList *inlineList )
 		/* Need to recurse into longest match items. */
 		if ( item->type == InlineItem::LmSwitch ) {
 			TokenRegion *lm = item->tokenRegion;
-			for ( TokenDefList::Iter lmi = lm->tokenDefList; lmi.lte(); lmi++ ) {
+			for ( TokenDefListReg::Iter lmi = lm->tokenDefList; lmi.lte(); lmi++ ) {
 				if ( lmi->action != 0 )
 					analyzeAction( action, lmi->action->inlineList );
 			}
diff --git a/colm/parsetree.cc b/colm/parsetree.cc
index 6d6407c8..6e2b4114 100644
--- a/colm/parsetree.cc
+++ b/colm/parsetree.cc
@@ -205,7 +205,7 @@ Action *TokenRegion::newAction( ParseData *pd, const InputLoc &loc,
 void TokenRegion::makeActions( ParseData *pd )
 {
 	/* Make actions that set the action id. */
-	for ( TokenDefList::Iter lmi = tokenDefList; lmi.lte(); lmi++ ) {
+	for ( TokenDefListReg::Iter lmi = tokenDefList; lmi.lte(); lmi++ ) {
 		/* For each part create actions for setting the match type.  We need
 		 * to do this so that the actions will go into the actionIndex. */
 		InlineList *inlineList = new InlineList;
@@ -217,7 +217,7 @@ void TokenRegion::makeActions( ParseData *pd )
 	}
 
 	/* Make actions that execute the user action and restart on the last character. */
-	for ( TokenDefList::Iter lmi = tokenDefList; lmi.lte(); lmi++ ) {
+	for ( TokenDefListReg::Iter lmi = tokenDefList; lmi.lte(); lmi++ ) {
 		/* For each part create actions for setting the match type.  We need
 		 * to do this so that the actions will go into the actionIndex. */
 		InlineList *inlineList = new InlineList;
@@ -231,7 +231,7 @@ void TokenRegion::makeActions( ParseData *pd )
 	/* Make actions that execute the user action and restart on the next
 	 * character.  These actions will set tokend themselves (it is the current
 	 * char). */
-	for ( TokenDefList::Iter lmi = tokenDefList; lmi.lte(); lmi++ ) {
+	for ( TokenDefListReg::Iter lmi = tokenDefList; lmi.lte(); lmi++ ) {
 		/* For each part create actions for setting the match type.  We need
 		 * to do this so that the actions will go into the actionIndex. */
 		InlineList *inlineList = new InlineList;
@@ -244,7 +244,7 @@ void TokenRegion::makeActions( ParseData *pd )
 
 	/* Make actions that execute the user action and restart at tokend. These
 	 * actions execute some time after matching the last char. */
-	for ( TokenDefList::Iter lmi = tokenDefList; lmi.lte(); lmi++ ) {
+	for ( TokenDefListReg::Iter lmi = tokenDefList; lmi.lte(); lmi++ ) {
 		/* For each part create actions for setting the match type.  We need
 		 * to do this so that the actions will go into the actionIndex. */
 		InlineList *inlineList = new InlineList;
@@ -292,7 +292,7 @@ void TokenRegion::makeNameTree( ParseData *pd )
 	regionNameInst = pd->curNameInst;
 
 	/* Recurse into all parts of the longest match operator. */
-	for ( TokenDefList::Iter td = tokenDefList; td.lte(); td++ ) {
+	for ( TokenDefListReg::Iter td = tokenDefList; td.lte(); td++ ) {
 		/* Watch out for patternless tokens. */
 		if ( td->join != 0 ) 
 			td->join->makeNameTree( pd );
@@ -314,7 +314,7 @@ void TokenRegion::resolveNameRefs( ParseData *pd )
 	NameFrame nameFrame = pd->enterNameScope( true, 1 );
 
 	/* Take an action reference for each longest match item and recurse. */
-	for ( TokenDefList::Iter lmi = tokenDefList; lmi.lte(); lmi++ ) {
+	for ( TokenDefListReg::Iter lmi = tokenDefList; lmi.lte(); lmi++ ) {
 		/* Watch out for patternless tokens. */
 		if ( lmi->join != 0 )
 			lmi->join->resolveNameRefs( pd );
@@ -520,7 +520,7 @@ FsmGraph *TokenRegion::walk( ParseData *pd )
 	/* Make each part of the longest match. */
 	int numParts = 0;
 	FsmGraph **parts = new FsmGraph*[tokenDefList.length()];
-	for ( TokenDefList::Iter lmi = tokenDefList; lmi.lte(); lmi++ ) {
+	for ( TokenDefListReg::Iter lmi = tokenDefList; lmi.lte(); lmi++ ) {
 		/* Watch out for patternless tokens. */
 		if ( lmi->join != 0 ) {
 			/* Create the machine and embed the setting of the longest match id. */
diff --git a/colm/parsetree.h b/colm/parsetree.h
index ea28a3a0..4ced99d1 100644
--- a/colm/parsetree.h
+++ b/colm/parsetree.h
@@ -31,6 +31,7 @@
 #include "vector.h"
 #include "dlist.h"
 #include "dlistval.h"
+#include "dlistmel.h"
 #include "astring.h"
 #include "bytecode.h"
 #include "avlbasic.h"
@@ -167,7 +168,8 @@ struct TokenRegion;
 struct Namespace;
 struct Context;
 struct TokenDef;
-struct TokenDefList;
+struct TokenDefListReg;
+struct TokenDefListNs;
 struct Range;
 struct KlangEl;
 
@@ -368,7 +370,20 @@ struct Context
 
 typedef Vector<ReCapture> ReCaptureVect;
 
+struct TokenDefPtr1
+{
+	TokenDef *prev, *next;
+};
+
+struct TokenDefPtr2
+{
+	TokenDef *prev, *next;
+};
+
 struct TokenDef
+:
+	public TokenDefPtr1, 
+	public TokenDefPtr2
 {
 	TokenDef( const String &name, const String &literal, Join *join,
 		KlangEl *token, InputLoc &semiLoc, 
@@ -396,12 +411,24 @@ struct TokenDef
 	Namespace *nspace;
 	TokenRegion *tokenRegion;
 	ReCaptureVect reCaptureVect;
+};
 
-	TokenDef *prev, *next;
+struct NtDef
+{
+	NtDef( const String &name, Namespace *nspace )
+	: 
+		name(name), nspace(nspace) {}
+
+	String name;
+	Namespace *nspace;
+
+	NtDef *prev, *next;
 };
 
 /* Declare a new type so that ptreetypes.h need not include dlist.h. */
-struct TokenDefList : DList<TokenDef> {};
+struct TokenDefListReg : DListMel<TokenDef, TokenDefPtr1> {};
+struct TokenDefListNs : DListMel<TokenDef, TokenDefPtr2> {};
+struct NtDefList : DList<NtDef> {};
 
 /* Symbol Map. */
 typedef AvlMap< String, KlangEl*, CmpStr > SymbolMap;
@@ -432,7 +459,7 @@ struct TokenRegion
 	void restart( FsmGraph *graph, FsmTrans *trans );
 
 	InputLoc loc;
-	TokenDefList tokenDefList;
+	TokenDefListReg tokenDefList;
 	String name;
 	int id;
 
@@ -534,6 +561,12 @@ struct Namespace
 	 * tokens. */
 	LiteralDict literalDict;
 
+	/* List of tokens defs in the namespace. */
+	TokenDefListNs tokenDefList;
+
+	/* List of nonterminal defs in the namespace. */
+	NtDefList ntDefList;
+
 	/* Dictionary of symbols within the region. */
 	SymbolMap symbolMap;
 	GenericList genericList;
diff --git a/colm/pdabuild.cc b/colm/pdabuild.cc
index 2a8b5e6e..290a46f2 100644
--- a/colm/pdabuild.cc
+++ b/colm/pdabuild.cc
@@ -1525,7 +1525,7 @@ void ParseData::makeRuntimeData()
 	/* Captured attributes. Loop over tokens and count first. */
 	long numCapturedAttr = 0;
 	for ( RegionList::Iter reg = regionList; reg.lte(); reg++ ) {
-		for ( TokenDefList::Iter td = reg->tokenDefList; td.lte(); td++ )
+		for ( TokenDefListReg::Iter td = reg->tokenDefList; td.lte(); td++ )
 			numCapturedAttr += td->reCaptureVect.length();
 	}
 	runtimeData->captureAttr = new CaptureAttr[numCapturedAttr];
@@ -1533,7 +1533,7 @@ void ParseData::makeRuntimeData()
 
 	count = 0;
 	for ( RegionList::Iter reg = regionList; reg.lte(); reg++ ) {
-		for ( TokenDefList::Iter td = reg->tokenDefList; td.lte(); td++ ) {
+		for ( TokenDefListReg::Iter td = reg->tokenDefList; td.lte(); td++ ) {
 			runtimeData->lelInfo[td->token->id].captureAttr = count;
 			runtimeData->lelInfo[td->token->id].numCaptureAttr = td->reCaptureVect.length();
 			for ( ReCaptureVect::Iter c = td->reCaptureVect; c.lte(); c++ ) {