Some cleanup.

3 files changed, 33 insertions, 168 deletions

diff --git a/colm/lmparse.kl b/colm/lmparse.kl
index 20bf2c3f..8bb66456 100644
--- a/colm/lmparse.kl
+++ b/colm/lmparse.kl
@@ -194,49 +194,22 @@ generic_def: KW_Map TK_Word '[' type_ref type_ref ']'
 	final {
 		Namespace *nspace = namespaceStack.top();
 
-//		KlangEl *langEl = getKlangEl( pd, nspace, $2->data );
-
-		/* Check that the element wasn't previously defined as something else. */
-//		if ( langEl->type != KlangEl::Unknown ) {
-//			error($1->loc) << "'" << $2->data << 
-//				"' already defined as something else" << endp;
-//		}
-//		langEl->type = KlangEl::NonTerm;
-
 		GenericType *generic = new GenericType( $2->data, GEN_MAP,
 				pd->nextGenericId++, 0/*langEl*/, $5->typeRef );
 		generic->keyTypeArg = $4->typeRef;
 
-		/* Add one empty production. */
-//		ProdElList *emptyList = new ProdElList;
-//		addProduction( $1->loc, $2->data, emptyList, false, 0, 0 );
-
 		nspace->genericList.append( generic );
-//		langEl->generic = generic;
 	};
 
 generic_def: KW_List TK_Word '[' type_ref ']'
 	final {
 		/* Get the language element. */
 		Namespace *nspace = namespaceStack.top();
-//		KlangEl *langEl = getKlangEl( pd, nspace, $2->data );
-//
-//		/* Check that the element wasn't previously defined as something else. */
-//		if ( langEl->type != KlangEl::Unknown ) {
-//			error($1->loc) << "'" << $2->data << 
-//				"' already defined as something else" << endp;
-//		}
-//		langEl->type = KlangEl::NonTerm;
 
 		GenericType *generic = new GenericType( $2->data, GEN_LIST, 
 				pd->nextGenericId++, 0/*langEl*/, $4->typeRef );
 
-		/* Add one empty production. */
-//		ProdElList *emptyList = new ProdElList;
-//		addProduction( $1->loc, $2->data, emptyList, false, 0, 0 );
-//
 		nspace->genericList.append( generic );
-//		langEl->generic = generic;
 	};
 
 generic_def: KW_Vector TK_Word '[' type_ref ']'
@@ -253,25 +226,11 @@ generic_def: KW_Accum TK_Word '[' type_ref ']'
 	final {
 		/* Get the language element. */
 		Namespace *nspace = namespaceStack.top();
-//		KlangEl *langEl = getKlangEl( pd, nspace, $2->data );
-//
-//		/* Check that the element wasn't previously defined as something else. */
-//		if ( langEl->type != KlangEl::Unknown ) {
-//			error($1->loc) << "'" << $2->data << 
-//				"' already defined as something else" << endp;
-//		}
-//
-//		langEl->type = KlangEl::NonTerm;
 
 		GenericType *generic = new GenericType( $2->data, GEN_PARSER, 
 				pd->nextGenericId++, 0/*langEl*/, $4->typeRef );
 
-		/* Add one empty production. */
-//		ProdElList *emptyList = new ProdElList;
-//		addProduction( $1->loc, $2->data, emptyList, false, 0, 0 );
-
 		nspace->genericList.append( generic );
-//		langEl->generic = generic;
 	};
 
 nonterm global_def uses statement;
@@ -345,12 +304,6 @@ pred_token:
 
 cfl_def: cfl_def_head obj_var_list properties_list cfl_prod_list
 	final {
-//		/* Get the language element. */
-//		KlangEl *langEl = $1->langEl;
-//		langEl->objectDef = pd->objectDef;
-//		if (  )
-//			langEl->reduceFirst = true;
-
 		Namespace *nspace = namespaceStack.top();
 		NtDef *ntDef = new NtDef( 
 				curDefineId,
@@ -366,18 +319,6 @@ cfl_def: cfl_def_head obj_var_list properties_list cfl_prod_list
 cfl_def_head: KW_Def TK_Word
 	final {
 		curDefineId = $2->data;
-
-//		/* Get the language element. */
-//		KlangEl *langEl = getKlangEl( pd, namespaceStack.top(), $2->data );
-//
-//		/* Check that the element wasn't previously defined as something else. */
-//		if ( langEl->type != KlangEl::Unknown ) {
-//			error($2->loc) << "'" << curDefineId << 
-//				"' has already been defined, maybe you want to use redef?" << endp;
-//		}
-//
-//		langEl->type = KlangEl::NonTerm;
-//		$$->langEl = langEl;
 	};
 
 nonterm cfl_prod_list
@@ -448,10 +389,6 @@ define_prod: '[' prod_el_list ']' opt_commit opt_reduce_code opt_prec
 		CodeBlock *redBlock = $5->codeBlock;
 		KlangEl *predOf = $6->predOf;
 
-		/* Get the language element. */
-//		KlangEl *prodName = getKlangEl( pd, namespaceStack.top(), name );
-//
-//		assert( prodName->type == KlangEl::NonTerm );
 		Namespace *nspace = namespaceStack.top();
 
 		Definition *newDef = new Definition( loc, 0/*prodName*/, 
@@ -459,15 +396,8 @@ define_prod: '[' prod_el_list ']' opt_commit opt_reduce_code opt_prec
 			pd->prodList.length(), Definition::Production );
 		newDef->predOf = predOf;
 		
-//		prodName->defList.append( newDef );
 		pd->prodList.append( newDef );
 
-//		if ( strcmp( name, nspace->name ) == 0 ) {
-//			/* Insert the name into the top of the region stack after popping the
-//			 * region just created. We need it in the parent. */
-//			nspace->parentNamespace->symbolMap.insert( name, prodName );
-//		}
-
 		$$->definition = newDef;
 	};
 
@@ -504,25 +434,11 @@ type_ref: KW_Map '<' type_ref type_ref '>'
 		static int mapId = 0;
 		String name( 36, "__map%d", mapId++ );
 
-//		KlangEl *langEl = getKlangEl( pd, nspace, name );
-//
-//		/* Check that the element wasn't previously defined as something else. */
-//		if ( langEl->type != KlangEl::Unknown ) {
-//			error($1->loc) << "'" << name  << 
-//				"' already defined as something else" << endp;
-//		}
-//		langEl->type = KlangEl::NonTerm;
-
 		GenericType *generic = new GenericType( name, GEN_MAP,
 				pd->nextGenericId++, 0, $4->typeRef );
 		generic->keyTypeArg = $3->typeRef;
 
-		/* Add one empty production. */
-//		ProdElList *emptyList = new ProdElList;
-//		addProduction( $1->loc, String(), emptyList, false, 0, 0 );
-
 		nspace->genericList.append( generic );
-//		langEl->generic = generic;
 
 		NamespaceQual *nspaceQual = new NamespaceQual( namespaceStack.top(), regionStack.top() );
 		$$->typeRef = new TypeRef( InputLoc(), nspaceQual, name );
@@ -986,21 +902,13 @@ literal_item: TK_Literal
 		if ( ldel != 0 )
 			error( $1->loc ) << "literal already defined in this namespace" << endp;
 		else {
-			/* Create a token for the literal. */
-//			KlangEl *newLangEl = getKlangEl( pd, nspace, name );
-//			assert( newLangEl->type == KlangEl::Unknown );
-//			newLangEl->type = KlangEl::Term;
-//			newLangEl->lit = $1->data;
-//			newLangEl->isLiteral = true;
-
 			Join *join = new Join( new Expression( new Term( new FactorWithAug(
 				new FactorWithRep( $1->loc, new FactorWithNeg( $1->loc, new Factor(
 				new Literal( $1->loc, $1->data, 
 					Literal::LitString ) ) ) ) ) ) ) );
 			
 			TokenDef *tokenDef = new TokenDef( name, $1->data, true, false, join, 
-					0, 0 /*newLangEl*/, $1->loc, pd->nextTokenId, nspace, region, 0, 0, 0 );
-//			newLangEl->tokenDef = tokenDef;
+					0, $1->loc, pd->nextTokenId, nspace, region, 0, 0, 0 );
 
 			region->tokenDefList.append( tokenDef );
 			ldel = nspace->literalDict.insert( interp, tokenDef );
@@ -1054,26 +962,16 @@ token_def:
 		if ( name == 0 )
 			name.setAs( 32, "_ignore_%.4x", pd->nextTokenId );
 
-//		/* Create the token. */
-//		KlangEl *tokEl = getKlangEl( pd, namespaceStack.top(), name );
-//		if ( tokEl->type != KlangEl::Unknown )
-//			error($1->loc) << "'" << name << "' already defined" << endp;
-
-//		tokEl->type = KlangEl::Term;
-//		tokEl->ignore = ignore;
-//		tokEl->transBlock = transBlock;
-
 		Namespace *nspace = namespaceStack.top();
 		TokenRegion *region = regionStack.top();
 
 		TokenDef *tokenDef = new TokenDef( name, String(), false, ignore, join, 
-				transBlock, 0 /*tokEl*/, $1->loc, pd->nextTokenId++, nspace, region,
+				transBlock, $1->loc, pd->nextTokenId++, nspace, region,
 				&reCaptureVect, pd->objectDef,
 				contextStack.length() > 0 ? contextStack.top() : 0 );
 
 		region->tokenDefList.append( tokenDef );
 		nspace->tokenDefList.append( tokenDef );
-//		tokEl->tokenDef = tokenDef;
 
 		reCaptureVect.empty();
 
@@ -1089,9 +987,6 @@ token_def:
 			JoinOrLm *joinOrLm = new JoinOrLm( join );
 			addRegularDef( $1->loc, namespaceStack.top(), name, joinOrLm, false );
 		}
-
-//		if ( contextStack.length() > 0 )
-//			tokEl->contextIn = contextStack.top();
 	};
 
 nonterm token_or_ignore
@@ -1242,13 +1137,6 @@ start_reduce:
 	block_open
 	final {
 		$$->localFrame = $1->localFrame;
-
-		/* Get the language element. */
-//		KlangEl *prodName = getKlangEl( pd, namespaceStack.top(), curDefineId );
-
-		/* References to the reduce item. */
-//		pd->addProdRedObjectVar( pd->curLocalFrame, prodName );
-//		pd->addProdRHSVars( pd->curLocalFrame, curProdElList );
 	};
 
 nonterm lang_stmt_list
@@ -1812,25 +1700,11 @@ code_factor: KW_Parse opt_capture type_ref '(' opt_code_expr_list ')'
 
 		/* Get the language element. */
 		Namespace *nspace = namespaceStack.top();
-//		KlangEl *langEl = getKlangEl( pd, nspace, parserName );
-//
-//		/* Check that the element wasn't previously defined as something else. */
-//		if ( langEl->type != KlangEl::Unknown ) {
-//			error($1->loc) << "'" << parserName << 
-//				"' already defined as something else" << endp;
-//		}
-//
-//		langEl->type = KlangEl::NonTerm;
 
 		GenericType *generic = new GenericType( parserName, GEN_PARSER, 
 				pd->nextGenericId++, 0/*langEl*/, $3->typeRef );
 
-//		/* Add one empty production. */
-//		ProdElList *emptyList = new ProdElList;
-//		addProduction( $1->loc, parserName, emptyList, false, 0, 0 );
-
 		nspace->genericList.append( generic );
-//		langEl->generic = generic;
 
 		Replacement *replacement = new Replacement( $1->loc, nspace, pd->rootRegion,
 				new ReplItemList, pd->nextPatReplId++ );
@@ -1863,25 +1737,11 @@ code_factor: KW_ParseStop opt_capture type_ref '(' opt_code_expr_list ')'
 
 		/* Get the language element. */
 		Namespace *nspace = namespaceStack.top();
-//		KlangEl *langEl = getKlangEl( pd, nspace, parserName );
-//
-//		/* Check that the element wasn't previously defined as something else. */
-//		if ( langEl->type != KlangEl::Unknown ) {
-//			error($1->loc) << "'" << parserName << 
-//				"' already defined as something else" << endp;
-//		}
-//
-//		langEl->type = KlangEl::NonTerm;
 
 		GenericType *generic = new GenericType( parserName, GEN_PARSER, 
 				pd->nextGenericId++, 0/*langEl*/, $3->typeRef );
 
-//		/* Add one empty production. */
-//		ProdElList *emptyList = new ProdElList;
-//		addProduction( $1->loc, parserName, emptyList, false, 0, 0 );
-
 		nspace->genericList.append( generic );
-//		langEl->generic = generic;
 
 		Replacement *replacement = new Replacement( $1->loc, nspace, pd->rootRegion,
 				new ReplItemList, pd->nextPatReplId++ );
@@ -2548,19 +2408,13 @@ void Parser::addArgvList()
 {
 	/* Get the language element. */
 	Namespace *nspace = namespaceStack.top();
-//	KlangEl *langEl = getKlangEl( pd, nspace, "argv_list" );
-//	langEl->type = KlangEl::NonTerm;
 
 	NamespaceQual *nspaceQual = new NamespaceQual( namespaceStack.top(), regionStack.top() );
 	TypeRef *typeRef = new TypeRef( InputLoc(), nspaceQual, "str" );
 	GenericType *generic = new GenericType( "argv_list", GEN_LIST, 
 			pd->nextGenericId++, 0/*langEl*/, typeRef );
-	/* Add one empty production. */
-//	ProdElList *emptyList = new ProdElList;
-//	addProduction( InputLoc(), "argv_list", emptyList, false, 0, 0 );
 
 	nspace->genericList.append( generic );
-//	langEl->generic = generic;
 
 	pd->argvList = generic;
 
diff --git a/colm/parsedata.cc b/colm/parsedata.cc
index c18c6317..2128006e 100644
--- a/colm/parsedata.cc
+++ b/colm/parsedata.cc
@@ -785,6 +785,7 @@ Action *ParseData::newAction( const String &name, InlineList *inlineList )
 	InputLoc loc;
 	loc.line = 1;
 	loc.col = 1;
+	loc.fileName = 0;
 
 	Action *action = new Action( loc, name, inlineList );
 	actionList.append( action );
@@ -1076,7 +1077,7 @@ FsmGraph *ParseData::makeFsmGraph( Join *join )
 
 void ParseData::createDefaultScanner()
 {
-	InputLoc loc;
+	InputLoc loc = { 0, 0, 0 };
 
 	const char *name = "___DEFAULT_SCANNER";
 
@@ -1099,18 +1100,21 @@ void ParseData::createDefaultScanner()
 	newEl->isInstance = true;
 	instanceList.append( newEl );
 
+	Join *join = new Join( new Expression( BT_Any ) );
+		
+	TokenDef *tokenDef = new TokenDef( name, String(), false, false, 
+			join, 0, loc, nextTokenId++, 
+			rootNamespace, defaultRegion, 0, 0, 0 );
+
+	defaultRegion->tokenDefList.append( tokenDef );
+
 	/* Now create the one and only token -> "<chr>" / any /  */
 	name = "___DEFAULT_SCANNER_CHR";
 	defaultCharKlangEl = getKlangEl( this, defaultNamespace, name );
 	assert( defaultCharKlangEl->type == KlangEl::Unknown );
 	defaultCharKlangEl->type = KlangEl::Term;
 
-	Join *join = new Join( new Expression( BT_Any ) );
-		
-	TokenDef *tokenDef = new TokenDef( name, String(), false, false, 
-			join, 0, defaultCharKlangEl, loc, nextTokenId++, 
-			rootNamespace, defaultRegion, 0, 0, 0 );
-	defaultRegion->tokenDefList.append( tokenDef );
+	tokenDef->token = defaultCharKlangEl;
 	defaultCharKlangEl->tokenDef = tokenDef;
 }
 
@@ -1459,18 +1463,22 @@ void ParseData::initEmptyScanners()
 {
 	for ( RegionList::Iter reg = regionList; reg.lte(); reg++ ) {
 		if ( reg->tokenDefList.length() == 0 ) {
-			InputLoc loc;
+			InputLoc loc = { 0, 0, 0 };
 			String name( reg->name.length() + 16, "__%s_DEF_PAT", reg->name.data );
 
-			KlangEl *lel = getKlangEl( this, rootNamespace, name );
-			assert( lel->type == KlangEl::Unknown );
-			lel->type = KlangEl::Term;
-
 			Join *join = new Join( new Expression( BT_Any ) );
 				
 			TokenDef *tokenDef = new TokenDef( name, String(), false, false, join, 
-					0, lel, loc, nextTokenId++, rootNamespace, reg, 0, 0, 0 );
+					0, loc, nextTokenId++, rootNamespace, reg, 0, 0, 0 );
 			reg->tokenDefList.append( tokenDef );
+
+			/* These do not go in the namespace so so they cannot get declared
+			 * in the declare pass. */
+			KlangEl *lel = getKlangEl( this, rootNamespace, name );
+			assert( lel->type == KlangEl::Unknown );
+			lel->type = KlangEl::Term;
+
+			tokenDef->token = lel;
 			lel->tokenDef = tokenDef;
 		}
 	}
@@ -1583,13 +1591,18 @@ void ParseData::semanticAnalysis()
 
 	rootNamespace->declare( this );
 
+	/* Fill any empty scanners with a default token. */
+	initEmptyScanners();
+
+	/* Create the default scanner which will return single characters for us
+	 * when we have no other scanner */
+	createDefaultScanner();
+
 	/* Resolve uses statements. */
 	resolveUses();
 	
-	/* Init the longest match data and create the default scanner which will
-	 * return single characters for us when we have no other scanner */
+	/* Init the longest match data */
 	initLongestMatchData();
-	createDefaultScanner();
 
 	/* Resolve pattern and replacement elements. */
 	resolvePatternEls();
@@ -1601,8 +1614,6 @@ void ParseData::semanticAnalysis()
 	/* This needs to happen before the scanner is built. */
 	resolveProductionEls();
 
-	/* Fill any empty scanners with a default token. */
-	initEmptyScanners();
 
 	FsmGraph *fsmGraph = makeScanner();
 
diff --git a/colm/parsetree.h b/colm/parsetree.h
index 4c76a24f..3634995b 100644
--- a/colm/parsetree.h
+++ b/colm/parsetree.h
@@ -386,12 +386,12 @@ struct TokenDef
 	public TokenDefPtr2
 {
 	TokenDef( const String &name, const String &literal, bool isLiteral, bool ignore,
-		Join *join, CodeBlock *codeBlock, KlangEl *token, InputLoc &semiLoc, 
+		Join *join, CodeBlock *codeBlock, InputLoc &semiLoc, 
 		int longestMatchId, Namespace *nspace, TokenRegion *tokenRegion,
 		ReCaptureVect *pReCaptureVect, ObjectDef *objectDef, Context *contextIn )
 	: 
 		name(name), literal(literal), isLiteral(isLiteral), ignore(ignore), join(join), action(0),
-		codeBlock(codeBlock), token(token), semiLoc(semiLoc), 
+		codeBlock(codeBlock), token(0), semiLoc(semiLoc), 
 		longestMatchId(longestMatchId), inLmSelect(false), 
 		nspace(nspace), tokenRegion(tokenRegion), objectDef(objectDef),
 		contextIn(contextIn)