Now translating only the parseTree->id to termDup from nonterminals. The data

tree id always remains as the nonterminal.

The nested comm output needs to be adjusted because it is now functioning
correctly. A nonterm was sent as a comment and was not translated back to the
nonterm from the dup when the comment was attached. Now that translation
doesn't happen at all and so it is correctly a nonterminal.

4 files changed, 13 insertions, 44 deletions

diff --git a/colm/bytecode.c b/colm/bytecode.c
index 76d1df22..fa7c9462 100644
--- a/colm/bytecode.c
+++ b/colm/bytecode.c
@@ -217,12 +217,7 @@ Word streamAppend( Program *prg, Tree **sp, Tree *input, InputStream *inputStrea
 	}
 	else {
 		input = prepParseTree( prg, sp, input );
-
-		if ( input->id >= prg->rtd->firstNonTermId )
-			input->id = prg->rtd->lelInfo[input->id].termDupId;
-
 		input->flags |= AF_ARTIFICIAL;
-
 		treeUpref( input );
 		appendTree( inputStream, input );
 	}
@@ -385,10 +380,6 @@ long streamPush( Program *prg, Tree **sp, FsmRun *fsmRun, InputStream *in, Tree
 	}
 	else {
 		tree = prepParseTree( prg, sp, tree );
-
-		if ( tree->id >= prg->rtd->firstNonTermId )
-			tree->id = prg->rtd->lelInfo[tree->id].termDupId;
-
 		tree->flags |= AF_ARTIFICIAL;
 		treeUpref( tree );
 		streamPushTree( fsmRun, in, tree, ignore );
diff --git a/colm/ctinput.cc b/colm/ctinput.cc
index be3bbdcd..40e80f29 100644
--- a/colm/ctinput.cc
+++ b/colm/ctinput.cc
@@ -391,8 +391,6 @@ void sendNamedLangEl( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, I
 	long length;
 
 	LangEl *klangEl = consumeLangEl( inputStream, &bindId, &data, &length );
-	if ( klangEl->termDup != 0 )
-		klangEl = klangEl->termDup;
 	
 	#ifdef COLM_LOG_PARSE
 	if ( colm_log_parse ) {
diff --git a/colm/pdarun.c b/colm/pdarun.c
index 2f44f996..af52cef0 100644
--- a/colm/pdarun.c
+++ b/colm/pdarun.c
@@ -526,9 +526,7 @@ static void reportParseError( Program *prg, Tree **sp, PdaRun *pdaRun )
 
 static void attachIgnoreRight( Program *prg, Tree **sp, PdaRun *pdaRun )
 {
-	if ( pdaRun->stackTop->id < prg->rtd->firstNonTermId || 
-			(pdaRun->stackTop->flags & AF_TERM_DUP) )
-	{
+	if ( pdaRun->stackTop->id < prg->rtd->firstNonTermId ) {
 		/* OK, do it */
 		Tree *alter = pdaRun->stackTop->shadow->tree;
 
@@ -1260,6 +1258,14 @@ case PcrGeneration:
 		if ( pdaRun->parseInput != 0 )
 			transferReverseCode( pdaRun, pdaRun->parseInput );
 
+		if ( pdaRun->parseInput != 0 ) {
+			/* If it's a nonterminal with a termdup then flip the parse tree to the terminal. */
+			if ( pdaRun->parseInput->id >= prg->rtd->firstNonTermId ) {
+				pdaRun->parseInput->id = prg->rtd->lelInfo[pdaRun->parseInput->id].termDupId;
+				pdaRun->parseInput->flags |= AF_TERM_DUP;
+			}
+		}
+
 		long pcr = parseToken( prg, sp, pdaRun, fsmRun, inputStream, PcrStart );
 		
 		while ( pcr != PcrDone ) {
@@ -1745,17 +1751,6 @@ again:
 			pdaRun->tokenList = ref;
 		}
 
-		/* If shifting a termDup then change it to the nonterm. */
-		if ( pdaRun->lel->id < prg->rtd->firstNonTermId &&
-				prg->rtd->lelInfo[pdaRun->lel->id].termDupId > 0 )
-		{
-			pdaRun->lel->id = prg->rtd->lelInfo[pdaRun->lel->id].termDupId;
-			pdaRun->lel->flags |= AF_TERM_DUP;
-
-			pdaRun->lel->shadow->tree->id = 
-					prg->rtd->lelInfo[pdaRun->lel->shadow->tree->id].termDupId;
-		}
-
 		if ( action[1] == 0 )
 			pdaRun->lel->retryLower = 0;
 		else {
@@ -1981,9 +1976,7 @@ case PcrReverse:
 		else if ( pdaRun->parseInput != 0 ) {
 			/* Either we are dealing with a terminal that was
 			 * shifted or a nonterminal that was reduced. */
-			if ( pdaRun->parseInput->id < prg->rtd->firstNonTermId || 
-					(pdaRun->parseInput->flags & AF_TERM_DUP) )
-			{
+			if ( pdaRun->parseInput->id < prg->rtd->firstNonTermId ) {
 				assert( pdaRun->parseInput->retryUpper == 0 );
 
 				if ( pdaRun->parseInput->retryLower != 0 ) {
@@ -2006,8 +1999,7 @@ case PcrReverse:
 
 					/* Either we are dealing with a terminal that was
 					 * shifted or a nonterminal that was reduced. */
-					assert( !(pdaRun->stackTop->id < prg->rtd->firstNonTermId || 
-							(pdaRun->stackTop->flags & AF_TERM_DUP) ) );
+					assert( !(pdaRun->stackTop->id < prg->rtd->firstNonTermId) );
 
 					debug( REALM_PARSE, "backing up over non-terminal: %s\n",
 							prg->rtd->lelInfo[pdaRun->stackTop->id].name );
@@ -2129,25 +2121,13 @@ case PcrReverse:
 
 			/* Either we are dealing with a terminal that was
 			 * shifted or a nonterminal that was reduced. */
-			if ( pdaRun->stackTop->id < prg->rtd->firstNonTermId || 
-					(pdaRun->stackTop->flags & AF_TERM_DUP) )
-			{
+			if ( pdaRun->stackTop->id < prg->rtd->firstNonTermId ) {
 				debug( REALM_PARSE, "backing up over effective terminal: %s\n",
 							prg->rtd->lelInfo[pdaRun->stackTop->id].name );
 
 				/* Pop the item from the stack. */
 				pdaRun->stackTop = pdaRun->stackTop->next;
 
-				/* Undo the translation from termDup. */
-				if ( pdaRun->undoLel->flags & AF_TERM_DUP ) {
-					pdaRun->undoLel->id = prg->rtd->lelInfo[pdaRun->undoLel->id].termDupId;
-					pdaRun->undoLel->flags &= ~AF_TERM_DUP;
-
-					pdaRun->undoLel->shadow->tree->id = 
-							prg->rtd->lelInfo[pdaRun->undoLel->shadow->tree->id].termDupId;
-					pdaRun->undoLel->shadow->tree->flags &= ~AF_TERM_DUP;
-				}
-
 				/* Queue it as next parseInput item. */
 				pdaRun->undoLel->next = pdaRun->parseInput;
 				pdaRun->parseInput = pdaRun->undoLel;
diff --git a/test/nestedcomm.exp b/test/nestedcomm.exp
index 38a6245f..bfb6bc98 100644
--- a/test/nestedcomm.exp
+++ b/test/nestedcomm.exp
@@ -1 +1 @@
-hello there ( (this is a nested comment /*sdf;asd_++_stuff) ) and this is not<nested><_repeat_id><id>hello</id><id>there</id><id>and</id><id>this</id><id>is</id><id>not</id></_repeat_id></nested><nested><_repeat_id><id>hello</id><_ignore_0001> </_ignore_0001><id>there</id><_ignore_0001> </_ignore_0001><_T_nested_comment><_literal_0004>(</_literal_0004><_repeat_nc_item><nc_item><nc_data> </nc_data></nc_item><nc_item><nested_comment><_literal_0004>(</_literal_0004><_repeat_nc_item><nc_item><nc_data>this is a nested comment /*sdf;asd_++_stuff</nc_data></nc_item></_repeat_nc_item><_literal_0005>)</_literal_0005></nested_comment></nc_item><nc_item><nc_data> </nc_data></nc_item></_repeat_nc_item><_literal_0005>)</_literal_0005></_T_nested_comment><_ignore_0001> </_ignore_0001><id>and</id><_ignore_0001> </_ignore_0001><id>this</id><_ignore_0001> </_ignore_0001><id>is</id><_ignore_0001> </_ignore_0001><id>not</id></_repeat_id></nested>hello there ( (this is a nested comment /*sdf;asd_++_stuff) ) and this is not
+hello there ( (this is a nested comment /*sdf;asd_++_stuff) ) and this is not<nested><_repeat_id><id>hello</id><id>there</id><id>and</id><id>this</id><id>is</id><id>not</id></_repeat_id></nested><nested><_repeat_id><id>hello</id><_ignore_0001> </_ignore_0001><id>there</id><_ignore_0001> </_ignore_0001><nested_comment><_literal_0004>(</_literal_0004><_repeat_nc_item><nc_item><nc_data> </nc_data></nc_item><nc_item><nested_comment><_literal_0004>(</_literal_0004><_repeat_nc_item><nc_item><nc_data>this is a nested comment /*sdf;asd_++_stuff</nc_data></nc_item></_repeat_nc_item><_literal_0005>)</_literal_0005></nested_comment></nc_item><nc_item><nc_data> </nc_data></nc_item></_repeat_nc_item><_literal_0005>)</_literal_0005></nested_comment><_ignore_0001> </_ignore_0001><id>and</id><_ignore_0001> </_ignore_0001><id>this</id><_ignore_0001> </_ignore_0001><id>is</id><_ignore_0001> </_ignore_0001><id>not</id></_repeat_id></nested>hello there ( (this is a nested comment /*sdf;asd_++_stuff) ) and this is not