Now attaching ignores when tokens are shifted. Allows better error reporting

and also simplifies the backtracker. There is only one place we need to send
back ignores and it can happen in the main parse loop. 

The region used is now going into ignore tokens, which means ignores (at least
those with retries) need to be parse trees. Previously had every parsed item
storing the region, not so anymore, only the first time the retry is
encountered.

refs #333.

5 files changed, 136 insertions, 133 deletions

diff --git a/colm/bytecode.c b/colm/bytecode.c
index 3265c033..30218e37 100644
--- a/colm/bytecode.c
+++ b/colm/bytecode.c
@@ -254,9 +254,7 @@ return pcr;
 case PcrReduction:
 case PcrGeneration:
 case PcrPreEof:
-case PcrRevIgnore1:
-case PcrRevIgnore2:
-case PcrRevIgnore3:
+case PcrRevIgnore:
 case PcrRevToken:
 case PcrRevReduction:
 
@@ -291,9 +289,7 @@ return pcr;
 case PcrReduction:
 case PcrGeneration:
 case PcrPreEof:
-case PcrRevIgnore1:
-case PcrRevIgnore2:
-case PcrRevIgnore3:
+case PcrRevIgnore:
 case PcrRevToken:
 case PcrRevReduction:
 
@@ -345,9 +341,7 @@ return pcr;
 case PcrReduction:
 case PcrGeneration:
 case PcrPreEof:
-case PcrRevIgnore1:
-case PcrRevIgnore2:
-case PcrRevIgnore3:
+case PcrRevIgnore:
 case PcrRevToken:
 case PcrRevReduction:
 
@@ -967,9 +961,7 @@ void callParseBlock( Code **pinstr, Tree ***psp, long pcr, Program *prg,
 			break;
 		}
 
-		case PcrRevIgnore1:
-		case PcrRevIgnore2:
-		case PcrRevIgnore3: {
+		case PcrRevIgnore: {
 
 			initReverseExecution( pdaRun->exec, prg, &pdaRun->rcodeCollect, 
 					pdaRun, fsmRun, -1, 0, 0, 0, 0, 0 );
diff --git a/colm/fsmrun.c b/colm/fsmrun.c
index b67adbb3..4b2d97c5 100644
--- a/colm/fsmrun.c
+++ b/colm/fsmrun.c
@@ -34,6 +34,7 @@
 
 #define false 0
 #define true 1
+#define pt(var) ((ParseTree*)(var))
 
 void listPrepend( List *list, ListEl *new_el) { listAddBefore(list, list->head, new_el); }
 void listAppend( List *list, ListEl *new_el)  { listAddAfter(list, list->tail, new_el); }
@@ -320,41 +321,32 @@ switch ( entry ) {
 case PcrStart:
 	pdaRun->ignore4 = ignoreKidList;
 
-	while ( pdaRun->ignore4 != 0 ) {
-		#ifdef COLM_LOG
-		LangElInfo *lelInfo = prg->rtd->lelInfo;
-		debug( REALM_PARSE, "sending back: %s%s\n",
-			lelInfo[pdaRun->ignore4->tree->id].name, 
-			pdaRun->ignore4->tree->flags & AF_ARTIFICIAL ? " (artificial)" : "" );
-		#endif
+	#ifdef COLM_LOG
+	LangElInfo *lelInfo = prg->rtd->lelInfo;
+	debug( REALM_PARSE, "sending back: %s%s\n",
+		lelInfo[pdaRun->ignore4->tree->id].name, 
+		pdaRun->ignore4->tree->flags & AF_ARTIFICIAL ? " (artificial)" : "" );
+	#endif
 
-		Head *head = pdaRun->ignore4->tree->tokdata;
-		int artificial = pdaRun->ignore4->tree->flags & AF_ARTIFICIAL;
+	Head *head = pdaRun->ignore4->tree->tokdata;
+	int artificial = pdaRun->ignore4->tree->flags & AF_ARTIFICIAL;
 
-		if ( head != 0 && !artificial )
-			sendBackText( fsmRun, inputStream, stringData( head ), head->length );
+	if ( head != 0 && !artificial )
+		sendBackText( fsmRun, inputStream, stringData( head ), head->length );
 
-		decrementConsumed( pdaRun );
+	decrementConsumed( pdaRun );
 
-		/* Check for reverse code. */
-		if ( pdaRun->ignore4->tree->flags & AF_HAS_RCODE ) {
+	/* Check for reverse code. */
+	if ( pdaRun->ignore4->tree->flags & AF_HAS_RCODE ) {
 
 return PcrRevIgnore;
 case PcrRevIgnore:
 
-			pdaRun->ignore4->tree->flags &= ~AF_HAS_RCODE;
-		}
-
-		pdaRun->ignore4 = pdaRun->ignore4->next;
+		pdaRun->ignore4->tree->flags &= ~AF_HAS_RCODE;
 	}
 
-	Kid *ignore = ignoreKidList;
-	while ( ignore != 0 ) {
-		Kid *next = ignore->next;
-		treeDownref( prg, sp, ignore->tree );
-		kidFree( prg, ignore );
-		ignore = next;
-	}
+	treeDownref( prg, sp, pdaRun->ignore4->tree );
+	kidFree( prg, pdaRun->ignore4 );
 
 case PcrDone:
 break; }
@@ -362,12 +354,13 @@ break; }
 	return PcrDone;
 }
 
-void detachIgnores( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, InputStream *inputStream, Kid *input )
+void detachIgnores( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, Kid *input )
 {
 	assert( pdaRun->accumIgnore == 0 );
 
-	/* Detach right. */
-	pdaRun->rightIgnore = 0;
+	/* Right ignore are immediately discarded since they are copies of
+	 * left-ignores. */
+	Tree *rightIgnore = 0;
 	if ( pdaRun->tokenList != 0 && pdaRun->tokenList->kid->tree->flags & AF_RIGHT_IL_ATTACHED ) {
 		Kid *riKid = treeRightIgnoreKid( prg, pdaRun->tokenList->kid->tree );
 
@@ -377,21 +370,23 @@ void detachIgnores( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, Inp
 		if ( li != 0 ) {
 			treeUpref( li->tree );
 			removeLeftIgnore( prg, sp, riKid->tree );
-			pdaRun->rightIgnore = riKid->tree;
-			treeUpref( pdaRun->rightIgnore );
+			rightIgnore = riKid->tree;
+			treeUpref( rightIgnore );
 			riKid->tree = li->tree;
 		}
 		else  {
-			pdaRun->rightIgnore = riKid->tree;
-			treeUpref( pdaRun->rightIgnore );
+			rightIgnore = riKid->tree;
+			treeUpref( rightIgnore );
 			removeRightIgnore( prg, sp, pdaRun->tokenList->kid->tree );
 			pdaRun->tokenList->kid->tree->flags &= ~AF_RIGHT_IL_ATTACHED;
 		}
 
 	}
 
+	treeDownref( prg, sp, rightIgnore );
+
 	/* Detach left. */
-	pdaRun->leftIgnore = 0;
+	Tree *leftIgnore = 0;
 	if ( input->tree->flags & AF_LEFT_IL_ATTACHED ) {
 		Kid *liKid = treeLeftIgnoreKid( prg, input->tree );
 
@@ -401,22 +396,27 @@ void detachIgnores( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, Inp
 		if ( ri != 0 ) {
 			treeUpref( ri->tree );
 			removeRightIgnore( prg, sp, liKid->tree );
-			pdaRun->leftIgnore = liKid->tree;
-			treeUpref( pdaRun->leftIgnore );
+			leftIgnore = liKid->tree;
+			treeUpref( leftIgnore );
 			liKid->tree = ri->tree;
 		}
 		else {
-			pdaRun->leftIgnore = liKid->tree;
-			treeUpref( pdaRun->leftIgnore );
+			leftIgnore = liKid->tree;
+			treeUpref( leftIgnore );
 			removeLeftIgnore( prg, sp, input->tree );
 			input->tree->flags &= ~AF_LEFT_IL_ATTACHED;
 		}
+	}
 
+	if ( leftIgnore != 0 ) {
+		pdaRun->accumIgnore = reverseKidList( leftIgnore->child );
+		leftIgnore->child = 0;
 	}
+
+	treeDownref( prg, sp, leftIgnore );
 }
 
 /* Stops on:
- *   PcrRevIgnore3
  *   PcrRevToken
  */
 
@@ -446,11 +446,6 @@ case PcrStart:
 
 	decrementConsumed( pdaRun );
 
-	/*
-	 * Detach ignores.
-	 */
-	detachIgnores( prg, sp, pdaRun, fsmRun, inputStream, input );
-
 	/* Artifical were not parsed, instead sent in as items. */
 	if ( input->tree->flags & AF_ARTIFICIAL ) {
 		treeUpref( input->tree );
@@ -479,20 +474,6 @@ case PcrRevToken:
 		unbind( prg, sp, pdaRun, input->tree );
 	}
 
-	if ( pdaRun->leftIgnore != 0 ) {
-		pdaRun->ignore5 = reverseKidList( pdaRun->leftIgnore->child );
-		pdaRun->leftIgnore->child = 0;
-
-		long pcr = sendBackIgnore( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->ignore5, PcrStart );
-		while ( pcr != PcrDone ) {
-
-return PcrRevIgnore3;
-case PcrRevIgnore3:
-
-			pcr = sendBackIgnore( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->ignore5, PcrRevIgnore );
-		}
-	}
-
 	if ( pdaRun->consumed == pdaRun->targetConsumed ) {
 		debug( REALM_PARSE, "trigger parse stop, consumed = target = %d", pdaRun->targetConsumed );
 		pdaRun->stop = true;
@@ -500,8 +481,6 @@ case PcrRevIgnore3:
 
 	/* Downref the tree that was sent back and free the kid. */
 	treeDownref( prg, sp, input->tree );
-	treeDownref( prg, sp, pdaRun->rightIgnore );
-	treeDownref( prg, sp, pdaRun->leftIgnore );
 	kidFree( prg, input );
 
 case PcrDone:
@@ -512,6 +491,16 @@ break; }
 
 void ignoreTree( Program *prg, PdaRun *pdaRun, Tree *tree )
 {
+	if ( pdaRun->accumIgnore == 0 ) {
+		/* Recording the next region. */
+		pt(tree)->region = pdaRun->nextRegionInd;
+		if ( pdaRun->tables->tokenRegions[pt(tree)->region+1] != 0 )
+			pdaRun->numRetry += 1;
+	}
+	else {
+		pt(tree)->region = -1;
+	}
+
 	/* Add the ignore string to the head of the ignore list. */
 	Kid *ignore = kidAllocate( prg );
 	ignore->tree = tree;
@@ -658,6 +647,7 @@ static void reportParseError( Program *prg, Tree **sp, PdaRun *pdaRun )
 
 void attachIgnore( Program *prg, Tree **sp, PdaRun *pdaRun, Kid *input )
 {
+
 	/* Need to preserve the layout under a tree:
 	 *    attributes, ignore tokens, grammar children. */
 
@@ -669,6 +659,8 @@ void attachIgnore( Program *prg, Tree **sp, PdaRun *pdaRun, Kid *input )
 	Kid *ignoreKid = extractIgnore( pdaRun );
 	
 	if ( ignoreKid != 0 ) {
+		debug( REALM_PARSE, "attaching ignore\n" );
+
 		ignoreKid = reverseKidList( ignoreKid );
 
 		/* Copy the ignore list first if we need to attach it as a right
@@ -759,11 +751,16 @@ void sendIgnore( Program *prg, Tree **sp, InputStream *inputStream, FsmRun *fsmR
 	Head *ignoreStr = extractMatch( prg, fsmRun, inputStream );
 	updatePosition( inputStream, fsmRun->tokstart, ignoreStr->length );
 	
-	Tree *tree = treeAllocate( prg );
+	debug( REALM_PARSE, "ignoring: %.*s\n", ignoreStr->length, ignoreStr->data );
+
+	Tree *tree = (Tree*)parseTreeAllocate( prg );
+	tree->flags |= AF_PARSE_TREE;
 	tree->refs = 1;
 	tree->id = id;
 	tree->tokdata = ignoreStr;
 
+	pt(tree)->region = pdaRun->nextRegionInd;
+
 	incrementConsumed( pdaRun );
 
 	/* Send it to the pdaRun. */
@@ -1053,12 +1050,10 @@ void sendTreeIgnore( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, In
 /*
  * Stops on:
  *   PcrPreEof
- *   PcrRevIgnore1
- *   PcrREvIgnore2
  *   PcrGeneration
  *   PcrReduction
  *   PcrRevReduction
- *   PcrRevIgnore3
+ *   PcrRevIgnore
  *   PcrRevToken
  */
 
@@ -1073,13 +1068,17 @@ case PcrStart:
 	pdaRun->stop = false;
 
 	while ( true ) {
+		debug( REALM_PARSE, "parse loop start\n" );
+
 		/* Pull the current scanner from the parser. This can change during
 		 * parsing due to inputStream pushes, usually for the purpose of includes.
 		 * */
 		pdaRun->tokenId = scanToken( prg, pdaRun, fsmRun, inputStream );
 
-		if ( pdaRun->tokenId == SCAN_TRY_AGAIN_LATER )
+		if ( pdaRun->tokenId == SCAN_TRY_AGAIN_LATER ) {
+			debug( REALM_PARSE, "scanner says try again later\n" );
 			break;
+		}
 
 		pdaRun->input2 = 0;
 
@@ -1107,22 +1106,9 @@ case PcrPreEof:
 		}
 		else if ( pdaRun->tokenId == SCAN_ERROR ) {
 			/* Scanner error, maybe retry. */
-			if ( pdaRunGetNextRegion( pdaRun, 1 ) != 0 ) {
+			if ( pdaRun->accumIgnore == 0 && pdaRunGetNextRegion( pdaRun, 1 ) != 0 ) {
 				debug( REALM_PARSE, "scanner failed, trying next region\n" );
 
-				/* May have accumulated ignore tokens from a previous region.
-				 * need to rescan them since we won't be sending tokens from
-				 * this region. */
-				pdaRun->ignore2 = extractIgnore( pdaRun );
-				long pcr = sendBackIgnore( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->ignore2, PcrStart );
-				while ( pcr != PcrDone ) {
-
-return PcrRevIgnore1;
-case PcrRevIgnore1:
-
-					pcr = sendBackIgnore( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->ignore2, PcrRevIgnore );
-				}
-
 				pdaRun->nextRegionInd += 1;
 				goto skipSend;
 			}
@@ -1131,21 +1117,10 @@ case PcrRevIgnore1:
 
 				/* Fall through to send null (error). */
 				pushBtPoint( prg, pdaRun, 0 );
-
-				/* Send back any accumulated ignore tokens, then trigger error
-				 * in the the parser. */
-				pdaRun->ignore3 = extractIgnore( pdaRun );
-				long pcr = sendBackIgnore( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->ignore3, PcrStart );
-				while ( pcr != PcrDone ) {
-
-return PcrRevIgnore2;
-case PcrRevIgnore2:
-
-					pcr = sendBackIgnore( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->ignore3, PcrRevIgnore );
-				}
 			}
 			else {
 				debug( REALM_PARSE, "no alternate scanning regions\n" );
+
 				/* There are no alternative scanning regions to try, nor are
 				 * there any alternatives stored in the current parse tree. No
 				 * choice but to end the parse. */
@@ -1163,7 +1138,7 @@ case PcrRevIgnore2:
 			pdaRun->input2 = sendNamedLangEl( prg, sp, pdaRun, fsmRun, inputStream );
 		}
 		else if ( pdaRun->tokenId == SCAN_TREE ) {
-			debug( REALM_PARSE, "sending an tree\n" );
+			debug( REALM_PARSE, "sending a tree\n" );
 
 			/* A tree already built. */
 			pdaRun->input2 = sendTree( prg, sp, pdaRun, fsmRun, inputStream );
@@ -1221,9 +1196,16 @@ case PcrGeneration:
 			pdaRun->input2 = sendToken( prg, sp, inputStream, fsmRun, pdaRun, pdaRun->tokenId );
 		}
 
-		if ( pdaRun->input2 != 0 ) {
-			/* Send the token to the parser. */
-			attachIgnore( prg, sp, pdaRun, pdaRun->input2 );
+		if ( pdaRun->input2 != 0 && pdaRun->cs >= 0 ) {
+			if ( pdaRun->accumIgnore == 0 ) {
+				/* Recording the next region. */
+				pt(pdaRun->input2->tree)->region = pdaRun->nextRegionInd;
+				if ( pdaRun->tables->tokenRegions[pt(pdaRun->input2->tree)->region+1] != 0 )
+					pdaRun->numRetry += 1;
+			}
+			else
+				pt(pdaRun->input2->tree)->region = -1;
+
 		}
 
 		assert( pdaRun->input1 == 0 );
@@ -1237,8 +1219,7 @@ case PcrGeneration:
 return pcr;
 case PcrReduction:
 case PcrRevReduction:
-case PcrRevIgnore3:
-case PcrRevIgnore4:
+case PcrRevIgnore:
 case PcrRevToken:
 
 			pcr = parseToken( prg, sp, pdaRun, fsmRun, inputStream, entry );
diff --git a/colm/input.c b/colm/input.c
index 7a380d1f..0b1a55e7 100644
--- a/colm/input.c
+++ b/colm/input.c
@@ -22,6 +22,7 @@
 #include <colm/input.h>
 #include <colm/fsmrun.h>
 #include <colm/pdarun.h>
+#include <colm/debug.h>
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -384,6 +385,7 @@ int inputStreamStringGetDataImpl( InputStream *is, char *dest, int length )
 
 	if ( is->offset == is->dlen ) {
 		//eof = true;
+		debug( REALM_PARSE, "setting later = true\n" );
 		is->later = true;
 	}
 
@@ -475,8 +477,10 @@ void initFdFuncs()
 
 int inputStreamAccumTryAgainLater( InputStream *is )
 {
-	if ( is->later || ( !is->flush && is->queue == 0 ))
+	if ( is->later || ( !is->flush && is->queue == 0 )) {
+		debug( REALM_PARSE, "try again later %d %d %d\n", is->later, is->flush, is->queue );
 		return true;
+	}
 
 	return false;
 }
diff --git a/colm/pdarun.c b/colm/pdarun.c
index 7e285545..2262ecdf 100644
--- a/colm/pdarun.c
+++ b/colm/pdarun.c
@@ -40,7 +40,6 @@
 #define act_rb 0x2
 #define lower 0x0000ffff
 #define upper 0xffff0000
-#define reject() induceReject = 1
 #define pt(var) ((ParseTree*)(var))
 
 #define read_word_p( i, p ) do { \
@@ -385,7 +384,6 @@ void pushBtPoint( Program *prg, PdaRun *pdaRun, Tree *tree )
 /* Stops on:
  *   PcrReduction
  *   PcrRevToken
- *   PcrRevIgnore3
  *   PcrRevIgnore4
  *   PcrRevReduction
  */
@@ -417,10 +415,8 @@ case PcrStart:
 	if ( pdaRun->cs < 0 )
 		return PcrDone;
 
-	pt(pdaRun->input1->tree)->region = pdaRun->nextRegionInd;
+	/* Record the state in the parse tree. */
 	pt(pdaRun->input1->tree)->state = pdaRun->cs;
-	if ( pdaRun->tables->tokenRegions[pt(pdaRun->input1->tree)->region+1] != 0 )
-		pdaRun->numRetry += 1;
 
 again:
 	if ( pdaRun->input1 == 0 )
@@ -490,6 +486,10 @@ again:
 	if ( pt(pdaRun->lel->tree)->retry_lower )
 		action += pt(pdaRun->lel->tree)->retry_lower;
 
+	/*
+	 * Shift
+	 */
+
 	if ( *action & act_sb ) {
 		debug( REALM_PARSE, "shifted: %s\n", 
 				prg->rtd->lelInfo[pt(pdaRun->lel->tree)->id].name );
@@ -500,8 +500,11 @@ again:
 		pdaRun->lel->next = pdaRun->stackTop;
 		pdaRun->stackTop = pdaRun->lel;
 
-		/* Record the last shifted token. Need this for attaching ignores. */
+		/* If its a token then attach ignores and record it in the token list
+		 * of the next ignore attachment to use. */
 		if ( pdaRun->lel->tree->id < prg->rtd->firstNonTermId ) {
+			attachIgnore( prg, sp, pdaRun, pdaRun->lel );
+
 			Ref *ref = (Ref*)kidAllocate( prg );
 			ref->kid = pdaRun->lel;
 			//treeUpref( pdaRun->lel->tree );
@@ -528,6 +531,10 @@ again:
 		}
 	}
 
+	/* 
+	 * Commit
+	 */
+
 	if ( pdaRun->tables->commitLen[pos] != 0 ) {
 		long causeReduce = 0;
 		if ( pdaRun->input1 != 0 ) { 
@@ -537,6 +544,10 @@ again:
 		commitFull( prg, sp, pdaRun, causeReduce );
 	}
 
+	/*
+	 * Reduce
+	 */
+
 	if ( *action & act_rb ) {
 		int r, objectLength;
 		Kid *last, *child, *attrs;
@@ -665,8 +676,10 @@ case PcrReduction:
 parseError:
 	debug( REALM_PARSE, "hit error, backtracking\n" );
 
-	if ( pdaRun->numRetry == 0 )
+	if ( pdaRun->numRetry == 0 ) {
+		debug( REALM_PARSE, "out of retries failing parse\n" );
 		goto fail;
+	}
 
 	while ( 1 ) {
 		if ( pdaRun->input1 != 0 ) {
@@ -683,19 +696,19 @@ parseError:
 			/* If there is no retry and there are no reductions caused by the
 			 * current input1 token then we are finished with it. Send it back. */
 			if ( pt(pdaRun->input1->tree)->causeReduce == 0 ) {
-				pdaRun->next = pt(pdaRun->input1->tree)->region + 1;
+				long region = pt(pdaRun->input1->tree)->region;
+				pdaRun->next = region >= 0 ? region + 1 : -1;
 
 				long pcr = sendBack( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->input1, PcrStart );
 				while ( pcr != PcrDone ) {
-
 return pcr;
 case PcrRevToken:
-case PcrRevIgnore3:
 					pcr = sendBack( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->input1, entry );
 				}
 
 				pdaRun->input1 = 0;
-				if ( pdaRun->tables->tokenRegions[pdaRun->next] != 0 ) {
+
+				if ( pdaRun->next >= 0 && pdaRun->tables->tokenRegions[pdaRun->next] != 0 ) {
 					debug( REALM_PARSE, "found a new region\n" );
 					pdaRun->numRetry -= 1;
 					pdaRun->cs = stackTopTarget( prg, pdaRun );
@@ -712,17 +725,33 @@ case PcrRevIgnore3:
 			}
 		}
 
-		if ( pdaRun->accumIgnore != 0 ) {
+		while ( pdaRun->input1 == 0 && pdaRun->accumIgnore != 0 ) {
+			debug( REALM_PARSE, "have accumulated ignore to undo\n" );
+
 			/* Send back any accumulated ignore tokens, then trigger error
 			 * in the the parser. */
-			pdaRun->ignore6 = extractIgnore( pdaRun );
+			pdaRun->ignore6 = pdaRun->accumIgnore;
+			pdaRun->accumIgnore = pdaRun->accumIgnore->next;
+			pdaRun->ignore6->next = 0;
+			
+			long region = pt(pdaRun->ignore6->tree)->region;
+			pdaRun->next = region >= 0 ? region + 1 : -1;
+			
 			long pcr = sendBackIgnore( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->ignore6, PcrStart );
 			while ( pcr != PcrDone ) {
 
-return PcrRevIgnore4;
-case PcrRevIgnore4:
+return pcr;
+case PcrRevIgnore:
 
-				pcr = sendBackIgnore( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->ignore6, PcrRevIgnore );
+				pcr = sendBackIgnore( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->ignore6, entry );
+			}
+
+			if ( pdaRun->next >= 0 && pdaRun->tables->tokenRegions[pdaRun->next] != 0 ) {
+				debug( REALM_PARSE, "found a new region\n" );
+				pdaRun->numRetry -= 1;
+				pdaRun->cs = stackTopTarget( prg, pdaRun );
+				pdaRun->nextRegionInd = pdaRun->next;
+				return PcrDone;
 			}
 		}
 
@@ -758,11 +787,12 @@ case PcrRevIgnore4:
 			pdaRun->undoLel->next = pdaRun->input1;
 			pdaRun->input1 = pdaRun->undoLel;
 
-			/* Record the last shifted token. Need this for attaching ignores. */
+			/* Pop from the token list. */
 			Ref *ref = pdaRun->tokenList;
 			pdaRun->tokenList = ref->next;
-			//treeDownref( prg, sp, kid->tree );
 			kidFree( prg, (Kid*)ref );
+
+			detachIgnores( prg, sp, pdaRun, fsmRun, pdaRun->input1 );
 		}
 		else {
 			debug( REALM_PARSE, "backing up over non-terminal: %s\n",
diff --git a/colm/pdarun.h b/colm/pdarun.h
index a57ea8e6..f039e325 100644
--- a/colm/pdarun.h
+++ b/colm/pdarun.h
@@ -254,8 +254,6 @@ typedef struct _PdaRun
 	Head *tokdata;
 	int frameId;
 	Kid *input2;
-	Tree *rightIgnore;
-	Tree *leftIgnore;
 	Kid *ignore2;
 	Kid *ignore3;
 	int next;
@@ -334,10 +332,6 @@ int pdaRunGetNextRegion( PdaRun *pdaRun, int offset );
 #define PcrGeneration   4
 #define PcrPreEof       5
 #define PcrRevIgnore    6
-#define PcrRevIgnore1   7
-#define PcrRevIgnore2   8
-#define PcrRevIgnore3   9
-#define PcrRevIgnore4   10
 #define PcrRevToken     11
 #define PcrRevReduction 12
 
@@ -383,6 +377,8 @@ Tree *getParsedRoot( PdaRun *pdaRun, int stop );
 void pushBtPoint( struct ColmProgram *prg, PdaRun *pdaRun, Tree *tree );
 void undoParseStream( struct ColmProgram *prg, Tree **sp, InputStream *inputStream, FsmRun *fsmRun, 
 		PdaRun *pdaRun, long consumed );
+void attachIgnore( struct ColmProgram *prg, Tree **sp, PdaRun *pdaRun, Kid *input );
+void detachIgnores( struct ColmProgram *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, Kid *input );
 
 #ifdef __cplusplus
 }