Added another point where we will sending back ignore tokens. refs #333.

3 files changed, 60 insertions, 36 deletions

diff --git a/colm/fsmrun.c b/colm/fsmrun.c
index 9ea1b79d..b67adbb3 100644
--- a/colm/fsmrun.c
+++ b/colm/fsmrun.c
@@ -362,40 +362,9 @@ break; }
 	return PcrDone;
 }
 
-/* Stops on:
- *   PcrRevIgnore3
- *   PcrRevToken
- */
-
-long sendBack( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, InputStream *inputStream, Kid *input, long entry )
+void detachIgnores( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, InputStream *inputStream, Kid *input )
 {
-	#ifdef COLM_LOG
-	LangElInfo *lelInfo = prg->rtd->lelInfo;
-	debug( REALM_PARSE, "sending back: %s  text: %.*s%s\n", 
-			lelInfo[input->tree->id].name, 
-			stringLength( input->tree->tokdata ), 
-			stringData( input->tree->tokdata ), 
-			input->tree->flags & AF_ARTIFICIAL ? " (artificial)" : "" );
-	#endif
-
-switch ( entry ) {
-case PcrStart:
-
-	if ( input->tree->flags & AF_NAMED ) {
-		/* Send back anything in the buffer that has not been parsed. */
-		if ( fsmRun->p == fsmRun->runBuf->data )
-			sendBackRunBufHead( fsmRun, inputStream );
-
-		/* Send the named lang el back first, then send back any leading
-		 * whitespace. */
-		inputStream->funcs->pushBackNamed( inputStream );
-	}
-
-	decrementConsumed( pdaRun );
-
-	/*
-	 * Detach ignores.
-	 */
+	assert( pdaRun->accumIgnore == 0 );
 
 	/* Detach right. */
 	pdaRun->rightIgnore = 0;
@@ -444,6 +413,43 @@ case PcrStart:
 		}
 
 	}
+}
+
+/* Stops on:
+ *   PcrRevIgnore3
+ *   PcrRevToken
+ */
+
+long sendBack( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, InputStream *inputStream, Kid *input, long entry )
+{
+	#ifdef COLM_LOG
+	LangElInfo *lelInfo = prg->rtd->lelInfo;
+	debug( REALM_PARSE, "sending back: %s  text: %.*s%s\n", 
+			lelInfo[input->tree->id].name, 
+			stringLength( input->tree->tokdata ), 
+			stringData( input->tree->tokdata ), 
+			input->tree->flags & AF_ARTIFICIAL ? " (artificial)" : "" );
+	#endif
+
+switch ( entry ) {
+case PcrStart:
+
+	if ( input->tree->flags & AF_NAMED ) {
+		/* Send back anything in the buffer that has not been parsed. */
+		if ( fsmRun->p == fsmRun->runBuf->data )
+			sendBackRunBufHead( fsmRun, inputStream );
+
+		/* Send the named lang el back first, then send back any leading
+		 * whitespace. */
+		inputStream->funcs->pushBackNamed( inputStream );
+	}
+
+	decrementConsumed( pdaRun );
+
+	/*
+	 * Detach ignores.
+	 */
+	detachIgnores( prg, sp, pdaRun, fsmRun, inputStream, input );
 
 	/* Artifical were not parsed, instead sent in as items. */
 	if ( input->tree->flags & AF_ARTIFICIAL ) {
@@ -1227,10 +1233,12 @@ case PcrGeneration:
 		long pcr = parseToken( prg, sp, pdaRun, fsmRun, inputStream, PcrStart );
 		
 		while ( pcr != PcrDone ) {
+
 return pcr;
 case PcrReduction:
 case PcrRevReduction:
 case PcrRevIgnore3:
+case PcrRevIgnore4:
 case PcrRevToken:
 
 			pcr = parseToken( prg, sp, pdaRun, fsmRun, inputStream, entry );
diff --git a/colm/pdarun.c b/colm/pdarun.c
index f23aa3d0..7e285545 100644
--- a/colm/pdarun.c
+++ b/colm/pdarun.c
@@ -386,6 +386,7 @@ void pushBtPoint( Program *prg, PdaRun *pdaRun, Tree *tree )
  *   PcrReduction
  *   PcrRevToken
  *   PcrRevIgnore3
+ *   PcrRevIgnore4
  *   PcrRevReduction
  */
 long parseToken( Program *prg, Tree **sp, PdaRun *pdaRun,
@@ -693,7 +694,6 @@ case PcrRevIgnore3:
 					pcr = sendBack( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->input1, entry );
 				}
 
-
 				pdaRun->input1 = 0;
 				if ( pdaRun->tables->tokenRegions[pdaRun->next] != 0 ) {
 					debug( REALM_PARSE, "found a new region\n" );
@@ -712,6 +712,20 @@ case PcrRevIgnore3:
 			}
 		}
 
+		if ( pdaRun->accumIgnore != 0 ) {
+			/* Send back any accumulated ignore tokens, then trigger error
+			 * in the the parser. */
+			pdaRun->ignore6 = extractIgnore( pdaRun );
+			long pcr = sendBackIgnore( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->ignore6, PcrStart );
+			while ( pcr != PcrDone ) {
+
+return PcrRevIgnore4;
+case PcrRevIgnore4:
+
+				pcr = sendBackIgnore( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->ignore6, PcrRevIgnore );
+			}
+		}
+
 		/* Now it is time to undo something. Pick an element from the top of
 		 * the stack. */
 		pdaRun->undoLel = pdaRun->stackTop;
diff --git a/colm/pdarun.h b/colm/pdarun.h
index 1d736018..a57ea8e6 100644
--- a/colm/pdarun.h
+++ b/colm/pdarun.h
@@ -262,6 +262,7 @@ typedef struct _PdaRun
 	Kid *undoLel;
 	Kid *ignore4;
 	Kid *ignore5;
+	Kid *ignore6;
 } PdaRun;
 
 void rtCodeVectReplace( RtCodeVect *vect, long pos, const Code *val, long len );
@@ -336,8 +337,9 @@ int pdaRunGetNextRegion( PdaRun *pdaRun, int offset );
 #define PcrRevIgnore1   7
 #define PcrRevIgnore2   8
 #define PcrRevIgnore3   9
-#define PcrRevToken     10
-#define PcrRevReduction 11
+#define PcrRevIgnore4   10
+#define PcrRevToken     11
+#define PcrRevReduction 12
 
 long parseToken( struct ColmProgram *prg, Tree **sp, PdaRun *pdaRun, 
 		FsmRun *fsmRun, InputStream *inputStream, long entry );