Need to clear any buffered data when dropping back into the bytecode loop from

the parser. refs #341.

4 files changed, 62 insertions, 18 deletions

diff --git a/colm/bytecode.c b/colm/bytecode.c
index 0626d77e..6d50134d 100644
--- a/colm/bytecode.c
+++ b/colm/bytecode.c
@@ -228,7 +228,7 @@ case PcrStart:
 		long pcr = parseLoop( prg, sp, accum->pdaRun, accum->fsmRun, accum->accumStream->in, entry );
 
 		while ( pcr != PcrDone ) {
-//			sendBackBuffered( accum->fsmRun, accum->accumStream->in );
+			clearBuffered( accum->fsmRun );
 
 return pcr;
 case PcrReduction:
@@ -248,7 +248,7 @@ case PcrRevReduction2:
 case PcrDone:
 break; }
 
-//	sendBackBuffered( accum->fsmRun, accum->accumStream->in );
+	clearBuffered( accum->fsmRun );
 
 	return PcrDone;
 }
@@ -267,7 +267,7 @@ case PcrStart:
 
 		 	while ( pcr != PcrDone ) {
 
-//				sendBackBuffered( accum->fsmRun, accum->accumStream->in );
+				clearBuffered( accum->fsmRun );
 
 return pcr;
 case PcrReduction:
@@ -300,7 +300,7 @@ case PcrRevReduction2:
 case PcrDone:
 break; }
 
-//	sendBackBuffered( accum->fsmRun, accum->accumStream->in );
+	clearBuffered( accum->fsmRun );
 
 	return PcrDone;
 }
diff --git a/colm/ctinput.cc b/colm/ctinput.cc
index 179cad03..c7b431cc 100644
--- a/colm/ctinput.cc
+++ b/colm/ctinput.cc
@@ -138,15 +138,31 @@ int inputStreamPatternConsumeData( SourceStream *is, int length )
 {
 	debug( REALM_INPUT, "consuming %ld bytes\n", length );
 
-	is->offset += length;
-	if ( is->offset == is->patItem->data.length() ) {
-		/* Read up to the end of the data. Advance the
-		 * pattern item. */
-		is->patItem = is->patItem->next;
-		is->offset = 0;
+	int consumed = 0;
+
+	while ( true ) {
+		if ( is->patItem == 0 )
+			break;
+
+		int avail = is->patItem->data.length() - is->offset;
+
+		if ( length >= avail ) {
+			/* Read up to the end of the data. Advance the
+			 * pattern item. */
+			is->patItem = is->patItem->next;
+			is->offset = 0;
+
+			length -= avail;
+			consumed += avail;
+		}
+		else {
+			is->offset += length;
+			consumed += length;
+			break;
+		}
 	}
 
-	return length;
+	return consumed;
 }
 
 int inputStreamPatternUndoConsumeData( SourceStream *is, const char *data, int length )
@@ -292,16 +308,31 @@ void inputStreamReplUndoConsumeLangEl( SourceStream *is )
 
 int inputStreamReplConsumeData( SourceStream *is, int length )
 {
-	is->offset += length;
+	int consumed = 0;
 
-	if ( is->offset == is->replItem->data.length() ) {
-		/* Read up to the end of the data. Advance the
-		 * pattern item. */
-		is->replItem = is->replItem->next;
-		is->offset = 0;
+	while ( true ) {
+		if ( is->replItem == 0 )
+			break;
+
+		int avail = is->replItem->data.length() - is->offset;
+
+		if ( length >= avail ) {
+			/* Read up to the end of the data. Advance the
+			 * pattern item. */
+			is->replItem = is->replItem->next;
+			is->offset = 0;
+
+			length -= avail;
+			consumed += avail;
+		}
+		else {
+			is->offset += length;
+			consumed += length;
+			break;
+		}
 	}
 
-	return length;
+	return consumed;
 }
 
 int inputStreamReplUndoConsumeData( SourceStream *is, const char *data, int length )
diff --git a/colm/pdarun.c b/colm/pdarun.c
index 0cad30b5..b901570d 100644
--- a/colm/pdarun.c
+++ b/colm/pdarun.c
@@ -389,10 +389,21 @@ void detachIgnores( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, Kid
 	treeDownref( prg, sp, leftIgnore );
 }
 
+void clearBuffered( FsmRun *fsmRun )
+{
+	/* If there is data in the current buffer then send the whole send back
+	 * should be in this buffer. */
+	if ( fsmRun->tokstart != 0 ) 
+		fsmRun->p = fsmRun->pe = fsmRun->tokstart;
+	else
+		fsmRun->pe = fsmRun->p;
+}
+
 /* Stops on:
  *   PcrRevToken
  */
 
+
 static long sendBack( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, 
 		InputStream *inputStream, Kid *input, long entry )
 {
diff --git a/colm/pdarun.h b/colm/pdarun.h
index 6e9335bd..79b679e6 100644
--- a/colm/pdarun.h
+++ b/colm/pdarun.h
@@ -449,6 +449,8 @@ void undoParseStream( struct ColmProgram *prg, Tree **sp, InputStream *inputStre
 void attachIgnore( struct ColmProgram *prg, Tree **sp, PdaRun *pdaRun, Kid *input );
 void detachIgnores( struct ColmProgram *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, Kid *input );
 
+void clearBuffered( FsmRun *fsmRun );
+
 #ifdef __cplusplus
 }
 #endif