put the data fetch at the top of the scanToken loop

Now that we are copying into FSM on consume, it makes sense to get buffer
blocks at the top of the scan token loop.

1 files changed, 34 insertions, 40 deletions

diff --git a/colm/pdarun.c b/colm/pdarun.c
index 73cc5b5c..ec868c7a 100644
--- a/colm/pdarun.c
+++ b/colm/pdarun.c
@@ -192,7 +192,6 @@ Head *peekMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is )
 	return head;
 }
 
-
 Head *streamPull( Program *prg, FsmRun *fsmRun, StreamImpl *is, long length )
 {
 	RunBuf *runBuf = fsmRun->consumeBuf;
@@ -1010,40 +1009,6 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is )
 		return SCAN_UNDO;
 
 	while ( true ) {
-		fsmExecute( fsmRun, is );
-
-		/* First check if scanning stopped because we have a token. */
-		if ( fsmRun->matchedToken > 0 ) {
-			/* If the token has a marker indicating the end (due to trailing
-			 * context) then adjust data now. */
-			LangElInfo *lelInfo = prg->rtd->lelInfo;
-			if ( lelInfo[fsmRun->matchedToken].markId >= 0 )
-				fsmRun->p = fsmRun->mark[lelInfo[fsmRun->matchedToken].markId];
-
-			return fsmRun->matchedToken;
-		}
-
-		/* Check for error. */
-		if ( fsmRun->cs == fsmRun->tables->errorState ) {
-			/* If a token was started, but not finished (tokstart != 0) then
-			 * restore data to the beginning of that token. */
-			if ( fsmRun->tokstart != 0 )
-				fsmRun->p = fsmRun->tokstart;
-
-			/* Check for a default token in the region. If one is there
-			 * then send it and continue with the processing loop. */
-			if ( prg->rtd->regionInfo[fsmRun->region].defaultToken >= 0 ) {
-				fsmRun->toklen = 0;
-				return prg->rtd->regionInfo[fsmRun->region].defaultToken;
-			}
-
-			return SCAN_ERROR;
-		}
-
-		/* Got here because the state machine didn't match a token or encounter
-		 * an error. Must be because we got to the end of the buffer data. */
-		assert( fsmRun->p == fsmRun->pe );
-
 		char *pd = 0;
 		int len = 0;
 		int type = is->funcs->getParseBlock( fsmRun, is, fsmRun->toklen, &pd, &len );
@@ -1056,18 +1021,13 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is )
 
 			case INPUT_EOS:
 				fsmRun->p = fsmRun->pe = 0;
-				//fsmRun->have = 0;
 				if ( fsmRun->tokstart != 0 )
 					fsmRun->eof = 1;
 				debug( REALM_SCAN, "EOS *******************\n" );
-				//else {
-				//	return SCAN_EOS;
-				//}
 				break;
 
 			case INPUT_EOF:
 				fsmRun->p = fsmRun->pe = 0;
-				//fsmRun->have = 0;
 				if ( fsmRun->tokstart != 0 )
 					fsmRun->eof = 1;
 				else 
@@ -1098,6 +1058,40 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is )
 					return SCAN_IGNORE;
 				break;
 		}
+
+		fsmExecute( fsmRun, is );
+
+		/* First check if scanning stopped because we have a token. */
+		if ( fsmRun->matchedToken > 0 ) {
+			/* If the token has a marker indicating the end (due to trailing
+			 * context) then adjust data now. */
+			LangElInfo *lelInfo = prg->rtd->lelInfo;
+			if ( lelInfo[fsmRun->matchedToken].markId >= 0 )
+				fsmRun->p = fsmRun->mark[lelInfo[fsmRun->matchedToken].markId];
+
+			return fsmRun->matchedToken;
+		}
+
+		/* Check for error. */
+		if ( fsmRun->cs == fsmRun->tables->errorState ) {
+			/* If a token was started, but not finished (tokstart != 0) then
+			 * restore data to the beginning of that token. */
+			if ( fsmRun->tokstart != 0 )
+				fsmRun->p = fsmRun->tokstart;
+
+			/* Check for a default token in the region. If one is there
+			 * then send it and continue with the processing loop. */
+			if ( prg->rtd->regionInfo[fsmRun->region].defaultToken >= 0 ) {
+				fsmRun->toklen = 0;
+				return prg->rtd->regionInfo[fsmRun->region].defaultToken;
+			}
+
+			return SCAN_ERROR;
+		}
+
+		/* Got here because the state machine didn't match a token or encounter
+		 * an error. Must be because we got to the end of the buffer data. */
+		assert( fsmRun->p == fsmRun->pe );
 	}
 
 	/* Should not be reached. */