renamed FsmRun "have" to toklen, compute it in fsmExecute

4 files changed, 28 insertions, 21 deletions

diff --git a/colm/fsmcodegen.cc b/colm/fsmcodegen.cc
index 30abf796..aded4148 100644
--- a/colm/fsmcodegen.cc
+++ b/colm/fsmcodegen.cc
@@ -194,7 +194,7 @@ void FsmCodeGen::LM_SWITCH( ostream &ret, InlineItem *item,
 	ret << 
 		"	}\n"
 		"\t"
-		" return;\n";
+		"	goto out;\n";
 }
 
 void FsmCodeGen::LM_ON_LAST( ostream &ret, InlineItem *item )
@@ -203,7 +203,7 @@ void FsmCodeGen::LM_ON_LAST( ostream &ret, InlineItem *item )
 
 	ret << "	" << P() << " += 1;\n";
 	EMIT_TOKEN( ret, item->longestMatchPart->tdLangEl );
-	ret << "	return;\n";
+	ret << "	goto out;\n";
 }
 
 void FsmCodeGen::LM_ON_NEXT( ostream &ret, InlineItem *item )
@@ -211,7 +211,7 @@ void FsmCodeGen::LM_ON_NEXT( ostream &ret, InlineItem *item )
 	assert( item->longestMatchPart->tdLangEl != 0 );
 
 	EMIT_TOKEN( ret, item->longestMatchPart->tdLangEl );
-	ret << "	return;\n";
+	ret << "	goto out;\n";
 }
 
 void FsmCodeGen::LM_ON_LAG_BEHIND( ostream &ret, InlineItem *item )
@@ -220,7 +220,7 @@ void FsmCodeGen::LM_ON_LAG_BEHIND( ostream &ret, InlineItem *item )
 
 //	ret << "	" << P() << " = " << TOKEND() << ";\n";
 	EMIT_TOKEN( ret, item->longestMatchPart->tdLangEl );
-	ret << "	return;\n";
+	ret << "	goto out;\n";
 }
 
 
@@ -848,6 +848,7 @@ void FsmCodeGen::writeExec()
 	out <<
 		"void fsmExecute( FsmRun *fsmRun, StreamImpl *inputStream )\n"
 		"{\n"
+		"	char *start = fsmRun->p;\n"
 		"/*_resume:*/\n";
 
 	if ( redFsm->errState != 0 ) {
@@ -872,7 +873,9 @@ void FsmCodeGen::writeExec()
 		"	}\n";
 
 	out <<
-		"	out: {}\n"
+		"	out:\n"
+		"	if ( fsmRun->p != 0 )\n"
+		"		fsmRun->toklen += fsmRun->p - start;\n"
 		"}\n"
 		"\n";
 }
diff --git a/colm/fsmexec.cc b/colm/fsmexec.cc
index bb24e086..ac9cd2d8 100644
--- a/colm/fsmexec.cc
+++ b/colm/fsmexec.cc
@@ -99,6 +99,8 @@ void fsmExecute( FsmRun *fsmRun, StreamImpl *inputStream )
 	const long *_acts;
 	unsigned int _nacts;
 	const char *_keys;
+		
+	char *start = fsmRun->p;
 
 	/* Init the token match to nothing (the sentinal). */
 	fsmRun->matchedToken = 0;
@@ -176,7 +178,7 @@ _match:
 	while ( _nacts-- > 0 )
 		execAction( fsmRun, fsmRun->tables->actionSwitch[*_acts++] );
 	if ( fsmRun->returnResult )
-		return;
+		goto final;
 
 _again:
 	_acts = fsmRun->tables->actions + fsmRun->tables->toStateActions[fsmRun->cs];
@@ -201,8 +203,13 @@ out:
 		while ( _nacts-- > 0 )
 			execAction( fsmRun, fsmRun->tables->actionSwitch[*_acts++] );
 		if ( fsmRun->returnResult )
-			return;
+			goto final;
 	}
+
+final:
+
+	if ( fsmRun->p != 0 )
+		fsmRun->toklen += fsmRun->p - start;
 }
 
 
diff --git a/colm/pdarun.c b/colm/pdarun.c
index 2f071a3c..cf15e2f6 100644
--- a/colm/pdarun.c
+++ b/colm/pdarun.c
@@ -64,7 +64,7 @@ void initFsmRun( FsmRun *fsmRun, Program *prg )
 	fsmRun->consumeBuf = 0;
 
 	fsmRun->p = fsmRun->pe = 0;
-	fsmRun->have = 0;
+	fsmRun->toklen = 0;
 	fsmRun->peof = (char*)-1;
 
 	fsmRun->preRegion = -1;
@@ -140,7 +140,7 @@ Head *streamPull( Program *prg, FsmRun *fsmRun, StreamImpl *is, long length )
 	is->funcs->consumeData( is, length );
 
 	fsmRun->p = fsmRun->pe = 0;
-	fsmRun->have = 0;
+	fsmRun->toklen = 0;
 	//fsmRun->peof = (char*)-1;
 
 	Head *tokdata = stringAllocPointer( prg, runBuf->data, length );
@@ -269,7 +269,7 @@ void resetToken( FsmRun *fsmRun )
 	 * must first backup over it. */
 	if ( fsmRun->tokstart != 0 ) {
 		fsmRun->p = fsmRun->pe = 0;
-		fsmRun->have = 0;
+		fsmRun->toklen = 0;
 		fsmRun->peof = (char*)-1;
 	}
 }
@@ -758,7 +758,7 @@ void sendIgnore( Program *prg, Tree **sp, StreamImpl *is, FsmRun *fsmRun, PdaRun
 /* Doesn't consume. */
 Head *peekMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is )
 {
-	long length = fsmRun->have;
+	long length = fsmRun->toklen;
 
 	RunBuf *runBuf = newRunBuf();
 	runBuf->next = fsmRun->consumeBuf;
@@ -768,7 +768,7 @@ Head *peekMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is )
 	is->funcs->getData( fsmRun, is, 0, runBuf->data, length, &lenCopied );
 
 	fsmRun->p = fsmRun->pe = 0;
-	fsmRun->have = 0;
+	fsmRun->toklen = 0;
 	//fsmRun->peof = (char*)-1;
 
 	Head *head = stringAllocPointer( prg, runBuf->data, length );
@@ -786,7 +786,7 @@ Head *peekMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is )
 /* Consumes. */
 Head *extractMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is )
 {
-	long length = fsmRun->have;
+	long length = fsmRun->toklen;
 
 	debug( REALM_PARSE, "extracting token of length: %ld\n", length );
 
@@ -806,7 +806,7 @@ Head *extractMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is )
 	is->funcs->consumeData( is, length );
 
 	fsmRun->p = fsmRun->pe = 0;
-	fsmRun->have = 0;
+	fsmRun->toklen = 0;
 	fsmRun->tokstart = 0;
 	//fsmRun->peof = (char*)-1;
 
@@ -942,7 +942,7 @@ static void sendEof( Program *prg, Tree **sp, StreamImpl *is, FsmRun *fsmRun, Pd
 void newToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun )
 {
 	fsmRun->p = fsmRun->pe = 0;
-	fsmRun->have = 0;
+	fsmRun->toklen = 0;
 	fsmRun->peof = (char*)-1;
 
 	/* Init the scanner vars. */
@@ -1008,10 +1008,7 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is )
 		return SCAN_UNDO;
 
 	while ( true ) {
-		char *start = fsmRun->p;
 		fsmExecute( fsmRun, is );
-		if ( fsmRun->p != 0 )
-			fsmRun->have += fsmRun->p - start;
 
 		/* First check if scanning stopped because we have a token. */
 		if ( fsmRun->matchedToken > 0 ) {
@@ -1047,7 +1044,7 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is )
 
 		char *pd = 0;
 		int len = 0;
-		int type = is->funcs->getParseBlock( fsmRun, is, fsmRun->have, &pd, &len );
+		int type = is->funcs->getParseBlock( fsmRun, is, fsmRun->toklen, &pd, &len );
 
 		switch ( type ) {
 			case INPUT_DATA:
@@ -1239,7 +1236,7 @@ case PcrPreEof:
 			 * data is pulled from the inputStream. */
 
 			fsmRun->p = fsmRun->pe = 0;
-			fsmRun->have = 0;
+			fsmRun->toklen = 0;
 			fsmRun->peof = (char*)-1;
 
 			pdaRun->fi = &prg->rtd->frameInfo[prg->rtd->lelInfo[pdaRun->tokenId].frameId];
diff --git a/colm/pdarun.h b/colm/pdarun.h
index ac08889e..5d8b3cea 100644
--- a/colm/pdarun.h
+++ b/colm/pdarun.h
@@ -79,8 +79,8 @@ typedef struct _FsmRun
 	long region, preRegion;
 	long cs, ncs, act;
 	char *tokstart, *tokend;
+	long toklen;
 	char *p, *pe, *peof;
-	int have;
 	int returnResult;
 	char *mark[MARK_SLOTS];
 	long matchedToken;