keep tokend as an int, not pointer

Since we are no longer maintaining a contiguous token buffer during scanning we
cannot use a pointer for tokend. Turn it into an offset (toklen).

5 files changed, 37 insertions, 21 deletions

diff --git a/colm/fsmcodegen.cc b/colm/fsmcodegen.cc
index aded4148..708922de 100644
--- a/colm/fsmcodegen.cc
+++ b/colm/fsmcodegen.cc
@@ -146,7 +146,7 @@ void FsmCodeGen::SET_ACT( ostream &ret, InlineItem *item )
 void FsmCodeGen::SET_TOKEND( ostream &ret, InlineItem *item )
 {
 	/* The tokend action sets tokend. */
-	ret << TOKEND() << " = " << P() << "+1;";
+	ret << "{ " << TOKEND() << " = " << TOKLEN() << " + ( " << P() << " - " << BLOCK_START() << " ) + 1; }";
 }
 void FsmCodeGen::INIT_TOKSTART( ostream &ret, InlineItem *item )
 {
@@ -172,7 +172,7 @@ void FsmCodeGen::LM_SWITCH( ostream &ret, InlineItem *item,
 		int targState, int inFinish )
 {
 	ret << 
-		//"	" << P() << " = " << TOKEND() << ";\n"
+		"	" << TOKLEN() << " = " << TOKEND() << ";\n"
 		"	switch( " << ACT() << " ) {\n";
 
 	/* If the switch handles error then we also forced the error state. It
@@ -194,7 +194,7 @@ void FsmCodeGen::LM_SWITCH( ostream &ret, InlineItem *item,
 	ret << 
 		"	}\n"
 		"\t"
-		"	goto out;\n";
+		"	goto skip_toklen;\n";
 }
 
 void FsmCodeGen::LM_ON_LAST( ostream &ret, InlineItem *item )
@@ -218,9 +218,9 @@ void FsmCodeGen::LM_ON_LAG_BEHIND( ostream &ret, InlineItem *item )
 {
 	assert( item->longestMatchPart->tdLangEl != 0 );
 
-//	ret << "	" << P() << " = " << TOKEND() << ";\n";
+	ret << "	" << TOKLEN() << " = " << TOKEND() << ";\n";
 	EMIT_TOKEN( ret, item->longestMatchPart->tdLangEl );
-	ret << "	goto out;\n";
+	ret << "	goto skip_toklen;\n";
 }
 
 
@@ -848,7 +848,7 @@ void FsmCodeGen::writeExec()
 	out <<
 		"void fsmExecute( FsmRun *fsmRun, StreamImpl *inputStream )\n"
 		"{\n"
-		"	char *start = fsmRun->p;\n"
+		"	" << BLOCK_START() << " = fsmRun->p;\n"
 		"/*_resume:*/\n";
 
 	if ( redFsm->errState != 0 ) {
@@ -873,9 +873,11 @@ void FsmCodeGen::writeExec()
 		"	}\n";
 
 	out <<
-		"	out:\n"
-		"	if ( fsmRun->p != 0 )\n"
-		"		fsmRun->toklen += fsmRun->p - start;\n"
+		"out:\n"
+		"	if ( " << P() << " != 0 )\n"
+		"		" << TOKLEN() << " += " << P() << " - " << BLOCK_START() << ";\n"
+		"skip_toklen:\n"
+		"	{}\n"
 		"}\n"
 		"\n";
 }
diff --git a/colm/fsmcodegen.h b/colm/fsmcodegen.h
index 1b004f5e..d1f23166 100644
--- a/colm/fsmcodegen.h
+++ b/colm/fsmcodegen.h
@@ -105,6 +105,8 @@ protected:
 	string TOP() { return ACCESS() + "top"; }
 	string TOKSTART() { return ACCESS() + "tokstart"; }
 	string TOKEND() { return ACCESS() + "tokend"; }
+	string BLOCK_START() { return ACCESS() + "start"; }
+	string TOKLEN() { return ACCESS() + "toklen"; }
 	string ACT() { return ACCESS() + "act"; }
 	string MATCHED_TOKEN() { return ACCESS() + "matchedToken"; }
 
diff --git a/colm/fsmexec.cc b/colm/fsmexec.cc
index ac9cd2d8..94d2e18e 100644
--- a/colm/fsmexec.cc
+++ b/colm/fsmexec.cc
@@ -42,7 +42,7 @@ void execAction( FsmRun *fsmRun, GenAction *genAction )
 			fsmRun->act = item->longestMatchPart->longestMatchId;
 			break;
 		case InlineItem::LmSetTokEnd:
-			fsmRun->tokend = fsmRun->p + 1;
+			fsmRun->tokend = fsmRun->toklen + ( fsmRun->p - fsmRun->start ) + 1;
 			break;
 		case InlineItem::LmInitTokStart:
 			assert(false);
@@ -56,9 +56,8 @@ void execAction( FsmRun *fsmRun, GenAction *genAction )
 		case InlineItem::LmSwitch:
 			/* If the switch handles error then we also forced the error state. It
 			 * will exist. */
-			//fsmRun->p = fsmRun->tokend;
+			fsmRun->toklen = fsmRun->tokend;
 			if ( item->tokenRegion->lmSwitchHandlesError && fsmRun->act == 0 ) {
-				//fsmRun->p = fsmRun->tokstart;
 				fsmRun->cs = fsmRun->tables->errorState;
 			}
 			else {
@@ -70,6 +69,7 @@ void execAction( FsmRun *fsmRun, GenAction *genAction )
 				}
 			}
 			fsmRun->returnResult = true;
+			fsmRun->skipToklen = true;
 			break;
 		case InlineItem::LmOnLast:
 			fsmRun->p += 1;
@@ -81,9 +81,10 @@ void execAction( FsmRun *fsmRun, GenAction *genAction )
 			fsmRun->returnResult = true;
 			break;
 		case InlineItem::LmOnLagBehind:
-			//fsmRun->p = fsmRun->tokend;
+			fsmRun->toklen = fsmRun->tokend;
 			fsmRun->matchedToken = item->longestMatchPart->tdLangEl->id;
 			fsmRun->returnResult = true;
+			fsmRun->skipToklen = true;
 			break;
 		}
 	}
@@ -100,7 +101,7 @@ void fsmExecute( FsmRun *fsmRun, StreamImpl *inputStream )
 	unsigned int _nacts;
 	const char *_keys;
 		
-	char *start = fsmRun->p;
+	fsmRun->start = fsmRun->p;
 
 	/* Init the token match to nothing (the sentinal). */
 	fsmRun->matchedToken = 0;
@@ -173,12 +174,16 @@ _match:
 		goto _again;
 
 	fsmRun->returnResult = false;
+	fsmRun->skipToklen = false;
 	_acts = fsmRun->tables->actions + fsmRun->tables->transActionsWI[_trans];
 	_nacts = (unsigned int) *_acts++;
 	while ( _nacts-- > 0 )
 		execAction( fsmRun, fsmRun->tables->actionSwitch[*_acts++] );
-	if ( fsmRun->returnResult )
+	if ( fsmRun->returnResult ) {
+		if ( fsmRun->skipToklen )
+			goto skip_toklen;
 		goto final;
+	}
 
 _again:
 	_acts = fsmRun->tables->actions + fsmRun->tables->toStateActions[fsmRun->cs];
@@ -194,6 +199,7 @@ _again:
 out:
 	if ( fsmRun->p == fsmRun->peof ) {
 		fsmRun->returnResult = false;
+		fsmRun->skipToklen = false;
 		_acts = fsmRun->tables->actions + fsmRun->tables->eofActions[fsmRun->cs];
 		_nacts = (unsigned int) *_acts++;
 
@@ -202,14 +208,17 @@ out:
 
 		while ( _nacts-- > 0 )
 			execAction( fsmRun, fsmRun->tables->actionSwitch[*_acts++] );
-		if ( fsmRun->returnResult )
+		if ( fsmRun->returnResult ) {
+			if ( fsmRun->skipToklen )
+				goto skip_toklen;
 			goto final;
+		}
 	}
 
 final:
 
 	if ( fsmRun->p != 0 )
-		fsmRun->toklen += fsmRun->p - start;
+		fsmRun->toklen += fsmRun->p - fsmRun->start;
+skip_toklen:
+	{}
 }
-
-
diff --git a/colm/pdarun.c b/colm/pdarun.c
index cf15e2f6..85c9148a 100644
--- a/colm/pdarun.c
+++ b/colm/pdarun.c
@@ -1031,7 +1031,7 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is )
 			/* Check for a default token in the region. If one is there
 			 * then send it and continue with the processing loop. */
 			if ( prg->rtd->regionInfo[fsmRun->region].defaultToken >= 0 ) {
-				fsmRun->tokstart = fsmRun->tokend = fsmRun->p;
+				fsmRun->toklen = 0;
 				return prg->rtd->regionInfo[fsmRun->region].defaultToken;
 			}
 
diff --git a/colm/pdarun.h b/colm/pdarun.h
index 5d8b3cea..350c10fb 100644
--- a/colm/pdarun.h
+++ b/colm/pdarun.h
@@ -78,10 +78,13 @@ typedef struct _FsmRun
 	/* FsmRun State. */
 	long region, preRegion;
 	long cs, ncs, act;
-	char *tokstart, *tokend;
+	char *start;
+	char *tokstart;
+	long tokend;
 	long toklen;
 	char *p, *pe, *peof;
 	int returnResult;
+	int skipToklen;
 	char *mark[MARK_SLOTS];
 	long matchedToken;
 } FsmRun;