keep tokend as an int, not pointer

Since we are no longer maintaining a contiguous token buffer during scanning we
cannot use a pointer for tokend. Turn it into an offset (toklen).

1 files changed, 4 insertions, 1 deletions

diff --git a/colm/pdarun.h b/colm/pdarun.h
index 5d8b3cea..350c10fb 100644
--- a/colm/pdarun.h
+++ b/colm/pdarun.h
@@ -78,10 +78,13 @@ typedef struct _FsmRun
 	/* FsmRun State. */
 	long region, preRegion;
 	long cs, ncs, act;
-	char *tokstart, *tokend;
+	char *start;
+	char *tokstart;
+	long tokend;
 	long toklen;
 	char *p, *pe, *peof;
 	int returnResult;
+	int skipToklen;
 	char *mark[MARK_SLOTS];
 	long matchedToken;
 } FsmRun;