set new token location in the consume function

This way we get the location from the leaf stream that it was pulled from.

4 files changed, 24 insertions, 14 deletions

diff --git a/colm/ctinput.cc b/colm/ctinput.cc
index 0bee62cc..2a3ff833 100644
--- a/colm/ctinput.cc
+++ b/colm/ctinput.cc
@@ -178,7 +178,7 @@ void inputStreamPatternUndoConsumeLangEl( StreamImpl *ss )
 	ss->offset = ss->patItem->data.length();
 }
 
-int inputStreamPatternConsumeData( StreamImpl *ss, int length )
+int inputStreamPatternConsumeData( StreamImpl *ss, int length, Location *loc )
 {
 	//debug( REALM_INPUT, "consuming %ld bytes\n", length );
 
@@ -395,7 +395,7 @@ void inputStreamConsUndoConsumeLangEl( StreamImpl *ss )
 	ss->offset = ss->consItem->data.length();
 }
 
-int inputStreamConsConsumeData( StreamImpl *ss, int length )
+int inputStreamConsConsumeData( StreamImpl *ss, int length, Location *loc )
 {
 	int consumed = 0;
 
diff --git a/colm/input.c b/colm/input.c
index e12772df..b4bb94e8 100644
--- a/colm/input.c
+++ b/colm/input.c
@@ -270,7 +270,7 @@ int fdGetData( StreamImpl *ss, char *dest, int length )
 	return copied;
 }
 
-int fdConsumeData( StreamImpl *ss, int length )
+int fdConsumeData( StreamImpl *ss, int length, Location *loc )
 {
 	int consumed = 0;
 
@@ -286,6 +286,12 @@ int fdConsumeData( StreamImpl *ss, int length )
 		else if ( buf->type == RunBufIgnoreType )
 			break;
 		else {
+			if ( loc != 0 && loc->line == 0 ) {
+				loc->line = ss->line;
+				loc->column = ss->column;
+				loc->byte = ss->byte;
+			}
+
 			/* Anything available in the current buffer. */
 			int avail = buf->length - buf->offset;
 			if ( avail > 0 ) {
@@ -620,7 +626,7 @@ static int _getData( StreamImpl *is, char *dest, int length )
 	return copied;
 }
 
-static int _consumeData( StreamImpl *is, int length )
+static int _consumeData( StreamImpl *is, int length, Location *loc )
 {
 	//debug( REALM_INPUT, "consuming %d bytes\n", length );
 
@@ -635,7 +641,7 @@ static int _consumeData( StreamImpl *is, int length )
 
 		if ( buf->type == RunBufSourceType ) {
 			Stream *stream = (Stream*)buf->tree;
-			int slen = stream->in->funcs->consumeData( stream->in, length );
+			int slen = stream->in->funcs->consumeData( stream->in, length, loc );
 			//debug( REALM_INPUT, " got %d bytes from source\n", slen );
 
 			consumed += slen;
@@ -800,7 +806,8 @@ static int _undoPrependData( StreamImpl *is, int length )
 
 		if ( buf->type == RunBufSourceType ) {
 			Stream *stream = (Stream*)buf->tree;
-			int slen = stream->in->funcs->consumeData( stream->in, length );
+			/* FIXME: provide real impl. */
+			int slen = stream->in->funcs->consumeData( stream->in, length, 0 );
 
 			consumed += slen;
 			length -= slen;
diff --git a/colm/input.h b/colm/input.h
index 00f674c8..314dc32a 100644
--- a/colm/input.h
+++ b/colm/input.h
@@ -86,6 +86,7 @@ typedef struct _RunBuf
 RunBuf *newRunBuf();
 
 typedef struct _StreamImpl StreamImpl;
+typedef struct _Location Location;
 
 struct StreamFuncs
 {
@@ -93,7 +94,7 @@ struct StreamFuncs
 
 	int (*getData)( StreamImpl *ss, char *dest, int length );
 
-	int (*consumeData)( StreamImpl *ss, int length );
+	int (*consumeData)( StreamImpl *ss, int length, Location *loc );
 	int (*undoConsumeData)( StreamImpl *ss, const char *data, int length );
 
 	struct ColmTree *(*consumeTree)( StreamImpl *ss );
diff --git a/colm/pdarun.c b/colm/pdarun.c
index 2bd37c07..0049b7a5 100644
--- a/colm/pdarun.c
+++ b/colm/pdarun.c
@@ -109,7 +109,8 @@ Head *extractMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is )
 	char *dest = runBuf->data + runBuf->length;
 
 	is->funcs->getData( is, dest, length );
-	is->funcs->consumeData( is, length );
+	Location *location = locationAllocate( prg );
+	is->funcs->consumeData( is, length, location );
 
 	runBuf->length += length;
 
@@ -119,10 +120,7 @@ Head *extractMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is )
 
 	Head *head = stringAllocPointer( prg, dest, length );
 
-	head->location = locationAllocate( prg );
-	head->location->line = is->line;
-	head->location->column = is->column;
-	head->location->byte = is->byte;
+	head->location = location;
 
 	debug( prg, REALM_PARSE, "location byte: %d\n", is->byte );
 
@@ -173,7 +171,8 @@ Head *streamPull( Program *prg, PdaRun *pdaRun, StreamImpl *is, long length )
 		char *dest = runBuf->data + runBuf->length;
 
 		is->funcs->getData( is, dest, length );
-		is->funcs->consumeData( is, length );
+		Location *loc = locationAllocate( prg );
+		is->funcs->consumeData( is, length, loc );
 
 		runBuf->length += length;
 
@@ -181,6 +180,7 @@ Head *streamPull( Program *prg, PdaRun *pdaRun, StreamImpl *is, long length )
 		fsmRun->toklen = 0;
 
 		Head *tokdata = stringAllocPointer( prg, dest, length );
+		tokdata->location = loc;
 
 		return tokdata;
 	}
@@ -189,7 +189,9 @@ Head *streamPull( Program *prg, PdaRun *pdaRun, StreamImpl *is, long length )
 		char *dest = (char*)head->data;
 
 		is->funcs->getData( is, dest, length );
-		is->funcs->consumeData( is, length );
+		Location *loc = locationAllocate( prg );
+		is->funcs->consumeData( is, length, loc );
+		head->location = loc;
 
 		return head;
 	}