Merge branch 'copy-on-consume' into eos

11 files changed, 159 insertions, 93 deletions

diff --git a/colm/ctinput.cc b/colm/ctinput.cc
index bde9f424..a10d03f5 100644
--- a/colm/ctinput.cc
+++ b/colm/ctinput.cc
@@ -67,8 +67,6 @@ int inputStreamPatternGetData( FsmRun *fsmRun, StreamImpl *ss, int skip, char *d
 	PatternItem *buf = ss->patItem;
 	int offset = ss->offset;
 
-	attachStream( fsmRun, ss );
-
 	while ( true ) {
 		if ( buf == 0 )
 			return INPUT_EOD;
@@ -244,8 +242,6 @@ int inputStreamConsGetData( FsmRun *fsmRun, StreamImpl *ss, int skip, char *dest
 	ConsItem *buf = ss->consItem;
 	int offset = ss->offset;
 
-	attachStream( fsmRun, ss );
-
 	while ( true ) {
 		if ( buf == 0 )
 			return INPUT_EOD;
diff --git a/colm/input.c b/colm/input.c
index 6aef03c2..14c1cb1b 100644
--- a/colm/input.c
+++ b/colm/input.c
@@ -233,8 +233,6 @@ int fdGetData( FsmRun *fsmRun, StreamImpl *ss, int skip, char *dest, int length,
 		buf = buf->next;
 	}
 
-	attachStream( fsmRun, ss );
-
 	return ret;
 }
 
@@ -452,8 +450,6 @@ int _getData( FsmRun *fsmRun, StreamImpl *is, int skip, char *dest, int length,
 	int ret = 0;
 	*copied = 0;
 
-	attachStream( fsmRun, is );
-
 	/* Move over skip bytes. */
 	RunBuf *buf = is->queue;
 	while ( true ) {
@@ -606,8 +602,7 @@ int _undoConsumeData( FsmRun *fsmRun, StreamImpl *is, const char *data, int leng
 		Stream *stream = (Stream*)is->queue->tree;
 		int len = stream->in->funcs->undoConsumeData( fsmRun, stream->in, data, length );
 
-		if ( stream->in->attached != 0 )
-			detachStream( stream->in->attached, stream->in );
+		clearBuffered( fsmRun );
 
 		return len;
 	}
@@ -617,8 +612,7 @@ int _undoConsumeData( FsmRun *fsmRun, StreamImpl *is, const char *data, int leng
 		memcpy( newBuf->data, data, length );
 		inputStreamPrepend( is, newBuf );
 
-		if ( is->attached != 0 )
-			detachStream( is->attached, is );
+		clearBuffered( fsmRun );
 
 		return length;
 	}
@@ -645,9 +639,6 @@ Tree *_consumeTree( StreamImpl *is )
 
 void _undoConsumeTree( StreamImpl *is, Tree *tree, int ignore )
 {
-	if ( is->attached != 0 )
-		detachStream( is->attached, is );
-
 	/* Create a new buffer for the data. This is the easy implementation.
 	 * Something better is needed here. It puts a max on the amount of
 	 * data that can be pushed back to the inputStream. */
@@ -681,9 +672,6 @@ void _undoConsumeLangEl( StreamImpl *is )
 
 void _prependData( StreamImpl *is, const char *data, long length )
 {
-	if ( is->attached != 0 )
-		detachStream( is->attached, is );
-
 	if ( isSourceStream( is ) && ((Stream*)is->queue->tree)->in->funcs == &streamFuncs ) {
 		Stream *stream = (Stream*)is->queue->tree;
 
@@ -705,9 +693,6 @@ void _prependData( StreamImpl *is, const char *data, long length )
 
 void _prependTree( StreamImpl *is, Tree *tree, int ignore )
 {
-	if ( is->attached != 0 )
-		detachStream( is->attached, is );
-
 	/* Create a new buffer for the data. This is the easy implementation.
 	 * Something better is needed here. It puts a max on the amount of
 	 * data that can be pushed back to the inputStream. */
@@ -730,9 +715,6 @@ void _prependStream( StreamImpl *in, struct ColmTree *tree )
 
 int _undoPrependData( StreamImpl *is, int length )
 {
-	if ( is->attached != 0 )
-		detachStream( is->attached, is );
-
 	debug( REALM_INPUT, "consuming %d bytes\n", length );
 
 	int consumed = 0;
@@ -779,9 +761,6 @@ int _undoPrependData( StreamImpl *is, int length )
 
 Tree *_undoPrependTree( StreamImpl *is )
 {
-	if ( is->attached != 0 )
-		detachStream( is->attached, is );
-
 	while ( is->queue != 0 && is->queue->type == RunBufDataType && is->queue->offset == is->queue->length ) {
 		RunBuf *runBuf = inputStreamPopHead( is );
 		free( runBuf );
@@ -819,9 +798,6 @@ void _appendData( StreamImpl *is, const char *data, long len )
 
 Tree *_undoAppendData( StreamImpl *is, int length )
 {
-	if ( is->attached != 0 )
-		detachStream( is->attached, is );
-
 	int consumed = 0;
 
 	/* Move over skip bytes. */
@@ -881,9 +857,6 @@ void _appendStream( StreamImpl *in, struct ColmTree *tree )
 
 Tree *_undoAppendStream( StreamImpl *is )
 {
-	if ( is->attached != 0 )
-		detachStream( is->attached, is );
-
 	RunBuf *runBuf = inputStreamPopTail( is );
 	Tree *tree = runBuf->tree;
 	free( runBuf );
@@ -892,9 +865,6 @@ Tree *_undoAppendStream( StreamImpl *is )
 
 Tree *_undoAppendTree( StreamImpl *is )
 {
-	if ( is->attached != 0 )
-		detachStream( is->attached, is );
-
 	RunBuf *runBuf = inputStreamPopTail( is );
 	Tree *tree = runBuf->tree;
 	free( runBuf );
diff --git a/colm/input.h b/colm/input.h
index 7a8007c2..13cd48c0 100644
--- a/colm/input.h
+++ b/colm/input.h
@@ -128,7 +128,6 @@ struct StreamFuncs
 struct _StreamImpl
 {
 	struct StreamFuncs *funcs;
-	struct _FsmRun *attached;
 
 	char eofSent;
 	char eof;
diff --git a/colm/pdarun.c b/colm/pdarun.c
index 68fae8d9..e655d9e4 100644
--- a/colm/pdarun.c
+++ b/colm/pdarun.c
@@ -60,14 +60,13 @@
 void initFsmRun( FsmRun *fsmRun, Program *prg )
 {
 	fsmRun->tables = prg->rtd->fsmTables;
-	fsmRun->runBuf = 0;
 
-	/* Run buffers need to stick around because 
-	 * token strings point into them. */
-	fsmRun->runBuf = newRunBuf();
-	fsmRun->runBuf->next = 0;
+	fsmRun->scanBuf = newRunBuf();
+	fsmRun->scanBuf->next = 0;
 
-	fsmRun->p = fsmRun->pe = fsmRun->runBuf->data;
+	fsmRun->consumeBuf = 0;
+
+	fsmRun->p = fsmRun->pe = fsmRun->scanBuf->data;
 	fsmRun->peof = 0;
 
 	fsmRun->preRegion = -1;
@@ -75,9 +74,9 @@ void initFsmRun( FsmRun *fsmRun, Program *prg )
 
 void clearFsmRun( Program *prg, FsmRun *fsmRun )
 {
-	if ( fsmRun->runBuf != 0 ) {
+	if ( fsmRun->consumeBuf != 0 ) {
 		/* Transfer the run buf list to the program */
-		RunBuf *head = fsmRun->runBuf;
+		RunBuf *head = fsmRun->consumeBuf;
 		RunBuf *tail = head;
 		while ( tail->next != 0 )
 			tail = tail->next;
@@ -135,13 +134,15 @@ Head *streamPull( Program *prg, FsmRun *fsmRun, StreamImpl *is, long length )
 	assert( fsmRun->tokstart == 0 );
 
 	RunBuf *runBuf = newRunBuf();
-	runBuf->next = fsmRun->runBuf;
-	fsmRun->runBuf = runBuf;
+	runBuf->next = fsmRun->consumeBuf;
+	fsmRun->consumeBuf = runBuf;
 
 	int lenCopied = 0;
 	is->funcs->getData( fsmRun, is, 0, runBuf->data, length, &lenCopied );
 	is->funcs->consumeData( is, length );
-	fsmRun->p = fsmRun->pe = runBuf->data + length;
+
+	fsmRun->p = fsmRun->pe = fsmRun->scanBuf->data;
+	//fsmRun->peof = 0;
 
 	Head *tokdata = stringAllocPointer( prg, runBuf->data, length );
 	updatePosition( is, runBuf->data, length );
@@ -250,33 +251,6 @@ static void sendBackIgnore( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsm
 	}
 }
 
-void attachStream( FsmRun *fsmRun, StreamImpl *is )
-{
-	if ( is->attached != 0 && is->attached != fsmRun )
-		detachStream( is->attached, is );
-
-	if ( is->attached != fsmRun ) {
-		debug( REALM_INPUT, "attaching FsmRun to stream:  %p %p\n", fsmRun, is );
-		is->attached = fsmRun;
-	}
-}
-
-void detachStream( FsmRun *fsmRun, StreamImpl *is )
-{
-	debug( REALM_INPUT, "detaching FsmRun from stream:  %p %p\n", fsmRun, is );
-
-	is->attached = 0;
-	clearBuffered( fsmRun );
-}
-
-void detachSource( FsmRun *fsmRun, StreamImpl *is )
-{
-	debug( REALM_INPUT, "detaching fsm run from source stream: %p %p\n", fsmRun, is );
-
-	is->attached = 0;
-	clearBuffered( fsmRun );
-}
-
 void clearBuffered( FsmRun *fsmRun )
 {
 	if ( fsmRun->tokstart != 0 ) {
@@ -787,12 +761,14 @@ Head *peekMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is )
 	long length = fsmRun->p - fsmRun->tokstart;
 
 	RunBuf *runBuf = newRunBuf();
-	runBuf->next = fsmRun->runBuf;
-	fsmRun->runBuf = runBuf;
+	runBuf->next = fsmRun->consumeBuf;
+	fsmRun->consumeBuf = runBuf;
 
 	int lenCopied = 0;
 	is->funcs->getData( fsmRun, is, 0, runBuf->data, length, &lenCopied );
-	fsmRun->p = fsmRun->pe = runBuf->data + length;
+
+	fsmRun->p = fsmRun->pe = fsmRun->scanBuf->data;
+	//fsmRun->peof = 0;
 
 	Head *head = stringAllocPointer( prg, runBuf->data, length );
 
@@ -812,13 +788,22 @@ Head *extractMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is )
 	long length = fsmRun->p - fsmRun->tokstart;
 
 	RunBuf *runBuf = newRunBuf();
-	runBuf->next = fsmRun->runBuf;
-	fsmRun->runBuf = runBuf;
+	runBuf->next = fsmRun->consumeBuf;
+	fsmRun->consumeBuf = runBuf;
 
 	int lenCopied = 0;
+	int total = 0;
 	is->funcs->getData( fsmRun, is, 0, runBuf->data, length, &lenCopied );
+	total += lenCopied;
+	while ( total < length ) {
+		is->funcs->getData( fsmRun, is, total, runBuf->data+total, length-total, &lenCopied );
+		total += lenCopied;
+	}
+
 	is->funcs->consumeData( is, length );
-	fsmRun->p = fsmRun->pe = runBuf->data + length;
+
+	fsmRun->p = fsmRun->pe = fsmRun->scanBuf->data;
+	//fsmRun->peof = 0;
 
 	Head *head = stringAllocPointer( prg, runBuf->data, length );
 
@@ -1051,7 +1036,7 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is )
 
 		/* There may be space left in the current buffer. If not then we need
 		 * to make some. */
-		long space = fsmRun->runBuf->data + FSM_BUFSIZE - fsmRun->pe;
+		long space = fsmRun->scanBuf->data + FSM_BUFSIZE - fsmRun->pe;
 		if ( space == 0 ) {
 			/* Create a new run buf. */
 			RunBuf *newBuf = newRunBuf();
@@ -1061,15 +1046,15 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is )
 
 			if ( fsmRun->tokstart == 0 ) {
 				/* No prefix. We filled the previous buffer. */
-				fsmRun->runBuf->length = FSM_BUFSIZE;
+				fsmRun->scanBuf->length = FSM_BUFSIZE;
 			}
 			else {
 				int i;
 
 				debug( REALM_SCAN, "copying data over to new buffer\n" );
-				assert( fsmRun->runBuf->offset == 0 );
+				assert( fsmRun->scanBuf->offset == 0 );
 
-				if ( fsmRun->tokstart == fsmRun->runBuf->data ) {
+				if ( fsmRun->tokstart == fsmRun->scanBuf->data ) {
 					/* A token is started and it is already at the beginning
 					 * of the current buffer. This means buffer is full and it
 					 * must be grown. Probably need to do this sooner. */
@@ -1081,7 +1066,7 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is )
 				memcpy( newBuf->data, fsmRun->tokstart, have );
 
 				/* Compute the length of the previous buffer. */
-				fsmRun->runBuf->length = FSM_BUFSIZE - have;
+				fsmRun->scanBuf->length = FSM_BUFSIZE - have;
 
 				/* Compute tokstart and tokend. */
 				long dist = fsmRun->tokstart - newBuf->data;
@@ -1099,12 +1084,12 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is )
 			fsmRun->p = fsmRun->pe = newBuf->data + have;
 			fsmRun->peof = 0;
 
-			newBuf->next = fsmRun->runBuf;
-			fsmRun->runBuf = newBuf;
+			newBuf->next = fsmRun->scanBuf;
+			fsmRun->scanBuf = newBuf;
 		}
 
 		/* We don't have any data. What is next in the input inputStream? */
-		space = fsmRun->runBuf->data + FSM_BUFSIZE - fsmRun->pe;
+		space = fsmRun->scanBuf->data + FSM_BUFSIZE - fsmRun->pe;
 		assert( space > 0 );
 			
 		/* Get more data. */
diff --git a/colm/pdarun.h b/colm/pdarun.h
index 4b37c5cd..81d376c4 100644
--- a/colm/pdarun.h
+++ b/colm/pdarun.h
@@ -73,7 +73,8 @@ typedef struct _FsmRun
 {
 	FsmTables *tables;
 
-	RunBuf *runBuf;
+	RunBuf *scanBuf;
+	RunBuf *consumeBuf;
 
 	/* FsmRun State. */
 	long region, preRegion;
@@ -461,9 +462,6 @@ void undoParseStream( struct ColmProgram *prg, Tree **sp, StreamImpl *inputStrea
 void clearBuffered( FsmRun *fsmRun );
 void resetToken( FsmRun *fsmRun );
 
-void detachStream( FsmRun *fsmRun, StreamImpl *is );
-void attachStream( FsmRun *fsmRun, StreamImpl *is );
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/test/include1.exp b/test/include1.exp
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/include1.exp
diff --git a/test/include1.in b/test/include1.in
new file mode 100644
index 00000000..9c7aa806
--- /dev/null
+++ b/test/include1.in
@@ -0,0 +1,14 @@
+
+hello;
+
+#include "include1a.in"
+
+there;
+
+#include "include1b.in"
+
+dude;
+
+#include "include1c.in"
+
+and dudettes;
diff --git a/test/include1.lm b/test/include1.lm
new file mode 100644
index 00000000..03a3b046
--- /dev/null
+++ b/test/include1.lm
@@ -0,0 +1,98 @@
+namespace string
+	lex
+		literal '"'
+		token data /[^"\\]+/
+		token escape /'\\' any/
+	end
+
+	def string_data
+		[data]
+	|	[escape]
+
+	def string 
+		['"' string_data* '"']
+	
+	str unquote( S: string )
+	{
+		match S ['"' DL: string_data* '"']
+		for E: escape in DL
+			E.data = 'x'
+		return $DL
+	}
+
+end string
+		
+namespace hash
+
+	lex
+		literal 'define', 'include'
+		literal '#', '\n' ni
+
+		token id /[a-zA-Z_][a-zA-Z_0-9]*/
+		token number /[0-9]+/
+
+		ignore /[ \t]/
+	end
+
+	def hash
+		['#' 'define' Id: id number '\n']
+	|	['#' 'include' Inc: string::string '\n']
+
+end hash
+
+token rest_of_line /[^\n]* '\n'/
+
+namespace lang
+
+	lex
+		ignore /space/
+		literal '*', '(', ')', ';'
+		token id /[a-zA-Z_][a-zA-Z_0-9]*/
+		token number /[0-9]+/
+
+		token hash /'#'/ {
+			parse_stop H: hash::hash[ input ]
+			if ( H.tree ) {
+				if ( H.tree.Inc ) {
+					FN: str = unquote( H.tree.Inc )
+					print( 'opening ' FN '\n' )
+					IS: stream = open( FN 'r' )
+					if ( ! IS ) {
+						print( 'ERROR: failed to open ' FN '\n' )
+						exit(1)
+					}
+					input.push( IS )
+				}
+			}
+			else {
+				parse_stop L: rest_of_line[ input ]
+				if ! L.tree {
+					print( "ERROR: stuck: " L.error )
+					exit(1)
+				}
+				print( "ERROR: failed to parse # directive: " L.tree )
+			}
+		}
+	end
+
+	def item 
+		[id]
+	|	['(' item* ')']
+
+	def statement
+		[item* ';']
+
+	def start 
+		[statement*]
+
+end lang
+
+parse Input: lang::start[ stdin ]
+
+if ! Input.tree 
+	print( Input.error '\n' )
+else {
+	#print( Input.tree '\n' )
+	S: lang::start = Input.tree
+	print( Input.tree '\n' )
+}
diff --git a/test/include1a.in b/test/include1a.in
new file mode 100644
index 00000000..26da0afa
--- /dev/null
+++ b/test/include1a.in
@@ -0,0 +1,2 @@
+a;
+b;
diff --git a/test/include1b.in b/test/include1b.in
new file mode 100644
index 00000000..6c574323
--- /dev/null
+++ b/test/include1b.in
@@ -0,0 +1,2 @@
+c;
+d;
diff --git a/test/include1c.in b/test/include1c.in
new file mode 100644
index 00000000..5373832d
--- /dev/null
+++ b/test/include1c.in
@@ -0,0 +1,2 @@
+e;
+f;