experimenting with EOS, very rough

3 files changed, 32 insertions, 9 deletions

diff --git a/colm/input.c b/colm/input.c
index ba144d94..6aef03c2 100644
--- a/colm/input.c
+++ b/colm/input.c
@@ -467,9 +467,16 @@ int _getData( FsmRun *fsmRun, StreamImpl *is, int skip, char *dest, int length,
 			Stream *stream = (Stream*)buf->tree;
 			int type = stream->in->funcs->getData( fsmRun, stream->in, skip, dest, length, copied );
 
-			if ( type == INPUT_EOD && is->eof ) {
-				ret = INPUT_EOF;
-				break;
+//			if ( type == INPUT_EOD && !stream->in->eosSent ) {
+//				stream->in->eosSent = 1;
+//				ret = INPUT_EOS;
+//				continue;
+//			}
+
+			if ( type == INPUT_EOD || type == INPUT_EOF ) {
+				debug( REALM_INPUT, "skipping over input\n" );
+				buf = buf->next;
+				continue;
 			}
 
 			ret = type;
@@ -558,6 +565,7 @@ int _consumeData( StreamImpl *is, int length )
 		if ( buf->type == RunBufSourceType ) {
 			Stream *stream = (Stream*)buf->tree;
 			int slen = stream->in->funcs->consumeData( stream->in, length );
+			debug( REALM_INPUT, " got %d bytes from source\n", slen );
 
 			consumed += slen;
 			length -= slen;
@@ -578,8 +586,10 @@ int _consumeData( StreamImpl *is, int length )
 			}
 		}
 
-		if ( length == 0 )
+		if ( length == 0 ) {
+			debug( REALM_INPUT, "exiting consume\n", length );
 			break;
+		}
 
 		RunBuf *runBuf = inputStreamPopHead( is );
 		free( runBuf );
diff --git a/colm/input.h b/colm/input.h
index 3bfc43c5..7a8007c2 100644
--- a/colm/input.h
+++ b/colm/input.h
@@ -35,9 +35,10 @@ extern "C" {
 /* This is for data sources to return, not for the wrapper. */
 #define INPUT_EOD      2
 #define INPUT_EOF      3
-#define INPUT_LANG_EL  4
-#define INPUT_TREE     5
-#define INPUT_IGNORE   6
+#define INPUT_EOS      4
+#define INPUT_LANG_EL  5
+#define INPUT_TREE     6
+#define INPUT_IGNORE   7
 
 /*
  * pdaRun <- fsmRun <- stream 
@@ -131,6 +132,7 @@ struct _StreamImpl
 
 	char eofSent;
 	char eof;
+	char eosSent;
 
 	RunBuf *queue;
 	RunBuf *queueTail;
diff --git a/colm/pdarun.c b/colm/pdarun.c
index 5bd0553a..68fae8d9 100644
--- a/colm/pdarun.c
+++ b/colm/pdarun.c
@@ -280,8 +280,10 @@ void detachSource( FsmRun *fsmRun, StreamImpl *is )
 void clearBuffered( FsmRun *fsmRun )
 {
 	if ( fsmRun->tokstart != 0 ) {
-		fsmRun->p = fsmRun->pe = fsmRun->tokstart;
-		fsmRun->tokstart = 0;
+		//fsmRun->p = fsmRun->pe = fsmRun->tokstart;
+		//fsmRun->tokstart = 0;
+
+		fsmRun->pe = fsmRun->p;
 	}
 	else {
 		fsmRun->pe = fsmRun->p;
@@ -1116,6 +1118,15 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is )
 				fsmRun->pe = fsmRun->p + len;
 				break;
 
+			case INPUT_EOS:
+				if ( fsmRun->tokstart != 0 )
+					fsmRun->peof = fsmRun->pe;
+				debug( REALM_SCAN, "EOS *******************\n" );
+//				else {
+//					return SCAN_EOS;
+//				}
+				break;
+
 			case INPUT_EOF:
 				if ( fsmRun->tokstart != 0 )
 					fsmRun->peof = fsmRun->pe;