diff options
-rw-r--r-- | colm/input.c | 18 | ||||
-rw-r--r-- | colm/input.h | 8 | ||||
-rw-r--r-- | colm/pdarun.c | 15 |
3 files changed, 32 insertions, 9 deletions
diff --git a/colm/input.c b/colm/input.c index ba144d94..6aef03c2 100644 --- a/colm/input.c +++ b/colm/input.c @@ -467,9 +467,16 @@ int _getData( FsmRun *fsmRun, StreamImpl *is, int skip, char *dest, int length, Stream *stream = (Stream*)buf->tree; int type = stream->in->funcs->getData( fsmRun, stream->in, skip, dest, length, copied ); - if ( type == INPUT_EOD && is->eof ) { - ret = INPUT_EOF; - break; +// if ( type == INPUT_EOD && !stream->in->eosSent ) { +// stream->in->eosSent = 1; +// ret = INPUT_EOS; +// continue; +// } + + if ( type == INPUT_EOD || type == INPUT_EOF ) { + debug( REALM_INPUT, "skipping over input\n" ); + buf = buf->next; + continue; } ret = type; @@ -558,6 +565,7 @@ int _consumeData( StreamImpl *is, int length ) if ( buf->type == RunBufSourceType ) { Stream *stream = (Stream*)buf->tree; int slen = stream->in->funcs->consumeData( stream->in, length ); + debug( REALM_INPUT, " got %d bytes from source\n", slen ); consumed += slen; length -= slen; @@ -578,8 +586,10 @@ int _consumeData( StreamImpl *is, int length ) } } - if ( length == 0 ) + if ( length == 0 ) { + debug( REALM_INPUT, "exiting consume\n", length ); break; + } RunBuf *runBuf = inputStreamPopHead( is ); free( runBuf ); diff --git a/colm/input.h b/colm/input.h index 3bfc43c5..7a8007c2 100644 --- a/colm/input.h +++ b/colm/input.h @@ -35,9 +35,10 @@ extern "C" { /* This is for data sources to return, not for the wrapper. */ #define INPUT_EOD 2 #define INPUT_EOF 3 -#define INPUT_LANG_EL 4 -#define INPUT_TREE 5 -#define INPUT_IGNORE 6 +#define INPUT_EOS 4 +#define INPUT_LANG_EL 5 +#define INPUT_TREE 6 +#define INPUT_IGNORE 7 /* * pdaRun <- fsmRun <- stream @@ -131,6 +132,7 @@ struct _StreamImpl char eofSent; char eof; + char eosSent; RunBuf *queue; RunBuf *queueTail; diff --git a/colm/pdarun.c b/colm/pdarun.c index 5bd0553a..68fae8d9 100644 --- a/colm/pdarun.c +++ b/colm/pdarun.c @@ -280,8 +280,10 @@ void detachSource( FsmRun *fsmRun, StreamImpl *is ) void clearBuffered( FsmRun *fsmRun ) { if ( fsmRun->tokstart != 0 ) { - fsmRun->p = fsmRun->pe = fsmRun->tokstart; - fsmRun->tokstart = 0; + //fsmRun->p = fsmRun->pe = fsmRun->tokstart; + //fsmRun->tokstart = 0; + + fsmRun->pe = fsmRun->p; } else { fsmRun->pe = fsmRun->p; @@ -1116,6 +1118,15 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is ) fsmRun->pe = fsmRun->p + len; break; + case INPUT_EOS: + if ( fsmRun->tokstart != 0 ) + fsmRun->peof = fsmRun->pe; + debug( REALM_SCAN, "EOS *******************\n" ); +// else { +// return SCAN_EOS; +// } + break; + case INPUT_EOF: if ( fsmRun->tokstart != 0 ) fsmRun->peof = fsmRun->pe; |