diff options
-rw-r--r-- | colm/ctinput.cc | 4 | ||||
-rw-r--r-- | colm/input.c | 34 | ||||
-rw-r--r-- | colm/input.h | 1 | ||||
-rw-r--r-- | colm/pdarun.c | 89 | ||||
-rw-r--r-- | colm/pdarun.h | 6 | ||||
-rw-r--r-- | test/include1.exp | 0 | ||||
-rw-r--r-- | test/include1.in | 14 | ||||
-rw-r--r-- | test/include1.lm | 98 | ||||
-rw-r--r-- | test/include1a.in | 2 | ||||
-rw-r--r-- | test/include1b.in | 2 | ||||
-rw-r--r-- | test/include1c.in | 2 |
11 files changed, 159 insertions, 93 deletions
diff --git a/colm/ctinput.cc b/colm/ctinput.cc index bde9f424..a10d03f5 100644 --- a/colm/ctinput.cc +++ b/colm/ctinput.cc @@ -67,8 +67,6 @@ int inputStreamPatternGetData( FsmRun *fsmRun, StreamImpl *ss, int skip, char *d PatternItem *buf = ss->patItem; int offset = ss->offset; - attachStream( fsmRun, ss ); - while ( true ) { if ( buf == 0 ) return INPUT_EOD; @@ -244,8 +242,6 @@ int inputStreamConsGetData( FsmRun *fsmRun, StreamImpl *ss, int skip, char *dest ConsItem *buf = ss->consItem; int offset = ss->offset; - attachStream( fsmRun, ss ); - while ( true ) { if ( buf == 0 ) return INPUT_EOD; diff --git a/colm/input.c b/colm/input.c index 6aef03c2..14c1cb1b 100644 --- a/colm/input.c +++ b/colm/input.c @@ -233,8 +233,6 @@ int fdGetData( FsmRun *fsmRun, StreamImpl *ss, int skip, char *dest, int length, buf = buf->next; } - attachStream( fsmRun, ss ); - return ret; } @@ -452,8 +450,6 @@ int _getData( FsmRun *fsmRun, StreamImpl *is, int skip, char *dest, int length, int ret = 0; *copied = 0; - attachStream( fsmRun, is ); - /* Move over skip bytes. */ RunBuf *buf = is->queue; while ( true ) { @@ -606,8 +602,7 @@ int _undoConsumeData( FsmRun *fsmRun, StreamImpl *is, const char *data, int leng Stream *stream = (Stream*)is->queue->tree; int len = stream->in->funcs->undoConsumeData( fsmRun, stream->in, data, length ); - if ( stream->in->attached != 0 ) - detachStream( stream->in->attached, stream->in ); + clearBuffered( fsmRun ); return len; } @@ -617,8 +612,7 @@ int _undoConsumeData( FsmRun *fsmRun, StreamImpl *is, const char *data, int leng memcpy( newBuf->data, data, length ); inputStreamPrepend( is, newBuf ); - if ( is->attached != 0 ) - detachStream( is->attached, is ); + clearBuffered( fsmRun ); return length; } @@ -645,9 +639,6 @@ Tree *_consumeTree( StreamImpl *is ) void _undoConsumeTree( StreamImpl *is, Tree *tree, int ignore ) { - if ( is->attached != 0 ) - detachStream( is->attached, is ); - /* Create a new buffer for the data. This is the easy implementation. * Something better is needed here. It puts a max on the amount of * data that can be pushed back to the inputStream. */ @@ -681,9 +672,6 @@ void _undoConsumeLangEl( StreamImpl *is ) void _prependData( StreamImpl *is, const char *data, long length ) { - if ( is->attached != 0 ) - detachStream( is->attached, is ); - if ( isSourceStream( is ) && ((Stream*)is->queue->tree)->in->funcs == &streamFuncs ) { Stream *stream = (Stream*)is->queue->tree; @@ -705,9 +693,6 @@ void _prependData( StreamImpl *is, const char *data, long length ) void _prependTree( StreamImpl *is, Tree *tree, int ignore ) { - if ( is->attached != 0 ) - detachStream( is->attached, is ); - /* Create a new buffer for the data. This is the easy implementation. * Something better is needed here. It puts a max on the amount of * data that can be pushed back to the inputStream. */ @@ -730,9 +715,6 @@ void _prependStream( StreamImpl *in, struct ColmTree *tree ) int _undoPrependData( StreamImpl *is, int length ) { - if ( is->attached != 0 ) - detachStream( is->attached, is ); - debug( REALM_INPUT, "consuming %d bytes\n", length ); int consumed = 0; @@ -779,9 +761,6 @@ int _undoPrependData( StreamImpl *is, int length ) Tree *_undoPrependTree( StreamImpl *is ) { - if ( is->attached != 0 ) - detachStream( is->attached, is ); - while ( is->queue != 0 && is->queue->type == RunBufDataType && is->queue->offset == is->queue->length ) { RunBuf *runBuf = inputStreamPopHead( is ); free( runBuf ); @@ -819,9 +798,6 @@ void _appendData( StreamImpl *is, const char *data, long len ) Tree *_undoAppendData( StreamImpl *is, int length ) { - if ( is->attached != 0 ) - detachStream( is->attached, is ); - int consumed = 0; /* Move over skip bytes. */ @@ -881,9 +857,6 @@ void _appendStream( StreamImpl *in, struct ColmTree *tree ) Tree *_undoAppendStream( StreamImpl *is ) { - if ( is->attached != 0 ) - detachStream( is->attached, is ); - RunBuf *runBuf = inputStreamPopTail( is ); Tree *tree = runBuf->tree; free( runBuf ); @@ -892,9 +865,6 @@ Tree *_undoAppendStream( StreamImpl *is ) Tree *_undoAppendTree( StreamImpl *is ) { - if ( is->attached != 0 ) - detachStream( is->attached, is ); - RunBuf *runBuf = inputStreamPopTail( is ); Tree *tree = runBuf->tree; free( runBuf ); diff --git a/colm/input.h b/colm/input.h index 7a8007c2..13cd48c0 100644 --- a/colm/input.h +++ b/colm/input.h @@ -128,7 +128,6 @@ struct StreamFuncs struct _StreamImpl { struct StreamFuncs *funcs; - struct _FsmRun *attached; char eofSent; char eof; diff --git a/colm/pdarun.c b/colm/pdarun.c index 68fae8d9..e655d9e4 100644 --- a/colm/pdarun.c +++ b/colm/pdarun.c @@ -60,14 +60,13 @@ void initFsmRun( FsmRun *fsmRun, Program *prg ) { fsmRun->tables = prg->rtd->fsmTables; - fsmRun->runBuf = 0; - /* Run buffers need to stick around because - * token strings point into them. */ - fsmRun->runBuf = newRunBuf(); - fsmRun->runBuf->next = 0; + fsmRun->scanBuf = newRunBuf(); + fsmRun->scanBuf->next = 0; - fsmRun->p = fsmRun->pe = fsmRun->runBuf->data; + fsmRun->consumeBuf = 0; + + fsmRun->p = fsmRun->pe = fsmRun->scanBuf->data; fsmRun->peof = 0; fsmRun->preRegion = -1; @@ -75,9 +74,9 @@ void initFsmRun( FsmRun *fsmRun, Program *prg ) void clearFsmRun( Program *prg, FsmRun *fsmRun ) { - if ( fsmRun->runBuf != 0 ) { + if ( fsmRun->consumeBuf != 0 ) { /* Transfer the run buf list to the program */ - RunBuf *head = fsmRun->runBuf; + RunBuf *head = fsmRun->consumeBuf; RunBuf *tail = head; while ( tail->next != 0 ) tail = tail->next; @@ -135,13 +134,15 @@ Head *streamPull( Program *prg, FsmRun *fsmRun, StreamImpl *is, long length ) assert( fsmRun->tokstart == 0 ); RunBuf *runBuf = newRunBuf(); - runBuf->next = fsmRun->runBuf; - fsmRun->runBuf = runBuf; + runBuf->next = fsmRun->consumeBuf; + fsmRun->consumeBuf = runBuf; int lenCopied = 0; is->funcs->getData( fsmRun, is, 0, runBuf->data, length, &lenCopied ); is->funcs->consumeData( is, length ); - fsmRun->p = fsmRun->pe = runBuf->data + length; + + fsmRun->p = fsmRun->pe = fsmRun->scanBuf->data; + //fsmRun->peof = 0; Head *tokdata = stringAllocPointer( prg, runBuf->data, length ); updatePosition( is, runBuf->data, length ); @@ -250,33 +251,6 @@ static void sendBackIgnore( Program *prg, Tree **sp, PdaRun *pdaRun, FsmRun *fsm } } -void attachStream( FsmRun *fsmRun, StreamImpl *is ) -{ - if ( is->attached != 0 && is->attached != fsmRun ) - detachStream( is->attached, is ); - - if ( is->attached != fsmRun ) { - debug( REALM_INPUT, "attaching FsmRun to stream: %p %p\n", fsmRun, is ); - is->attached = fsmRun; - } -} - -void detachStream( FsmRun *fsmRun, StreamImpl *is ) -{ - debug( REALM_INPUT, "detaching FsmRun from stream: %p %p\n", fsmRun, is ); - - is->attached = 0; - clearBuffered( fsmRun ); -} - -void detachSource( FsmRun *fsmRun, StreamImpl *is ) -{ - debug( REALM_INPUT, "detaching fsm run from source stream: %p %p\n", fsmRun, is ); - - is->attached = 0; - clearBuffered( fsmRun ); -} - void clearBuffered( FsmRun *fsmRun ) { if ( fsmRun->tokstart != 0 ) { @@ -787,12 +761,14 @@ Head *peekMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is ) long length = fsmRun->p - fsmRun->tokstart; RunBuf *runBuf = newRunBuf(); - runBuf->next = fsmRun->runBuf; - fsmRun->runBuf = runBuf; + runBuf->next = fsmRun->consumeBuf; + fsmRun->consumeBuf = runBuf; int lenCopied = 0; is->funcs->getData( fsmRun, is, 0, runBuf->data, length, &lenCopied ); - fsmRun->p = fsmRun->pe = runBuf->data + length; + + fsmRun->p = fsmRun->pe = fsmRun->scanBuf->data; + //fsmRun->peof = 0; Head *head = stringAllocPointer( prg, runBuf->data, length ); @@ -812,13 +788,22 @@ Head *extractMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is ) long length = fsmRun->p - fsmRun->tokstart; RunBuf *runBuf = newRunBuf(); - runBuf->next = fsmRun->runBuf; - fsmRun->runBuf = runBuf; + runBuf->next = fsmRun->consumeBuf; + fsmRun->consumeBuf = runBuf; int lenCopied = 0; + int total = 0; is->funcs->getData( fsmRun, is, 0, runBuf->data, length, &lenCopied ); + total += lenCopied; + while ( total < length ) { + is->funcs->getData( fsmRun, is, total, runBuf->data+total, length-total, &lenCopied ); + total += lenCopied; + } + is->funcs->consumeData( is, length ); - fsmRun->p = fsmRun->pe = runBuf->data + length; + + fsmRun->p = fsmRun->pe = fsmRun->scanBuf->data; + //fsmRun->peof = 0; Head *head = stringAllocPointer( prg, runBuf->data, length ); @@ -1051,7 +1036,7 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is ) /* There may be space left in the current buffer. If not then we need * to make some. */ - long space = fsmRun->runBuf->data + FSM_BUFSIZE - fsmRun->pe; + long space = fsmRun->scanBuf->data + FSM_BUFSIZE - fsmRun->pe; if ( space == 0 ) { /* Create a new run buf. */ RunBuf *newBuf = newRunBuf(); @@ -1061,15 +1046,15 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is ) if ( fsmRun->tokstart == 0 ) { /* No prefix. We filled the previous buffer. */ - fsmRun->runBuf->length = FSM_BUFSIZE; + fsmRun->scanBuf->length = FSM_BUFSIZE; } else { int i; debug( REALM_SCAN, "copying data over to new buffer\n" ); - assert( fsmRun->runBuf->offset == 0 ); + assert( fsmRun->scanBuf->offset == 0 ); - if ( fsmRun->tokstart == fsmRun->runBuf->data ) { + if ( fsmRun->tokstart == fsmRun->scanBuf->data ) { /* A token is started and it is already at the beginning * of the current buffer. This means buffer is full and it * must be grown. Probably need to do this sooner. */ @@ -1081,7 +1066,7 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is ) memcpy( newBuf->data, fsmRun->tokstart, have ); /* Compute the length of the previous buffer. */ - fsmRun->runBuf->length = FSM_BUFSIZE - have; + fsmRun->scanBuf->length = FSM_BUFSIZE - have; /* Compute tokstart and tokend. */ long dist = fsmRun->tokstart - newBuf->data; @@ -1099,12 +1084,12 @@ long scanToken( Program *prg, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *is ) fsmRun->p = fsmRun->pe = newBuf->data + have; fsmRun->peof = 0; - newBuf->next = fsmRun->runBuf; - fsmRun->runBuf = newBuf; + newBuf->next = fsmRun->scanBuf; + fsmRun->scanBuf = newBuf; } /* We don't have any data. What is next in the input inputStream? */ - space = fsmRun->runBuf->data + FSM_BUFSIZE - fsmRun->pe; + space = fsmRun->scanBuf->data + FSM_BUFSIZE - fsmRun->pe; assert( space > 0 ); /* Get more data. */ diff --git a/colm/pdarun.h b/colm/pdarun.h index 4b37c5cd..81d376c4 100644 --- a/colm/pdarun.h +++ b/colm/pdarun.h @@ -73,7 +73,8 @@ typedef struct _FsmRun { FsmTables *tables; - RunBuf *runBuf; + RunBuf *scanBuf; + RunBuf *consumeBuf; /* FsmRun State. */ long region, preRegion; @@ -461,9 +462,6 @@ void undoParseStream( struct ColmProgram *prg, Tree **sp, StreamImpl *inputStrea void clearBuffered( FsmRun *fsmRun ); void resetToken( FsmRun *fsmRun ); -void detachStream( FsmRun *fsmRun, StreamImpl *is ); -void attachStream( FsmRun *fsmRun, StreamImpl *is ); - #ifdef __cplusplus } #endif diff --git a/test/include1.exp b/test/include1.exp new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/include1.exp diff --git a/test/include1.in b/test/include1.in new file mode 100644 index 00000000..9c7aa806 --- /dev/null +++ b/test/include1.in @@ -0,0 +1,14 @@ + +hello; + +#include "include1a.in" + +there; + +#include "include1b.in" + +dude; + +#include "include1c.in" + +and dudettes; diff --git a/test/include1.lm b/test/include1.lm new file mode 100644 index 00000000..03a3b046 --- /dev/null +++ b/test/include1.lm @@ -0,0 +1,98 @@ +namespace string + lex + literal '"' + token data /[^"\\]+/ + token escape /'\\' any/ + end + + def string_data + [data] + | [escape] + + def string + ['"' string_data* '"'] + + str unquote( S: string ) + { + match S ['"' DL: string_data* '"'] + for E: escape in DL + E.data = 'x' + return $DL + } + +end string + +namespace hash + + lex + literal 'define', 'include' + literal '#', '\n' ni + + token id /[a-zA-Z_][a-zA-Z_0-9]*/ + token number /[0-9]+/ + + ignore /[ \t]/ + end + + def hash + ['#' 'define' Id: id number '\n'] + | ['#' 'include' Inc: string::string '\n'] + +end hash + +token rest_of_line /[^\n]* '\n'/ + +namespace lang + + lex + ignore /space/ + literal '*', '(', ')', ';' + token id /[a-zA-Z_][a-zA-Z_0-9]*/ + token number /[0-9]+/ + + token hash /'#'/ { + parse_stop H: hash::hash[ input ] + if ( H.tree ) { + if ( H.tree.Inc ) { + FN: str = unquote( H.tree.Inc ) + print( 'opening ' FN '\n' ) + IS: stream = open( FN 'r' ) + if ( ! IS ) { + print( 'ERROR: failed to open ' FN '\n' ) + exit(1) + } + input.push( IS ) + } + } + else { + parse_stop L: rest_of_line[ input ] + if ! L.tree { + print( "ERROR: stuck: " L.error ) + exit(1) + } + print( "ERROR: failed to parse # directive: " L.tree ) + } + } + end + + def item + [id] + | ['(' item* ')'] + + def statement + [item* ';'] + + def start + [statement*] + +end lang + +parse Input: lang::start[ stdin ] + +if ! Input.tree + print( Input.error '\n' ) +else { + #print( Input.tree '\n' ) + S: lang::start = Input.tree + print( Input.tree '\n' ) +} diff --git a/test/include1a.in b/test/include1a.in new file mode 100644 index 00000000..26da0afa --- /dev/null +++ b/test/include1a.in @@ -0,0 +1,2 @@ +a; +b; diff --git a/test/include1b.in b/test/include1b.in new file mode 100644 index 00000000..6c574323 --- /dev/null +++ b/test/include1b.in @@ -0,0 +1,2 @@ +c; +d; diff --git a/test/include1c.in b/test/include1c.in new file mode 100644 index 00000000..5373832d --- /dev/null +++ b/test/include1c.in @@ -0,0 +1,2 @@ +e; +f; |