From 2ccf3932d99d1a82f12e8cccbd2c1e7f54e63bb4 Mon Sep 17 00:00:00 2001 From: Adrian Thurston Date: Sun, 10 Feb 2013 11:11:34 -0500 Subject: support stream pull outside of a parser --- colm/bytecode.c | 22 +++++++++++++++++++--- colm/bytecode.h | 2 ++ colm/pdarun.c | 45 +++++++++++++++++++++++++++++---------------- colm/pdarun.h | 2 +- colm/string.c | 2 -- colm/synthesis.cc | 2 +- test/pull1.exp | 1 + test/pull1.in | 1 + test/pull1.lm | 2 ++ 9 files changed, 56 insertions(+), 23 deletions(-) create mode 100644 test/pull1.exp create mode 100644 test/pull1.in create mode 100644 test/pull1.lm diff --git a/colm/bytecode.c b/colm/bytecode.c index d167013c..4d93c0f7 100644 --- a/colm/bytecode.c +++ b/colm/bytecode.c @@ -300,10 +300,10 @@ break; } return PcrDone; } -Tree *streamPullBc( Program *prg, FsmRun *fsmRun, StreamImpl *in, Tree *length ) +Tree *streamPullBc( Program *prg, PdaRun *pdaRun, StreamImpl *in, Tree *length ) { long len = ((Int*)length)->value; - Head *tokdata = streamPull( prg, fsmRun, in, len ); + Head *tokdata = streamPull( prg, pdaRun, in, len ); return constructString( prg, tokdata ); } @@ -2456,7 +2456,8 @@ again: Stream *accumStream = (Stream*)vm_pop(); Tree *len = vm_pop(); - Tree *string = streamPullBc( prg, exec->parser->pdaRun->fsmRun, accumStream->in, len ); + PdaRun *pdaRun = exec->parser != 0 ? exec->parser->pdaRun : 0; + Tree *string = streamPullBc( prg, pdaRun, accumStream->in, len ); treeUpref( string ); vm_push( string ); @@ -2470,6 +2471,21 @@ again: treeDownref( prg, sp, len ); break; } + + case IN_INPUT_PULL_WC: { + debug( REALM_BYTECODE, "IN_INPUT_PULL_WC\n" ); + + Stream *accumStream = (Stream*)vm_pop(); + Tree *len = vm_pop(); + PdaRun *pdaRun = exec->parser != 0 ? exec->parser->pdaRun : 0; + Tree *string = streamPullBc( prg, pdaRun, accumStream->in, len ); + treeUpref( string ); + vm_push( string ); + + treeDownref( prg, sp, (Tree*)accumStream ); + treeDownref( prg, sp, len ); + break; + } case IN_INPUT_PULL_BKT: { Tree *string; read_tree( string ); diff --git a/colm/bytecode.h b/colm/bytecode.h index cb777481..c84ccb6b 100644 --- a/colm/bytecode.h +++ b/colm/bytecode.h @@ -223,6 +223,7 @@ typedef unsigned char uchar; #define IN_CONSTRUCT_TERM 0x9d #define IN_INPUT_PULL_WV 0x9e +#define IN_INPUT_PULL_WC 0xe1 #define IN_INPUT_PULL_BKT 0x9f #define IN_PARSE_SAVE_STEPS 0xa0 @@ -458,6 +459,7 @@ typedef struct _Execution long stringLength( Head *str ); const char *stringData( Head *str ); Head *stringAllocFull( struct ColmProgram *prg, const char *data, long length ); +Head *initStrSpace( long length ); Head *stringCopy( struct ColmProgram *prg, Head *head ); void stringFree( struct ColmProgram *prg, Head *head ); void stringShorten( Head *tokdata, long newlen ); diff --git a/colm/pdarun.c b/colm/pdarun.c index d116a70d..6ac5b256 100644 --- a/colm/pdarun.c +++ b/colm/pdarun.c @@ -191,29 +191,42 @@ Head *peekMatch( Program *prg, FsmRun *fsmRun, StreamImpl *is ) return head; } -Head *streamPull( Program *prg, FsmRun *fsmRun, StreamImpl *is, long length ) +Head *streamPull( Program *prg, PdaRun *pdaRun, StreamImpl *is, long length ) { - RunBuf *runBuf = fsmRun->consumeBuf; - if ( length > ( FSM_BUFSIZE - runBuf->length ) ) { - runBuf = newRunBuf(); - runBuf->next = fsmRun->consumeBuf; - fsmRun->consumeBuf = runBuf; - } + if ( pdaRun != 0 ) { + FsmRun *fsmRun = pdaRun->fsmRun; + RunBuf *runBuf = fsmRun->consumeBuf; + if ( length > ( FSM_BUFSIZE - runBuf->length ) ) { + runBuf = newRunBuf(); + runBuf->next = fsmRun->consumeBuf; + fsmRun->consumeBuf = runBuf; + } - char *dest = runBuf->data + runBuf->length; + char *dest = runBuf->data + runBuf->length; - is->funcs->getData( is, dest, length ); - is->funcs->consumeData( is, length ); + is->funcs->getData( is, dest, length ); + is->funcs->consumeData( is, length ); - runBuf->length += length; + runBuf->length += length; - fsmRun->p = fsmRun->pe = 0; - fsmRun->toklen = 0; + fsmRun->p = fsmRun->pe = 0; + fsmRun->toklen = 0; - Head *tokdata = stringAllocPointer( prg, dest, length ); - updatePosition( is, dest, length ); + Head *tokdata = stringAllocPointer( prg, dest, length ); + updatePosition( is, dest, length ); - return tokdata; + return tokdata; + } + else { + Head *head = initStrSpace( length ); + char *dest = (char*)head->data; + + is->funcs->getData( is, dest, length ); + is->funcs->consumeData( is, length ); + + updatePosition( is, dest, length ); + return head; + } } void undoStreamPull( StreamImpl *is, const char *data, long length ) diff --git a/colm/pdarun.h b/colm/pdarun.h index 17fa3aa7..00f07885 100644 --- a/colm/pdarun.h +++ b/colm/pdarun.h @@ -433,7 +433,7 @@ long parseToken( struct ColmProgram *prg, Tree **sp, PdaRun *pdaRun, long undoParse( Tree **sp, PdaRun *pdaRun, FsmRun *fsmRun, StreamImpl *inputStream, Tree *tree ); -Head *streamPull( struct ColmProgram *prg, FsmRun *fsmRun, StreamImpl *inputStream, long length ); +Head *streamPull( struct ColmProgram *prg, PdaRun *pdaRun, StreamImpl *is, long length ); Head *stringAllocPointer( struct ColmProgram *prg, const char *data, long length ); void streamPushText( StreamImpl *inputStream, const char *data, long length ); diff --git a/colm/string.c b/colm/string.c index d670b68c..21c3aac4 100644 --- a/colm/string.c +++ b/colm/string.c @@ -89,8 +89,6 @@ Head *initStrSpace( long length ) { /* Find the length and allocate the space for the shared string. */ Head *head = (Head*) malloc( sizeof(Head) + length ); - //if ( head == 0 ) - // throw std::bad_alloc(); /* Init the header. */ head->data = (char*)(head+1); diff --git a/colm/synthesis.cc b/colm/synthesis.cc index ad065343..f164e1ed 100644 --- a/colm/synthesis.cc +++ b/colm/synthesis.cc @@ -2531,7 +2531,7 @@ void Compiler::initStreamObject( ) streamLangEl->objectDef = streamObj; initFunction( uniqueTypeStr, streamObj, "pull", - IN_INPUT_PULL_WV, IN_INPUT_PULL_WV, uniqueTypeInt, false ); + IN_INPUT_PULL_WV, IN_INPUT_PULL_WC, uniqueTypeInt, false ); initFunction( uniqueTypeStr, streamObj, "push", IN_INPUT_PUSH_WV, IN_INPUT_PUSH_WV, uniqueTypeAny, false ); initFunction( uniqueTypeStr, streamObj, "push_ignore", diff --git a/test/pull1.exp b/test/pull1.exp new file mode 100644 index 00000000..aa3a0fe0 --- /dev/null +++ b/test/pull1.exp @@ -0,0 +1 @@ +this is in diff --git a/test/pull1.in b/test/pull1.in new file mode 100644 index 00000000..f4d2e4a0 --- /dev/null +++ b/test/pull1.in @@ -0,0 +1 @@ +this is input for a non-parse pull diff --git a/test/pull1.lm b/test/pull1.lm new file mode 100644 index 00000000..bc559671 --- /dev/null +++ b/test/pull1.lm @@ -0,0 +1,2 @@ +String: str = stdin.pull( 10 ) +print( String '\n' ) -- cgit v1.2.1