From 988d379eb082423274381656863c804b00f82427 Mon Sep 17 00:00:00 2001 From: Joseph Langley Date: Wed, 9 Jun 2021 20:38:57 -0400 Subject: fix(skel): Add state buffer size handling to reentrant scanner skeletons. --- src/c99-flex.skl | 30 ++++++++++++++++++++++++++---- src/cpp-flex.skl | 30 +++++++++++++++++++++++++++--- src/go-flex.skl | 31 ++++++++++++++++++++++++++----- 3 files changed, 79 insertions(+), 12 deletions(-) diff --git a/src/c99-flex.skl b/src/c99-flex.skl index 502337a..b01913d 100755 --- a/src/c99-flex.skl +++ b/src/c99-flex.skl @@ -268,9 +268,11 @@ typedef struct yyguts_t *yyscan_t; /* Action number for EOF rule of a given start state. */ m4_define([[YY_STATE_EOF]], [[YY_END_OF_BUFFER + $1 + 1]]) -/* The state buf must be large enough to hold one state per character in the main buffer. +/* The state buf must be large enough to hold one state per character in the main buffer, + * plus the start state, plus the two end-of-buffer byte states. */ -m4_define([[YY_STATE_BUF_SIZE]], [[((YY_BUF_SIZE + 2) * sizeof(yy_state_type))]]) +m4_define([[YY_STATE_BUF_EXTRA_SPACE]], [[3]]) +m4_define([[YY_STATE_BUF_SIZE]], [[(YY_BUF_SIZE + YY_STATE_BUF_EXTRA_SPACE)]]) const bool FLEX_DEBUG = m4_ifdef([[M4_MODE_DEBUG]], [[true]], [[false]]); @@ -516,6 +518,7 @@ struct yyguts_t { m4_ifdef( [[M4_MODE_USES_REJECT]], [[ yy_state_type *yy_state_buf; yy_state_type *yy_state_ptr; + size_t yy_state_buf_max; char *yy_full_match; int yy_lp; @@ -1593,7 +1596,9 @@ bool yyatbol(yyscan_t yyscanner) { */ void yy_switch_to_buffer(yybuffer new_buffer, yyscan_t yyscanner) { - + size_t new_size = 0; + yy_state_type *new_state_buf = 0; + /* TODO. We should be able to replace this entire function body * with * yypop_buffer_state(); @@ -1613,6 +1618,23 @@ void yy_switch_to_buffer(yybuffer new_buffer, yyscan_t yyscanner) yyscanner->yy_buffer_stack[yyscanner->yy_buffer_stack_top] = new_buffer; yy_load_buffer_state( yyscanner ); +m4_ifdef( [[M4_MODE_USES_REJECT]], [[ + /* Ensure the reject state buffer is large enough. + */ + if ( yyscanner->yy_state_buf_max < (yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) { + new_size = yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE; + new_state_buf = (yy_state_type *)yyrealloc( yyscanner->yy_state_buf, (new_size * sizeof(yy_state_type)), yyscanner ); + + if ( new_state_buf == NULL ) { + YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); + } + else { + yyscanner->yy_state_buf = new_state_buf; + yyscanner->yy_state_buf_max = new_size; + } + } +]] ) + /* We don't actually know whether we did this switch during * EOF (yywrap()) processing, but the only time this flag * is looked at is after yywrap() is called, so it's safe @@ -2010,7 +2032,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], } } - if ( ! yyscanner->yy_state_buf ) { + if ( yyscanner->yy_state_buf == NULL ) { yypanic( "out of dynamic memory in yylex()", yyscanner ); } else if ( yy_current_buffer(yyscanner) == NULL ) { diff --git a/src/cpp-flex.skl b/src/cpp-flex.skl index eefe106..fe9344f 100755 --- a/src/cpp-flex.skl +++ b/src/cpp-flex.skl @@ -422,9 +422,11 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]], m4_ifdef( [[M4_YY_NOT_IN_HEADER]], [[ -/* The state buf must be large enough to hold one state per character in the main buffer. +/* The state buf must be large enough to hold one state per character in the main buffer, + * plus the start state, plus the two end-of-buffer byte states. */ -#define YY_STATE_BUF_SIZE ((YY_BUF_SIZE + 2) * sizeof(yy_state_type)) +#define YY_STATE_BUF_EXTRA_SPACE 3 +#define YY_STATE_BUF_SIZE (YY_BUF_SIZE + YY_STATE_BUF_EXTRA_SPACE) ]]) @@ -1227,6 +1229,7 @@ struct yyguts_t { m4_ifdef( [[M4_MODE_USES_REJECT]], [[ yy_state_type *yy_state_buf; yy_state_type *yy_state_ptr; + size_t yy_state_buf_max; char *yy_full_match; int yy_lp; @@ -2324,6 +2327,7 @@ void yyFlexLexer::ctor_common() { m4_ifdef( [[M4_MODE_USES_REJECT]], [[ yy_state_buf = new yy_state_type[YY_STATE_BUF_SIZE]; + yy_state_buf_max = YY_STATE_BUF_SIZE; ]], [[ yy_state_buf = 0; @@ -2842,7 +2846,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], [[ */ if ( YY_G(yy_state_buf_max) < (yy_current_buffer()->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) { new_size = yy_current_buffer()->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE; - new_state_buf = (yy_state_type*) yyrealloc( YY_G(yy_state_buf), (new_size * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG ); + new_state_buf = (yy_state_type *)yyrealloc( YY_G(yy_state_buf), (new_size * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG ); if ( new_state_buf == NULL ) { YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); @@ -2889,6 +2893,8 @@ void yyFlexLexer::yy_switch_to_buffer( yybuffer new_buffer ) ]]) { M4_YY_DECL_GUTS_VAR(); + size_t new_size = 0; + yy_state_type *new_state_buf = 0; /* TODO. We should be able to replace this entire function body * with @@ -2908,6 +2914,23 @@ void yyFlexLexer::yy_switch_to_buffer( yybuffer new_buffer ) YY_CURRENT_BUFFER_LVALUE = new_buffer; yy_load_buffer_state( M4_YY_CALL_ONLY_ARG ); + +m4_ifdef( [[M4_MODE_USES_REJECT]], [[ + /* Ensure the reject state buffer is large enough. + */ + if ( YY_G(yy_state_buf_max) < (YY_CURRENT_BUFFER_LVALUE->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) { + new_size = YY_CURRENT_BUFFER_LVALUE->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE; + new_state_buf = (yy_state_type *)yyrealloc( YY_G(yy_state_buf), (new_size * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG ); + + if ( new_state_buf == NULL ) { + YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); + } + else { + YY_G(yy_state_buf) = new_state_buf; + YY_G(yy_state_buf_max) = new_size; + } + } +]] ) /* We don't actually know whether we did this switch during * EOF (yywrap()) processing, but the only time this flag @@ -3741,6 +3764,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], [[ YY_G(yy_state_buf) = 0; YY_G(yy_state_ptr) = 0; + YY_G(yy_state_buf_max) = 0; YY_G(yy_full_match) = 0; YY_G(yy_lp) = 0; ]]) diff --git a/src/go-flex.skl b/src/go-flex.skl index 431cde8..e9ce1f1 100755 --- a/src/go-flex.skl +++ b/src/go-flex.skl @@ -194,9 +194,11 @@ m4_define([[YY_SC_TO_UI]], [[((YY_CHAR)($1))]]) /* Action number for EOF rule of a given start state. */ m4_define([[YY_STATE_EOF]], [[YY_END_OF_BUFFER + $1 + 1]]) -/* The state buf must be large enough to hold one state per character in the main buffer. +/* The state buf must be large enough to hold one state per character in the main buffer, + * plus the start state, plus the two end-of-buffer byte states. */ -m4_define([[YY_STATE_BUF_SIZE]], [[((flexInputBufferSize + 2) * sizeof(yyStateType))]]) +m4_define([[YY_STATE_BUF_EXTRA_SPACE]], [[3]]) +m4_define([[YY_STATE_BUF_SIZE]], [[(YY_BUF_SIZE + YY_STATE_BUF_EXTRA_SPACE)]]) const bool FLEX_DEBUG = m4_ifdef([[M4_MODE_DEBUG]], [[true]], [[false]]); @@ -1444,6 +1446,8 @@ bool yyatbol(FlexLexer *yyscanner) { */ void yy_switch_to_buffer(yybuffer newBuffer, FlexLexer *yyscanner) { + size_t newSize = 0; + yyStateType *newStateBuf = 0; /* TODO. We should be able to replace this entire function body * with @@ -1463,6 +1467,23 @@ void yy_switch_to_buffer(yybuffer newBuffer, FlexLexer *yyscanner) yyscanner->yyBufferStack[yyscanner->yyBufferStackTop] = newBuffer; yy_load_buffer_state(yyscanner); + +m4_ifdef( [[M4_MODE_USES_REJECT]], [[ + /* Ensure the reject state buffer is large enough. + */ + if ( yyscanner->yyStateBufMax < (yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE) ) { + newSize = yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE; + newStateBuf = (yyStateType *)yyrealloc( yyscanner->yyStateBuf, (new_size * sizeof(yyStateType)), yyscanner ); + + if ( newStateBuf == NULL ) { + YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); + } + else { + yyscanner->yyStateBuf = newStateBuf; + yyscanner->yyStateBufMax = newSize; + } + } +]] ) /* We don't actually know whether we did this switch during * EOF (yywrap()) processing, but the only time this flag @@ -1861,14 +1882,14 @@ m4_ifdef([[M4_MODE_USES_REJECT]], } } - if ( ! yyscanner->yyStateBuf ) { + if ( yyscanner->yyStateBuf == NULL ) { yypanic( "out of dynamic memory in yylex()", yyscanner ); } else if ( yy_current_buffer(yyscanner) == NULL ) { yyscanner->yyStateBufMax = YY_STATE_BUF_SIZE; } - if (yyscanner->yyStateBuf == NULL) { - yypanic("out of dynamic memory in yylex()", yyscanner); + else { + yyscanner->yyStateBufMax = (yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE); } ]]) -- cgit v1.2.1