fix(skel): Add state buffer size handling to non-reentrant scanner skeletons.

3 files changed, 109 insertions, 5 deletions

diff --git a/src/c99-flex.skl b/src/c99-flex.skl
index 1e04afe..502337a 100644..100755
--- a/src/c99-flex.skl
+++ b/src/c99-flex.skl
@@ -847,6 +847,8 @@ void yypop_buffer_state (yyscan_t yyscanner)
  */
 void yyrestart(FILE * input_file, yyscan_t yyscanner)
 {
+	size_t new_size = 0;
+	yy_state_type *new_state_buf = 0;
 
 	if ( yy_current_buffer(yyscanner) == NULL ) {
 		yyensure_buffer_stack (yyscanner);
@@ -856,6 +858,30 @@ void yyrestart(FILE * input_file, yyscan_t yyscanner)
 
 	yy_init_buffer( yyscanner->yy_buffer_stack[yyscanner->yy_buffer_stack_top], input_file, yyscanner);
 	yy_load_buffer_state( yyscanner );
+	
+m4_ifdef( [[M4_MODE_USES_REJECT]], [[        
+	/* Ensure the reject state buffer is large enough.
+	 */
+	if ( yyscanner->yy_state_buf_max < (yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) {
+		new_size = yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE;
+		new_state_buf = (yy_state_type*) yyrealloc( yyscanner->yy_state_buf, (new_size * sizeof(yy_state_type)), yyscanner );
+		
+		if ( new_state_buf == NULL ) {
+			yypanic( "out of dynamic memory in yylex()", yyscanner );
+		}
+		else {
+			yyscanner->yy_state_buf = new_state_buf;
+			yyscanner->yy_state_buf_max = new_size;
+		}
+	}
+]] )
+
+	/* We don't actually know whether we did this switch during
+	 * EOF (yywrap()) processing, but the only time this flag
+	 * is looked at is after yywrap() is called, so it's safe
+	 * to go ahead and always set it.
+	 */
+	yyscanner->yy_did_buffer_switch_on_eof = true;
 }
 
 static void yybumpline( yyscan_t yyscanner) {
@@ -1976,10 +2002,22 @@ m4_ifdef( [[M4_MODE_USES_REJECT]],
 [[
 		/* Create the reject buffer large enough to save one state per allowed character. */
 		if ( yyscanner->yy_state_buf  == NULL ) {
-			yyscanner->yy_state_buf = (yy_state_type *)yyalloc(YY_STATE_BUF_SIZE, yyscanner);
+			if( yy_current_buffer(yyscanner) == NULL ) {
+				yyscanner->yy_state_buf = (yy_state_type *)yyalloc((YY_STATE_BUF_SIZE * sizeof(yy_state_type)), yyscanner);
+			}
+			else {
+				yyscanner->yy_state_buf = (yy_state_type *)yyalloc(((yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) * sizeof(yy_state_type)), yyscanner);
+			}
+		}
+
+		if ( ! yyscanner->yy_state_buf ) {
+			yypanic( "out of dynamic memory in yylex()", yyscanner );
+		}
+		else if ( yy_current_buffer(yyscanner) == NULL ) {
+			yyscanner->yy_state_buf_max = YY_STATE_BUF_SIZE;
 		}
-		if ( yyscanner->yy_state_buf == NULL) {
-			yypanic( "out of dynamic memory in yylex()", yyscanner);
+		else {
+			yyscanner->yy_state_buf_max = (yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE);
 		}
 ]])
 
diff --git a/src/cpp-flex.skl b/src/cpp-flex.skl
index 1db0446..eefe106 100644..100755
--- a/src/cpp-flex.skl
+++ b/src/cpp-flex.skl
@@ -1074,6 +1074,7 @@ m4_ifdef( [[M4_YY_NOT_REENTRANT]], [[
 m4_ifdef( [[M4_MODE_C_ONLY]], [[
 /* Declare state buffer variables. */
 static yy_state_type *yy_state_buf=0, *yy_state_ptr=0;
+static size_t yy_state_buf_max=0;
 static char *yy_full_match;
 static int yy_lp;
 m4_ifdef( [[M4_MODE_VARIABLE_TRAILING_CONTEXT_RULES]], [[
@@ -2824,6 +2825,8 @@ void yyFlexLexer::yyrestart( std::istream& input_file )
 ]])
 {
 	M4_YY_DECL_GUTS_VAR();
+	size_t new_size = 0;
+	yy_state_type *new_state_buf = 0;
 
 	if ( yy_current_buffer() == NULL ) {
 		yyensure_buffer_stack (M4_YY_CALL_ONLY_ARG);
@@ -2833,6 +2836,30 @@ void yyFlexLexer::yyrestart( std::istream& input_file )
 
 	yy_init_buffer( YY_CURRENT_BUFFER_LVALUE, input_file M4_YY_CALL_LAST_ARG);
 	yy_load_buffer_state( M4_YY_CALL_ONLY_ARG );
+	
+m4_ifdef( [[M4_MODE_USES_REJECT]], [[        
+	/* Ensure the reject state buffer is large enough.
+	 */
+	if ( YY_G(yy_state_buf_max) < (yy_current_buffer()->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) {
+		new_size = yy_current_buffer()->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE;
+		new_state_buf = (yy_state_type*) yyrealloc( YY_G(yy_state_buf), (new_size * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG );
+		
+		if ( new_state_buf == NULL ) {
+			YY_FATAL_ERROR( "out of dynamic memory in yylex()" );
+		}
+		else {
+			YY_G(yy_state_buf) = new_state_buf;
+			YY_G(yy_state_buf_max) = new_size;
+		}
+	}
+]] )
+
+	/* We don't actually know whether we did this switch during
+	 * EOF (yywrap()) processing, but the only time this flag
+	 * is looked at is after yywrap() is called, so it's safe
+	 * to go ahead and always set it.
+	 */
+	YY_G(yy_did_buffer_switch_on_eof) = 1;
 }
 
 m4_ifdef([[M4_MODE_CXX_ONLY]], [[
diff --git a/src/go-flex.skl b/src/go-flex.skl
index f2807dd..431cde8 100644..100755
--- a/src/go-flex.skl
+++ b/src/go-flex.skl
@@ -756,7 +756,10 @@ void yypop_buffer_state (FlexLexer *yyscanner)
  */
 void yyrestart(FILE * input_file, FlexLexer *yyscanner)
 {
-
+	
+	size_t newSize = 0;
+	yyStateType *newStateBuf = 0;
+	
 	if (yy_current_buffer(yyscanner) == NULL) {
 		yyensure_buffer_stack (yyscanner);
 		yyscanner->yyBufferStack[yyscanner->yyBufferStackTop] =
@@ -765,6 +768,30 @@ void yyrestart(FILE * input_file, FlexLexer *yyscanner)
 
 	yy_init_buffer(yyscanner->yyBufferStack[yyscanner->yyBufferStackTop], input_file, yyscanner);
 	yy_load_buffer_state(yyscanner);
+	
+m4_ifdef( [[M4_MODE_USES_REJECT]], [[        
+	/* Ensure the reject state buffer is large enough.
+	 */
+	if ( yyscanner->yyStateBufMax < (yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE) ) {
+		newSize = yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE;
+		newStateBuf = (yyStateType *)yyrealloc( yyscanner->yyStateBuf, (newSize * sizeof(yyStateType)), yyscanner );
+		
+		if ( newStateBuf == NULL ) {
+			yypanic( "out of dynamic memory in yylex()", yyscanner );
+		}
+		else {
+			yyscanner->yyStateBuf = newStateBuf;
+			yyscanner->yyStateBufMax = newSize;
+		}
+	}
+]] )
+
+	/* We don't actually know whether we did this switch during
+	 * EOF (yywrap()) processing, but the only time this flag
+	 * is looked at is after yywrap() is called, so it's safe
+	 * to go ahead and always set it.
+	 */
+	yyscanner->yyDidBufferSwitchOnEof = true;
 }
 
 static void yybumpline(FlexLexer *yyscanner) {
@@ -1826,7 +1853,19 @@ m4_ifdef([[M4_MODE_USES_REJECT]],
 [[
 		/* Create the reject buffer large enough to save one state per allowed character. */
 		if (yyscanner->yyStateBuf  == NULL) {
-			yyscanner->yyStateBuf = (yyStateType *)yyalloc(YY_STATE_BUF_SIZE, yyscanner);
+			if (yy_current_buffer(yyscanner) == NULL) {
+				yyscanner->yyStateBuf = (yyStateType *)yyalloc((YY_STATE_BUF_SIZE * sizeof(yyStateType)), yyscanner);
+			} 
+			else {
+				yyscanner->yyStateBuf = (yyStateType *)yyalloc((yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE) * sizeof(yyStateType), yyscanner);
+			}
+		}
+		
+		if ( ! yyscanner->yyStateBuf ) {
+			yypanic( "out of dynamic memory in yylex()", yyscanner );
+		}
+		else if ( yy_current_buffer(yyscanner) == NULL ) {
+			yyscanner->yyStateBufMax = YY_STATE_BUF_SIZE;
 		}
 		if (yyscanner->yyStateBuf == NULL) {
 			yypanic("out of dynamic memory in yylex()", yyscanner);