fix(skel): Add state buffer size handling to reentrant scanner skeletons.

3 files changed, 79 insertions, 12 deletions

diff --git a/src/c99-flex.skl b/src/c99-flex.skl
index 502337a..b01913d 100755
--- a/src/c99-flex.skl
+++ b/src/c99-flex.skl
@@ -268,9 +268,11 @@ typedef struct yyguts_t *yyscan_t;
 /* Action number for EOF rule of a given start state. */
 m4_define([[YY_STATE_EOF]], [[YY_END_OF_BUFFER + $1 + 1]])
 
-/* The state buf must be large enough to hold one state per character in the main buffer.
+/* The state buf must be large enough to hold one state per character in the main buffer,
+ * plus the start state, plus the two end-of-buffer byte states.
  */
-m4_define([[YY_STATE_BUF_SIZE]], [[((YY_BUF_SIZE + 2) * sizeof(yy_state_type))]])
+m4_define([[YY_STATE_BUF_EXTRA_SPACE]], [[3]])
+m4_define([[YY_STATE_BUF_SIZE]], [[(YY_BUF_SIZE + YY_STATE_BUF_EXTRA_SPACE)]])
 
 const bool FLEX_DEBUG = m4_ifdef([[M4_MODE_DEBUG]], [[true]], [[false]]);
 
@@ -516,6 +518,7 @@ struct yyguts_t {
 m4_ifdef( [[M4_MODE_USES_REJECT]], [[
 	yy_state_type *yy_state_buf;
 	yy_state_type *yy_state_ptr;
+	size_t yy_state_buf_max;
 	char *yy_full_match;
 	int yy_lp;
 
@@ -1593,7 +1596,9 @@ bool yyatbol(yyscan_t yyscanner) {
  */
 void yy_switch_to_buffer(yybuffer new_buffer, yyscan_t yyscanner)
 {
-
+	size_t new_size = 0;
+	yy_state_type *new_state_buf = 0;
+	
 	/* TODO. We should be able to replace this entire function body
 	 * with
 	 *		yypop_buffer_state();
@@ -1613,6 +1618,23 @@ void yy_switch_to_buffer(yybuffer new_buffer, yyscan_t yyscanner)
 	yyscanner->yy_buffer_stack[yyscanner->yy_buffer_stack_top] = new_buffer;
 	yy_load_buffer_state( yyscanner );
 
+m4_ifdef( [[M4_MODE_USES_REJECT]], [[
+	/* Ensure the reject state buffer is large enough.
+	 */
+	if ( yyscanner->yy_state_buf_max < (yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) {
+		new_size = yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE;
+		new_state_buf = (yy_state_type *)yyrealloc( yyscanner->yy_state_buf, (new_size * sizeof(yy_state_type)), yyscanner );
+		
+		if ( new_state_buf == NULL ) {
+			YY_FATAL_ERROR( "out of dynamic memory in yylex()" );
+		}
+		else {
+			yyscanner->yy_state_buf = new_state_buf;
+			yyscanner->yy_state_buf_max = new_size;
+		}
+	}
+]] )
+
 	/* We don't actually know whether we did this switch during
 	 * EOF (yywrap()) processing, but the only time this flag
 	 * is looked at is after yywrap() is called, so it's safe
@@ -2010,7 +2032,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]],
 			}
 		}
 
-		if ( ! yyscanner->yy_state_buf ) {
+		if ( yyscanner->yy_state_buf == NULL ) {
 			yypanic( "out of dynamic memory in yylex()", yyscanner );
 		}
 		else if ( yy_current_buffer(yyscanner) == NULL ) {
diff --git a/src/cpp-flex.skl b/src/cpp-flex.skl
index eefe106..fe9344f 100755
--- a/src/cpp-flex.skl
+++ b/src/cpp-flex.skl
@@ -422,9 +422,11 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 
 m4_ifdef( [[M4_YY_NOT_IN_HEADER]],
 [[
-/* The state buf must be large enough to hold one state per character in the main buffer.
+/* The state buf must be large enough to hold one state per character in the main buffer,
+ * plus the start state, plus the two end-of-buffer byte states.
  */
-#define YY_STATE_BUF_SIZE   ((YY_BUF_SIZE + 2) * sizeof(yy_state_type))
+#define YY_STATE_BUF_EXTRA_SPACE 3
+#define YY_STATE_BUF_SIZE   (YY_BUF_SIZE + YY_STATE_BUF_EXTRA_SPACE)
 ]])
 
 
@@ -1227,6 +1229,7 @@ struct yyguts_t {
 m4_ifdef( [[M4_MODE_USES_REJECT]], [[
 	yy_state_type *yy_state_buf;
 	yy_state_type *yy_state_ptr;
+	size_t yy_state_buf_max;
 	char *yy_full_match;
 	int yy_lp;
 
@@ -2324,6 +2327,7 @@ void yyFlexLexer::ctor_common() {
 m4_ifdef( [[M4_MODE_USES_REJECT]],
 [[
 	yy_state_buf = new yy_state_type[YY_STATE_BUF_SIZE];
+	yy_state_buf_max = YY_STATE_BUF_SIZE;
 ]],
 [[
 	yy_state_buf = 0;
@@ -2842,7 +2846,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], [[
 	 */
 	if ( YY_G(yy_state_buf_max) < (yy_current_buffer()->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) {
 		new_size = yy_current_buffer()->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE;
-		new_state_buf = (yy_state_type*) yyrealloc( YY_G(yy_state_buf), (new_size * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG );
+		new_state_buf = (yy_state_type *)yyrealloc( YY_G(yy_state_buf), (new_size * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG );
 		
 		if ( new_state_buf == NULL ) {
 			YY_FATAL_ERROR( "out of dynamic memory in yylex()" );
@@ -2889,6 +2893,8 @@ void yyFlexLexer::yy_switch_to_buffer( yybuffer new_buffer )
 ]])
 {
 	M4_YY_DECL_GUTS_VAR();
+	size_t new_size = 0;
+	yy_state_type *new_state_buf = 0;
 
 	/* TODO. We should be able to replace this entire function body
 	 * with
@@ -2908,6 +2914,23 @@ void yyFlexLexer::yy_switch_to_buffer( yybuffer new_buffer )
 
 	YY_CURRENT_BUFFER_LVALUE = new_buffer;
 	yy_load_buffer_state( M4_YY_CALL_ONLY_ARG );
+	
+m4_ifdef( [[M4_MODE_USES_REJECT]], [[
+	/* Ensure the reject state buffer is large enough.
+	 */
+	if ( YY_G(yy_state_buf_max) < (YY_CURRENT_BUFFER_LVALUE->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) {
+		new_size = YY_CURRENT_BUFFER_LVALUE->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE;
+		new_state_buf = (yy_state_type *)yyrealloc( YY_G(yy_state_buf), (new_size * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG );
+		
+		if ( new_state_buf == NULL ) {
+			YY_FATAL_ERROR( "out of dynamic memory in yylex()" );
+		}
+		else {
+			YY_G(yy_state_buf) = new_state_buf;
+			YY_G(yy_state_buf_max) = new_size;
+		}
+	}
+]] )
 
 	/* We don't actually know whether we did this switch during
 	 * EOF (yywrap()) processing, but the only time this flag
@@ -3741,6 +3764,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]],
 [[
 	YY_G(yy_state_buf) = 0;
 	YY_G(yy_state_ptr) = 0;
+	YY_G(yy_state_buf_max) = 0;
 	YY_G(yy_full_match) = 0;
 	YY_G(yy_lp) = 0;
 ]])
diff --git a/src/go-flex.skl b/src/go-flex.skl
index 431cde8..e9ce1f1 100755
--- a/src/go-flex.skl
+++ b/src/go-flex.skl
@@ -194,9 +194,11 @@ m4_define([[YY_SC_TO_UI]], [[((YY_CHAR)($1))]])
 /* Action number for EOF rule of a given start state. */
 m4_define([[YY_STATE_EOF]], [[YY_END_OF_BUFFER + $1 + 1]])
 
-/* The state buf must be large enough to hold one state per character in the main buffer.
+/* The state buf must be large enough to hold one state per character in the main buffer,
+ * plus the start state, plus the two end-of-buffer byte states.
  */
-m4_define([[YY_STATE_BUF_SIZE]], [[((flexInputBufferSize + 2) * sizeof(yyStateType))]])
+m4_define([[YY_STATE_BUF_EXTRA_SPACE]], [[3]])
+m4_define([[YY_STATE_BUF_SIZE]], [[(YY_BUF_SIZE + YY_STATE_BUF_EXTRA_SPACE)]])
 
 const bool FLEX_DEBUG = m4_ifdef([[M4_MODE_DEBUG]], [[true]], [[false]]);
 
@@ -1444,6 +1446,8 @@ bool yyatbol(FlexLexer *yyscanner) {
  */
 void yy_switch_to_buffer(yybuffer newBuffer, FlexLexer *yyscanner)
 {
+	size_t newSize = 0;
+	yyStateType *newStateBuf = 0;
 
 	/* TODO. We should be able to replace this entire function body
 	 * with
@@ -1463,6 +1467,23 @@ void yy_switch_to_buffer(yybuffer newBuffer, FlexLexer *yyscanner)
 
 	yyscanner->yyBufferStack[yyscanner->yyBufferStackTop] = newBuffer;
 	yy_load_buffer_state(yyscanner);
+	
+m4_ifdef( [[M4_MODE_USES_REJECT]], [[
+	/* Ensure the reject state buffer is large enough.
+	 */
+	if ( yyscanner->yyStateBufMax < (yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE) ) {
+		newSize = yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE;
+		newStateBuf = (yyStateType *)yyrealloc( yyscanner->yyStateBuf, (new_size * sizeof(yyStateType)), yyscanner );
+		
+		if ( newStateBuf == NULL ) {
+			YY_FATAL_ERROR( "out of dynamic memory in yylex()" );
+		}
+		else {
+			yyscanner->yyStateBuf = newStateBuf;
+			yyscanner->yyStateBufMax = newSize;
+		}
+	}
+]] )
 
 	/* We don't actually know whether we did this switch during
 	 * EOF (yywrap()) processing, but the only time this flag
@@ -1861,14 +1882,14 @@ m4_ifdef([[M4_MODE_USES_REJECT]],
 			}
 		}
 		
-		if ( ! yyscanner->yyStateBuf ) {
+		if ( yyscanner->yyStateBuf == NULL ) {
 			yypanic( "out of dynamic memory in yylex()", yyscanner );
 		}
 		else if ( yy_current_buffer(yyscanner) == NULL ) {
 			yyscanner->yyStateBufMax = YY_STATE_BUF_SIZE;
 		}
-		if (yyscanner->yyStateBuf == NULL) {
-			yypanic("out of dynamic memory in yylex()", yyscanner);
+		else {
+			yyscanner->yyStateBufMax = (yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE);
 		}
 ]])