put reducer code in its own file, use local parse tree pool

11 files changed, 140 insertions, 36 deletions

diff --git a/src/bytecode.c b/src/bytecode.c
index 68c7d148..eae30651 100644
--- a/src/bytecode.c
+++ b/src/bytecode.c
@@ -842,14 +842,6 @@ again:
 			break;
 		}
 
-		case IN_PARSER_NOT_USED: {
-			debug( prg, REALM_BYTECODE, "IN_PARSER_NOT_USED\n" );
-
-			parser_t *parser = vm_pop_parser();
-			parser->pda_run->not_used = 1;
-			vm_push_parser( parser );
-			break;
-		}
 		case IN_SET_PARSER_CONTEXT: {
 			debug( prg, REALM_BYTECODE, "IN_SET_PARSER_CTX_WC\n" );
 
@@ -870,10 +862,11 @@ again:
 			/* If there are captures (this is a translate block) then copy them into
 			 * the local frame now. */
 			struct lang_el_info *lel_info = prg->rtd->lel_info;
-			char **mark = exec->parser->pda_run->mark;
+			struct pda_run *pda_run = exec->parser->pda_run;
+			char **mark = pda_run->mark;
 
-			int i;
-			for ( i = 0; i < lel_info[exec->parser->pda_run->token_id].num_capture_attr; i++ ) {
+			int i, num_capture_attr = lel_info[pda_run->token_id].num_capture_attr;
+			for ( i = 0; i < num_capture_attr; i++ ) {
 				struct lang_el_info *lei = &lel_info[exec->parser->pda_run->token_id];
 				CaptureAttr *ca = &prg->rtd->capture_attr[lei->capture_attr + i];
 				head_t *data = string_alloc_full( prg, mark[ca->mark_enter],
@@ -2740,6 +2733,18 @@ again:
 			vm_push_struct( gen );
 			break;
 		}
+		case IN_CONS_REDUCER: {
+			half_t generic_id;
+			read_half( generic_id );
+
+			debug( prg, REALM_BYTECODE, "IN_CONS_REDUCER %hd\n", generic_id );
+
+			struct_t *gen = colm_construct_generic( prg, generic_id );
+			parser_t *parser = (parser_t*)gen;
+			parser->pda_run->reducer = 1;
+			vm_push_struct( gen );
+			break;
+		}
 		case IN_CONS_OBJECT: {
 			half_t lang_el_id;
 			read_half( lang_el_id );
diff --git a/src/bytecode.h b/src/bytecode.h
index 668602d2..793b3381 100644
--- a/src/bytecode.h
+++ b/src/bytecode.h
@@ -211,8 +211,7 @@ typedef unsigned long colm_value_t;
 #define IN_GET_VLIST_MEM_WV      0x70
 #define IN_GET_VLIST_MEM_BKT     0x5c
 
-/* This should be a bit in the parser generic. */
-#define IN_PARSER_NOT_USED       0x76
+#define IN_CONS_REDUCER          0x76
 
 #define IN_DONE                  0x78
 
diff --git a/src/compiler.cc b/src/compiler.cc
index 008788c5..d2588d14 100644
--- a/src/compiler.cc
+++ b/src/compiler.cc
@@ -993,7 +993,7 @@ pda_run *Compiler::parsePattern( program_t *prg, tree_t **sp, const InputLoc &lo
 	struct stream_impl *in = colm_impl_new_generic( "<internal>" );
 
 	struct pda_run *pdaRun = new pda_run;
-	colm_pda_init( prg, pdaRun, pdaTables, parserId, 0, false, 0 );
+	colm_pda_init( prg, pdaRun, pdaTables, parserId, 0, false, 0, false );
 
 	stream_t *stream = colm_stream_new_struct( prg );
 	stream->impl = sourceStream;
@@ -1138,9 +1138,35 @@ void Compiler::writeHostCall()
 
 }
 
+void Compiler::writeCommitStub()
+{
+	*outStream <<
+		"void commit_forward_recurse( program_t *prg, tree_t **root,\n"
+		"		struct pda_run *pda_run, parse_tree_t *pt )\n"
+		"{\n"
+		"	commit_clear_parse_tree( prg, root, pda_run, pt->child );\n"
+		"}\n";
+}
+
 void Compiler::writeCommit()
 {
 	*outStream <<
+		"#include <colm/pdarun.h>\n"
+		"#include <colm/debug.h>\n"
+		"#include <colm/bytecode.h>\n"
+		"#include <colm/config.h>\n"
+		"#include <colm/defs.h>\n"
+		"#include <colm/input.h>\n"
+		"#include <colm/tree.h>\n"
+		"#include <colm/program.h>\n"
+		"#include <colm/colm.h>\n"
+		"\n"
+		"#include <stdio.h>\n"
+		"#include <stdlib.h>\n"
+		"#include <string.h>\n"
+		"#include <assert.h>\n"
+		"\n"
+		"#include <iostream>\n"
 		"\n"
 		"void commit_forward_recurse( program_t *prg, tree_t **root,\n"
 		"		struct pda_run *pda_run, parse_tree_t *pt )\n"
@@ -1203,7 +1229,7 @@ void Compiler::writeCommit()
 }
 
 
-void Compiler::generateOutput( long activeRealm )
+void Compiler::generateOutput( long activeRealm, bool includeCommit )
 {
 	FsmCodeGen *fsmGen = new FsmCodeGen( *outStream, redFsm, fsmTables );
 
@@ -1220,7 +1246,9 @@ void Compiler::generateOutput( long activeRealm )
 	pdaGen->writeRuntimeData( runtimeData, pdaTables );
 
 	writeHostCall();
-	writeCommit();
+
+	if ( includeCommit )
+		writeCommitStub();
 
 	if ( !gblLibrary ) 
 		fsmGen->writeMain( activeRealm );
diff --git a/src/compiler.h b/src/compiler.h
index a2fb554f..611fac6c 100644
--- a/src/compiler.h
+++ b/src/compiler.h
@@ -820,7 +820,7 @@ struct Compiler
 	void compileByteCode();
 
 	void resolveUses();
-	void generateOutput( long activeRealm );
+	void generateOutput( long activeRealm, bool includeCommit );
 	void compile();
 
 	void openNameSpace( ostream &out, Namespace *nspace );
@@ -1011,6 +1011,7 @@ struct Compiler
 
 	void writeHostCall();
 	void writeCommit();
+	void writeCommitStub();
 };
 
 void afterOpMinimize( FsmGraph *fsm, bool lastInSeq = true );
diff --git a/src/main.cc b/src/main.cc
index 61914bfa..6805f57f 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -83,6 +83,7 @@ const char *intermedFn = 0;
 const char *binaryFn = 0;
 const char *exportHeaderFn = 0;
 const char *exportCodeFn = 0;
+const char *commitCodeFn = 0;
 bool exportCode = false;
 
 bool generateGraphviz = false;
@@ -182,7 +183,8 @@ void usage()
 "   -c                   compile only (don't produce binary)\n"
 "   -e <file>            write C++ export header to <file>\n"
 "   -x <file>            write C++ export code to <file>\n"
-"	-a <file>            additional code file to include in output program\n"
+"   -m <file>            write C++ commit code to <file>\n"
+"   -a <file>            additional code file to include in output program\n"
 	;	
 }
 
@@ -370,6 +372,31 @@ void openExportsImpl( )
 	}
 }
 
+void openCommit( )
+{
+	/* Make sure we are not writing to the same file as the input file. */
+	if ( inputFn != 0 && commitCodeFn != 0 && strcmp( inputFn, commitCodeFn  ) == 0 ) {
+		error() << "output file \"" << commitCodeFn  << 
+				"\" is the same as the input file" << endl;
+	}
+
+	if ( commitCodeFn != 0 ) {
+		/* Open the output stream, attaching it to the filter. */
+		ofstream *outFStream = new ofstream( commitCodeFn );
+
+		if ( !outFStream->is_open() ) {
+			error() << "error opening " << commitCodeFn << " for writing" << endl;
+			exit(1);
+		}
+
+		outStream = outFStream;
+	}
+	else {
+		/* Writing out ot std out. */
+		outStream = &cout;
+	}
+}
+
 void compileOutputCommand( const char *command )
 {
 	//cout << "compiling with: " << command << endl;
@@ -482,7 +509,7 @@ bool inSourceTree( const char *argv0 )
 
 void processArgs( int argc, const char **argv )
 {
-	ParamCheck pc( "cD:e:x:I:vdlio:S:M:vHh?-:sVa:", argc, argv );
+	ParamCheck pc( "cD:e:x:I:vdlio:S:M:vHh?-:sVa:m:", argc, argv );
 
 	while ( pc.check() ) {
 		switch ( pc.state ) {
@@ -551,6 +578,9 @@ void processArgs( int argc, const char **argv )
 			case 'a':
 				additionalCodeFiles.append( pc.parameterArg );
 				break;
+			case 'm':
+				commitCodeFn = pc.parameterArg;
+				break;
 
 			case 'D':
 #if DEBUG
@@ -679,7 +709,7 @@ int main(int argc, const char **argv)
 		else
 			openOutputCompiled();
 
-		pd->generateOutput( gblActiveRealm );
+		pd->generateOutput( gblActiveRealm, ( commitCodeFn == 0 ) );
 		if ( outStream != 0 )
 			delete outStream;
 
@@ -690,16 +720,22 @@ int main(int argc, const char **argv)
 				compileOutputInstalled( argv[0] );
 		}
 
-		if ( exportHeaderFn != 0 )  {
+		if ( exportHeaderFn != 0 ) {
 			openExports();
 			pd->generateExports();
 			delete outStream;
 		}
-		if ( exportCodeFn != 0 )  {
+		if ( exportCodeFn != 0 ) {
 			openExportsImpl();
 			pd->generateExportsImpl();
 			delete outStream;
 		}
+		if ( commitCodeFn != 0 ) {
+			openCommit();
+			pd->writeCommit();
+			delete outStream;
+
+		}
 	}
 
 	delete parser;
diff --git a/src/pdarun.c b/src/pdarun.c
index d06659f3..79a403ba 100644
--- a/src/pdarun.c
+++ b/src/pdarun.c
@@ -1178,10 +1178,18 @@ void colm_pda_clear( program_t *prg, tree_t **sp, struct pda_run *pda_run )
 	colm_rt_code_vect_empty( &pda_run->rcode_collect );
 
 	colm_tree_downref( prg, sp, pda_run->parse_error_text );
+
+	if ( pda_run->reducer ) {
+		long local_lost = pool_alloc_num_lost( &pda_run->local_pool );
+
+		if ( local_lost )
+			message( "warning: reducer local lost parse trees: %ld\n", local_lost );
+		pool_alloc_clear( &pda_run->local_pool );
+	}
 }
 
 void colm_pda_init( program_t *prg, struct pda_run *pda_run, struct pda_tables *tables,
-		int parser_id, long stop_target, int revert_on, struct_t *context )
+		int parser_id, long stop_target, int revert_on, struct_t *context, int reducer )
 {
 	memset( pda_run, 0, sizeof(struct pda_run) );
 
@@ -1190,6 +1198,13 @@ void colm_pda_init( program_t *prg, struct pda_run *pda_run, struct pda_tables *
 	pda_run->stop_target = stop_target;
 	pda_run->revert_on = revert_on;
 	pda_run->target_steps = -1;
+	pda_run->reducer = reducer;
+
+	if ( reducer ) {
+		init_pool_alloc( &pda_run->local_pool, sizeof(parse_tree_t) + sizeof(void*) * 8 );
+		pda_run->parse_tree_pool = &pda_run->local_pool;
+	}
+
 	pda_run->parse_tree_pool = &prg->parse_tree_pool;
 
 	debug( prg, REALM_PARSE, "initializing struct pda_run\n" );
@@ -1382,9 +1397,8 @@ again:
 		pda_run->commit_shift_count = pda_run->shift_count;
 
 		/* Not in a reverting context and the parser result is not used. */
-		if ( pda_run->not_used ) {
+		if ( pda_run->reducer )
 			commit_reduce( prg, sp, pda_run );
-		}
 	}
 
 	/*
@@ -2137,7 +2151,7 @@ long colm_parse_finish( tree_t **result, program_t *prg, tree_t **sp,
 	//		streamToImpl( input )->eofSent );
 
 	/* Flush out anything not committed. */
-	if ( pda_run->not_used )
+	if ( pda_run->reducer )
 		commit_reduce( prg, sp, pda_run );
 
 	if ( !revert_on )
@@ -2145,7 +2159,7 @@ long colm_parse_finish( tree_t **result, program_t *prg, tree_t **sp,
 	
 	tree_t *tree = get_parsed_root( pda_run, pda_run->stop_target > 0 );
 
-	if ( pda_run->not_used ) {
+	if ( pda_run->reducer ) {
 		*result = 0;
 	}
 	else {
diff --git a/src/pdarun.h b/src/pdarun.h
index cb442c58..02cd7e14 100644
--- a/src/pdarun.h
+++ b/src/pdarun.h
@@ -355,15 +355,16 @@ struct pda_run
 	int rc_block_count;
 
 	tree_t *parse_error_text;
-	char not_used;
+	char reducer;
 	parse_tree_t *last_final;
 
 	struct pool_alloc *parse_tree_pool;
+	struct pool_alloc local_pool;
 };
 
 void colm_pda_init( struct colm_program *prg, struct pda_run *pda_run,
 		struct pda_tables *tables, int parser_id, long stop_target,
-		int revert_on, struct colm_struct *context );
+		int revert_on, struct colm_struct *context, int reducer );
 
 void colm_pda_clear( struct colm_program *prg, struct colm_tree **sp,
 		struct pda_run *pda_run );
diff --git a/src/pool.h b/src/pool.h
index e9b498d2..ec098aca 100644
--- a/src/pool.h
+++ b/src/pool.h
@@ -61,6 +61,9 @@ void location_free( program_t *prg, location_t *el );
 void location_clear( program_t *prg );
 long location_num_lost( program_t *prg );
 
+void pool_alloc_clear( struct pool_alloc *pool_alloc );
+long pool_alloc_num_lost( struct pool_alloc *pool_alloc );
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/struct.c b/src/struct.c
index d1a6bdcf..51ac7315 100644
--- a/src/struct.c
+++ b/src/struct.c
@@ -76,13 +76,13 @@ void colm_parser_destroy( program_t *prg, tree_t **sp, struct colm_struct *s )
 	colm_tree_downref( prg, sp, parser->result );
 }
 
-parser_t *colm_parser_new( program_t *prg, struct generic_info *gi )
+parser_t *colm_parser_new( program_t *prg, struct generic_info *gi, bool reducer )
 {
 	struct pda_run *pda_run = malloc( sizeof(struct pda_run) );
 
 	/* Start off the parsing process. */
 	colm_pda_init( prg, pda_run, prg->rtd->pda_tables, 
-			gi->parser_id, 0, 0, 0 );
+			gi->parser_id, 0, 0, 0, reducer );
 	
 	size_t memsize = sizeof(struct colm_parser);
 	struct colm_parser *parser = (struct colm_parser*) malloc( memsize );
@@ -137,7 +137,7 @@ struct_t *colm_construct_generic( program_t *prg, long generic_id )
 			break;
 		}
 		case GEN_PARSER: {
-			parser_t *parser = colm_parser_new( prg, generic_info );
+			parser_t *parser = colm_parser_new( prg, generic_info, 0 );
 			parser->input = colm_stream_new( prg );
 			new_generic = (struct_t*) parser;
 			break;
@@ -146,3 +146,15 @@ struct_t *colm_construct_generic( program_t *prg, long generic_id )
 
 	return new_generic;
 }
+
+struct_t *colm_construct_reducer( program_t *prg, long generic_id )
+{
+	struct generic_info *generic_info = &prg->rtd->generic_info[generic_id];
+	struct_t *new_generic = 0;
+
+	parser_t *parser = colm_parser_new( prg, generic_info, 1 );
+	parser->input = colm_stream_new( prg );
+	new_generic = (struct_t*) parser;
+
+	return new_generic;
+}
diff --git a/src/struct.h b/src/struct.h
index ce3d374a..25000fbb 100644
--- a/src/struct.h
+++ b/src/struct.h
@@ -5,6 +5,8 @@
 extern "C" {
 #endif
 
+#include <stdbool.h>
+
 typedef void (*colm_destructor_t)( struct colm_program *prg,
 		tree_t **sp, struct colm_struct *s );
 
@@ -128,7 +130,8 @@ struct colm_struct *colm_struct_inbuilt( struct colm_program *prg, int size,
 #define colm_struct_to_map_el( prg, obj, genId ) \
 	colm_struct_get_addr( obj, map_el_t*, prg->rtd->generic_info[genId].el_offset )
 
-parser_t *colm_parser_new( struct colm_program *prg, struct generic_info *gi );
+parser_t *colm_parser_new( struct colm_program *prg,
+		struct generic_info *gi, bool reducer );
 stream_t *colm_stream_new( struct colm_program *prg );
 stream_t *colm_stream_new_struct( struct colm_program *prg );
 
@@ -148,6 +151,7 @@ struct colm_struct *colm_map_get( struct colm_program *prg, map_t *map,
 		word_t gen_id, word_t field );
 
 struct colm_struct *colm_construct_generic( struct colm_program *prg, long generic_id );
+struct colm_struct *colm_construct_reducer( struct colm_program *prg, long generic_id );
 
 #define STRUCT_INBUILT_ID -1
 
diff --git a/src/synthesis.cc b/src/synthesis.cc
index f8279590..4f262ce9 100644
--- a/src/synthesis.cc
+++ b/src/synthesis.cc
@@ -1399,11 +1399,12 @@ UniqueType *LangTerm::evaluateParse( Compiler *pd, CodeVect &code,
 	}
 
 	/* Construct the parser. */
-	code.append( IN_CONS_GENERIC );
-	code.appendHalf( parserUT->generic->id );
 
 	if ( parserText->reduce )
-		code.append( IN_PARSER_NOT_USED );
+		code.append( IN_CONS_REDUCER );
+	else
+		code.append( IN_CONS_GENERIC );
+	code.appendHalf( parserUT->generic->id );
 
 	/*
 	 * First load the context into the parser.