pushing the trim decision to run-time

The trim and no-trim operators are both now in the expression. In places where
we want the print mechanism to decide if it trims or not, we lift the trim
operator out of the expression and into the send expression, then pass it to
the print and send instructions. This way the stream or input (parser) can
control the trim (auto_trim feature) with the expression overriding the
default.

8 files changed, 139 insertions, 36 deletions

diff --git a/src/bytecode.c b/src/bytecode.c
index c5f8f313..fe0026fc 100644
--- a/src/bytecode.c
+++ b/src/bytecode.c
@@ -200,7 +200,7 @@ static head_t *tree_to_str_postfix( program_t *prg, tree_t **sp, tree_t *tree, i
 }
 
 
-static word_t stream_append_text( program_t *prg, tree_t **sp, input_t *dest, tree_t *input )
+static word_t stream_append_text( program_t *prg, tree_t **sp, input_t *dest, tree_t *input, int trim )
 {
 	long length = 0;
 	struct input_impl *impl = input_to_impl( dest );
@@ -209,12 +209,10 @@ static word_t stream_append_text( program_t *prg, tree_t **sp, input_t *dest, tr
 		assert(false);
 	}
 	else {
-		int auto_trim = impl->funcs->get_option( prg, impl, 0 );
-
 		/* Collect the tree data. */
 		str_collect_t collect;
 		init_str_collect( &collect );
-		colm_print_tree_collect( prg, sp, &collect, input, auto_trim );
+		colm_print_tree_collect( prg, sp, &collect, input, trim );
 
 		/* Load it into the input. */
 		impl->funcs->append_data( prg, impl, collect.data, collect.length );
@@ -2323,13 +2321,24 @@ again:
 		}
 
 		case IN_PRINT_TREE: {
-			debug( prg, REALM_BYTECODE, "IN_PRINT_TREE\n" );
+			uchar trim;
+			read_byte( trim );
+
+			debug( prg, REALM_BYTECODE, "IN_PRINT_TREE %d\n", (int)trim );
 
 			tree_t *to_send = vm_pop_tree();
 			stream_t *stream = vm_pop_stream();
 
 			struct stream_impl *si = stream_to_impl( stream );
-			int auto_trim = si->funcs->get_option( prg, si, 0 );
+
+			int auto_trim;
+			if ( trim == TRIM_YES )
+				auto_trim = true;
+			else if ( trim == TRIM_NO )
+				auto_trim = false;
+			else 
+				auto_trim = si->funcs->get_option( prg, si, 0 );
+
 			si->funcs->print_tree( prg, sp, si, to_send, auto_trim );
 			vm_push_stream( stream );
 			colm_tree_downref( prg, sp, to_send );
@@ -2337,12 +2346,25 @@ again:
 		}
 
 		case IN_SEND_TEXT_W: {
-			debug( prg, REALM_BYTECODE, "IN_SEND_TEXT_W\n" );
+			uchar trim;
+			read_byte( trim );
+
+			debug( prg, REALM_BYTECODE, "IN_SEND_TEXT_W %d\n", (int)trim );
 
 			tree_t *to_send = vm_pop_tree();
 			parser_t *parser = vm_pop_parser();
 
-			word_t len = stream_append_text( prg, sp, parser->input, to_send );
+			struct input_impl *si = input_to_impl( parser->input );
+
+			int auto_trim;
+			if ( trim == TRIM_YES )
+				auto_trim = true;
+			else if ( trim == TRIM_NO )
+				auto_trim = false;
+			else 
+				auto_trim = si->funcs->get_option( prg, si, 0 );
+
+			word_t len = stream_append_text( prg, sp, parser->input, to_send, auto_trim );
 
 			vm_push_parser( parser );
 
@@ -2380,11 +2402,27 @@ again:
 		}
 
 		case IN_SEND_TREE_W: {
-			debug( prg, REALM_BYTECODE, "IN_SEND_TREE_W\n" );
+			uchar trim;
+			read_byte( trim );
+
+			debug( prg, REALM_BYTECODE, "IN_SEND_TREE_W %d\n", (int)trim );
 
 			tree_t *to_send = vm_pop_tree();
 			parser_t *parser = vm_pop_parser();
 
+			struct input_impl *si = input_to_impl( parser->input );
+
+			int auto_trim;
+			if ( trim == TRIM_YES )
+				auto_trim = true;
+			else if ( trim == TRIM_NO )
+				auto_trim = false;
+			else 
+				auto_trim = si->funcs->get_option( prg, si, 0 );
+
+			if ( auto_trim )
+				to_send = tree_trim( prg, sp, to_send );
+
 			word_t len = stream_append_tree( prg, sp, parser->input, to_send );
 
 			vm_push_parser( parser );
diff --git a/src/bytecode.h b/src/bytecode.h
index 7197cc36..02cd78f4 100644
--- a/src/bytecode.h
+++ b/src/bytecode.h
@@ -424,6 +424,10 @@ typedef unsigned char uchar;
 #define FN_PREFIX                0x3b
 #define FN_SUFFIX                0x3c
 
+#define TRIM_DEFAULT 0x01
+#define TRIM_YES     0x02
+#define TRIM_NO      0x03
+
 /* Types of Generics. */
 enum GEN {
 	GEN_PARSER   = 0x14,
diff --git a/src/colm.lm b/src/colm.lm
index a8e398cf..3875babd 100644
--- a/src/colm.lm
+++ b/src/colm.lm
@@ -616,6 +616,7 @@ def code_unary
 |	[DOLLAR code_factor] :Dollar
 |	[DOLLAR DOLLAR code_factor] :DollarDollar
 |	[CARET code_factor] :Caret
+|	[AT code_factor] :At
 |	[PERCENT code_factor] :Percent
 |	[code_factor] :Base
 
@@ -732,10 +733,6 @@ def pattern
 # Constructor List
 #
 
-def opt_no_trim
-	[AT] :At
-|	[] :Trim
-
 def E1 []
 def E2 []
 def E3 []
@@ -746,7 +743,7 @@ def cons_el
 |	[E1 DQ LitConsElList: lit_cons_el* dq_lit_term] :Dq
 |	[E1 SQ SqConsDataList: sq_cons_data* sq_lit_term] :Sq
 |	[E1 TILDE opt_tilde_data TILDE_NL] :Tilde
-|	[E2 opt_no_trim code_expr] :CodeExpr
+|	[E2 code_expr] :CodeExpr
 
 def lit_cons_el
 	[lit_dq_data] :ConsData
@@ -772,7 +769,7 @@ def accum_el
 	[E1 DQ LitAccumElList: lit_accum_el* dq_lit_term] :Dq
 |	[E1 SQ SqConsDataList: sq_cons_data* sq_lit_term] :Sq
 |	[E1 TILDE opt_tilde_data TILDE_NL] :Tilde
-|	[E2 opt_no_trim code_expr] :CodeExpr
+|	[E2 code_expr] :CodeExpr
 
 def lit_accum_el
 	[lit_dq_data] :ConsData
@@ -799,7 +796,7 @@ def string_el
 	[E1 DQ LitStringElList: lit_string_el* dq_lit_term] :Dq
 |	[E1 SQ SqConsDataList: sq_cons_data* sq_lit_term] :Sq
 |	[E1 TILDE opt_tilde_data TILDE_NL] :Tilde
-|	[E2 opt_no_trim code_expr] :CodeExpr
+|	[E2 code_expr] :CodeExpr
 
 def lit_string_el
 	[lit_dq_data] :ConsData
diff --git a/src/consinit.cc b/src/consinit.cc
index 1c0814ea..995e24f1 100644
--- a/src/consinit.cc
+++ b/src/consinit.cc
@@ -785,7 +785,7 @@ void ConsInit::parseInput( StmtList *stmtList )
 	LangExpr *open = LangExpr::cons( LangTerm::cons( InputLoc(), openRef, openArgs ) );
 
 	/* Construct a list containing the open stream. */
-	ConsItem *consItem = ConsItem::cons( internal, ConsItem::ExprType, open, false );
+	ConsItem *consItem = ConsItem::cons( internal, ConsItem::ExprType, open, ConsItem::TrimDefault );
 	ConsItemList *list = ConsItemList::cons( consItem );
 
 	/* Will capture the parser to "P" */
diff --git a/src/loadcolm.cc b/src/loadcolm.cc
index 1b0dba85..21f0217a 100644
--- a/src/loadcolm.cc
+++ b/src/loadcolm.cc
@@ -1272,6 +1272,23 @@ struct LoadColm
 		return callArgVect;
 	}
 
+	LangExpr *liftTrim( LangExpr *expr, ConsItem::Trim &trim )
+	{
+		if ( expr->type == LangExpr::UnaryType ) {
+			if ( expr->op == '^' ) {
+				trim = ConsItem::TrimYes;
+				expr = expr->right;
+				std::cerr << "trim" << std::endl;
+			}
+			else if ( expr->op == '@' ) {
+				trim = ConsItem::TrimNo;
+				expr = expr->right;
+				std::cerr << "notrim" << std::endl;
+			}
+		}
+		return expr;
+	}
+
 	ConsItemList *walkCallArgSeqAccum( call_arg_seq callArgSeq )
 	{
 		ConsItemList *consItemList = new ConsItemList;
@@ -1281,8 +1298,9 @@ struct LoadColm
 //			LangExpr *expr = walkCodeExpr( codeExpr );
 //			callArgVect->append( new CallArg(expr) );
 
-			bool trim = false; //walkTrim( consEl.opt_no_trim() );
+			ConsItem::Trim trim = ConsItem::TrimDefault;
 			LangExpr *consExpr = walkCodeExpr( codeExpr );
+
 			ConsItem *consItem = ConsItem::cons( consExpr->loc,
 					ConsItem::ExprType, consExpr, trim );
 			consItemList->append( consItem );
@@ -1430,12 +1448,6 @@ struct LoadColm
 		return list;
 	}
 
-	/* Returns the trim status. (true for trim, false otherwise) */
-	bool walkTrim( opt_no_trim OptNoTrim )
-	{
-		return OptNoTrim.prodName() != opt_no_trim::At;
-	}
-
 	ConsItemList *walkConsEl( cons_el consEl, TypeRef *consTypeRef )
 	{
 		ConsItemList *list = 0;
@@ -1460,7 +1472,7 @@ struct LoadColm
 			break;
 		}
 		case cons_el::CodeExpr: {
-			bool trim = walkTrim( consEl.opt_no_trim() );
+			ConsItem::Trim trim = ConsItem::TrimDefault;
 			LangExpr *consExpr = walkCodeExpr( consEl.code_expr() );
 			ConsItem *consItem = ConsItem::cons( consExpr->loc,
 					ConsItem::ExprType, consExpr, trim );
@@ -1589,8 +1601,9 @@ struct LoadColm
 			break;
 		}
 		case string_el::CodeExpr: {
-			bool trim = walkTrim( stringEl.opt_no_trim() );
+			ConsItem::Trim trim = ConsItem::TrimDefault;
 			LangExpr *consExpr = walkCodeExpr( stringEl.code_expr() );
+			consExpr = liftTrim( consExpr, trim );
 			ConsItem *consItem = ConsItem::cons( consExpr->loc,
 					ConsItem::ExprType, consExpr, trim );
 			list = ConsItemList::cons( consItem );
@@ -1714,8 +1727,9 @@ struct LoadColm
 			break;
 		}
 		case accum_el::CodeExpr: {
-			bool trim = walkTrim( accumEl.opt_no_trim() );
+			ConsItem::Trim trim = ConsItem::TrimDefault;
 			LangExpr *accumExpr = walkCodeExpr( accumEl.code_expr() );
+			accumExpr = liftTrim( accumExpr, trim );
 			ConsItem *consItem = ConsItem::cons( accumExpr->loc,
 					ConsItem::ExprType, accumExpr, trim );
 			list = ConsItemList::cons( consItem );
@@ -2072,6 +2086,11 @@ struct LoadColm
 			expr = LangExpr::cons( unary.CARET().loc(), '^', factor );
 			break;
 		}
+		case code_unary::At: {
+			LangExpr *factor = walkCodeFactor( unary.code_factor() );
+			expr = LangExpr::cons( unary.AT().loc(), '@', factor );
+			break;
+		}
 		case code_unary::Percent: {
 			LangExpr *factor = walkCodeFactor( unary.code_factor() );
 			expr = LangExpr::cons( unary.PERCENT().loc(), '%', factor );
diff --git a/src/loadinit.cc b/src/loadinit.cc
index 4bc99d18..fc75906c 100644
--- a/src/loadinit.cc
+++ b/src/loadinit.cc
@@ -317,7 +317,7 @@ void LoadInit::consParseStmt( StmtList *stmtList )
 	LangExpr *open = LangExpr::cons( LangTerm::cons( InputLoc(), openRef, openArgs ) );
 
 	/* Construct a list containing the open stream. */
-	ConsItem *consItem = ConsItem::cons( internal, ConsItem::ExprType, open, false );
+	ConsItem *consItem = ConsItem::cons( internal, ConsItem::ExprType, open, ConsItem::TrimDefault );
 	ConsItemList *list = ConsItemList::cons( consItem );
 
 	/* Will capture the parser to "P" */
diff --git a/src/parsetree.h b/src/parsetree.h
index 599c16b7..4bf657d7 100644
--- a/src/parsetree.h
+++ b/src/parsetree.h
@@ -1734,6 +1734,12 @@ struct PatternItemList
 
 struct ConsItem
 {
+	enum Trim {
+		TrimYes,
+		TrimNo,
+		TrimDefault
+	};
+
 	enum Type { 
 		InputText, 
 		ExprType,
@@ -1747,7 +1753,7 @@ struct ConsItem
 		langEl(0),
 		prodEl(0),
 		bindId(-1),
-		trim(false)
+		trim(TrimDefault)
 	{
 	}
 
@@ -1760,7 +1766,7 @@ struct ConsItem
 		return r;
 	}
 
-	static ConsItem *cons( const InputLoc &loc, Type type, LangExpr *expr, bool trim )
+	static ConsItem *cons( const InputLoc &loc, Type type, LangExpr *expr, Trim trim )
 	{
 		ConsItem *r = new ConsItem;
 		r->loc = loc;
@@ -1787,7 +1793,8 @@ struct ConsItem
 	LangEl *langEl;
 	ProdEl *prodEl;
 	long bindId;
-	bool trim;
+	Trim trim;
+
 	ConsItem *prev, *next;
 };
 
diff --git a/src/synthesis.cc b/src/synthesis.cc
index 5a47d56e..e1d192b6 100644
--- a/src/synthesis.cc
+++ b/src/synthesis.cc
@@ -1401,7 +1401,7 @@ UniqueType *LangTerm::evaluateConstruct( Compiler *pd, CodeVect &code ) const
 			}
 
 			if ( !isStr( ut ) ) {
-				if ( item->trim ) 
+				if ( item->trim == ConsItem::TrimYes ) 
 					code.append( IN_TREE_TRIM );
 			}
 
@@ -1493,6 +1493,9 @@ UniqueType *LangTerm::evaluateParse( Compiler *pd, CodeVect &code,
 			if ( ut->typeId != TYPE_TREE )
 				error() << "variables used in replacements must be trees" << endp;
 
+			if ( item->trim == ConsItem::TrimYes ) 
+				code.append( IN_TREE_TRIM );
+
 			item->langEl = ut->langEl;
 		}
 	}
@@ -1542,6 +1545,8 @@ UniqueType *LangTerm::evaluateParse( Compiler *pd, CodeVect &code,
 	else {
 		for ( ConsItemList::Iter item = *parserText->list; item.lte(); item++ ) {
 			bool isStream = false;
+			uchar trim = TRIM_DEFAULT;
+
 			switch ( item->type ) {
 			case ConsItem::LiteralType: {
 				String result;
@@ -1584,15 +1589,24 @@ UniqueType *LangTerm::evaluateParse( Compiler *pd, CodeVect &code,
 				if ( ut == pd->uniqueTypeStream )
 					isStream = true;
 
+				if ( item->trim == ConsItem::TrimYes )
+					trim = TRIM_YES;
+				else if ( item->trim == ConsItem::TrimNo )
+					trim = TRIM_NO;
+
 				break;
 			}}
 
 			if ( isStream )
 				code.append( IN_SEND_STREAM_W );
-			else if ( tree )
+			else if ( tree ) {
 				code.append( IN_SEND_TREE_W );
-			else
+				code.append( trim );
+			}
+			else {
 				code.append( IN_SEND_TEXT_W );
+				code.append( trim );
+			}
 
 			/* Parse instruction, dependent on whether or not we are producing
 			 * revert or commit code. */
@@ -1641,6 +1655,8 @@ void LangTerm::evaluateSendStream( Compiler *pd, CodeVect &code ) const
 
 	/* Assign bind ids to the variables in the replacement. */
 	for ( ConsItemList::Iter item = *parserText->list; item.lte(); item++ ) {
+		uchar trim = TRIM_DEFAULT;
+
 		switch ( item->type ) {
 		case ConsItem::LiteralType: {
 			String result;
@@ -1679,10 +1695,16 @@ void LangTerm::evaluateSendStream( Compiler *pd, CodeVect &code ) const
 			if ( ut->typeId == TYPE_INT || ut->typeId == TYPE_BOOL )
 				code.append( IN_INT_TO_STR );
 
+			if ( item->trim == ConsItem::TrimYes )
+				trim = TRIM_YES;
+			else if ( item->trim == ConsItem::TrimNo )
+				trim = TRIM_NO;
+
 			break;
 		}}
 
 		code.append( IN_PRINT_TREE );
+		code.append( trim );
 	}
 }
 
@@ -1710,6 +1732,8 @@ void LangTerm::evaluateSendParser( Compiler *pd, CodeVect &code, bool strings )
 		/* Assign bind ids to the variables in the replacement. */
 		for ( ConsItemList::Iter item = *parserText->list; item.lte(); item++ ) {
 			bool isStream = false;
+			uchar trim = TRIM_DEFAULT;
+
 			switch ( item->type ) {
 			case ConsItem::LiteralType: {
 				String result;
@@ -1748,6 +1772,11 @@ void LangTerm::evaluateSendParser( Compiler *pd, CodeVect &code, bool strings )
 				if ( ut == pd->uniqueTypeStream )
 					isStream = true;
 
+				if ( item->trim == ConsItem::TrimYes )
+					trim = TRIM_YES;
+				else if ( item->trim == ConsItem::TrimNo )
+					trim = TRIM_NO;
+
 				if ( ut->typeId == TYPE_INT || ut->typeId == TYPE_BOOL )
 					code.append( IN_INT_TO_STR );
 
@@ -1756,10 +1785,14 @@ void LangTerm::evaluateSendParser( Compiler *pd, CodeVect &code, bool strings )
 
 			if ( isStream )
 				code.append( IN_SEND_STREAM_W );
-			else if ( !strings )
+			else if ( !strings ) {
 				code.append( IN_SEND_TREE_W );
-			else
+				code.append( trim );
+			}
+			else {
 				code.append( IN_SEND_TEXT_W );
+				code.append( trim );
+			}
 
 			parseFrag( pd, code, 0 );
 		}
@@ -2240,6 +2273,11 @@ UniqueType *LangExpr::evaluate( Compiler *pd, CodeVect &code ) const
 					code.append( IN_TREE_TRIM );
 					return rt;
 				}
+				case '@': {
+					UniqueType *rt = right->evaluate( pd, code );
+					//code.append( IN_TREE_TRIM );
+					return rt;
+				}
 				default: 
 					assert(false);
 			}