added with_ignore(tree) iterator

This is identical to the basic tree iterator, except it visits ignore tokens.
This makes it possible to rewrite comments, albeit, primitively. Constructors
for tokens don't work right. Though we can set the data field.

The following code can be used to turn comments into newlines:

  for I1: comment in with_ignore(Flux) {
      I1.data = "\n"
  }

8 files changed, 73 insertions, 12 deletions

diff --git a/src/bytecode.c b/src/bytecode.c
index 39aee070..f9a90d4f 100644
--- a/src/bytecode.c
+++ b/src/bytecode.c
@@ -2155,7 +2155,19 @@ again:
 			debug( prg, REALM_BYTECODE, "IN_TRITER_ADVANCE\n" );
 
 			tree_iter_t *iter = (tree_iter_t*) vm_get_plocal(exec, field);
-			tree_t *res = tree_iter_advance( prg, &sp, iter );
+			tree_t *res = tree_iter_advance( prg, &sp, iter, false );
+			//colm_tree_upref( prg, res );
+			vm_push_tree( res );
+			break;
+		}
+		case IN_TRITER_WIG_ADVANCE: {
+			short field;
+			read_half( field );
+
+			debug( prg, REALM_BYTECODE, "IN_TRITER_WIG_ADVANCE\n" );
+
+			tree_iter_t *iter = (tree_iter_t*) vm_get_plocal(exec, field);
+			tree_t *res = tree_iter_advance( prg, &sp, iter, true );
 			//colm_tree_upref( prg, res );
 			vm_push_tree( res );
 			break;
diff --git a/src/bytecode.h b/src/bytecode.h
index 02cd78f4..06d463e9 100644
--- a/src/bytecode.h
+++ b/src/bytecode.h
@@ -155,6 +155,7 @@ typedef unsigned char uchar;
 
 #define IN_TRITER_FROM_REF       0x41
 #define IN_TRITER_ADVANCE        0x42
+#define IN_TRITER_WIG_ADVANCE    0x84
 #define IN_TRITER_NEXT_CHILD     0x43
 #define IN_TRITER_GET_CUR_R      0x44
 #define IN_TRITER_GET_CUR_WC     0x45
diff --git a/src/declare.cc b/src/declare.cc
index 5102746c..5d153a9b 100644
--- a/src/declare.cc
+++ b/src/declare.cc
@@ -693,6 +693,16 @@ void Compiler::makeDefaultIterators()
 		objMethod->iterDef = triter;
 	}
 
+	{
+		UniqueType *anyRefUT = findUniqueType( TYPE_REF, anyLangEl );
+		ObjectMethod *objMethod = initFunction( uniqueTypeAny, rootNamespace, globalObjectDef, 
+				ObjectMethod::Call, "with_ignore", IN_HALT, IN_HALT, anyRefUT, true );
+
+		IterDef *triter = findIterDef( IterDef::WithIgnore );
+		objMethod->iterDef = triter;
+	}
+
+
 	/* Child iterator. */
 	{
 		UniqueType *anyRefUT = findUniqueType( TYPE_REF, anyLangEl );
diff --git a/src/iter.c b/src/iter.c
index 66974f4a..f3df16af 100644
--- a/src/iter.c
+++ b/src/iter.c
@@ -354,7 +354,7 @@ void split_iter_cur( program_t *prg, tree_t ***psp, tree_iter_t *iter )
 	split_ref( prg, psp, &iter->ref );
 }
 
-void iter_find( program_t *prg, tree_t ***psp, tree_iter_t *iter, int try_first )
+void iter_find( program_t *prg, tree_t ***psp, tree_iter_t *iter, int try_first, int with_ignore )
 {
 	int any_tree = iter->search_id == prg->rtd->any_id;
 	tree_t **top = iter->stack_root;
@@ -367,7 +367,7 @@ rec_call:
 		return;
 	}
 	else {
-		child = tree_child( prg, iter->ref.kid->tree );
+		child = tree_child_maybe_ignore( prg, iter->ref.kid->tree, with_ignore );
 		if ( child != 0 ) {
 			vm_contiguous( 2 );
 			vm_push_ref( iter->ref.next );
@@ -392,7 +392,7 @@ rec_call:
 	*psp = sp;
 }
 
-tree_t *tree_iter_advance( program_t *prg, tree_t ***psp, tree_iter_t *iter )
+tree_t *tree_iter_advance( program_t *prg, tree_t ***psp, tree_iter_t *iter, int with_ignore )
 {
 	tree_t **sp = *psp;
 	assert( iter->yield_size == (vm_ssize() - iter->root_size) );
@@ -400,11 +400,11 @@ tree_t *tree_iter_advance( program_t *prg, tree_t ***psp, tree_iter_t *iter )
 	if ( iter->ref.kid == 0 ) {
 		/* kid_t is zero, start from the root. */
 		iter->ref = iter->root_ref;
-		iter_find( prg, psp, iter, true );
+		iter_find( prg, psp, iter, true, with_ignore );
 	}
 	else {
 		/* Have a previous item, continue searching from there. */
-		iter_find( prg, psp, iter, false );
+		iter_find( prg, psp, iter, false, with_ignore );
 	}
 
 	sp = *psp;
@@ -643,6 +643,3 @@ tree_t *tree_iter_prev_repeat( program_t *prg, tree_t ***psp, tree_iter_t *iter
 
 	return (iter->ref.kid ? prg->true_val : prg->false_val );
 }
-
-
-
diff --git a/src/parsetree.h b/src/parsetree.h
index 1890bbae..14e92c24 100644
--- a/src/parsetree.h
+++ b/src/parsetree.h
@@ -1934,7 +1934,7 @@ struct IterDef
 {
 	enum Type { Tree, Child, RevChild, Repeat,
 			RevRepeat, User, ListEl, 
-			RevListVal, MapEl };
+			RevListVal, MapEl, WithIgnore };
 
 	IterDef( Type type, Function *func );
 	IterDef( Type type );
@@ -1948,7 +1948,7 @@ struct IterImpl
 {
 	enum Type { Tree, Child, RevChild, Repeat,
 			RevRepeat, User, ListEl, ListVal,
-			RevListVal, MapEl, MapVal };
+			RevListVal, MapEl, MapVal, WithIgnore };
 
 	IterImpl( Type type, Function *func );
 	IterImpl( Type type );
diff --git a/src/synthesis.cc b/src/synthesis.cc
index d049e40d..bac12b20 100644
--- a/src/synthesis.cc
+++ b/src/synthesis.cc
@@ -201,6 +201,19 @@ IterImpl::IterImpl( Type type ) :
 		useGenericId = true;
 		break;
 
+	case WithIgnore:
+		inCreateWV = IN_TRITER_FROM_REF;
+		inCreateWC = IN_TRITER_FROM_REF;
+		inUnwind =   IN_TRITER_UNWIND;
+		inDestroy =  IN_TRITER_DESTROY;
+		inAdvance =  IN_TRITER_WIG_ADVANCE;
+
+		inGetCurR =  IN_TRITER_GET_CUR_R;
+		inGetCurWC = IN_TRITER_GET_CUR_WC;
+		inSetCurWC = IN_TRITER_SET_CUR_WC;
+		inRefFromCur = IN_TRITER_REF_FROM_CUR;
+		useSearchUT = true;
+		break;
 	case User:
 		assert(false);
 	}
@@ -331,6 +344,7 @@ long sizeOfField( UniqueType *fieldUT )
 		/* Select on the iterator type. */
 		switch ( fieldUT->iterDef->type ) {
 			case IterDef::Tree:
+			case IterDef::WithIgnore:
 			case IterDef::Child:
 			case IterDef::Repeat:
 			case IterDef::RevRepeat:
@@ -2476,6 +2490,9 @@ void LangStmt::compileForIter( Compiler *pd, CodeVect &code ) const
 			iterImpl = iterCall->langTerm->varRef->chooseTriterCall( pd,
 					searchUT, iterCall->langTerm->args );
 			break;
+		case IterDef::WithIgnore:
+			iterImpl = new IterImpl( IterImpl::WithIgnore );
+			break;
 		case IterDef::Child:
 			iterImpl = new IterImpl( IterImpl::Child );
 			break;
@@ -2760,6 +2777,7 @@ void Compiler::findLocals( ObjectDef *localFrame, CodeBlock *block )
 				LocalType type = LT_Tree;
 				switch ( ut->iterDef->type ) {
 					case IterDef::Tree:
+					case IterDef::WithIgnore:
 					case IterDef::Child:
 					case IterDef::Repeat:
 					case IterDef::RevRepeat:
diff --git a/src/tree.c b/src/tree.c
index e05681b9..c355720d 100644
--- a/src/tree.c
+++ b/src/tree.c
@@ -1039,6 +1039,28 @@ kid_t *tree_child( program_t *prg, const tree_t *tree )
 	return kid;
 }
 
+/* Find the first child of a tree. */
+kid_t *tree_child_maybe_ignore( program_t *prg, const tree_t *tree, int with_ignore )
+{
+	struct lang_el_info *lel_info = prg->rtd->lel_info;
+	kid_t *kid = tree->child;
+
+	if ( !with_ignore ) {
+		if ( tree->flags & AF_LEFT_IGNORE )
+			kid = kid->next;
+		if ( tree->flags & AF_RIGHT_IGNORE )
+			kid = kid->next;
+	}
+
+	/* Skip over attributes. */
+	long object_length = lel_info[tree->id].object_length;
+	long a;
+	for ( a = 0; a < object_length; a++ )
+		kid = kid->next;
+
+	return kid;
+}
+
 /* Detach at the first real child of a tree. */
 kid_t *tree_extract_child( program_t *prg, tree_t *tree )
 {
diff --git a/src/tree.h b/src/tree.h
index 97833c6f..f12eeb5a 100644
--- a/src/tree.h
+++ b/src/tree.h
@@ -234,6 +234,7 @@ tree_t *tree_right_ignore( struct colm_program *prg, tree_t *tree );
 kid_t *tree_left_ignore_kid( struct colm_program *prg, tree_t *tree );
 kid_t *tree_right_ignore_kid( struct colm_program *prg, tree_t *tree );
 kid_t *tree_child( struct colm_program *prg, const tree_t *tree );
+kid_t *tree_child_maybe_ignore( struct colm_program *prg, const tree_t *tree, int with_ignore );
 kid_t *tree_attr( struct colm_program *prg, const tree_t *tree );
 kid_t *kid_list_concat( kid_t *list1, kid_t *list2 );
 kid_t *tree_extract_child( struct colm_program *prg, tree_t *tree );
@@ -292,7 +293,7 @@ tree_t *list_remove_head( struct colm_program *prg, list_t *list );
 tree_t *get_list_mem_split( struct colm_program *prg, list_t *list, word_t field );
 tree_t *get_parser_mem( parser_t *parser, word_t field );
 
-tree_t *tree_iter_advance( struct colm_program *prg, tree_t ***psp, tree_iter_t *iter );
+tree_t *tree_iter_advance( struct colm_program *prg, tree_t ***psp, tree_iter_t *iter, int with_ignore );
 tree_t *tree_iter_next_child( struct colm_program *prg, tree_t ***psp, tree_iter_t *iter );
 tree_t *tree_rev_iter_prev_child( struct colm_program *prg, tree_t ***psp, rev_tree_iter_t *iter );
 tree_t *tree_iter_next_repeat( struct colm_program *prg, tree_t ***psp, tree_iter_t *iter );