Now using the accumulated ignore stack when reporting some backtracking points.

The error reporting skips over the deepst token, on the assumption that it is
the deepest, successfully parsed token and the error is therefore after. 
refs #326.

Bumped the version to release candidate 5.

Added a test case for exports. Only setting them.

3 files changed, 67 insertions, 28 deletions

diff --git a/colm/bytecode.c b/colm/bytecode.c
index 3f176b09..3265c033 100644
--- a/colm/bytecode.c
+++ b/colm/bytecode.c
@@ -1026,7 +1026,7 @@ Tree **executeCode( Execution *exec, Tree **sp, Code *instr )
 {
 	/* When we exit we are going to verify that we did not eat up any stack
 	 * space. */
-	//Tree **root = sp;
+	Tree **root = sp;
 	Program *prg = exec->prg;
 	Code c;
 
@@ -3746,7 +3746,6 @@ again:
 			treeDownref( prg, sp, global );
 			treeUpref( prg->lastParseError );
 			vm_push( prg->lastParseError );
-			treeDownref( prg, sp, global );
 			break;
 		}
 		case IN_OPEN_FILE: {
@@ -3811,10 +3810,8 @@ again:
 	goto again;
 
 out:
-	if ( ! prg->induceExit ) {
-		// FIXME: bring this back.
-		//assert( sp == root );
-	}
+	if ( ! prg->induceExit )
+		assert( sp == root );
 	return sp;
 }
 
diff --git a/colm/fsmrun.c b/colm/fsmrun.c
index 2175ad51..9ea1b79d 100644
--- a/colm/fsmrun.c
+++ b/colm/fsmrun.c
@@ -609,27 +609,42 @@ void clearIgnoreList( Program *prg, Tree **sp, Kid *kid )
 static void reportParseError( Program *prg, Tree **sp, PdaRun *pdaRun )
 {
 	Kid *kid = pdaRun->btPoint;
-	Location *deepest = 0;
+	Head *deepest = 0;
 	while ( kid != 0 ) {
 		Head *head = kid->tree->tokdata;
-		Location *location = head != 0 ? head->location : 0;
-		if ( location && ( deepest == 0 || location->byte > deepest->byte ) )
-			deepest = location;
+		if ( head != 0 && head->location != 0 ) {
+			if ( deepest == 0 || head->location->byte > deepest->location->byte ) 
+				deepest = head;
+		}
 		kid = kid->next;
 	}
 
-	Head *head = 0;
+	Head *errorHead = 0;
 
 	/* If there are no error points on record assume the error occurred at the beginning of the stream. */
 	if ( deepest == 0 ) 
-		head = stringAllocFull( prg, "PARSE ERROR at 1:1", 18 );
+		errorHead = stringAllocFull( prg, "PARSE ERROR at 1:1", 18 );
 	else {
+		debug( REALM_PARSE, "deepest location byte: %d\n", deepest->location->byte );
+
+		long line = deepest->location->line;
+		long i, column = deepest->location->column;
+
+		for ( i = 0; i < deepest->length; i++ ) {
+			if ( deepest->data[i] != '\n' )
+				column += 1;
+			else {
+				line += 1;
+				column = 1;
+			}
+		}
+
 		char formatted[128];
-		sprintf( formatted, "PARSE ERROR at %ld:%ld", deepest->line, deepest->column );
-		head = stringAllocFull( prg, formatted, strlen(formatted) );
+		sprintf( formatted, "PARSE ERROR at %ld:%ld", line, column );
+		errorHead = stringAllocFull( prg, formatted, strlen(formatted) );
 	}
 
-	Tree *tree = constructString( prg, head );
+	Tree *tree = constructString( prg, errorHead );
 	treeDownref( prg, sp, prg->lastParseError );
 	prg->lastParseError = tree;
 	treeUpref( prg->lastParseError );
@@ -757,6 +772,8 @@ Head *extractMatch( Program *prg, FsmRun *fsmRun, InputStream *inputStream )
 	head->location->line = inputStream->line;
 	head->location->column = inputStream->column;
 	head->location->byte = inputStream->byte;
+
+	debug( REALM_PARSE, "location byte: %d\n", inputStream->byte );
 	return head;
 }
 
@@ -1080,6 +1097,7 @@ case PcrPreEof:
 		}
 		else if ( pdaRun->tokenId == SCAN_UNDO ) {
 			/* Fall through with input2 = 0. FIXME: Do we need to send back ignore? */
+			debug( REALM_PARSE, "invoking undo from the scanner\n" );
 		}
 		else if ( pdaRun->tokenId == SCAN_ERROR ) {
 			/* Scanner error, maybe retry. */
@@ -1105,6 +1123,9 @@ case PcrRevIgnore1:
 			else if ( pdaRun->numRetry > 0 ) {
 				debug( REALM_PARSE, "invoking parse error from the scanner\n" );
 
+				/* Fall through to send null (error). */
+				pushBtPoint( prg, pdaRun, 0 );
+
 				/* Send back any accumulated ignore tokens, then trigger error
 				 * in the the parser. */
 				pdaRun->ignore3 = extractIgnore( pdaRun );
@@ -1116,15 +1137,13 @@ case PcrRevIgnore2:
 
 					pcr = sendBackIgnore( prg, sp, pdaRun, fsmRun, inputStream, pdaRun->ignore3, PcrRevIgnore );
 				}
-
-				/* Fall through to send null (error). */
 			}
 			else {
+				debug( REALM_PARSE, "no alternate scanning regions\n" );
 				/* There are no alternative scanning regions to try, nor are
 				 * there any alternatives stored in the current parse tree. No
 				 * choice but to end the parse. */
-				if ( pdaRun->tokenList != 0 ) 
-					pushBtPoint( prg, pdaRun, pdaRun->tokenList->kid->tree );
+				pushBtPoint( prg, pdaRun, 0 );
 
 				reportParseError( prg, sp, pdaRun );
 				pdaRun->parseError = 1;
@@ -1132,14 +1151,20 @@ case PcrRevIgnore2:
 			}
 		}
 		else if ( pdaRun->tokenId == SCAN_LANG_EL ) {
+			debug( REALM_PARSE, "sending an named lang el\n" );
+
 			/* A named language element (parsing colm program). */
 			pdaRun->input2 = sendNamedLangEl( prg, sp, pdaRun, fsmRun, inputStream );
 		}
 		else if ( pdaRun->tokenId == SCAN_TREE ) {
+			debug( REALM_PARSE, "sending an tree\n" );
+
 			/* A tree already built. */
 			pdaRun->input2 = sendTree( prg, sp, pdaRun, fsmRun, inputStream );
 		}
 		else if ( pdaRun->tokenId == SCAN_IGNORE ) {
+			debug( REALM_PARSE, "sending an ignore token\n" );
+
 			/* A tree to ignore. */
 			sendTreeIgnore( prg, sp, pdaRun, fsmRun, inputStream );
 			goto skipSend;
@@ -1175,11 +1200,17 @@ case PcrGeneration:
 			goto skipSend;
 		}
 		else if ( lelInfo[pdaRun->tokenId].ignore ) {
+			debug( REALM_PARSE, "sending an ignore token: %s\n", 
+					prg->rtd->lelInfo[pdaRun->tokenId].name );
+
 			/* Is an ignore token. */
 			sendIgnore( prg, sp, inputStream, fsmRun, pdaRun, pdaRun->tokenId );
 			goto skipSend;
 		}
 		else {
+			debug( REALM_PARSE, "sending an a plain old token: %s\n", 
+					prg->rtd->lelInfo[pdaRun->tokenId].name );
+
 			/* Is a plain token. */
 			pdaRun->input2 = sendToken( prg, sp, inputStream, fsmRun, pdaRun, pdaRun->tokenId );
 		}
diff --git a/colm/pdarun.c b/colm/pdarun.c
index 6c730ffb..c21ea44b 100644
--- a/colm/pdarun.c
+++ b/colm/pdarun.c
@@ -353,13 +353,27 @@ void commitFull( Program *prg, Tree **sp, PdaRun *pdaRun, long causeReduce )
 		rcodeDownrefAll( prg, sp, &pdaRun->reverseCode );
 }
 
+/* Tree null means compute from what we find in the parser. */
 void pushBtPoint( Program *prg, PdaRun *pdaRun, Tree *tree )
 {
-	Kid *kid = kidAllocate( prg );
-	kid->tree = tree;
-	treeUpref( tree );
-	kid->next = pdaRun->btPoint;
-	pdaRun->btPoint = kid;
+	if ( tree == 0 ) {
+		if ( pdaRun->accumIgnore != 0 ) 
+			tree = pdaRun->accumIgnore->tree;
+		else if ( pdaRun->tokenList != 0 )
+			tree = pdaRun->tokenList->kid->tree;
+	}
+
+	if ( tree != 0 ) {
+		debug( REALM_PARSE, "pushing bt point with location byte %d\n", 
+				( tree != 0 && tree->tokdata != 0 && tree->tokdata->location != 0 ) ? 
+				tree->tokdata->location->byte : 0 );
+
+		Kid *kid = kidAllocate( prg );
+		kid->tree = tree;
+		treeUpref( tree );
+		kid->next = pdaRun->btPoint;
+		pdaRun->btPoint = kid;
+	}
 }
 
 /*
@@ -389,11 +403,8 @@ switch ( entry ) {
 case PcrStart:
 
 	/* The scanner will send a null token if it can't find a token. */
-	if ( pdaRun->input1 == 0 ) {
-		/* Grab the most recently accepted item. */
-		pushBtPoint( prg, pdaRun, pdaRun->tokenList->kid->tree );
+	if ( pdaRun->input1 == 0 )
 		goto parseError;
-	}
 
 	/* The tree we are given must be * parse tree size. It also must have at
 	 * least one reference. */