fix for optimized append wherby we were not respecting run_buf->offset

3 files changed, 74 insertions, 46 deletions

diff --git a/src/input.c b/src/input.c
index ea3529ab..9d93fb8e 100644
--- a/src/input.c
+++ b/src/input.c
@@ -550,7 +550,7 @@ static tree_t *input_undo_prepend_stream( struct colm_program *prg, struct input
 	return 0;
 }
 
-//#define OPTIM_APPEND
+#define OPTIM_APPEND
 
 static void input_append_data( struct colm_program *prg, struct input_impl_seq *si, const char *data, long length )
 {
diff --git a/src/input.h b/src/input.h
index 9d53b2ca..fc100313 100644
--- a/src/input.h
+++ b/src/input.h
@@ -53,35 +53,6 @@ struct colm_stream;
 struct input_impl;
 struct stream_impl;
 
-enum seq_buf_type {
-	SB_TOKEN = 1,
-	SB_IGNORE,
-	SB_SOURCE,
-	SB_ACCUM
-};
-
-struct seq_buf
-{
-	enum seq_buf_type type;
-	char own_si;
-	struct colm_tree *tree;
-	struct stream_impl *si;
-	struct seq_buf *next, *prev;
-};
-
-struct run_buf
-{
-	long length;
-	long offset;
-	struct run_buf *next, *prev;
-
-	/* Must be at the end. We will grow this struct to add data if the input
-	 * demands it. */
-	char data[FSM_BUFSIZE];
-};
-
-struct run_buf *new_run_buf( int sz );
-
 #define DEF_INPUT_FUNCS( input_funcs, _input_impl ) \
 struct input_funcs \
 { \
@@ -145,6 +116,22 @@ struct stream_impl
 	struct stream_funcs *funcs;
 };
 
+enum seq_buf_type {
+	SB_TOKEN = 1,
+	SB_IGNORE,
+	SB_SOURCE,
+	SB_ACCUM
+};
+
+struct seq_buf
+{
+	enum seq_buf_type type;
+	char own_si;
+	struct colm_tree *tree;
+	struct stream_impl *si;
+	struct seq_buf *next, *prev;
+};
+
 /* List of source streams. Enables streams to be pushed/popped. */
 struct input_impl_seq
 {
@@ -164,7 +151,19 @@ struct input_impl_seq
 	int consumed;
 };
 
-/* List of source streams. Enables streams to be pushed/popped. */
+struct run_buf
+{
+	long length;
+	long offset;
+	struct run_buf *next, *prev;
+
+	/* Must be at the end. We will grow this struct to add data if the input
+	 * demands it. */
+	char data[FSM_BUFSIZE];
+};
+
+struct run_buf *new_run_buf( int sz );
+
 struct stream_impl_data
 {
 	struct stream_funcs *funcs;
diff --git a/src/stream.c b/src/stream.c
index 11ba9b71..c2d416d9 100644
--- a/src/stream.c
+++ b/src/stream.c
@@ -42,6 +42,19 @@ DEF_STREAM_FUNCS( stream_funcs_data, stream_impl_data );
 extern struct stream_funcs_data file_funcs;
 extern struct stream_funcs_data accum_funcs;
 
+static void dump_contents( struct colm_program *prg, struct stream_impl_data *sid )
+{
+	struct run_buf *rb = sid->queue.head;
+	while ( rb != 0 ) {
+		debug( prg, REALM_INPUT, " %p contents |%d|%d|%d|%.*s|\n", sid,
+				rb->offset, rb->length,
+				rb->length - rb->offset,
+				(int)rb->length - rb->offset,
+				rb->data + rb->offset );
+		rb = rb->next;
+	}
+}
+
 static bool loc_set( location_t *loc )
 {
 	return loc->line != 0;
@@ -240,6 +253,10 @@ int data_append_data( struct colm_program *prg, struct stream_impl_data *sid, co
 	memcpy( tail->data + tail->length, data, length );
 	tail->length += length;
 
+#ifdef DEBUG
+	dump_contents( prg, sid );
+#endif
+
 	return length;
 }
 
@@ -256,7 +273,7 @@ int data_undo_append_data( struct colm_program *prg, struct stream_impl_data *si
 			break;
 
 		/* Anything available in the current buffer. */
-		int avail = buf->length; // - buf->offset;
+		int avail = buf->length - buf->offset;
 		if ( avail > 0 ) {
 			/* The source data from the current buffer. */
 			int slen = avail <= remaining ? avail : remaining;
@@ -276,6 +293,10 @@ int data_undo_append_data( struct colm_program *prg, struct stream_impl_data *si
 	debug( prg, REALM_INPUT, "data_undo_append_data: stream %p "
 			"ask: %d, consumed: %d, now: %d\n", sid, length, consumed );
 
+#ifdef DEBUG
+	dump_contents( prg, sid );
+#endif
+
 	return consumed;
 
 }
@@ -398,20 +419,20 @@ static int data_get_parse_block( struct colm_program *prg, struct stream_impl_da
 	return ret;
 }
 
-static int data_consume_data( struct colm_program *prg, struct stream_impl_data *si, int length, location_t *loc )
+static int data_consume_data( struct colm_program *prg, struct stream_impl_data *sid, int length, location_t *loc )
 {
 	int consumed = 0;
 	int remaining = length;
 
 	/* Move over skip bytes. */
 	while ( true ) {
-		struct run_buf *buf = si->queue.head;
+		struct run_buf *buf = sid->queue.head;
 
 		if ( buf == 0 )
 			break;
 
 		if ( !loc_set( loc ) )
-			data_transfer_loc( prg, loc, si );
+			data_transfer_loc( prg, loc, sid );
 
 		/* Anything available in the current buffer. */
 		int avail = buf->length - buf->offset;
@@ -420,41 +441,49 @@ static int data_consume_data( struct colm_program *prg, struct stream_impl_data
 			int slen = avail <= remaining ? avail : remaining;
 			consumed += slen;
 			remaining -= slen;
-			update_position_data( si, buf->data + buf->offset, slen );
+			update_position_data( sid, buf->data + buf->offset, slen );
 			buf->offset += slen;
-			si->consumed += slen;
+			sid->consumed += slen;
 		}
 
 		if ( remaining == 0 )
 			break;
 
-		struct run_buf *run_buf = si_data_pop_head( si );
+		struct run_buf *run_buf = si_data_pop_head( sid );
 		free( run_buf );
 	}
 
 	debug( prg, REALM_INPUT, "data_consume_data: stream %p "
-			"ask: %d, consumed: %d, now: %d\n", si, length, consumed, si->consumed );
+			"ask: %d, consumed: %d, now: %d\n", sid, length, consumed, sid->consumed );
+
+#ifdef DEBUG
+	dump_contents( prg, sid );
+#endif
 
 	return consumed;
 }
 
-static int data_undo_consume_data( struct colm_program *prg, struct stream_impl_data *si, const char *data, int length )
+static int data_undo_consume_data( struct colm_program *prg, struct stream_impl_data *sid, const char *data, int length )
 {
 	int amount = length;
-	if ( amount > si->consumed )
-		amount = si->consumed;
+	if ( amount > sid->consumed )
+		amount = sid->consumed;
 
 	if ( amount > 0 ) {
 		struct run_buf *new_buf = new_run_buf( 0 );
 		new_buf->length = amount;
 		memcpy( new_buf->data, data + ( length - amount ), amount );
-		si_data_push_head( si, new_buf );
-		undo_position_data( si, data, amount );
-		si->consumed -= amount;
+		si_data_push_head( sid, new_buf );
+		undo_position_data( sid, data, amount );
+		sid->consumed -= amount;
 	}
 
 	debug( prg, REALM_INPUT, "data_undo_consume_data: stream %p "
-			"undid consume %d of %d bytes, consumed now %d, \n", si, amount, length, si->consumed );
+			"undid consume %d of %d bytes, consumed now %d, \n", sid, amount, length, sid->consumed );
+
+#ifdef DEBUG
+	dump_contents( prg, sid );
+#endif
 
 	return amount;
 }