Update diff to use iterators

This is a major reorganization of the diff code.  This changes
the diff functions to use the iterators for traversing the
content.  This allowed a lot of code to be simplified.  Also,
this moved the functions relating to outputting a diff into a
new file (diff_output.c).

This includes a number of other changes - adding utility
functions, extending iterators, etc. plus more tests for the
diff code.  This also takes the example diff.c program much
further in terms of emulating git-diff command line options.

5 files changed, 266 insertions, 30 deletions

diff --git a/tests-clar/diff/diff_helpers.c b/tests-clar/diff/diff_helpers.c
index cd5a0f9af..67a4f1c51 100644
--- a/tests-clar/diff/diff_helpers.c
+++ b/tests-clar/diff/diff_helpers.c
@@ -29,12 +29,14 @@ int diff_file_fn(
 	diff_expects *e = cb_data;
 	(void)progress;
 	e->files++;
-	if (delta->old_attr == 0)
-		e->file_adds++;
-	else if (delta->new_attr == 0)
-		e->file_dels++;
-	else
-		e->file_mods++;
+	switch (delta->status) {
+	case GIT_STATUS_ADDED: e->file_adds++; break;
+	case GIT_STATUS_DELETED: e->file_dels++; break;
+	case GIT_STATUS_MODIFIED: e->file_mods++; break;
+	case GIT_STATUS_IGNORED: e->file_ignored++; break;
+	case GIT_STATUS_UNTRACKED: e->file_untracked++; break;
+	default: break;
+	}
 	return 0;
 }
 
diff --git a/tests-clar/diff/diff_helpers.h b/tests-clar/diff/diff_helpers.h
index 035c000f5..010d156fa 100644
--- a/tests-clar/diff/diff_helpers.h
+++ b/tests-clar/diff/diff_helpers.h
@@ -9,6 +9,8 @@ typedef struct {
 	int file_adds;
 	int file_dels;
 	int file_mods;
+	int file_ignored;
+	int file_untracked;
 
 	int hunks;
 	int hunk_new_lines;
diff --git a/tests-clar/diff/iterator.c b/tests-clar/diff/iterator.c
index 185a37a53..1ad126ca8 100644
--- a/tests-clar/diff/iterator.c
+++ b/tests-clar/diff/iterator.c
@@ -338,7 +338,7 @@ static void workdir_iterator_test(
 
 void test_diff_iterator__workdir_0(void)
 {
-	workdir_iterator_test("attr", 24, 2, NULL, "ign");
+	workdir_iterator_test("attr", 25, 2, NULL, "ign");
 }
 
 static const char *status_paths[] = {
@@ -351,10 +351,10 @@ static const char *status_paths[] = {
 	"staged_delete_modified_file",
 	"staged_new_file",
 	"staged_new_file_modified_file",
+	"subdir.txt",
 	"subdir/current_file",
 	"subdir/modified_file",
 	"subdir/new_file",
-	"subdir.txt",
 	NULL
 };
 
diff --git a/tests-clar/diff/tree.c b/tests-clar/diff/tree.c
index eb4092af8..f5fdfba16 100644
--- a/tests-clar/diff/tree.c
+++ b/tests-clar/diff/tree.c
@@ -100,7 +100,7 @@ void test_diff_tree__options(void)
 
 	git_diff_options opts = {0};
 	git_diff_list *diff = NULL;
-	diff_expects exp;
+	diff_expects actual;
 	int test_ab_or_cd[] = { 0, 0, 0, 0, 1, 1, 1, 1, 1 };
 	git_diff_options test_options[] = {
 		/* a vs b tests */
@@ -123,25 +123,26 @@ void test_diff_tree__options(void)
 	 */
 	diff_expects test_expects[] = {
 		/* a vs b tests */
-		{ 5, 3, 0, 2, 4, 0, 0, 51, 2, 46, 3 },
-		{ 5, 3, 0, 2, 4, 0, 0, 53, 4, 46, 3 },
-		{ 5, 0, 3, 2, 4, 0, 0, 52, 3, 3, 46 },
-		{ 5, 3, 0, 2, 5, 0, 0, 54, 3, 48, 3 },
+		{ 5, 3, 0, 2, 0, 0, 4, 0, 0, 51, 2, 46, 3 },
+		{ 5, 3, 0, 2, 0, 0, 4, 0, 0, 53, 4, 46, 3 },
+		{ 5, 0, 3, 2, 0, 0, 4, 0, 0, 52, 3, 3, 46 },
+		{ 5, 3, 0, 2, 0, 0, 5, 0, 0, 54, 3, 48, 3 },
 		/* c vs d tests */
-		{ 1, 0, 0, 1, 1, 0, 0, 22, 9, 10, 3 },
-		{ 1, 0, 0, 1, 1, 0, 0, 19, 12, 7, 0 },
-		{ 1, 0, 0, 1, 1, 0, 0, 20, 11, 8, 1 },
-		{ 1, 0, 0, 1, 1, 0, 0, 20, 11, 8, 1 },
-		{ 1, 0, 0, 1, 1, 0, 0, 18, 11, 0, 7 },
+		{ 1, 0, 0, 1, 0, 0, 1, 0, 0, 22, 9, 10, 3 },
+		{ 1, 0, 0, 1, 0, 0, 1, 0, 0, 19, 12, 7, 0 },
+		{ 1, 0, 0, 1, 0, 0, 1, 0, 0, 20, 11, 8, 1 },
+		{ 1, 0, 0, 1, 0, 0, 1, 0, 0, 20, 11, 8, 1 },
+		{ 1, 0, 0, 1, 0, 0, 1, 0, 0, 18, 11, 0, 7 },
 		{ 0 },
 	};
+	diff_expects *expected;
 	int i;
 
 	cl_assert(a);
 	cl_assert(b);
 
 	for (i = 0; test_expects[i].files > 0; i++) {
-		memset(&exp, 0, sizeof(exp)); /* clear accumulator */
+		memset(&actual, 0, sizeof(actual)); /* clear accumulator */
 		opts = test_options[i];
 
 		if (test_ab_or_cd[i] == 0)
@@ -150,17 +151,18 @@ void test_diff_tree__options(void)
 			cl_git_pass(git_diff_tree_to_tree(g_repo, &opts, c, d, &diff));
 
 		cl_git_pass(git_diff_foreach(
-			diff, &exp, diff_file_fn, diff_hunk_fn, diff_line_fn));
-
-		cl_assert(exp.files     == test_expects[i].files);
-		cl_assert(exp.file_adds == test_expects[i].file_adds);
-		cl_assert(exp.file_dels == test_expects[i].file_dels);
-		cl_assert(exp.file_mods == test_expects[i].file_mods);
-		cl_assert(exp.hunks     == test_expects[i].hunks);
-		cl_assert(exp.lines     == test_expects[i].lines);
-		cl_assert(exp.line_ctxt == test_expects[i].line_ctxt);
-		cl_assert(exp.line_adds == test_expects[i].line_adds);
-		cl_assert(exp.line_dels == test_expects[i].line_dels);
+			diff, &actual, diff_file_fn, diff_hunk_fn, diff_line_fn));
+
+		expected = &test_expects[i];
+		cl_assert_intequal(actual.files,     expected->files);
+		cl_assert_intequal(actual.file_adds, expected->file_adds);
+ 		cl_assert_intequal(actual.file_dels, expected->file_dels);
+		cl_assert_intequal(actual.file_mods, expected->file_mods);
+		cl_assert_intequal(actual.hunks,     expected->hunks);
+		cl_assert_intequal(actual.lines,     expected->lines);
+		cl_assert_intequal(actual.line_ctxt, expected->line_ctxt);
+		cl_assert_intequal(actual.line_adds, expected->line_adds);
+		cl_assert_intequal(actual.line_dels, expected->line_dels);
 
 		git_diff_list_free(diff);
 		diff = NULL;
diff --git a/tests-clar/diff/workdir.c b/tests-clar/diff/workdir.c
new file mode 100644
index 000000000..7312a72a7
--- /dev/null
+++ b/tests-clar/diff/workdir.c
@@ -0,0 +1,230 @@
+#include "clar_libgit2.h"
+#include "diff_helpers.h"
+
+static git_repository *g_repo = NULL;
+
+void test_diff_workdir__initialize(void)
+{
+	cl_fixture_sandbox("status");
+	cl_git_pass(p_rename("status/.gitted", "status/.git"));
+	cl_git_pass(git_repository_open(&g_repo, "status/.git"));
+}
+
+void test_diff_workdir__cleanup(void)
+{
+	git_repository_free(g_repo);
+	g_repo = NULL;
+	cl_fixture_cleanup("status");
+}
+
+void test_diff_workdir__to_index(void)
+{
+	git_diff_options opts = {0};
+	git_diff_list *diff = NULL;
+	diff_expects exp;
+
+	opts.context_lines = 3;
+	opts.interhunk_lines = 1;
+	opts.flags |= GIT_DIFF_INCLUDE_IGNORED | GIT_DIFF_INCLUDE_UNTRACKED;
+
+	memset(&exp, 0, sizeof(exp));
+
+	cl_git_pass(git_diff_workdir_to_index(g_repo, &opts, &diff));
+
+	cl_git_pass(git_diff_foreach(
+		diff, &exp, diff_file_fn, diff_hunk_fn, diff_line_fn));
+
+	/* to generate these values:
+	 * - cd to tests/resources/status,
+	 * - mv .gitted .git
+	 * - git diff --name-status
+	 * - git diff
+	 * - mv .git .gitted
+	 */
+	cl_assert_intequal(12, exp.files);
+	cl_assert_intequal(0, exp.file_adds);
+	cl_assert_intequal(4, exp.file_dels);
+	cl_assert_intequal(4, exp.file_mods);
+	cl_assert_intequal(1, exp.file_ignored);
+	cl_assert_intequal(3, exp.file_untracked);
+
+	cl_assert_intequal(8, exp.hunks);
+
+	cl_assert_intequal(14, exp.lines);
+	cl_assert_intequal(5, exp.line_ctxt);
+	cl_assert_intequal(4, exp.line_adds);
+	cl_assert_intequal(5, exp.line_dels);
+
+	git_diff_list_free(diff);
+}
+
+void test_diff_workdir__to_tree(void)
+{
+	/* grabbed a couple of commit oids from the history of the attr repo */
+	const char *a_commit = "26a125ee1bf"; /* the current HEAD */
+	const char *b_commit = "0017bd4ab1ec3"; /* the start */
+	git_tree *a = resolve_commit_oid_to_tree(g_repo, a_commit);
+	git_tree *b = resolve_commit_oid_to_tree(g_repo, b_commit);
+	git_diff_options opts = {0};
+	git_diff_list *diff = NULL;
+	git_diff_list *diff2 = NULL;
+	diff_expects exp;
+
+	opts.context_lines = 3;
+	opts.interhunk_lines = 1;
+	opts.flags |= GIT_DIFF_INCLUDE_IGNORED | GIT_DIFF_INCLUDE_UNTRACKED;
+
+	memset(&exp, 0, sizeof(exp));
+
+	/* You can't really generate the equivalent of git_diff_workdir_to_tree()
+	 * using C git.  It really wants to interpose the index into the diff.
+	 *
+	 * To validate the following results with command line git, I ran the
+	 * following:
+	 * - git ls-tree 26a125
+	 * - find . ! -path ./.git/\* -a -type f | git hash-object --stdin-paths
+	 * The results are documented at the bottom of this file in the
+	 * long comment entitled "PREPARATION OF TEST DATA".
+	 */
+	cl_git_pass(git_diff_workdir_to_tree(g_repo, &opts, a, &diff));
+
+	cl_git_pass(git_diff_foreach(
+		diff, &exp, diff_file_fn, diff_hunk_fn, diff_line_fn));
+
+	cl_assert(exp.files == 13);
+	cl_assert(exp.file_adds == 0);
+	cl_assert(exp.file_dels == 4);
+	cl_assert(exp.file_mods == 4);
+	cl_assert(exp.file_ignored == 1);
+	cl_assert(exp.file_untracked == 4);
+
+	/* Since there is no git diff equivalent, let's just assume that the
+	 * text diffs produced by git_diff_foreach are accurate here.  We will
+	 * do more apples-to-apples test comparison below.
+	 */
+
+	git_diff_list_free(diff);
+	diff = NULL;
+	memset(&exp, 0, sizeof(exp));
+
+	/* This is a compatible emulation of "git diff <sha>" which looks like
+	 * a workdir to tree diff (even though it is not really).  This is what
+	 * you would get from "git diff --name-status 26a125ee1bf"
+	 */
+	cl_git_pass(git_diff_index_to_tree(g_repo, &opts, a, &diff));
+	cl_git_pass(git_diff_workdir_to_index(g_repo, &opts, &diff2));
+	cl_git_pass(git_diff_merge(diff, diff2));
+	git_diff_list_free(diff2);
+
+	cl_git_pass(git_diff_foreach(
+		diff, &exp, diff_file_fn, diff_hunk_fn, diff_line_fn));
+
+	cl_assert(exp.files == 14);
+	cl_assert(exp.file_adds == 2);
+	cl_assert(exp.file_dels == 5);
+	cl_assert(exp.file_mods == 4);
+	cl_assert(exp.file_ignored == 1);
+	cl_assert(exp.file_untracked == 2);
+
+	cl_assert(exp.hunks == 11);
+
+	cl_assert(exp.lines == 17);
+	cl_assert(exp.line_ctxt == 4);
+	cl_assert(exp.line_adds == 8);
+	cl_assert(exp.line_dels == 5);
+
+	git_diff_list_free(diff);
+	diff = NULL;
+	memset(&exp, 0, sizeof(exp));
+
+	/* Again, emulating "git diff <sha>" for testing purposes using
+	 * "git diff --name-status 0017bd4ab1ec3" instead.
+	 */
+	cl_git_pass(git_diff_index_to_tree(g_repo, &opts, b, &diff));
+	cl_git_pass(git_diff_workdir_to_index(g_repo, &opts, &diff2));
+	cl_git_pass(git_diff_merge(diff, diff2));
+	git_diff_list_free(diff2);
+
+	cl_git_pass(git_diff_foreach(
+		diff, &exp, diff_file_fn, diff_hunk_fn, diff_line_fn));
+
+	cl_assert(exp.files == 15);
+	cl_assert(exp.file_adds == 5);
+	cl_assert(exp.file_dels == 4);
+	cl_assert(exp.file_mods == 3);
+	cl_assert(exp.file_ignored == 1);
+	cl_assert(exp.file_untracked == 2);
+
+	cl_assert(exp.hunks == 12);
+
+	cl_assert(exp.lines == 19);
+	cl_assert(exp.line_ctxt == 3);
+	cl_assert(exp.line_adds == 12);
+	cl_assert(exp.line_dels == 4);
+
+	git_tree_free(a);
+	git_tree_free(b);
+}
+
+/* PREPARATION OF TEST DATA
+ *
+ * Since there is no command line equivalent of git_diff_workdir_to_tree,
+ * it was a bit of a pain to confirm that I was getting the expected
+ * results in the first part of this tests.  Here is what I ended up
+ * doing to set my expectation for the file counts and results:
+ *
+ * Running "git ls-tree 26a125" and "git ls-tree aa27a6" shows:
+ *
+ *  A a0de7e0ac200c489c41c59dfa910154a70264e6e	current_file
+ *  B 5452d32f1dd538eb0405e8a83cc185f79e25e80f	file_deleted
+ *  C 452e4244b5d083ddf0460acf1ecc74db9dcfa11a	modified_file
+ *  D 32504b727382542f9f089e24fddac5e78533e96c	staged_changes
+ *  E 061d42a44cacde5726057b67558821d95db96f19	staged_changes_file_deleted
+ *  F 70bd9443ada07063e7fbf0b3ff5c13f7494d89c2	staged_changes_modified_file
+ *  G e9b9107f290627c04d097733a10055af941f6bca	staged_delete_file_deleted
+ *  H dabc8af9bd6e9f5bbe96a176f1a24baf3d1f8916	staged_delete_modified_file
+ *  I 53ace0d1cc1145a5f4fe4f78a186a60263190733	subdir/current_file
+ *  J 1888c805345ba265b0ee9449b8877b6064592058	subdir/deleted_file
+ *  K a6191982709b746d5650e93c2acf34ef74e11504	subdir/modified_file
+ *  L e8ee89e15bbe9b20137715232387b3de5b28972e	subdir.txt
+ *
+ * --------
+ *
+ * find . ! -path ./.git/\* -a -type f | git hash-object --stdin-paths
+ *
+ *  A a0de7e0ac200c489c41c59dfa910154a70264e6e current_file
+ *  M 6a79f808a9c6bc9531ac726c184bbcd9351ccf11 ignored_file
+ *  C 0a539630525aca2e7bc84975958f92f10a64c9b6 modified_file
+ *  N d4fa8600b4f37d7516bef4816ae2c64dbf029e3a new_file
+ *  D 55d316c9ba708999f1918e9677d01dfcae69c6b9 staged_changes
+ *  F 011c3440d5c596e21d836aa6d7b10eb581f68c49 staged_changes_modified_file
+ *  H dabc8af9bd6e9f5bbe96a176f1a24baf3d1f8916 staged_delete_modified_file
+ *  O 529a16e8e762d4acb7b9636ff540a00831f9155a staged_new_file
+ *  P 8b090c06d14ffa09c4e880088ebad33893f921d1 staged_new_file_modified_file
+ *  I 53ace0d1cc1145a5f4fe4f78a186a60263190733 subdir/current_file
+ *  K 57274b75eeb5f36fd55527806d567b2240a20c57 subdir/modified_file
+ *  Q 80a86a6931b91bc01c2dbf5ca55bdd24ad1ef466 subdir/new_file
+ *  L e8ee89e15bbe9b20137715232387b3de5b28972e subdir.txt
+ *
+ * --------
+ *
+ *  A - current_file (UNMODIFIED) -> not in results
+ *  B D file_deleted
+ *  M I ignored_file (IGNORED)
+ *  C M modified_file
+ *  N U new_file (UNTRACKED)
+ *  D M staged_changes
+ *  E D staged_changes_file_deleted
+ *  F M staged_changes_modified_file
+ *  G D staged_delete_file_deleted
+ *  H - staged_delete_modified_file (UNMODIFIED) -> not in results
+ *  O U staged_new_file
+ *  P U staged_new_file_modified_file
+ *  I - subdir/current_file (UNMODIFIED) -> not in results
+ *  J D subdir/deleted_file
+ *  K M subdir/modified_file
+ *  Q U subdir/new_file
+ *  L - subdir.txt (UNMODIFIED) -> not in results
+ *
+ * Expect 13 files, 0 ADD, 4 DEL, 4 MOD, 1 IGN, 4 UNTR
+ */