Update diff to use iterators

This is a major reorganization of the diff code.  This changes
the diff functions to use the iterators for traversing the
content.  This allowed a lot of code to be simplified.  Also,
this moved the functions relating to outputting a diff into a
new file (diff_output.c).

This includes a number of other changes - adding utility
functions, extending iterators, etc. plus more tests for the
diff code.  This also takes the example diff.c program much
further in terms of emulating git-diff command line options.

18 files changed, 1311 insertions, 1019 deletions

diff --git a/src/diff.c b/src/diff.c
index cfa34c138..9e4105571 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -1,194 +1,203 @@
 /*
- * Copyright (C) 2009-2011 the libgit2 contributors
+ * Copyright (C) 2012 the libgit2 contributors
  *
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
  */
-
 #include "common.h"
 #include "git2/diff.h"
 #include "diff.h"
-#include "xdiff/xdiff.h"
-#include "blob.h"
-#include "ignore.h"
-#include <ctype.h>
+#include "fileops.h"
 
-static void file_delta_free(git_diff_delta *delta)
+static void diff_delta__free(git_diff_delta *delta)
 {
 	if (!delta)
 		return;
 
-	if (delta->new_path != delta->path) {
-		git__free((char *)delta->new_path);
-		delta->new_path = NULL;
+	if (delta->new.flags & GIT_DIFF_FILE_FREE_PATH) {
+		git__free((char *)delta->new.path);
+		delta->new.path = NULL;
 	}
 
-	git__free((char *)delta->path);
-	delta->path = NULL;
+	if (delta->old.flags & GIT_DIFF_FILE_FREE_PATH) {
+		git__free((char *)delta->old.path);
+		delta->old.path = NULL;
+	}
 
 	git__free(delta);
 }
 
-static int file_delta_new__from_one(
+static git_diff_delta *diff_delta__alloc(
 	git_diff_list *diff,
-	git_status_t   status,
-	mode_t         attr,
-	const git_oid *oid,
-	const char    *path)
+	git_status_t status,
+	const char *path)
 {
-	int error;
 	git_diff_delta *delta = git__calloc(1, sizeof(git_diff_delta));
-
-	/* This fn is just for single-sided diffs */
-	assert(status == GIT_STATUS_ADDED || status == GIT_STATUS_DELETED);
-
 	if (!delta)
-		return git__rethrow(GIT_ENOMEM, "Could not allocate diff record");
+		return NULL;
 
-	if ((delta->path = git__strdup(path)) == NULL) {
+	delta->old.path = git__strdup(path);
+	if (delta->old.path == NULL) {
 		git__free(delta);
-		return git__rethrow(GIT_ENOMEM, "Could not allocate diff record path");
+		return NULL;
 	}
+	delta->old.flags |= GIT_DIFF_FILE_FREE_PATH;
+	delta->new.path = delta->old.path;
 
-	if (diff->opts.flags & GIT_DIFF_REVERSE)
-		status = (status == GIT_STATUS_ADDED) ?
-			GIT_STATUS_DELETED : GIT_STATUS_ADDED;
-
+	if (diff->opts.flags & GIT_DIFF_REVERSE) {
+		switch (status) {
+		case GIT_STATUS_ADDED:   status = GIT_STATUS_DELETED; break;
+		case GIT_STATUS_DELETED: status = GIT_STATUS_ADDED; break;
+		default: break; /* leave other status values alone */
+		}
+	}
 	delta->status = status;
 
-	if (status == GIT_STATUS_ADDED) {
-		delta->new_attr = attr;
-		if (oid != NULL)
-			git_oid_cpy(&delta->new_oid, oid);
-	} else {
-		delta->old_attr = attr;
-		if (oid != NULL)
-			git_oid_cpy(&delta->old_oid, oid);
+	return delta;
+}
+
+static git_diff_delta *diff_delta__dup(const git_diff_delta *d)
+{
+	git_diff_delta *delta = git__malloc(sizeof(git_diff_delta));
+	if (!delta)
+		return NULL;
+
+	memcpy(delta, d, sizeof(git_diff_delta));
+
+	delta->old.path = git__strdup(d->old.path);
+	if (delta->old.path == NULL) {
+		git__free(delta);
+		return NULL;
 	}
+	delta->old.flags |= GIT_DIFF_FILE_FREE_PATH;
 
-	if ((error = git_vector_insert(&diff->files, delta)) < GIT_SUCCESS)
-		file_delta_free(delta);
+	if (d->new.path != d->old.path) {
+		delta->new.path = git__strdup(d->new.path);
+		if (delta->new.path == NULL) {
+			git__free(delta->old.path);
+			git__free(delta);
+			return NULL;
+		}
+		delta->new.flags |= GIT_DIFF_FILE_FREE_PATH;
+	} else {
+		delta->new.path = delta->old.path;
+		delta->new.flags &= ~GIT_DIFF_FILE_FREE_PATH;
+	}
 
-	return error;
+	return delta;
 }
 
-static int file_delta_new__from_tree_diff(
-	git_diff_list *diff,
-	const git_tree_diff_data *tdiff)
+static git_diff_delta *diff_delta__merge_like_cgit(
+	const git_diff_delta *a, const git_diff_delta *b)
 {
-	int error;
-	git_diff_delta *delta = git__calloc(1, sizeof(git_diff_delta));
+	git_diff_delta *dup = diff_delta__dup(a);
+	if (!dup)
+		return NULL;
 
-	if (!delta)
-		return git__rethrow(GIT_ENOMEM, "Could not allocate diff record");
+	if (git_oid_cmp(&dup->new.oid, &b->new.oid) == 0)
+		return dup;
 
-	if ((diff->opts.flags & GIT_DIFF_REVERSE) == 0) {
-		delta->status   = tdiff->status;
-		delta->old_attr = tdiff->old_attr;
-		delta->new_attr = tdiff->new_attr;
-		delta->old_oid  = tdiff->old_oid;
-		delta->new_oid  = tdiff->new_oid;
-	} else {
-		/* reverse the polarity of the neutron flow */
-		switch (tdiff->status) {
-		case GIT_STATUS_ADDED:   delta->status = GIT_STATUS_DELETED; break;
-		case GIT_STATUS_DELETED: delta->status = GIT_STATUS_ADDED; break;
-		default:                 delta->status = tdiff->status;
-		}
-		delta->old_attr = tdiff->new_attr;
-		delta->new_attr = tdiff->old_attr;
-		delta->old_oid  = tdiff->new_oid;
-		delta->new_oid  = tdiff->old_oid;
-	}
+	git_oid_cpy(&dup->new.oid, &b->new.oid);
 
-	delta->path = git__strdup(diff->pfx.ptr);
-	if (delta->path == NULL) {
-		git__free(delta);
-		return git__rethrow(GIT_ENOMEM, "Could not allocate diff record path");
-	}
+	dup->new.mode = b->new.mode;
+	dup->new.size = b->new.size;
+	dup->new.flags =
+		(dup->new.flags & GIT_DIFF_FILE_FREE_PATH) |
+		(b->new.flags & ~GIT_DIFF_FILE_FREE_PATH);
 
-	if ((error = git_vector_insert(&diff->files, delta)) < GIT_SUCCESS)
-		file_delta_free(delta);
+	/* Emulate C git for merging two diffs (a la 'git diff <sha>').
+	 *
+	 * When C git does a diff between the work dir and a tree, it actually
+	 * diffs with the index but uses the workdir contents.  This emulates
+	 * those choices so we can emulate the type of diff.
+	 */
+	if (git_oid_cmp(&dup->old.oid, &dup->new.oid) == 0) {
+		if (dup->status == GIT_STATUS_DELETED)
+			/* preserve pending delete info */;
+		else if (b->status == GIT_STATUS_UNTRACKED ||
+				 b->status == GIT_STATUS_IGNORED)
+			dup->status = b->status;
+		else
+			dup->status = GIT_STATUS_UNMODIFIED;
+	}
+	else if (dup->status == GIT_STATUS_UNMODIFIED ||
+			 b->status == GIT_STATUS_DELETED)
+		dup->status = b->status;
 
-	return error;
+	return dup;
 }
 
-static int create_diff_for_tree_entry(const char *root, git_tree_entry *entry, void *data)
+static int diff_delta__from_one(
+	git_diff_list *diff,
+	git_status_t   status,
+	const git_index_entry *entry)
 {
 	int error;
-	git_diff_list *diff = data;
-	ssize_t pfx_len = diff->pfx.size;
+	git_diff_delta *delta = diff_delta__alloc(diff, status, entry->path);
+	if (!delta)
+		return git__rethrow(GIT_ENOMEM, "Could not allocate diff record");
 
-	if (S_ISDIR(git_tree_entry_attributes(entry)))
-		return GIT_SUCCESS;
+	/* This fn is just for single-sided diffs */
+	assert(status != GIT_STATUS_MODIFIED);
 
-	/* join pfx, root, and entry->filename into one */
-	if ((error = git_buf_joinpath(&diff->pfx, diff->pfx.ptr, root)) ||
-		(error = git_buf_joinpath(
-			&diff->pfx, diff->pfx.ptr, git_tree_entry_name(entry))))
-		return error;
+	if (delta->status == GIT_STATUS_DELETED) {
+		delta->old.mode = entry->mode;
+		delta->old.size = entry->file_size;
+		git_oid_cpy(&delta->old.oid, &entry->oid);
+	} else /* ADDED, IGNORED, UNTRACKED */ {
+		delta->new.mode = entry->mode;
+		delta->new.size = entry->file_size;
+		git_oid_cpy(&delta->new.oid, &entry->oid);
+	}
 
-	error = file_delta_new__from_one(
-		diff, diff->status, git_tree_entry_attributes(entry),
-		git_tree_entry_id(entry), diff->pfx.ptr);
+	delta->old.flags |= GIT_DIFF_FILE_VALID_OID;
+	delta->new.flags |= GIT_DIFF_FILE_VALID_OID;
 
-	git_buf_truncate(&diff->pfx, pfx_len);
+	if ((error = git_vector_insert(&diff->deltas, delta)) < GIT_SUCCESS)
+		diff_delta__free(delta);
 
 	return error;
 }
 
-static int tree_to_tree_diff_cb(const git_tree_diff_data *tdiff, void *data)
+static int diff_delta__from_two(
+	git_diff_list *diff,
+	git_status_t   status,
+	const git_index_entry *old,
+	const git_index_entry *new,
+	git_oid *new_oid)
 {
 	int error;
-	git_diff_list *diff = data;
-	ssize_t pfx_len = diff->pfx.size;
+	git_diff_delta *delta;
 
-	error = git_buf_joinpath(&diff->pfx, diff->pfx.ptr, tdiff->path);
-	if (error < GIT_SUCCESS)
-		return error;
+	if ((diff->opts.flags & GIT_DIFF_REVERSE) != 0) {
+		const git_index_entry *temp = old;
+		old = new;
+		new = temp;
+	}
 
-	/* there are 4 tree related cases:
-	 * - diff tree to tree, which just means we recurse
-	 * - tree was deleted
-	 * - tree was added
-	 * - tree became non-tree or vice versa, which git_tree_diff
-	 *   will already have converted into two calls: an addition
-	 *   and a deletion (thank you, git_tree_diff!)
-	 * otherwise, this is a blob-to-blob diff
-	 */
-	if (S_ISDIR(tdiff->old_attr) && S_ISDIR(tdiff->new_attr)) {
-		git_tree *old = NULL, *new = NULL;
-
-		if (!(error = git_tree_lookup(&old, diff->repo, &tdiff->old_oid)) &&
-			!(error = git_tree_lookup(&new, diff->repo, &tdiff->new_oid)))
-			error = git_tree_diff(old, new, tree_to_tree_diff_cb, diff);
-
-		git_tree_free(old);
-		git_tree_free(new);
-	} else if (S_ISDIR(tdiff->old_attr) || S_ISDIR(tdiff->new_attr)) {
-		git_tree *tree     = NULL;
-		int added_dir      = S_ISDIR(tdiff->new_attr);
-		const git_oid *oid = added_dir ? &tdiff->new_oid : &tdiff->old_oid;
-		diff->status       = added_dir ? GIT_STATUS_ADDED : GIT_STATUS_DELETED;
-
-		if (!(error = git_tree_lookup(&tree, diff->repo, oid)))
-			error = git_tree_walk(
-				tree, create_diff_for_tree_entry, GIT_TREEWALK_POST, diff);
-		git_tree_free(tree);
-	} else
-		error = file_delta_new__from_tree_diff(diff, tdiff);
-
-	git_buf_truncate(&diff->pfx, pfx_len);
+	delta = diff_delta__alloc(diff, status, old->path);
+	if (!delta)
+		return git__rethrow(GIT_ENOMEM, "Could not allocate diff record");
+
+	delta->old.mode = old->mode;
+	git_oid_cpy(&delta->old.oid, &old->oid);
+	delta->old.flags |= GIT_DIFF_FILE_VALID_OID;
+
+	delta->new.mode = new->mode;
+	git_oid_cpy(&delta->new.oid, new_oid ? new_oid : &new->oid);
+	if (new_oid || !git_oid_iszero(&new->oid))
+		delta->new.flags |= GIT_DIFF_FILE_VALID_OID;
+
+	if ((error = git_vector_insert(&diff->deltas, delta)) < GIT_SUCCESS)
+		diff_delta__free(delta);
 
 	return error;
 }
 
-static char *git_diff_src_prefix_default = "a/";
-static char *git_diff_dst_prefix_default = "b/";
-#define PREFIX_IS_DEFAULT(A) \
-	((A) == git_diff_src_prefix_default || (A) == git_diff_dst_prefix_default)
+#define DIFF_SRC_PREFIX_DEFAULT "a/"
+#define DIFF_DST_PREFIX_DEFAULT "b/"
 
-static char *copy_prefix(const char *prefix)
+static char *diff_strdup_prefix(const char *prefix)
 {
 	size_t len = strlen(prefix);
 	char *str = git__malloc(len + 2);
@@ -203,6 +212,13 @@ static char *copy_prefix(const char *prefix)
 	return str;
 }
 
+static int diff_delta__cmp(const void *a, const void *b)
+{
+	const git_diff_delta *da = a, *db = b;
+	int val = strcmp(da->old.path, db->old.path);
+	return val ? val : ((int)da->status - (int)db->status);
+}
+
 static git_diff_list *git_diff_list_alloc(
 	git_repository *repo, const git_diff_options *opts)
 {
@@ -211,27 +227,35 @@ static git_diff_list *git_diff_list_alloc(
 		return NULL;
 
 	diff->repo = repo;
-	git_buf_init(&diff->pfx, 0);
 
 	if (opts == NULL)
 		return diff;
 
 	memcpy(&diff->opts, opts, sizeof(git_diff_options));
 
-	diff->opts.src_prefix = (opts->src_prefix == NULL) ?
-		git_diff_src_prefix_default : copy_prefix(opts->src_prefix);
-	diff->opts.dst_prefix = (opts->dst_prefix == NULL) ?
-		git_diff_dst_prefix_default : copy_prefix(opts->dst_prefix);
+	diff->opts.src_prefix = diff_strdup_prefix(
+		opts->src_prefix ? opts->src_prefix : DIFF_SRC_PREFIX_DEFAULT);
+	diff->opts.dst_prefix = diff_strdup_prefix(
+		opts->dst_prefix ? opts->dst_prefix : DIFF_DST_PREFIX_DEFAULT);
+
 	if (!diff->opts.src_prefix || !diff->opts.dst_prefix) {
 		git__free(diff);
 		return NULL;
 	}
+
 	if (diff->opts.flags & GIT_DIFF_REVERSE) {
 		char *swap = diff->opts.src_prefix;
 		diff->opts.src_prefix = diff->opts.dst_prefix;
 		diff->opts.dst_prefix = swap;
 	}
 
+	if (git_vector_init(&diff->deltas, 0, diff_delta__cmp) < GIT_SUCCESS) {
+		git__free(diff->opts.src_prefix);
+		git__free(diff->opts.dst_prefix);
+		git__free(diff);
+		return NULL;
+	}
+
 	/* do something safe with the pathspec strarray */
 
 	return diff;
@@ -245,930 +269,329 @@ void git_diff_list_free(git_diff_list *diff)
 	if (!diff)
 		return;
 
-	git_buf_free(&diff->pfx);
-	git_vector_foreach(&diff->files, i, delta) {
-		file_delta_free(delta);
-		diff->files.contents[i] = NULL;
-	}
-	git_vector_free(&diff->files);
-	if (!PREFIX_IS_DEFAULT(diff->opts.src_prefix)) {
-		git__free(diff->opts.src_prefix);
-		diff->opts.src_prefix = NULL;
-	}
-	if (!PREFIX_IS_DEFAULT(diff->opts.dst_prefix)) {
-		git__free(diff->opts.dst_prefix);
-		diff->opts.dst_prefix = NULL;
+	git_vector_foreach(&diff->deltas, i, delta) {
+		diff_delta__free(delta);
+		diff->deltas.contents[i] = NULL;
 	}
+	git_vector_free(&diff->deltas);
+	git__free(diff->opts.src_prefix);
+	git__free(diff->opts.dst_prefix);
 	git__free(diff);
 }
 
-int git_diff_tree_to_tree(
+static int oid_for_workdir_item(
 	git_repository *repo,
-	const git_diff_options *opts,
-	git_tree *old,
-	git_tree *new,
-	git_diff_list **diff_ptr)
+	const git_index_entry *item,
+	git_oid *oid)
 {
-	int error;
-	git_diff_list *diff = git_diff_list_alloc(repo, opts);
-	if (!diff)
-		return GIT_ENOMEM;
-
-	error = git_tree_diff(old, new, tree_to_tree_diff_cb, diff);
-	if (error == GIT_SUCCESS) {
-		git_buf_free(&diff->pfx);	/* don't need this anymore */
-		*diff_ptr = diff;
-	} else
-		git_diff_list_free(diff);
-
-	return error;
-}
-
-typedef struct {
-	git_diff_list *diff;
-	git_index     *index;
-	unsigned int  index_pos;
-	git_ignores   *ignores;
-} diff_callback_info;
-
-static int add_new_index_deltas(
-	diff_callback_info *info,
-	git_status_t status,
-	const char *stop_path)
-{
-	int error;
-	git_index_entry *idx_entry = git_index_get(info->index, info->index_pos);
-
-	while (idx_entry != NULL &&
-		(stop_path == NULL || strcmp(idx_entry->path, stop_path) < 0))
-	{
-		error = file_delta_new__from_one(
-			info->diff, status, idx_entry->mode,
-			&idx_entry->oid, idx_entry->path);
-		if (error < GIT_SUCCESS)
-			return error;
-
-		idx_entry = git_index_get(info->index, ++info->index_pos);
-	}
-
-	return GIT_SUCCESS;
-}
-
-static int diff_index_to_tree_cb(const char *root, git_tree_entry *tree_entry, void *data)
-{
-	int error;
-	diff_callback_info *info = data;
-	git_index_entry *idx_entry;
-
-	/* TODO: submodule support for GIT_OBJ_COMMITs in tree */
-	if (git_tree_entry_type(tree_entry) != GIT_OBJ_BLOB)
-		return GIT_SUCCESS;
-
-	error = git_buf_joinpath(&info->diff->pfx, root, git_tree_entry_name(tree_entry));
-	if (error < GIT_SUCCESS)
-		return error;
+	int error = GIT_SUCCESS;
+	git_buf full_path = GIT_BUF_INIT;
 
-	/* create add deltas for index entries that are not in the tree */
-	error = add_new_index_deltas(info, GIT_STATUS_ADDED, info->diff->pfx.ptr);
-	if (error < GIT_SUCCESS)
+	error = git_buf_joinpath(
+		&full_path, git_repository_workdir(repo), item->path);
+	if (error != GIT_SUCCESS)
 		return error;
 
-	/* create delete delta for tree entries that are not in the index */
-	idx_entry = git_index_get(info->index, info->index_pos);
-	if (idx_entry == NULL || strcmp(idx_entry->path, info->diff->pfx.ptr) > 0) {
-		return file_delta_new__from_one(
-			info->diff, GIT_STATUS_DELETED, git_tree_entry_attributes(tree_entry),
-			git_tree_entry_id(tree_entry), info->diff->pfx.ptr);
-	}
-
-	/* create modified delta for non-matching tree & index entries */
-	info->index_pos++;
-
-	if (git_oid_cmp(&idx_entry->oid, git_tree_entry_id(tree_entry)) ||
-		idx_entry->mode != git_tree_entry_attributes(tree_entry))
-	{
-		git_tree_diff_data tdiff;
-		tdiff.old_attr = git_tree_entry_attributes(tree_entry);
-		tdiff.new_attr = idx_entry->mode;
-		tdiff.status   = GIT_STATUS_MODIFIED;
-		tdiff.path     = idx_entry->path;
-		git_oid_cpy(&tdiff.old_oid, git_tree_entry_id(tree_entry));
-		git_oid_cpy(&tdiff.new_oid, &idx_entry->oid);
-
-		error = file_delta_new__from_tree_diff(info->diff, &tdiff);
-	}
-
-	return error;
-
-}
-
-int git_diff_index_to_tree(
-	git_repository *repo,
-	const git_diff_options *opts,
-	git_tree *old,
-	git_diff_list **diff_ptr)
-{
-	int error;
-	diff_callback_info info = {0};
-
-	if ((info.diff = git_diff_list_alloc(repo, opts)) == NULL)
-		return GIT_ENOMEM;
-
-	if ((error = git_repository_index(&info.index, repo)) == GIT_SUCCESS) {
-		error = git_tree_walk(
-			old, diff_index_to_tree_cb, GIT_TREEWALK_POST, &info);
-		if (error == GIT_SUCCESS)
-			error = add_new_index_deltas(&info, GIT_STATUS_ADDED, NULL);
-		git_index_free(info.index);
+	/* otherwise calculate OID for file */
+	if (S_ISLNK(item->mode))
+		error = git_odb__hashlink(oid, full_path.ptr);
+	else if (!git__is_sizet(item->file_size))
+		error = git__throw(GIT_ERROR, "File size overflow for 32-bit systems");
+	else {
+		int fd;
+
+		if ((fd = p_open(full_path.ptr, O_RDONLY)) < 0)
+			error = git__throw(
+				GIT_EOSERR, "Could not open '%s'", item->path);
+		else {
+			error = git_odb__hashfd(
+				oid, fd, (size_t)item->file_size, GIT_OBJ_BLOB);
+			p_close(fd);
+		}
 	}
-	git_buf_free(&info.diff->pfx);
 
-	if (error != GIT_SUCCESS)
-		git_diff_list_free(info.diff);
-	else
-		*diff_ptr = info.diff;
+	git_buf_free(&full_path);
 
 	return error;
 }
 
-typedef struct {
-	struct stat st;
-	mode_t mode;
-	char path[GIT_FLEX_ARRAY];
-} workdir_entry;
-
-#define MODE_PERMS_MASK 0777
-
-/* TODO: need equiv of core git's "trust_executable_bit" flag? */
-#define CANONICAL_PERMS(MODE) (((MODE) & 0100) ? 0755 : 0644)
-#define MODE_TYPE(MODE)  ((MODE) & ~MODE_PERMS_MASK)
-
-static mode_t canonical_mode(mode_t raw_mode)
-{
-	if (S_ISREG(raw_mode))
-		return S_IFREG | CANONICAL_PERMS(raw_mode);
-	else if (S_ISLNK(raw_mode))
-		return S_IFLNK;
-	else if (S_ISDIR(raw_mode))
-		return S_IFDIR;
-	else if (S_ISGITLINK(raw_mode))
-		return S_IFGITLINK;
-	else
-		return 0;
-}
-
-static int diff_workdir_insert(void *data, git_buf *dir)
-{
-	workdir_entry *wd_entry = git__malloc(sizeof(workdir_entry) + dir->size + 2);
-	if (wd_entry == NULL)
-		return GIT_ENOMEM;
-	if (p_lstat(dir->ptr, &wd_entry->st) < 0) {
-		git__free(wd_entry);
-		return GIT_EOSERR;
-	}
-	git_buf_copy_cstr(wd_entry->path, dir->size + 1, dir);
-	wd_entry->mode = canonical_mode(wd_entry->st.st_mode);
-	/* suffix directories with / to mimic tree/index sort order */
-	if (S_ISDIR(wd_entry->st.st_mode)) {
-		wd_entry->path[dir->size] = '/';
-		wd_entry->path[dir->size+1] = '\0';
-	}
-
-	return git_vector_insert((git_vector *)data, wd_entry);
-}
-
-static int diff_workdir_walk(
-	const char *dir,
-	diff_callback_info *info,
-	int (*cb)(diff_callback_info *, workdir_entry *))
+static int maybe_modified(
+	git_iterator *old,
+	const git_index_entry *oitem,
+	git_iterator *new,
+	const git_index_entry *nitem,
+	git_diff_list *diff)
 {
 	int error = GIT_SUCCESS;
-	git_vector files = GIT_VECTOR_INIT;
-	git_buf buf = GIT_BUF_INIT;
-	unsigned int i;
-	workdir_entry *wd_entry;
-	git_ignores ignores = {0}, *old_ignores = info->ignores;
-
-	if (!dir)
-		dir = git_repository_workdir(info->diff->repo);
-
-	if ((error = git_vector_init(&files, 0, git__strcmp_cb)) < GIT_SUCCESS ||
-		(error = git_buf_sets(&buf, dir)) < GIT_SUCCESS ||
-		(error = git_path_direach(&buf, diff_workdir_insert, &files)) < GIT_SUCCESS ||
-		(error = git_ignore__for_path(info->diff->repo, dir, &ignores)) < GIT_SUCCESS)
-		goto cleanup;
-
-	git_vector_sort(&files);
-	info->ignores = old_ignores;
+	git_oid noid, *use_noid = NULL;
 
-	git_vector_foreach(&files, i, wd_entry) {
-		if ((error = cb(info, wd_entry)) < GIT_SUCCESS)
-			goto cleanup;
-	}
-
-cleanup:
-	git_vector_foreach(&files, i, wd_entry)
-		git__free(wd_entry);
-	info->ignores = old_ignores;
-	git_ignore__free(&ignores);
-	git_vector_free(&files);
-	git_buf_free(&buf);
-
-	return error;
-}
-
-static int found_new_workdir_entry(
-	diff_callback_info *info, workdir_entry *wd_entry)
-{
-	int error;
-	int ignored = 0;
-	git_status_t status;
+	GIT_UNUSED_ARG(old);
 
-	/* skip file types that are not trackable */
-	if (wd_entry->mode == 0)
+	/* support "assume unchanged" & "skip worktree" bits */
+	if ((oitem->flags_extended & GIT_IDXENTRY_INTENT_TO_ADD) != 0 ||
+		(oitem->flags_extended & GIT_IDXENTRY_SKIP_WORKTREE) != 0)
 		return GIT_SUCCESS;
 
-	error = git_ignore__lookup(info->ignores, wd_entry->path, &ignored);
-	if (error < GIT_SUCCESS)
-		return error;
-	status = ignored ? GIT_STATUS_IGNORED : GIT_STATUS_UNTRACKED;
-
-	return file_delta_new__from_one(
-		info->diff, status, wd_entry->mode, NULL, wd_entry->path);
-}
-
-static int diff_workdir_to_index_cb(
-	diff_callback_info *info, workdir_entry *wd_entry)
-{
-	int error, modified;
-	git_index_entry *idx_entry;
-	git_oid new_oid;
-
-	/* Store index entries that precede this workdir entry */
-	error = add_new_index_deltas(info, GIT_STATUS_DELETED, wd_entry->path);
-	if (error < GIT_SUCCESS)
+	if (GIT_MODE_TYPE(oitem->mode) != GIT_MODE_TYPE(nitem->mode)) {
+		error = diff_delta__from_one(diff, GIT_STATUS_DELETED, oitem);
+		if (error == GIT_SUCCESS)
+			error = diff_delta__from_one(diff, GIT_STATUS_ADDED, nitem);
 		return error;
-
-	/* Process workdir entries that are not in the index.
-	 * These might be untracked, ignored, or special (dirs, etc).
-	 */
-	idx_entry = git_index_get(info->index, info->index_pos);
-	if (idx_entry == NULL || strcmp(idx_entry->path, wd_entry->path) > 0) {
-		git_buf dotgit = GIT_BUF_INIT;
-		int contains_dotgit;
-
-		if (!S_ISDIR(wd_entry->mode))
-			return found_new_workdir_entry(info, wd_entry);
-
-		error = git_buf_joinpath(&dotgit, wd_entry->path, DOT_GIT);
-		if (error < GIT_SUCCESS)
-			return error;
-		contains_dotgit = (git_path_exists(dotgit.ptr) == GIT_SUCCESS);
-		git_buf_free(&dotgit);
-
-		if (contains_dotgit)
-			/* TODO: deal with submodule or embedded repo */
-			return GIT_SUCCESS;
-		else if (git__prefixcmp(idx_entry->path, wd_entry->path) == GIT_SUCCESS)
-			/* there are entries in the directory in the index already,
-			 * so recurse into it.
-			 */
-			return diff_workdir_walk(wd_entry->path, info, diff_workdir_to_index_cb);
-		else
-			/* TODO: this is not the same behavior as core git.
-			 *
-			 * I don't recurse into the directory once I know that no files
-			 * in it are being tracked.  But core git does and only adds an
-			 * entry if there are non-directory entries contained under the
-			 * dir (although, interestingly, it only shows the dir, not the
-			 * individual entries).
-			 */
-			return found_new_workdir_entry(info, wd_entry);
-	}
-
-	/* create modified delta for non-matching tree & index entries */
-	info->index_pos++;
-
-	/* check for symlink/blob changes and split into add/del pair */
-	if (MODE_TYPE(wd_entry->mode) != MODE_TYPE(idx_entry->mode)) {
-		error = file_delta_new__from_one(
-			info->diff, GIT_STATUS_DELETED,
-			idx_entry->mode, &idx_entry->oid, idx_entry->path);
-		if (error < GIT_SUCCESS)
-			return error;
-
-		/* because of trailing slash, cannot have non-dir to dir transform */
-		assert(!S_ISDIR(wd_entry->mode));
-
-		return file_delta_new__from_one(
-			info->diff, GIT_STATUS_ADDED,
-			wd_entry->mode, NULL, wd_entry->path);
 	}
 
-	/* mode or size changed, so git blob has definitely changed */
-	if (wd_entry->mode != idx_entry->mode ||
-		wd_entry->st.st_size != idx_entry->file_size)
-	{
-		modified = 1;
-		memset(&new_oid, 0, sizeof(new_oid));
-	}
+	if (git_oid_cmp(&oitem->oid, &nitem->oid) == 0 &&
+		oitem->mode == nitem->mode)
+		return GIT_SUCCESS;
 
-	/* all other things are indicators there might be a change, so get oid */
-	if (!modified &&
-		((git_time_t)wd_entry->st.st_ctime != idx_entry->ctime.seconds ||
-		 (git_time_t)wd_entry->st.st_mtime != idx_entry->mtime.seconds ||
-		 (unsigned int)wd_entry->st.st_dev != idx_entry->dev ||
-		 (unsigned int)wd_entry->st.st_ino != idx_entry->ino ||
-		 /* TODO: need TRUST_UID_GID configs */
-		 (unsigned int)wd_entry->st.st_uid != idx_entry->uid ||
-		 (unsigned int)wd_entry->st.st_gid != idx_entry->gid))
-	{
-		/* calculate oid to confirm change */
-		if (S_ISLNK(wd_entry->st.st_mode))
-			error = git_odb__hashlink(&new_oid, wd_entry->path);
-		else {
-			int fd;
-			if ((fd = p_open(wd_entry->path, O_RDONLY)) < 0)
-				error = git__throw(
-					GIT_EOSERR, "Could not open '%s'", wd_entry->path);
-			else {
-				error = git_odb__hashfd(
-					&new_oid, fd, wd_entry->st.st_size, GIT_OBJ_BLOB);
-				p_close(fd);
-			}
-		}
+	if (git_oid_iszero(&nitem->oid) && new->type == GIT_ITERATOR_WORKDIR) {
+		/* if they files look exactly alike, then we'll assume the same */
+		if (oitem->file_size == nitem->file_size &&
+			oitem->ctime.seconds == nitem->ctime.seconds &&
+			oitem->mtime.seconds == nitem->mtime.seconds &&
+			oitem->dev == nitem->dev &&
+			oitem->ino == nitem->ino &&
+			oitem->uid == nitem->uid &&
+			oitem->gid == nitem->gid)
+			return GIT_SUCCESS;
 
-		if (error < GIT_SUCCESS)
+		/* TODO: check git attributes so we will not have to read the file
+		 * in if it is marked binary.
+		 */
+		error = oid_for_workdir_item(diff->repo, nitem, &noid);
+		if (error != GIT_SUCCESS)
 			return error;
 
-		modified = (git_oid_cmp(&new_oid, &idx_entry->oid) != 0);
-	}
-
-	/* TODO: check index flags for forced ignore changes */
-
-	if (modified) {
-		git_tree_diff_data tdiff;
-
-		tdiff.old_attr = idx_entry->mode;
-		tdiff.new_attr = wd_entry->mode;
-		tdiff.status   = GIT_STATUS_MODIFIED;
-		tdiff.path     = wd_entry->path;
-		git_oid_cpy(&tdiff.old_oid, &idx_entry->oid);
-		git_oid_cpy(&tdiff.new_oid, &new_oid);
+		if (git_oid_cmp(&oitem->oid, &noid) == 0 &&
+			oitem->mode == nitem->mode)
+			return GIT_SUCCESS;
 
-		error = file_delta_new__from_tree_diff(info->diff, &tdiff);
+		/* store calculated oid so we don't have to recalc later */
+		use_noid = &noid;
 	}
 
-	return error;
+	return diff_delta__from_two(
+		diff, GIT_STATUS_MODIFIED, oitem, nitem, use_noid);
 }
 
-int git_diff_workdir_to_index(
+static int diff_from_iterators(
 	git_repository *repo,
-	const git_diff_options *opts,
-	git_diff_list **diff)
+	const git_diff_options *opts, /**< can be NULL for defaults */
+	git_iterator *old,
+	git_iterator *new,
+	git_diff_list **diff_ptr)
 {
 	int error;
-	diff_callback_info info = {0};
-
-	if ((info.diff = git_diff_list_alloc(repo, opts)) == NULL)
-		return GIT_ENOMEM;
-
-	if ((error = git_repository_index(&info.index, repo)) == GIT_SUCCESS) {
-		error = diff_workdir_walk(NULL, &info, diff_workdir_to_index_cb);
-		if (error == GIT_SUCCESS)
-			error = add_new_index_deltas(&info, GIT_STATUS_DELETED, NULL);
-		git_index_free(info.index);
+	const git_index_entry *oitem, *nitem;
+	char *ignore_prefix = NULL;
+	git_diff_list *diff = git_diff_list_alloc(repo, opts);
+	if (!diff) {
+		error = GIT_ENOMEM;
+		goto cleanup;
 	}
-	git_buf_free(&info.diff->pfx);
-
-	if (error != GIT_SUCCESS)
-		git_diff_list_free(info.diff);
-	else
-		*diff = info.diff;
 
-	return error;
-}
+	diff->old_src = old->type;
+	diff->new_src = new->type;
 
-typedef struct {
-	git_diff_list *diff;
-	void *cb_data;
-	git_diff_hunk_fn hunk_cb;
-	git_diff_line_fn line_cb;
-	unsigned int index;
-	git_diff_delta *delta;
-} diff_output_info;
+	if ((error = git_iterator_current(old, &oitem)) < GIT_SUCCESS ||
+		(error = git_iterator_current(new, &nitem)) < GIT_SUCCESS)
+		goto cleanup;
 
-static int read_next_int(const char **str, int *value)
-{
-	const char *scan = *str;
-	int v = 0, digits = 0;
-	/* find next digit */
-	for (scan = *str; *scan && !isdigit(*scan); scan++);
-	/* parse next number */
-	for (; isdigit(*scan); scan++, digits++)
-		v = (v * 10) + (*scan - '0');
-	*str = scan;
-	*value = v;
-	return (digits > 0) ? GIT_SUCCESS : GIT_ENOTFOUND;
-}
+	/* run iterators building diffs */
+	while (!error && (oitem || nitem)) {
 
-static int diff_output_cb(void *priv, mmbuffer_t *bufs, int len)
-{
-	int err = GIT_SUCCESS;
-	diff_output_info *info = priv;
-
-	if (len == 1 && info->hunk_cb) {
-		git_diff_range range = { -1, 0, -1, 0 };
-
-		/* expect something of the form "@@ -%d[,%d] +%d[,%d] @@" */
-		if (bufs[0].ptr[0] == '@') {
-			const char *scan = bufs[0].ptr;
-			if (!(err = read_next_int(&scan, &range.old_start)) && *scan == ',')
-				err = read_next_int(&scan, &range.old_lines);
-			if (!err &&
-				!(err = read_next_int(&scan, &range.new_start)) && *scan == ',')
-				err = read_next_int(&scan, &range.new_lines);
-			if (!err && range.old_start >= 0 && range.new_start >= 0)
-				err = info->hunk_cb(
-					info->cb_data, info->delta, &range, bufs[0].ptr, bufs[0].size);
-		}
-	}
-	else if ((len == 2 || len == 3) && info->line_cb) {
-		int origin;
-
-		/* expect " "/"-"/"+", then data, then maybe newline */
-		origin =
-			(*bufs[0].ptr == '+') ? GIT_DIFF_LINE_ADDITION :
-			(*bufs[0].ptr == '-') ? GIT_DIFF_LINE_DELETION :
-			GIT_DIFF_LINE_CONTEXT;
-
-		err = info->line_cb(
-			info->cb_data, info->delta, origin, bufs[1].ptr, bufs[1].size);
-
-		/* deal with adding and removing newline at EOF */
-		if (err == GIT_SUCCESS && len == 3) {
-			if (origin == GIT_DIFF_LINE_ADDITION)
-				origin = GIT_DIFF_LINE_ADD_EOFNL;
-			else
-				origin = GIT_DIFF_LINE_DEL_EOFNL;
-
-			err = info->line_cb(
-				info->cb_data, info->delta, origin, bufs[2].ptr, bufs[2].size);
+		/* create DELETED records for old items not matched in new */
+		if (oitem && (!nitem || strcmp(oitem->path, nitem->path) < 0)) {
+			error = diff_delta__from_one(diff, GIT_STATUS_DELETED, oitem);
+			if (error == GIT_SUCCESS)
+				error = git_iterator_advance(old, &oitem);
+			continue;
 		}
-	}
-
-	return err;
-}
-
-static int set_file_is_binary(
-	git_repository *repo,
-	git_diff_delta *file,
-	mmfile_t *old,
-	mmfile_t *new)
-{
-	int error;
-	const char *value;
-
-	/* check diff attribute +, -, or 0 */
-	error = git_attr_get(repo, file->path, "diff", &value);
-	if (error != GIT_SUCCESS)
-		return error;
-
-	if (value == GIT_ATTR_TRUE) {
-		file->binary = 0;
-		return GIT_SUCCESS;
-	}
-	if (value == GIT_ATTR_FALSE) {
-		file->binary = 1;
-		return GIT_SUCCESS;
-	}
-
-	/* TODO: if value != NULL, implement diff drivers */
-	/* TODO: check if NUL byte appears in first bit */
-	GIT_UNUSED_ARG(old);
-	GIT_UNUSED_ARG(new);
-	file->binary = 0;
-	return GIT_SUCCESS;
-}
-
-static void setup_xdiff_options(
-	git_diff_options *opts, xdemitconf_t *cfg, xpparam_t *param)
-{
-	memset(cfg, 0, sizeof(xdemitconf_t));
-	memset(param, 0, sizeof(xpparam_t));
 
-	cfg->ctxlen =
-		(!opts || !opts->context_lines) ? 3 : opts->context_lines;
-	cfg->interhunkctxlen =
-		(!opts || !opts->interhunk_lines) ? 3 : opts->interhunk_lines;
-
-	if (!opts)
-		return;
-
-	if (opts->flags & GIT_DIFF_IGNORE_WHITESPACE)
-		param->flags |= XDF_WHITESPACE_FLAGS;
-	if (opts->flags & GIT_DIFF_IGNORE_WHITESPACE_CHANGE)
-		param->flags |= XDF_IGNORE_WHITESPACE_CHANGE;
-	if (opts->flags & GIT_DIFF_IGNORE_WHITESPACE_EOL)
-		param->flags |= XDF_IGNORE_WHITESPACE_AT_EOL;
-}
+		/* create ADDED, TRACKED, or IGNORED records for new items not
+		 * matched in old (and/or descend into directories as needed)
+		 */
+		if (nitem && (!oitem || strcmp(oitem->path, nitem->path) > 0)) {
+			int is_ignored;
+			git_status_t use_status = GIT_STATUS_ADDED;
 
-int git_diff_foreach(
-	git_diff_list *diff,
-	void *data,
-	git_diff_file_fn file_cb,
-	git_diff_hunk_fn hunk_cb,
-	git_diff_line_fn line_cb)
-{
-	int error = GIT_SUCCESS;
-	diff_output_info info;
-	git_diff_delta *delta;
-	xpparam_t    xdiff_params;
-	xdemitconf_t xdiff_config;
-	xdemitcb_t   xdiff_callback;
-
-	info.diff    = diff;
-	info.cb_data = data;
-	info.hunk_cb = hunk_cb;
-	info.line_cb = line_cb;
-
-	setup_xdiff_options(&diff->opts, &xdiff_config, &xdiff_params);
-	memset(&xdiff_callback, 0, sizeof(xdiff_callback));
-	xdiff_callback.outf = diff_output_cb;
-	xdiff_callback.priv = &info;
-
-	git_vector_foreach(&diff->files, info.index, delta) {
-		mmfile_t old_data, new_data;
-
-		/* map files */
-		if (hunk_cb || line_cb) {
-			/* TODO: Partial blob reading to defer loading whole blob.
-			 * I.e. I want a blob with just the first 4kb loaded, then
-			 * later on I will read the rest of the blob if needed.
-			 */
-
-			if (delta->status == GIT_STATUS_DELETED ||
-				delta->status == GIT_STATUS_MODIFIED)
+			/* contained in ignored parent directory, so this can be skipped. */
+			if (ignore_prefix != NULL &&
+				git__prefixcmp(nitem->path, ignore_prefix) == 0)
 			{
-				error = git_blob_lookup(
-					&delta->old_blob, diff->repo, &delta->old_oid);
-				old_data.ptr = (char *)git_blob_rawcontent(delta->old_blob);
-				old_data.size = git_blob_rawsize(delta->old_blob);
-			} else {
-				delta->old_blob = NULL;
-				old_data.ptr = "";
-				old_data.size = 0;
+				error = git_iterator_advance(new, &nitem);
+				continue;
 			}
 
-			if (delta->status == GIT_STATUS_ADDED ||
-				delta->status == GIT_STATUS_MODIFIED)
-			{
-				error = git_blob_lookup(
-					&delta->new_blob, diff->repo, &delta->new_oid);
-				new_data.ptr = (char *)git_blob_rawcontent(delta->new_blob);
-				new_data.size = git_blob_rawsize(delta->new_blob);
-			} else {
-				delta->new_blob = NULL;
-				new_data.ptr = "";
-				new_data.size = 0;
+			is_ignored = git_iterator_current_is_ignored(new);
+
+			if (S_ISDIR(nitem->mode)) {
+				if (git__prefixcmp(oitem->path, nitem->path) == 0) {
+					if (is_ignored)
+						ignore_prefix = nitem->path;
+					error = git_iterator_advance_into_directory(new, &nitem);
+					continue;
+				}
+				use_status = GIT_STATUS_UNTRACKED;
 			}
+			else if (is_ignored)
+				use_status = GIT_STATUS_IGNORED;
+			else if (new->type == GIT_ITERATOR_WORKDIR)
+				use_status = GIT_STATUS_UNTRACKED;
+
+			error = diff_delta__from_one(diff, use_status, nitem);
+			if (error == GIT_SUCCESS)
+				error = git_iterator_advance(new, &nitem);
+			continue;
 		}
 
-		if (diff->opts.flags & GIT_DIFF_FORCE_TEXT)
-			delta->binary = 0;
-		else if ((error = set_file_is_binary(
-			diff->repo, delta, &old_data, &new_data)) < GIT_SUCCESS)
-			break;
-
-		/* TODO: if ignore_whitespace is set, then we *must* do text
-		 * diffs to tell if a file has really been changed.
+		/* otherwise item paths match, so create MODIFIED record
+		 * (or ADDED and DELETED pair if type changed)
 		 */
+		assert(oitem && nitem && strcmp(oitem->path, nitem->path) == 0);
 
-		if (file_cb != NULL) {
-			error = file_cb(data, delta, (float)info.index / diff->files.length);
-			if (error != GIT_SUCCESS)
-				break;
-		}
-
-		/* don't do hunk and line diffs if file is binary */
-		if (delta->binary)
-			continue;
-
-		/* nothing to do if we did not get a blob */
-		if (!delta->old_blob && !delta->new_blob)
-			continue;
-
-		assert(hunk_cb || line_cb);
-
-		info.delta = delta;
-
-		xdl_diff(&old_data, &new_data,
-			&xdiff_params, &xdiff_config, &xdiff_callback);
-
-		git_blob_free(delta->old_blob);
-		delta->old_blob = NULL;
-
-		git_blob_free(delta->new_blob);
-		delta->new_blob = NULL;
+		error = maybe_modified(old, oitem, new, nitem, diff);
+		if (error == GIT_SUCCESS)
+			error = git_iterator_advance(old, &oitem);
+		if (error == GIT_SUCCESS)
+			error = git_iterator_advance(new, &nitem);
 	}
 
-	return error;
-}
-
-typedef struct {
-	git_diff_list *diff;
-	git_diff_output_fn print_cb;
-	void *cb_data;
-	git_buf *buf;
-} diff_print_info;
-
-static char pick_suffix(int mode)
-{
-	if (S_ISDIR(mode))
-		return '/';
-	else if (mode & 0100)
-		/* modes in git are not very flexible, so if this bit is set,
-		 * we must be dealwith with a 100755 type of file.
-		 */
-		return '*';
-	else
-		return ' ';
-}
+cleanup:
+	git_iterator_free(old);
+	git_iterator_free(new);
 
-static int print_compact(void *data, git_diff_delta *delta, float progress)
-{
-	diff_print_info *pi = data;
-	char code, old_suffix, new_suffix;
-
-	GIT_UNUSED_ARG(progress);
-
-	switch (delta->status) {
-	case GIT_STATUS_ADDED: code = 'A'; break;
-	case GIT_STATUS_DELETED: code = 'D'; break;
-	case GIT_STATUS_MODIFIED: code = 'M'; break;
-	case GIT_STATUS_RENAMED: code = 'R'; break;
-	case GIT_STATUS_COPIED: code = 'C'; break;
-	case GIT_STATUS_IGNORED: code = 'I'; break;
-	case GIT_STATUS_UNTRACKED: code = '?'; break;
-	default: code = 0;
+	if (error != GIT_SUCCESS) {
+		git_diff_list_free(diff);
+		diff = NULL;
 	}
 
-	if (!code)
-		return GIT_SUCCESS;
-
-	old_suffix = pick_suffix(delta->old_attr);
-	new_suffix = pick_suffix(delta->new_attr);
+	*diff_ptr = diff;
 
-	git_buf_clear(pi->buf);
-
-	if (delta->new_path != NULL)
-		git_buf_printf(pi->buf, "%c\t%s%c -> %s%c\n", code,
-			delta->path, old_suffix, delta->new_path, new_suffix);
-	else if (delta->old_attr != delta->new_attr &&
-		delta->old_attr != 0 && delta->new_attr != 0)
-		git_buf_printf(pi->buf, "%c\t%s%c (%o -> %o)\n", code,
-			delta->path, new_suffix, delta->old_attr, delta->new_attr);
-	else if (old_suffix != ' ')
-		git_buf_printf(pi->buf, "%c\t%s%c\n", code, delta->path, old_suffix);
-	else
-		git_buf_printf(pi->buf, "%c\t%s\n", code, delta->path);
-
-	if (git_buf_lasterror(pi->buf) != GIT_SUCCESS)
-		return git_buf_lasterror(pi->buf);
-
-	return pi->print_cb(pi->cb_data, GIT_DIFF_LINE_FILE_HDR, pi->buf->ptr);
+	return error;
 }
 
-int git_diff_print_compact(
-	git_diff_list *diff,
-	void *cb_data,
-	git_diff_output_fn print_cb)
+
+int git_diff_tree_to_tree(
+	git_repository *repo,
+	const git_diff_options *opts, /**< can be NULL for defaults */
+	git_tree *old,
+	git_tree *new,
+	git_diff_list **diff)
 {
 	int error;
-	git_buf buf = GIT_BUF_INIT;
-	diff_print_info pi;
+	git_iterator *a = NULL, *b = NULL;
 
-	pi.diff     = diff;
-	pi.print_cb = print_cb;
-	pi.cb_data  = cb_data;
-	pi.buf      = &buf;
+	assert(repo && old && new && diff);
 
-	error = git_diff_foreach(diff, &pi, print_compact, NULL, NULL);
-
-	git_buf_free(&buf);
+	if ((error = git_iterator_for_tree(repo, old, &a)) < GIT_SUCCESS ||
+		(error = git_iterator_for_tree(repo, new, &b)) < GIT_SUCCESS)
+		return error;
 
-	return error;
+	return diff_from_iterators(repo, opts, a, b, diff);
 }
 
-static int print_oid_range(diff_print_info *pi, git_diff_delta *delta)
+int git_diff_index_to_tree(
+	git_repository *repo,
+	const git_diff_options *opts,
+	git_tree *old,
+	git_diff_list **diff)
 {
-	char start_oid[8], end_oid[8];
+	int error;
+	git_iterator *a = NULL, *b = NULL;
 
-	/* TODO: Determine a good actual OID range to print */
-	git_oid_to_string(start_oid, sizeof(start_oid), &delta->old_oid);
-	git_oid_to_string(end_oid, sizeof(end_oid), &delta->new_oid);
+	assert(repo && old && diff);
 
-	/* TODO: Match git diff more closely */
-	if (delta->old_attr == delta->new_attr) {
-		git_buf_printf(pi->buf, "index %s..%s %o\n",
-			start_oid, end_oid, delta->old_attr);
-	} else {
-		if (delta->old_attr == 0) {
-			git_buf_printf(pi->buf, "new file mode %o\n", delta->new_attr);
-		} else if (delta->new_attr == 0) {
-			git_buf_printf(pi->buf, "deleted file mode %o\n", delta->old_attr);
-		} else {
-			git_buf_printf(pi->buf, "old mode %o\n", delta->old_attr);
-			git_buf_printf(pi->buf, "new mode %o\n", delta->new_attr);
-		}
-		git_buf_printf(pi->buf, "index %s..%s\n", start_oid, end_oid);
-	}
+	if ((error = git_iterator_for_tree(repo, old, &a)) < GIT_SUCCESS ||
+		(error = git_iterator_for_index(repo, &b)) < GIT_SUCCESS)
+		return error;
 
-	return git_buf_lasterror(pi->buf);
+	return diff_from_iterators(repo, opts, a, b, diff);
 }
 
-static int print_patch_file(void *data, git_diff_delta *delta, float progress)
+int git_diff_workdir_to_index(
+	git_repository *repo,
+	const git_diff_options *opts,
+	git_diff_list **diff)
 {
 	int error;
-	diff_print_info *pi = data;
-	const char *oldpfx = pi->diff->opts.src_prefix;
-	const char *oldpath = delta->path;
-	const char *newpfx = pi->diff->opts.dst_prefix;
-	const char *newpath = delta->new_path ? delta->new_path : delta->path;
-
-	GIT_UNUSED_ARG(progress);
-
-	git_buf_clear(pi->buf);
-	git_buf_printf(pi->buf, "diff --git %s%s %s%s\n", oldpfx, delta->path, newpfx, newpath);
-	if ((error = print_oid_range(pi, delta)) < GIT_SUCCESS)
-		return error;
-
-	if (delta->old_blob == NULL) {
-		oldpfx = "";
-		oldpath = "/dev/null";
-	}
-	if (delta->new_blob == NULL) {
-		oldpfx = "";
-		oldpath = "/dev/null";
-	}
-
-	if (!delta->binary) {
-		git_buf_printf(pi->buf, "--- %s%s\n", oldpfx, oldpath);
-		git_buf_printf(pi->buf, "+++ %s%s\n", newpfx, newpath);
-	}
+	git_iterator *a = NULL, *b = NULL;
 
-	if (git_buf_lasterror(pi->buf) != GIT_SUCCESS)
-		return git_buf_lasterror(pi->buf);
+	assert(repo && diff);
 
-	error = pi->print_cb(pi->cb_data, GIT_DIFF_LINE_FILE_HDR, pi->buf->ptr);
-	if (error != GIT_SUCCESS || !delta->binary)
+	if ((error = git_iterator_for_index(repo, &a)) < GIT_SUCCESS ||
+		(error = git_iterator_for_workdir(repo, &b)) < GIT_SUCCESS)
 		return error;
 
-	git_buf_clear(pi->buf);
-	git_buf_printf(
-		pi->buf, "Binary files %s%s and %s%s differ\n",
-		oldpfx, oldpath, newpfx, newpath);
-	if (git_buf_lasterror(pi->buf) != GIT_SUCCESS)
-		return git_buf_lasterror(pi->buf);
-
-	return pi->print_cb(pi->cb_data, GIT_DIFF_LINE_BINARY, pi->buf->ptr);
+	return diff_from_iterators(repo, opts, a, b, diff);
 }
 
-static int print_patch_hunk(
-	void *data,
-	git_diff_delta *d,
-	git_diff_range *r,
-	const char *header,
-	size_t header_len)
-{
-	diff_print_info *pi = data;
-
-	GIT_UNUSED_ARG(d);
-	GIT_UNUSED_ARG(r);
-
-	git_buf_clear(pi->buf);
-
-	if (git_buf_printf(pi->buf, "%.*s", (int)header_len, header) == GIT_SUCCESS)
-		return pi->print_cb(pi->cb_data, GIT_DIFF_LINE_HUNK_HDR, pi->buf->ptr);
-	else
-		return git_buf_lasterror(pi->buf);
-}
 
-static int print_patch_line(
-	void *data,
-	git_diff_delta *delta,
-	char line_origin, /* GIT_DIFF_LINE value from above */
-	const char *content,
-	size_t content_len)
+int git_diff_workdir_to_tree(
+	git_repository *repo,
+	const git_diff_options *opts,
+	git_tree *old,
+	git_diff_list **diff)
 {
-	diff_print_info *pi = data;
-
-	GIT_UNUSED_ARG(delta);
-
-	git_buf_clear(pi->buf);
+	int error;
+	git_iterator *a = NULL, *b = NULL;
 
-	if (line_origin == GIT_DIFF_LINE_ADDITION ||
-		line_origin == GIT_DIFF_LINE_DELETION ||
-		line_origin == GIT_DIFF_LINE_CONTEXT)
-		git_buf_printf(pi->buf, "%c%.*s", line_origin, (int)content_len, content);
-	else if (content_len > 0)
-		git_buf_printf(pi->buf, "%.*s", (int)content_len, content);
+	assert(repo && old && diff);
 
-	if (git_buf_lasterror(pi->buf) != GIT_SUCCESS)
-		return git_buf_lasterror(pi->buf);
+	if ((error = git_iterator_for_tree(repo, old, &a)) < GIT_SUCCESS ||
+		(error = git_iterator_for_workdir(repo, &b)) < GIT_SUCCESS)
+		return error;
 
-	return pi->print_cb(pi->cb_data, line_origin, pi->buf->ptr);
+	return diff_from_iterators(repo, opts, a, b, diff);
 }
 
-int git_diff_print_patch(
-	git_diff_list *diff,
-	void *cb_data,
-	git_diff_output_fn print_cb)
+int git_diff_merge(
+	git_diff_list *onto,
+	const git_diff_list *from)
 {
 	int error;
-	git_buf buf = GIT_BUF_INIT;
-	diff_print_info pi;
-
-	pi.diff     = diff;
-	pi.print_cb = print_cb;
-	pi.cb_data  = cb_data;
-	pi.buf      = &buf;
+	unsigned int i = 0, j = 0;
+	git_vector onto_new;
+	git_diff_delta *delta;
 
-	error = git_diff_foreach(
-		diff, &pi, print_patch_file, print_patch_hunk, print_patch_line);
+	error = git_vector_init(&onto_new, onto->deltas.length, diff_delta__cmp);
+	if (error < GIT_SUCCESS)
+		return error;
 
-	git_buf_free(&buf);
+	while (i < onto->deltas.length || j < from->deltas.length) {
+		git_diff_delta       *o = git_vector_get(&onto->deltas, i);
+		const git_diff_delta *f = git_vector_get_const(&from->deltas, j);
+		const char *opath = !o ? NULL : o->old.path ? o->old.path : o->new.path;
+		const char *fpath = !f ? NULL : f->old.path ? f->old.path : f->new.path;
+
+		if (opath && (!fpath || strcmp(opath, fpath) < 0)) {
+			delta = diff_delta__dup(o);
+			i++;
+		} else if (fpath && (!opath || strcmp(opath, fpath) > 0)) {
+			delta = diff_delta__dup(f);
+			j++;
+		} else {
+			delta = diff_delta__merge_like_cgit(o, f);
+			i++;
+			j++;
+		}
 
-	return error;
-}
+		if (!delta)
+			error = GIT_ENOMEM;
+		else
+			error = git_vector_insert(&onto_new, delta);
 
-int git_diff_blobs(
-	git_repository *repo,
-	git_blob *old_blob,
-	git_blob *new_blob,
-	git_diff_options *options,
-	void *cb_data,
-	git_diff_hunk_fn hunk_cb,
-	git_diff_line_fn line_cb)
-{
-	diff_output_info info;
-	git_diff_delta delta;
-	mmfile_t old, new;
-	xpparam_t xdiff_params;
-	xdemitconf_t xdiff_config;
-	xdemitcb_t xdiff_callback;
-
-	assert(repo);
-
-	if (options && (options->flags & GIT_DIFF_REVERSE)) {
-		git_blob *swap = old_blob;
-		old_blob = new_blob;
-		new_blob = swap;
+		if (error != GIT_SUCCESS)
+			break;
 	}
 
-	if (old_blob) {
-		old.ptr  = (char *)git_blob_rawcontent(old_blob);
-		old.size = git_blob_rawsize(old_blob);
-	} else {
-		old.ptr  = "";
-		old.size = 0;
+	if (error == GIT_SUCCESS) {
+		git_vector_swap(&onto->deltas, &onto_new);
+		onto->new_src = from->new_src;
 	}
 
-	if (new_blob) {
-		new.ptr  = (char *)git_blob_rawcontent(new_blob);
-		new.size = git_blob_rawsize(new_blob);
-	} else {
-		new.ptr  = "";
-		new.size = 0;
-	}
+	git_vector_foreach(&onto_new, i, delta)
+		diff_delta__free(delta);
+	git_vector_free(&onto_new);
 
-	/* populate a "fake" delta record */
-	delta.status = old.ptr ?
-		(new.ptr ? GIT_STATUS_MODIFIED : GIT_STATUS_DELETED) :
-		(new.ptr ? GIT_STATUS_ADDED : GIT_STATUS_UNTRACKED);
-	delta.old_attr = 0100644; /* can't know the truth from a blob alone */
-	delta.new_attr = 0100644;
-	git_oid_cpy(&delta.old_oid, git_object_id((const git_object *)old_blob));
-	git_oid_cpy(&delta.new_oid, git_object_id((const git_object *)new_blob));
-	delta.old_blob = old_blob;
-	delta.new_blob = new_blob;
-	delta.path = NULL;
-	delta.new_path = NULL;
-	delta.similarity = 0;
-	delta.binary = 0;
-
-	info.diff    = NULL;
-	info.delta   = &delta;
-	info.cb_data = cb_data;
-	info.hunk_cb = hunk_cb;
-	info.line_cb = line_cb;
-
-	setup_xdiff_options(options, &xdiff_config, &xdiff_params);
-	memset(&xdiff_callback, 0, sizeof(xdiff_callback));
-	xdiff_callback.outf = diff_output_cb;
-	xdiff_callback.priv = &info;
-
-	xdl_diff(&old, &new, &xdiff_params, &xdiff_config, &xdiff_callback);
-
-	return GIT_SUCCESS;
+	return error;
 }
diff --git a/src/diff.h b/src/diff.h
index b0f1ebbe8..7d69199ea 100644
--- a/src/diff.h
+++ b/src/diff.h
@@ -10,15 +10,15 @@
 #include <stdio.h>
 #include "vector.h"
 #include "buffer.h"
+#include "iterator.h"
+#include "repository.h"
 
 struct git_diff_list {
 	git_repository   *repo;
 	git_diff_options opts;
-	git_vector       files;    /* vector of git_diff_file_delta */
-
-	/* the following are just used while processing the diff list */
-	git_buf          pfx;
-	git_status_t     status;
+	git_vector       deltas;    /* vector of git_diff_file_delta */
+	git_iterator_type_t old_src;
+	git_iterator_type_t new_src;
 };
 
 #endif
diff --git a/src/diff_output.c b/src/diff_output.c
new file mode 100644
index 000000000..ac60e9822
--- /dev/null
+++ b/src/diff_output.c
@@ -0,0 +1,722 @@
+/*
+ * Copyright (C) 2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "common.h"
+#include "git2/diff.h"
+#include "git2/attr.h"
+#include "git2/blob.h"
+#include "xdiff/xdiff.h"
+#include <ctype.h>
+#include "diff.h"
+#include "map.h"
+#include "fileops.h"
+
+typedef struct {
+	git_diff_list *diff;
+	void *cb_data;
+	git_diff_hunk_fn hunk_cb;
+	git_diff_line_fn line_cb;
+	unsigned int index;
+	git_diff_delta *delta;
+} diff_output_info;
+
+static int read_next_int(const char **str, int *value)
+{
+	const char *scan = *str;
+	int v = 0, digits = 0;
+	/* find next digit */
+	for (scan = *str; *scan && !isdigit(*scan); scan++);
+	/* parse next number */
+	for (; isdigit(*scan); scan++, digits++)
+		v = (v * 10) + (*scan - '0');
+	*str = scan;
+	*value = v;
+	return (digits > 0) ? GIT_SUCCESS : GIT_ENOTFOUND;
+}
+
+static int diff_output_cb(void *priv, mmbuffer_t *bufs, int len)
+{
+	int err = GIT_SUCCESS;
+	diff_output_info *info = priv;
+
+	if (len == 1 && info->hunk_cb) {
+		git_diff_range range = { -1, 0, -1, 0 };
+
+		/* expect something of the form "@@ -%d[,%d] +%d[,%d] @@" */
+		if (bufs[0].ptr[0] == '@') {
+			const char *scan = bufs[0].ptr;
+			if (!(err = read_next_int(&scan, &range.old_start)) && *scan == ',')
+				err = read_next_int(&scan, &range.old_lines);
+			if (!err &&
+				!(err = read_next_int(&scan, &range.new_start)) && *scan == ',')
+				err = read_next_int(&scan, &range.new_lines);
+			if (!err && range.old_start >= 0 && range.new_start >= 0)
+				err = info->hunk_cb(
+					info->cb_data, info->delta, &range, bufs[0].ptr, bufs[0].size);
+		}
+	}
+	else if ((len == 2 || len == 3) && info->line_cb) {
+		int origin;
+
+		/* expect " "/"-"/"+", then data, then maybe newline */
+		origin =
+			(*bufs[0].ptr == '+') ? GIT_DIFF_LINE_ADDITION :
+			(*bufs[0].ptr == '-') ? GIT_DIFF_LINE_DELETION :
+			GIT_DIFF_LINE_CONTEXT;
+
+		err = info->line_cb(
+			info->cb_data, info->delta, origin, bufs[1].ptr, bufs[1].size);
+
+		/* deal with adding and removing newline at EOF */
+		if (err == GIT_SUCCESS && len == 3) {
+			if (origin == GIT_DIFF_LINE_ADDITION)
+				origin = GIT_DIFF_LINE_ADD_EOFNL;
+			else
+				origin = GIT_DIFF_LINE_DEL_EOFNL;
+
+			err = info->line_cb(
+				info->cb_data, info->delta, origin, bufs[2].ptr, bufs[2].size);
+		}
+	}
+
+	return err;
+}
+
+#define BINARY_DIFF_FLAGS (GIT_DIFF_FILE_BINARY|GIT_DIFF_FILE_NOT_BINARY)
+
+static int set_file_is_binary_by_attr(git_repository *repo, git_diff_file *file)
+{
+	const char *value;
+	int error = git_attr_get(repo, file->path, "diff", &value);
+	if (error != GIT_SUCCESS)
+		return error;
+	if (value == GIT_ATTR_FALSE)
+		file->flags |= GIT_DIFF_FILE_BINARY;
+	else if (value == GIT_ATTR_TRUE)
+		file->flags |= GIT_DIFF_FILE_NOT_BINARY;
+	/* otherwise leave file->flags alone */
+	return error;
+}
+
+static void set_delta_is_binary(git_diff_delta *delta)
+{
+	if ((delta->old.flags & GIT_DIFF_FILE_BINARY) != 0 ||
+		(delta->new.flags & GIT_DIFF_FILE_BINARY) != 0)
+		delta->binary = 1;
+	else if ((delta->old.flags & GIT_DIFF_FILE_NOT_BINARY) != 0 ||
+			 (delta->new.flags & GIT_DIFF_FILE_NOT_BINARY) != 0)
+		delta->binary = 0;
+	/* otherwise leave delta->binary value untouched */
+}
+
+static int file_is_binary_by_attr(
+	git_diff_list *diff,
+	git_diff_delta *delta)
+{
+	int error, mirror_new;
+
+	delta->binary = -1;
+
+	/* make sure files are conceivably mmap-able */
+	if ((git_off_t)((size_t)delta->old.size) != delta->old.size ||
+		(git_off_t)((size_t)delta->new.size) != delta->new.size)
+	{
+		delta->old.flags |= GIT_DIFF_FILE_BINARY;
+		delta->new.flags |= GIT_DIFF_FILE_BINARY;
+		delta->binary = 1;
+		return GIT_SUCCESS;
+	}
+
+	/* check if user is forcing us to text diff these files */
+	if (diff->opts.flags & GIT_DIFF_FORCE_TEXT) {
+		delta->old.flags |= GIT_DIFF_FILE_NOT_BINARY;
+		delta->new.flags |= GIT_DIFF_FILE_NOT_BINARY;
+		delta->binary = 0;
+		return GIT_SUCCESS;
+	}
+
+	/* check diff attribute +, -, or 0 */
+	error = set_file_is_binary_by_attr(diff->repo, &delta->old);
+	if (error != GIT_SUCCESS)
+		return error;
+
+	mirror_new = (delta->new.path == delta->old.path ||
+				  strcmp(delta->new.path, delta->old.path) == 0);
+	if (mirror_new)
+		delta->new.flags &= (delta->old.flags & BINARY_DIFF_FLAGS);
+	else
+		error = set_file_is_binary_by_attr(diff->repo, &delta->new);
+
+	set_delta_is_binary(delta);
+
+	return error;
+}
+
+static int file_is_binary_by_content(
+	git_diff_list *diff,
+	git_diff_delta *delta,
+	git_map *old_data,
+	git_map *new_data)
+{
+	GIT_UNUSED_ARG(diff);
+
+	if ((delta->old.flags & BINARY_DIFF_FLAGS) == 0) {
+		size_t search_len = min(old_data->len, 4000);
+		if (strnlen(old_data->data, search_len) != search_len)
+			delta->old.flags |= GIT_DIFF_FILE_BINARY;
+		else
+			delta->old.flags |= GIT_DIFF_FILE_NOT_BINARY;
+	}
+
+	if ((delta->new.flags & BINARY_DIFF_FLAGS) == 0) {
+		size_t search_len = min(new_data->len, 4000);
+		if (strnlen(new_data->data, search_len) != search_len)
+			delta->new.flags |= GIT_DIFF_FILE_BINARY;
+		else
+			delta->new.flags |= GIT_DIFF_FILE_NOT_BINARY;
+	}
+
+	set_delta_is_binary(delta);
+
+	/* TODO: if value != NULL, implement diff drivers */
+
+	return GIT_SUCCESS;
+}
+
+static void setup_xdiff_options(
+	git_diff_options *opts, xdemitconf_t *cfg, xpparam_t *param)
+{
+	memset(cfg, 0, sizeof(xdemitconf_t));
+	memset(param, 0, sizeof(xpparam_t));
+
+	cfg->ctxlen =
+		(!opts || !opts->context_lines) ? 3 : opts->context_lines;
+	cfg->interhunkctxlen =
+		(!opts || !opts->interhunk_lines) ? 3 : opts->interhunk_lines;
+
+	if (!opts)
+		return;
+
+	if (opts->flags & GIT_DIFF_IGNORE_WHITESPACE)
+		param->flags |= XDF_WHITESPACE_FLAGS;
+	if (opts->flags & GIT_DIFF_IGNORE_WHITESPACE_CHANGE)
+		param->flags |= XDF_IGNORE_WHITESPACE_CHANGE;
+	if (opts->flags & GIT_DIFF_IGNORE_WHITESPACE_EOL)
+		param->flags |= XDF_IGNORE_WHITESPACE_AT_EOL;
+}
+
+static int get_blob_content(
+	git_repository *repo,
+	const git_oid *oid,
+	git_map *map,
+	git_blob **blob)
+{
+	int error;
+
+	if (git_oid_iszero(oid))
+		return GIT_SUCCESS;
+
+	if ((error = git_blob_lookup(blob, repo, oid)) == GIT_SUCCESS) {
+		map->data = (void *)git_blob_rawcontent(*blob);
+		map->len  = git_blob_rawsize(*blob);
+	}
+
+	return error;
+}
+
+static int get_workdir_content(
+	git_repository *repo,
+	git_diff_file *file,
+	git_map *map)
+{
+	git_buf full_path = GIT_BUF_INIT;
+	int error = git_buf_joinpath(
+		&full_path, git_repository_workdir(repo), file->path);
+	if (error != GIT_SUCCESS)
+		return error;
+
+	if (S_ISLNK(file->mode)) {
+		file->flags |= GIT_DIFF_FILE_FREE_DATA;
+		file->flags |= GIT_DIFF_FILE_BINARY;
+
+		map->data = git__malloc((size_t)file->size + 1);
+		if (map->data == NULL)
+			error = GIT_ENOMEM;
+		else {
+			ssize_t read_len =
+				p_readlink(full_path.ptr, map->data, (size_t)file->size + 1);
+			if (read_len != (ssize_t)file->size)
+				error = git__throw(
+					GIT_EOSERR, "Failed to read symlink %s", file->path);
+			else
+				map->len = read_len;
+
+		}
+	}
+	else {
+		error = git_futils_mmap_ro_file(map, full_path.ptr);
+		file->flags |= GIT_DIFF_FILE_UNMAP_DATA;
+	}
+	git_buf_free(&full_path);
+	return error;
+}
+
+static void release_content(git_diff_file *file, git_map *map, git_blob *blob)
+{
+	if (blob != NULL)
+		git_blob_free(blob);
+
+	if (file->flags & GIT_DIFF_FILE_FREE_DATA) {
+		git__free(map->data);
+		map->data = NULL;
+		file->flags &= ~GIT_DIFF_FILE_FREE_DATA;
+	}
+	else if (file->flags & GIT_DIFF_FILE_UNMAP_DATA) {
+		git_futils_mmap_free(map);
+		map->data = NULL;
+		file->flags &= ~GIT_DIFF_FILE_UNMAP_DATA;
+	}
+}
+
+int git_diff_foreach(
+	git_diff_list *diff,
+	void *data,
+	git_diff_file_fn file_cb,
+	git_diff_hunk_fn hunk_cb,
+	git_diff_line_fn line_cb)
+{
+	int error = GIT_SUCCESS;
+	diff_output_info info;
+	git_diff_delta *delta;
+	xpparam_t    xdiff_params;
+	xdemitconf_t xdiff_config;
+	xdemitcb_t   xdiff_callback;
+
+	info.diff    = diff;
+	info.cb_data = data;
+	info.hunk_cb = hunk_cb;
+	info.line_cb = line_cb;
+
+	setup_xdiff_options(&diff->opts, &xdiff_config, &xdiff_params);
+	memset(&xdiff_callback, 0, sizeof(xdiff_callback));
+	xdiff_callback.outf = diff_output_cb;
+	xdiff_callback.priv = &info;
+
+	git_vector_foreach(&diff->deltas, info.index, delta) {
+		git_blob *old_blob = NULL, *new_blob = NULL;
+		git_map old_data, new_data;
+
+		if (delta->status == GIT_STATUS_UNMODIFIED)
+			continue;
+
+		if (delta->status == GIT_STATUS_IGNORED &&
+			(diff->opts.flags & GIT_DIFF_INCLUDE_IGNORED) == 0)
+			continue;
+
+		if (delta->status == GIT_STATUS_UNTRACKED &&
+			(diff->opts.flags & GIT_DIFF_INCLUDE_UNTRACKED) == 0)
+			continue;
+
+		error = file_is_binary_by_attr(diff, delta);
+		if (error < GIT_SUCCESS)
+			goto cleanup;
+
+		old_data.data = "";
+		old_data.len = 0;
+		new_data.data = "";
+		new_data.len  = 0;
+
+		/* TODO: Partial blob reading to defer loading whole blob.
+		 * I.e. I want a blob with just the first 4kb loaded, then
+		 * later on I will read the rest of the blob if needed.
+		 */
+
+		/* map files */
+		if (delta->binary != 1 &&
+			(hunk_cb || line_cb) &&
+			(delta->status == GIT_STATUS_DELETED ||
+			 delta->status == GIT_STATUS_MODIFIED))
+		{
+			if (diff->old_src == GIT_ITERATOR_WORKDIR)
+				error = get_workdir_content(diff->repo, &delta->old, &old_data);
+			else
+				error = get_blob_content(
+					diff->repo, &delta->old.oid, &old_data, &old_blob);
+			if (error != GIT_SUCCESS)
+				goto cleanup;
+		}
+
+		if (delta->binary != 1 &&
+			(hunk_cb || line_cb || git_oid_iszero(&delta->new.oid)) &&
+			(delta->status == GIT_STATUS_ADDED ||
+			 delta->status == GIT_STATUS_MODIFIED))
+		{
+			if (diff->new_src == GIT_ITERATOR_WORKDIR)
+				error = get_workdir_content(diff->repo, &delta->new, &new_data);
+			else
+				error = get_blob_content(
+					diff->repo, &delta->new.oid, &new_data, &new_blob);
+			if (error != GIT_SUCCESS)
+				goto cleanup;
+
+			if ((delta->new.flags | GIT_DIFF_FILE_VALID_OID) == 0) {
+				error = git_odb_hash(
+					&delta->new.oid, new_data.data, new_data.len, GIT_OBJ_BLOB);
+				if (error != GIT_SUCCESS)
+					goto cleanup;
+
+				/* since we did not have the definitive oid, we may have
+				 * incorrect status and need to skip this item.
+				 */
+				if (git_oid_cmp(&delta->old.oid, &delta->new.oid) == 0) {
+					delta->status = GIT_STATUS_UNMODIFIED;
+					goto cleanup;
+				}
+			}
+		}
+
+		/* if we have not already decided whether file is binary,
+		 * check the first 4K for nul bytes to decide...
+		 */
+		if (delta->binary == -1) {
+			error = file_is_binary_by_content(
+				diff, delta, &old_data, &new_data);
+			if (error < GIT_SUCCESS)
+				goto cleanup;
+		}
+
+		/* TODO: if ignore_whitespace is set, then we *must* do text
+		 * diffs to tell if a file has really been changed.
+		 */
+
+		if (file_cb != NULL) {
+			error = file_cb(data, delta, (float)info.index / diff->deltas.length);
+			if (error != GIT_SUCCESS)
+				goto cleanup;
+		}
+
+		/* don't do hunk and line diffs if file is binary */
+		if (delta->binary == 1)
+			goto cleanup;
+
+		/* nothing to do if we did not get data */
+		if (!old_data.len && !new_data.len)
+			goto cleanup;
+
+		assert(hunk_cb || line_cb);
+
+		info.delta = delta;
+
+		xdl_diff((mmfile_t *)&old_data, (mmfile_t *)&new_data,
+			&xdiff_params, &xdiff_config, &xdiff_callback);
+
+cleanup:
+		release_content(&delta->old, &old_data, old_blob);
+		release_content(&delta->new, &new_data, new_blob);
+
+		if (error != GIT_SUCCESS)
+			break;
+	}
+
+	return error;
+}
+
+
+typedef struct {
+	git_diff_list *diff;
+	git_diff_output_fn print_cb;
+	void *cb_data;
+	git_buf *buf;
+} diff_print_info;
+
+static char pick_suffix(int mode)
+{
+	if (S_ISDIR(mode))
+		return '/';
+	else if (mode & 0100)
+		/* in git, modes are very regular, so we must have 0100755 mode */
+		return '*';
+	else
+		return ' ';
+}
+
+static int print_compact(void *data, git_diff_delta *delta, float progress)
+{
+	diff_print_info *pi = data;
+	char code, old_suffix, new_suffix;
+
+	GIT_UNUSED_ARG(progress);
+
+	switch (delta->status) {
+	case GIT_STATUS_ADDED: code = 'A'; break;
+	case GIT_STATUS_DELETED: code = 'D'; break;
+	case GIT_STATUS_MODIFIED: code = 'M'; break;
+	case GIT_STATUS_RENAMED: code = 'R'; break;
+	case GIT_STATUS_COPIED: code = 'C'; break;
+	case GIT_STATUS_IGNORED: code = 'I'; break;
+	case GIT_STATUS_UNTRACKED: code = '?'; break;
+	default: code = 0;
+	}
+
+	if (!code)
+		return GIT_SUCCESS;
+
+	old_suffix = pick_suffix(delta->old.mode);
+	new_suffix = pick_suffix(delta->new.mode);
+
+	git_buf_clear(pi->buf);
+
+	if (delta->old.path != delta->new.path &&
+		strcmp(delta->old.path,delta->new.path) != 0)
+		git_buf_printf(pi->buf, "%c\t%s%c -> %s%c\n", code,
+			delta->old.path, old_suffix, delta->new.path, new_suffix);
+	else if (delta->old.mode != delta->new.mode &&
+		delta->old.mode != 0 && delta->new.mode != 0)
+		git_buf_printf(pi->buf, "%c\t%s%c (%o -> %o)\n", code,
+			delta->old.path, new_suffix, delta->old.mode, delta->new.mode);
+	else if (old_suffix != ' ')
+		git_buf_printf(pi->buf, "%c\t%s%c\n", code, delta->old.path, old_suffix);
+	else
+		git_buf_printf(pi->buf, "%c\t%s\n", code, delta->old.path);
+
+	if (git_buf_lasterror(pi->buf) != GIT_SUCCESS)
+		return git_buf_lasterror(pi->buf);
+
+	return pi->print_cb(pi->cb_data, GIT_DIFF_LINE_FILE_HDR, pi->buf->ptr);
+}
+
+int git_diff_print_compact(
+	git_diff_list *diff,
+	void *cb_data,
+	git_diff_output_fn print_cb)
+{
+	int error;
+	git_buf buf = GIT_BUF_INIT;
+	diff_print_info pi;
+
+	pi.diff     = diff;
+	pi.print_cb = print_cb;
+	pi.cb_data  = cb_data;
+	pi.buf      = &buf;
+
+	error = git_diff_foreach(diff, &pi, print_compact, NULL, NULL);
+
+	git_buf_free(&buf);
+
+	return error;
+}
+
+
+static int print_oid_range(diff_print_info *pi, git_diff_delta *delta)
+{
+	char start_oid[8], end_oid[8];
+
+	/* TODO: Determine a good actual OID range to print */
+	git_oid_to_string(start_oid, sizeof(start_oid), &delta->old.oid);
+	git_oid_to_string(end_oid, sizeof(end_oid), &delta->new.oid);
+
+	/* TODO: Match git diff more closely */
+	if (delta->old.mode == delta->new.mode) {
+		git_buf_printf(pi->buf, "index %s..%s %o\n",
+			start_oid, end_oid, delta->old.mode);
+	} else {
+		if (delta->old.mode == 0) {
+			git_buf_printf(pi->buf, "new file mode %o\n", delta->new.mode);
+		} else if (delta->new.mode == 0) {
+			git_buf_printf(pi->buf, "deleted file mode %o\n", delta->old.mode);
+		} else {
+			git_buf_printf(pi->buf, "old mode %o\n", delta->old.mode);
+			git_buf_printf(pi->buf, "new mode %o\n", delta->new.mode);
+		}
+		git_buf_printf(pi->buf, "index %s..%s\n", start_oid, end_oid);
+	}
+
+	return git_buf_lasterror(pi->buf);
+}
+
+static int print_patch_file(void *data, git_diff_delta *delta, float progress)
+{
+	int error;
+	diff_print_info *pi = data;
+	const char *oldpfx = pi->diff->opts.src_prefix;
+	const char *oldpath = delta->old.path;
+	const char *newpfx = pi->diff->opts.dst_prefix;
+	const char *newpath = delta->new.path;
+
+	GIT_UNUSED_ARG(progress);
+
+	git_buf_clear(pi->buf);
+	git_buf_printf(pi->buf, "diff --git %s%s %s%s\n", oldpfx, delta->old.path, newpfx, delta->new.path);
+	if ((error = print_oid_range(pi, delta)) < GIT_SUCCESS)
+		return error;
+
+	if (git_oid_iszero(&delta->old.oid)) {
+		oldpfx = "";
+		oldpath = "/dev/null";
+	}
+	if (git_oid_iszero(&delta->new.oid)) {
+		oldpfx = "";
+		oldpath = "/dev/null";
+	}
+
+	if (delta->binary != 1) {
+		git_buf_printf(pi->buf, "--- %s%s\n", oldpfx, oldpath);
+		git_buf_printf(pi->buf, "+++ %s%s\n", newpfx, newpath);
+	}
+
+	if (git_buf_lasterror(pi->buf) != GIT_SUCCESS)
+		return git_buf_lasterror(pi->buf);
+
+	error = pi->print_cb(pi->cb_data, GIT_DIFF_LINE_FILE_HDR, pi->buf->ptr);
+	if (error != GIT_SUCCESS || delta->binary != 1)
+		return error;
+
+	git_buf_clear(pi->buf);
+	git_buf_printf(
+		pi->buf, "Binary files %s%s and %s%s differ\n",
+		oldpfx, oldpath, newpfx, newpath);
+	if (git_buf_lasterror(pi->buf) != GIT_SUCCESS)
+		return git_buf_lasterror(pi->buf);
+
+	return pi->print_cb(pi->cb_data, GIT_DIFF_LINE_BINARY, pi->buf->ptr);
+}
+
+static int print_patch_hunk(
+	void *data,
+	git_diff_delta *d,
+	git_diff_range *r,
+	const char *header,
+	size_t header_len)
+{
+	diff_print_info *pi = data;
+
+	GIT_UNUSED_ARG(d);
+	GIT_UNUSED_ARG(r);
+
+	git_buf_clear(pi->buf);
+
+	if (git_buf_printf(pi->buf, "%.*s", (int)header_len, header) == GIT_SUCCESS)
+		return pi->print_cb(pi->cb_data, GIT_DIFF_LINE_HUNK_HDR, pi->buf->ptr);
+	else
+		return git_buf_lasterror(pi->buf);
+}
+
+static int print_patch_line(
+	void *data,
+	git_diff_delta *delta,
+	char line_origin, /* GIT_DIFF_LINE value from above */
+	const char *content,
+	size_t content_len)
+{
+	diff_print_info *pi = data;
+
+	GIT_UNUSED_ARG(delta);
+
+	git_buf_clear(pi->buf);
+
+	if (line_origin == GIT_DIFF_LINE_ADDITION ||
+		line_origin == GIT_DIFF_LINE_DELETION ||
+		line_origin == GIT_DIFF_LINE_CONTEXT)
+		git_buf_printf(pi->buf, "%c%.*s", line_origin, (int)content_len, content);
+	else if (content_len > 0)
+		git_buf_printf(pi->buf, "%.*s", (int)content_len, content);
+
+	if (git_buf_lasterror(pi->buf) != GIT_SUCCESS)
+		return git_buf_lasterror(pi->buf);
+
+	return pi->print_cb(pi->cb_data, line_origin, pi->buf->ptr);
+}
+
+int git_diff_print_patch(
+	git_diff_list *diff,
+	void *cb_data,
+	git_diff_output_fn print_cb)
+{
+	int error;
+	git_buf buf = GIT_BUF_INIT;
+	diff_print_info pi;
+
+	pi.diff     = diff;
+	pi.print_cb = print_cb;
+	pi.cb_data  = cb_data;
+	pi.buf      = &buf;
+
+	error = git_diff_foreach(
+		diff, &pi, print_patch_file, print_patch_hunk, print_patch_line);
+
+	git_buf_free(&buf);
+
+	return error;
+}
+
+
+int git_diff_blobs(
+	git_repository *repo,
+	git_blob *old_blob,
+	git_blob *new_blob,
+	git_diff_options *options,
+	void *cb_data,
+	git_diff_hunk_fn hunk_cb,
+	git_diff_line_fn line_cb)
+{
+	diff_output_info info;
+	git_diff_delta delta;
+	mmfile_t old, new;
+	xpparam_t xdiff_params;
+	xdemitconf_t xdiff_config;
+	xdemitcb_t xdiff_callback;
+
+	assert(repo);
+
+	if (options && (options->flags & GIT_DIFF_REVERSE)) {
+		git_blob *swap = old_blob;
+		old_blob = new_blob;
+		new_blob = swap;
+	}
+
+	if (old_blob) {
+		old.ptr  = (char *)git_blob_rawcontent(old_blob);
+		old.size = git_blob_rawsize(old_blob);
+	} else {
+		old.ptr  = "";
+		old.size = 0;
+	}
+
+	if (new_blob) {
+		new.ptr  = (char *)git_blob_rawcontent(new_blob);
+		new.size = git_blob_rawsize(new_blob);
+	} else {
+		new.ptr  = "";
+		new.size = 0;
+	}
+
+	/* populate a "fake" delta record */
+	delta.status = old.ptr ?
+		(new.ptr ? GIT_STATUS_MODIFIED : GIT_STATUS_DELETED) :
+		(new.ptr ? GIT_STATUS_ADDED : GIT_STATUS_UNTRACKED);
+	delta.old.mode = 0100644; /* can't know the truth from a blob alone */
+	delta.new.mode = 0100644;
+	git_oid_cpy(&delta.old.oid, git_object_id((const git_object *)old_blob));
+	git_oid_cpy(&delta.new.oid, git_object_id((const git_object *)new_blob));
+	delta.old.path = NULL;
+	delta.new.path = NULL;
+	delta.similarity = 0;
+
+	info.diff    = NULL;
+	info.delta   = &delta;
+	info.cb_data = cb_data;
+	info.hunk_cb = hunk_cb;
+	info.line_cb = line_cb;
+
+	setup_xdiff_options(options, &xdiff_config, &xdiff_params);
+	memset(&xdiff_callback, 0, sizeof(xdiff_callback));
+	xdiff_callback.outf = diff_output_cb;
+	xdiff_callback.priv = &info;
+
+	xdl_diff(&old, &new, &xdiff_params, &xdiff_config, &xdiff_callback);
+
+	return GIT_SUCCESS;
+}
diff --git a/src/fileops.c b/src/fileops.c
index d2b4af51e..6e45ff8a8 100644
--- a/src/fileops.c
+++ b/src/fileops.c
@@ -79,10 +79,6 @@ git_off_t git_futils_filesize(git_file fd)
 	return sb.st_size;
 }
 
-#define GIT_MODE_PERMS_MASK			0777
-#define GIT_CANONICAL_PERMS(MODE)	(((MODE) & 0100) ? 0755 : 0644)
-#define GIT_MODE_TYPE(MODE)			((MODE) & ~GIT_MODE_PERMS_MASK)
-
 mode_t git_futils_canonical_mode(mode_t raw_mode)
 {
 	if (S_ISREG(raw_mode))
@@ -181,6 +177,15 @@ int git_futils_mmap_ro(git_map *out, git_file fd, git_off_t begin, size_t len)
 	return p_mmap(out, len, GIT_PROT_READ, GIT_MAP_SHARED, fd, begin);
 }
 
+int git_futils_mmap_ro_file(git_map *out, const char *path)
+{
+	git_file fd = p_open(path, O_RDONLY /* | O_NOATIME */);
+	size_t len = git_futils_filesize(fd);
+	int result = git_futils_mmap_ro(out, fd, 0, len);
+	p_close(fd);
+	return result;
+}
+
 void git_futils_mmap_free(git_map *out)
 {
 	p_munmap(out);
diff --git a/src/fileops.h b/src/fileops.h
index 43ef21521..ab57b6f38 100644
--- a/src/fileops.h
+++ b/src/fileops.h
@@ -81,6 +81,10 @@ extern int git_futils_mv_withpath(const char *from, const char *to, const mode_t
  */
 extern git_off_t git_futils_filesize(git_file fd);
 
+#define GIT_MODE_PERMS_MASK			0777
+#define GIT_CANONICAL_PERMS(MODE)	(((MODE) & 0100) ? 0755 : 0644)
+#define GIT_MODE_TYPE(MODE)			((MODE) & ~GIT_MODE_PERMS_MASK)
+
 /**
  * Convert a mode_t from the OS to a legal git mode_t value.
  */
@@ -109,6 +113,19 @@ extern int git_futils_mmap_ro(
 	size_t len);
 
 /**
+ * Read-only map an entire file.
+ *
+ * @param out buffer to populate with the mapping information.
+ * @param path path to file to be opened.
+ * @return
+ * - GIT_SUCCESS on success;
+ * - GIT_EOSERR on an unspecified OS related error.
+ */
+extern int git_futils_mmap_ro_file(
+	git_map *out,
+	const char *path);
+
+/**
  * Release the memory associated with a previous memory mapping.
  * @param map the mapping description previously configured.
  */
diff --git a/src/iterator.c b/src/iterator.c
index 8255d4c9a..c026c3c09 100644
--- a/src/iterator.c
+++ b/src/iterator.c
@@ -143,6 +143,16 @@ static void tree_iterator__free(git_iterator *self)
 	git_buf_free(&ti->path);
 }
 
+static int tree_iterator__reset(git_iterator *self)
+{
+	tree_iterator *ti = (tree_iterator *)self;
+	while (ti->stack && ti->stack->next)
+		tree_iterator__pop_frame(ti);
+	if (ti->stack)
+		ti->stack->index = 0;
+	return tree_iterator__expand_tree(ti);
+}
+
 int git_iterator_for_tree(
 	git_repository *repo, git_tree *tree, git_iterator **iter)
 {
@@ -155,6 +165,7 @@ int git_iterator_for_tree(
 	ti->base.current = tree_iterator__current;
 	ti->base.at_end  = tree_iterator__at_end;
 	ti->base.advance = tree_iterator__advance;
+	ti->base.reset   = tree_iterator__reset;
 	ti->base.free    = tree_iterator__free;
 	ti->repo         = repo;
 	ti->stack        = tree_iterator__alloc_frame(tree);
@@ -199,6 +210,13 @@ static int index_iterator__advance(
 	return GIT_SUCCESS;
 }
 
+static int index_iterator__reset(git_iterator *self)
+{
+	index_iterator *ii = (index_iterator *)self;
+	ii->current = 0;
+	return GIT_SUCCESS;
+}
+
 static void index_iterator__free(git_iterator *self)
 {
 	index_iterator *ii = (index_iterator *)self;
@@ -217,6 +235,7 @@ int git_iterator_for_index(git_repository *repo, git_iterator **iter)
 	ii->base.current = index_iterator__current;
 	ii->base.at_end  = index_iterator__at_end;
 	ii->base.advance = index_iterator__advance;
+	ii->base.reset   = index_iterator__reset;
 	ii->base.free    = index_iterator__free;
 	ii->current      = 0;
 
@@ -251,7 +270,7 @@ static workdir_iterator_frame *workdir_iterator__alloc_frame(void)
 	workdir_iterator_frame *wf = git__calloc(1, sizeof(workdir_iterator_frame));
 	if (wf == NULL)
 		return wf;
-	if (git_vector_init(&wf->entries, 0, git__strcmp_cb) != GIT_SUCCESS) {
+	if (git_vector_init(&wf->entries, 0, git_path_with_stat_cmp) != GIT_SUCCESS) {
 		git__free(wf);
 		return NULL;
 	}
@@ -261,7 +280,7 @@ static workdir_iterator_frame *workdir_iterator__alloc_frame(void)
 static void workdir_iterator__free_frame(workdir_iterator_frame *wf)
 {
 	unsigned int i;
-	char *path;
+	git_path_with_stat *path;
 
 	git_vector_foreach(&wf->entries, i, path)
 		git__free(path);
@@ -278,10 +297,7 @@ static int workdir_iterator__expand_dir(workdir_iterator *wi)
 	if (wf == NULL)
 		return GIT_ENOMEM;
 
-	/* allocate dir entries with extra byte (the "1" param) so we
-	 * can suffix directory names with a "/".
-	 */
-	error = git_path_dirload(wi->path.ptr, wi->root_len, 1, &wf->entries);
+	error = git_path_dirload_with_stat(wi->path.ptr, wi->root_len, &wf->entries);
 	if (error < GIT_SUCCESS || wf->entries.length == 0) {
 		workdir_iterator__free_frame(wf);
 		return GIT_ENOTFOUND;
@@ -319,7 +335,7 @@ static int workdir_iterator__advance(
 	int error;
 	workdir_iterator *wi = (workdir_iterator *)self;
 	workdir_iterator_frame *wf;
-	const char *next;
+	git_path_with_stat *next;
 
 	if (entry != NULL)
 		*entry = NULL;
@@ -330,7 +346,7 @@ static int workdir_iterator__advance(
 	while ((wf = wi->stack) != NULL) {
 		next = git_vector_get(&wf->entries, ++wf->index);
 		if (next != NULL) {
-			if (strcmp(next, DOT_GIT) == 0)
+			if (strcmp(next->path, DOT_GIT "/") == 0)
 				continue;
 			/* else found a good entry */
 			break;
@@ -355,6 +371,20 @@ static int workdir_iterator__advance(
 	return error;
 }
 
+static int workdir_iterator__reset(git_iterator *self)
+{
+	workdir_iterator *wi = (workdir_iterator *)self;
+	while (wi->stack != NULL && wi->stack->next != NULL) {
+		workdir_iterator_frame *wf = wi->stack;
+		wi->stack = wf->next;
+		workdir_iterator__free_frame(wf);
+		git_ignore__pop_dir(&wi->ignores);
+	}
+	if (wi->stack)
+		wi->stack->index = 0;
+	return GIT_SUCCESS;
+}
+
 static void workdir_iterator__free(git_iterator *self)
 {
 	workdir_iterator *wi = (workdir_iterator *)self;
@@ -372,39 +402,35 @@ static void workdir_iterator__free(git_iterator *self)
 static int workdir_iterator__update_entry(workdir_iterator *wi)
 {
 	int error;
-	struct stat st;
-	char *relpath = git_vector_get(&wi->stack->entries, wi->stack->index);
+	git_path_with_stat *ps = git_vector_get(&wi->stack->entries, wi->stack->index);
 
-	error = git_buf_joinpath(
-		&wi->path, git_repository_workdir(wi->repo), relpath);
+	git_buf_truncate(&wi->path, wi->root_len);
+	error = git_buf_put(&wi->path, ps->path, ps->path_len);
 	if (error < GIT_SUCCESS)
 		return error;
 
 	memset(&wi->entry, 0, sizeof(wi->entry));
-	wi->entry.path = relpath;
+	wi->entry.path = ps->path;
 
 	/* skip over .git directory */
-	if (strcmp(relpath, DOT_GIT) == 0)
+	if (strcmp(ps->path, DOT_GIT "/") == 0)
 		return workdir_iterator__advance((git_iterator *)wi, NULL);
 
 	/* if there is an error processing the entry, treat as ignored */
 	wi->is_ignored = 1;
 
-	if (p_lstat(wi->path.ptr, &st) < 0)
-		return GIT_SUCCESS;
-
 	/* TODO: remove shared code for struct stat conversion with index.c */
-	wi->entry.ctime.seconds = (git_time_t)st.st_ctime;
-	wi->entry.mtime.seconds = (git_time_t)st.st_mtime;
-	wi->entry.dev  = st.st_rdev;
-	wi->entry.ino  = st.st_ino;
-	wi->entry.mode = git_futils_canonical_mode(st.st_mode);
-	wi->entry.uid  = st.st_uid;
-	wi->entry.gid  = st.st_gid;
-	wi->entry.file_size = st.st_size;
+	wi->entry.ctime.seconds = (git_time_t)ps->st.st_ctime;
+	wi->entry.mtime.seconds = (git_time_t)ps->st.st_mtime;
+	wi->entry.dev  = ps->st.st_rdev;
+	wi->entry.ino  = ps->st.st_ino;
+	wi->entry.mode = git_futils_canonical_mode(ps->st.st_mode);
+	wi->entry.uid  = ps->st.st_uid;
+	wi->entry.gid  = ps->st.st_gid;
+	wi->entry.file_size = ps->st.st_size;
 
 	/* if this is a file type we don't handle, treat as ignored */
-	if (st.st_mode == 0)
+	if (wi->entry.mode == 0)
 		return GIT_SUCCESS;
 
 	/* okay, we are far enough along to look up real ignore rule */
@@ -412,18 +438,10 @@ static int workdir_iterator__update_entry(workdir_iterator *wi)
 	if (error != GIT_SUCCESS)
 		return GIT_SUCCESS;
 
-	if (S_ISDIR(st.st_mode)) {
-		if (git_path_contains(&wi->path, DOT_GIT) == GIT_SUCCESS) {
-			/* create submodule entry */
-			wi->entry.mode = S_IFGITLINK;
-		} else {
-			/* create directory entry that can be advanced into as needed */
-			size_t pathlen = strlen(wi->entry.path);
-			wi->entry.path[pathlen] = '/';
-			wi->entry.path[pathlen + 1] = '\0';
-			wi->entry.mode = S_IFDIR;
-		}
-	}
+	/* detect submodules */
+	if (S_ISDIR(wi->entry.mode) &&
+		git_path_contains(&wi->path, DOT_GIT) == GIT_SUCCESS)
+		wi->entry.mode = S_IFGITLINK;
 
 	return GIT_SUCCESS;
 }
@@ -439,11 +457,14 @@ int git_iterator_for_workdir(git_repository *repo, git_iterator **iter)
 	wi->base.current = workdir_iterator__current;
 	wi->base.at_end  = workdir_iterator__at_end;
 	wi->base.advance = workdir_iterator__advance;
+	wi->base.reset   = workdir_iterator__reset;
 	wi->base.free    = workdir_iterator__free;
 	wi->repo         = repo;
 
 	error = git_buf_sets(&wi->path, git_repository_workdir(repo));
 	if (error == GIT_SUCCESS)
+		error = git_path_to_dir(&wi->path);
+	if (error == GIT_SUCCESS)
 		error = git_ignore__for_path(repo, "", &wi->ignores);
 	if (error != GIT_SUCCESS) {
 		git__free(wi);
diff --git a/src/iterator.h b/src/iterator.h
index ac30b4ded..aa78c9f29 100644
--- a/src/iterator.h
+++ b/src/iterator.h
@@ -23,6 +23,7 @@ struct git_iterator {
 	int (*current)(git_iterator *, const git_index_entry **);
 	int (*at_end)(git_iterator *);
 	int (*advance)(git_iterator *, const git_index_entry **);
+	int (*reset)(git_iterator *);
 	void (*free)(git_iterator *);
 };
 
@@ -60,6 +61,11 @@ GIT_INLINE(int) git_iterator_advance(
 	return iter->advance(iter, entry);
 }
 
+GIT_INLINE(int) git_iterator_reset(git_iterator *iter)
+{
+	return iter->reset(iter);
+}
+
 GIT_INLINE(void) git_iterator_free(git_iterator *iter)
 {
 	iter->free(iter);
diff --git a/src/oid.c b/src/oid.c
index 92d8d1e89..a1f010927 100644
--- a/src/oid.c
+++ b/src/oid.c
@@ -190,6 +190,16 @@ int git_oid_streq(const git_oid *a, const char *str)
 	return git_oid_cmp(a, &id) == 0 ? GIT_SUCCESS : GIT_ERROR;
 }
 
+int git_oid_iszero(const git_oid *oid_a)
+{
+	const unsigned char *a = oid_a->id;
+	unsigned int i;
+	for (i = 0; i < GIT_OID_RAWSZ; ++i, ++a)
+		if (*a != 0)
+			return 0;
+	return 1;
+}
+
 typedef short node_index;
 
 typedef union {
diff --git a/src/path.c b/src/path.c
index ec40f4b06..ceae2abcf 100644
--- a/src/path.c
+++ b/src/path.c
@@ -583,3 +583,46 @@ int git_path_dirload(
 	return GIT_SUCCESS;
 }
 
+int git_path_with_stat_cmp(const void *a, const void *b)
+{
+	const git_path_with_stat *psa = a, *psb = b;
+	return git__strcmp_cb(psa->path, psb->path);
+}
+
+int git_path_dirload_with_stat(
+	const char *path,
+	size_t prefix_len,
+	git_vector *contents)
+{
+	int error;
+	unsigned int i;
+	git_path_with_stat *ps;
+	git_buf full = GIT_BUF_INIT;
+
+	if ((error = git_buf_set(&full, path, prefix_len)) != GIT_SUCCESS)
+		return error;
+
+	if ((error = git_path_dirload(path, prefix_len,
+		sizeof(git_path_with_stat) + 1, contents)) != GIT_SUCCESS) {
+		git_buf_free(&full);
+		return error;
+	}
+
+	git_vector_foreach(contents, i, ps) {
+		size_t path_len = strlen((char *)ps);
+
+		memmove(ps->path, ps, path_len + 1);
+		ps->path_len = path_len;
+
+		git_buf_joinpath(&full, full.ptr, ps->path);
+		p_lstat(full.ptr, &ps->st);
+		git_buf_truncate(&full, prefix_len);
+
+		if (S_ISDIR(ps->st.st_mode)) {
+			ps->path[path_len] = '/';
+			ps->path[path_len + 1] = '\0';
+		}
+	}
+
+	return error;
+}
diff --git a/src/path.h b/src/path.h
index abe6c2217..981fdd6a4 100644
--- a/src/path.h
+++ b/src/path.h
@@ -246,4 +246,26 @@ extern int git_path_dirload(
 	size_t alloc_extra,
 	git_vector *contents);
 
+
+typedef struct {
+	struct stat st;
+	size_t      path_len;
+	char        path[GIT_FLEX_ARRAY];
+} git_path_with_stat;
+
+extern int git_path_with_stat_cmp(const void *a, const void *b);
+
+/**
+ * Load all directory entries along with stat info into a vector.
+ *
+ * This is just like git_path_dirload except that each entry in the
+ * vector is a git_path_with_stat structure that contains both the
+ * path and the stat info, plus directories will have a / suffixed
+ * to their path name.
+ */
+extern int git_path_dirload_with_stat(
+	const char *path,
+	size_t prefix_len,
+	git_vector *contents);
+
 #endif
diff --git a/src/posix.h b/src/posix.h
index 0cce1fe34..fb17cba6c 100644
--- a/src/posix.h
+++ b/src/posix.h
@@ -66,4 +66,6 @@ extern int p_rename(const char *from, const char *to);
 #	include "unix/posix.h"
 #endif
 
+#define p_readdir_r(d,e,r) readdir_r(d,e,r)
+
 #endif
diff --git a/src/status.c b/src/status.c
index 106e5005d..76ae83138 100644
--- a/src/status.c
+++ b/src/status.c
@@ -131,7 +131,7 @@ static int status_entry_update_ignore(struct status_entry *e, git_ignores *ignor
 	if ((error = git_ignore__lookup(ignores, path, &ignored)) == GIT_SUCCESS &&
 		ignored)
 		e->status_flags =
-			(e->status_flags & ~GIT_STATUS_WT_NEW) | GIT_STATUS_IGNORED;
+			(e->status_flags & ~GIT_STATUS_WT_NEW) | GIT_STATUS_WT_IGNORED;
 
 	return error;
 }
diff --git a/src/unix/posix.h b/src/unix/posix.h
index 2b0d85bb5..9973acf30 100644
--- a/src/unix/posix.h
+++ b/src/unix/posix.h
@@ -21,6 +21,5 @@
 #define p_snprintf(b, c, f, ...) snprintf(b, c, f, __VA_ARGS__)
 #define p_mkstemp(p) mkstemp(p)
 #define p_setenv(n,v,o) setenv(n,v,o)
-#define p_readdir_r(d,e,r) readdir_r(d,e,r)
 
 #endif
diff --git a/src/vector.c b/src/vector.c
index e109704ab..7513ea3f0 100644
--- a/src/vector.c
+++ b/src/vector.c
@@ -220,4 +220,14 @@ void git_vector_clear(git_vector *v)
 	v->sorted = 1;
 }
 
+void git_vector_swap(git_vector *a, git_vector *b)
+{
+	git_vector t;
+
+	if (!a || !b || a == b)
+		return;
 
+	memcpy(&t, a, sizeof(t));
+	memcpy(a, b, sizeof(t));
+	memcpy(b, &t, sizeof(t));
+}
diff --git a/src/vector.h b/src/vector.h
index 44635ae14..180edbf7c 100644
--- a/src/vector.h
+++ b/src/vector.h
@@ -24,6 +24,7 @@ typedef struct git_vector {
 int git_vector_init(git_vector *v, unsigned int initial_size, git_vector_cmp cmp);
 void git_vector_free(git_vector *v);
 void git_vector_clear(git_vector *v);
+void git_vector_swap(git_vector *a, git_vector *b);
 
 int git_vector_search(git_vector *v, const void *entry);
 int git_vector_search2(git_vector *v, git_vector_cmp cmp, const void *key);
@@ -38,6 +39,11 @@ GIT_INLINE(void *) git_vector_get(git_vector *v, unsigned int position)
 	return (position < v->length) ? v->contents[position] : NULL;
 }
 
+GIT_INLINE(const void *) git_vector_get_const(const git_vector *v, unsigned int position)
+{
+	return (position < v->length) ? v->contents[position] : NULL;
+}
+
 GIT_INLINE(void *) git_vector_last(git_vector *v)
 {
 	return (v->length > 0) ? git_vector_get(v, v->length - 1) : NULL;
diff --git a/src/win32/dir.c b/src/win32/dir.c
index 23bc55558..035e2b685 100644
--- a/src/win32/dir.c
+++ b/src/win32/dir.c
@@ -58,8 +58,11 @@ git__DIR *git__opendir(const char *dir)
 	return new;
 }
 
-int git__readdir_r(
-	git__DIR *d, struct git__dirent *entry, struct git__dirent **result)
+int git__readdir_ext(
+	git__DIR *d,
+	struct git__dirent *entry,
+	struct git__dirent **result,
+	int *is_dir)
 {
 	if (!d || !entry || !result || d->h == INVALID_HANDLE_VALUE)
 		return -1;
@@ -80,13 +83,16 @@ int git__readdir_r(
 		entry->d_name, GIT_PATH_MAX, NULL, NULL);
 
 	*result = entry;
+	if (is_dir != NULL)
+		*is_dir = ((d->f.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0);
+
 	return 0;
 }
 
 struct git__dirent *git__readdir(git__DIR *d)
 {
 	struct git__dirent *result;
-	if (git__readdir_r(d, &d->entry, &result) < 0)
+	if (git__readdir_ext(d, &d->entry, &result, NULL) < 0)
 		return NULL;
 	return result;
 }
diff --git a/src/win32/dir.h b/src/win32/dir.h
index fc54e2977..c816d79bb 100644
--- a/src/win32/dir.h
+++ b/src/win32/dir.h
@@ -24,7 +24,8 @@ typedef struct {
 
 extern git__DIR *git__opendir(const char *);
 extern struct git__dirent *git__readdir(git__DIR *);
-extern int git__readdir_r(git__DIR*, struct git__dirent*, struct git__dirent**);
+extern int git__readdir_ext(
+	git__DIR *, struct git__dirent *, struct git__dirent **, int *);
 extern void git__rewinddir(git__DIR *);
 extern int git__closedir(git__DIR *);
 
@@ -33,10 +34,9 @@ extern int git__closedir(git__DIR *);
 #	define DIR git__DIR
 #	define opendir	git__opendir
 #	define readdir	git__readdir
+#   define readdir_r(d,e,r) git__readdir_ext((d),(e),(r),NULL)
 #	define rewinddir git__rewinddir
 #	define closedir git__closedir
 # endif
 
-#define p_readdir_r(d,e,r) git__readdir_r(d,e,r)
-
 #endif /* INCLUDE_dir_h__ */
diff --git a/src/xdiff/xdiff.h b/src/xdiff/xdiff.h
index e18b4d2f6..cb8b235b5 100644
--- a/src/xdiff/xdiff.h
+++ b/src/xdiff/xdiff.h
@@ -69,12 +69,12 @@ extern "C" {
 
 typedef struct s_mmfile {
 	char *ptr;
-	long size;
+	size_t size;
 } mmfile_t;
 
 typedef struct s_mmbuffer {
 	char *ptr;
-	long size;
+	size_t size;
 } mmbuffer_t;
 
 typedef struct s_xpparam {