Add complex checkout test and then fix checkout

This started as a complex new test for checkout going through the
"typechanges" test repository, but that revealed numerous issues
with checkout, including:

* complete failure with submodules
* failure to create blobs with exec bits
* problems when replacing a tree with a blob because the tree
  "example/" sorts after the blob "example" so the delete was
  being processed after the single file blob was created

This fixes most of those problems and includes a number of other
minor changes that made it easier to do that, including improving
the TYPECHANGE support in diff/status, etc.

12 files changed, 324 insertions, 138 deletions

diff --git a/src/checkout.c b/src/checkout.c
index ee6e043d5..6f5cfffd7 100644
--- a/src/checkout.c
+++ b/src/checkout.c
@@ -30,6 +30,7 @@ struct checkout_diff_data
 	git_indexer_stats *stats;
 	git_repository *owner;
 	bool can_symlink;
+	bool create_submodules;
 	int error;
 };
 
@@ -40,18 +41,27 @@ static int buffer_to_file(
 	int file_open_flags,
 	mode_t file_mode)
 {
-	int fd, error_write, error_close;
+	int fd, error, error_close;
 
-	if (git_futils_mkpath2file(path, dir_mode) < 0)
-		return -1;
+	if ((error = git_futils_mkpath2file(path, dir_mode)) < 0)
+		return error;
 
 	if ((fd = p_open(path, file_open_flags, file_mode)) < 0)
-		return -1;
+		return fd;
+
+	error = p_write(fd, git_buf_cstr(buffer), git_buf_len(buffer));
 
-	error_write = p_write(fd, git_buf_cstr(buffer), git_buf_len(buffer));
 	error_close = p_close(fd);
 
-	return error_write ? error_write : error_close;
+	if (!error)
+		error = error_close;
+
+	if (!error &&
+		(file_mode & 0100) != 0 &&
+		(error = p_chmod(path, file_mode)) < 0)
+		giterr_set(GITERR_OS, "Failed to set permissions on '%s'", path);
+
+	return error;
 }
 
 static int blob_content_to_file(
@@ -125,107 +135,122 @@ static int blob_content_to_link(git_blob *blob, const char *path, bool can_symli
 	return error;
 }
 
+static int checkout_submodule(
+	struct checkout_diff_data *data,
+	const git_diff_file *file)
+{
+	if (git_futils_mkdir(
+			file->path, git_repository_workdir(data->owner),
+			data->checkout_opts->dir_mode, GIT_MKDIR_PATH) < 0)
+		return -1;
+
+	/* TODO: two cases:
+	 * 1 - submodule already checked out, but we need to move the HEAD
+	 *     to the new OID, or
+	 * 2 - submodule not checked out and we should recursively check it out
+	 *
+	 * Checkout will not execute a pull request on the submodule, but a
+	 * clone command should probably be able to.  Do we need a submodule
+	 * callback option?
+	 */
+
+	return 0;
+}
+
 static int checkout_blob(
-	git_repository *repo,
-	const git_oid *blob_oid,
-	const char *path,
-	mode_t filemode,
-	bool can_symlink,
-	git_checkout_opts *opts)
+	struct checkout_diff_data *data,
+	const git_diff_file *file)
 {
 	git_blob *blob;
 	int error;
 
-	if ((error = git_blob_lookup(&blob, repo, blob_oid)) < 0)
-		return error; /* Add an error message */
+	git_buf_truncate(data->path, data->workdir_len);
+	if (git_buf_joinpath(data->path, git_buf_cstr(data->path), file->path) < 0)
+		return -1;
+
+	if ((error = git_blob_lookup(&blob, data->owner, &file->oid)) < 0)
+		return error;
 
-	if (S_ISLNK(filemode))
-		error = blob_content_to_link(blob, path, can_symlink);
+	if (S_ISLNK(file->mode))
+		error = blob_content_to_link(
+			blob, git_buf_cstr(data->path), data->can_symlink);
 	else
-		error = blob_content_to_file(blob, path, filemode, opts);
+		error = blob_content_to_file(
+			blob, git_buf_cstr(data->path), file->mode, data->checkout_opts);
 
 	git_blob_free(blob);
 
 	return error;
 }
 
-static int checkout_diff_fn(
-	void *cb_data,
-	const git_diff_delta *delta,
-	float progress)
+static int checkout_remove_the_old(
+	void *cb_data, const git_diff_delta *delta, float progress)
 {
 	struct checkout_diff_data *data = cb_data;
-	int error = 0;
 	git_checkout_opts *opts = data->checkout_opts;
-	bool do_delete = false, do_checkout_blob = false;
-
-	data->stats->processed = (unsigned int)(data->stats->total * progress);
-
-	switch (delta->status) {
-	case GIT_DELTA_UNTRACKED:
-		if ((opts->checkout_strategy & GIT_CHECKOUT_REMOVE_UNTRACKED) != 0)
-			do_delete = true;
-		break;
-
-	case GIT_DELTA_MODIFIED:
-	case GIT_DELTA_TYPECHANGE:
-		if (!(opts->checkout_strategy & GIT_CHECKOUT_OVERWRITE_MODIFIED)) {
-
-			if (opts->skipped_notify_cb != NULL &&
-				opts->skipped_notify_cb(
-					delta->new_file.path,
-					&delta->old_file.oid,
-					delta->old_file.mode,
-					opts->notify_payload) != 0)
-			{
-				giterr_clear();
-				error = GIT_EUSER;
-			}
-
-			goto cleanup;
-		}
-
-		do_checkout_blob = true;
 
-		if (delta->status == GIT_DELTA_TYPECHANGE)
-			do_delete = true;
-		break;
+	GIT_UNUSED(progress);
+	data->stats->processed++;
+
+	if ((delta->status == GIT_DELTA_UNTRACKED &&
+		 (opts->checkout_strategy & GIT_CHECKOUT_REMOVE_UNTRACKED) != 0) ||
+		(delta->status == GIT_DELTA_TYPECHANGE &&
+		 (opts->checkout_strategy & GIT_CHECKOUT_OVERWRITE_MODIFIED) != 0))
+	{
+		data->error = git_futils_rmdir_r(
+			delta->new_file.path,
+			git_repository_workdir(data->owner),
+			GIT_DIRREMOVAL_FILES_AND_DIRS);
+	}
 
-	case GIT_DELTA_DELETED:
-		if ((opts->checkout_strategy & GIT_CHECKOUT_CREATE_MISSING) != 0)
-			do_checkout_blob = true;
-		break;
+	return data->error;
+}
 
-	default:
-		giterr_set(GITERR_INVALID, "Unexpected status (%d) for path '%s'.",
-			delta->status, delta->new_file.path);
-		error = -1;
-		goto cleanup;
+static int checkout_create_the_new(
+	void *cb_data, const git_diff_delta *delta, float progress)
+{
+	int error = 0;
+	struct checkout_diff_data *data = cb_data;
+	git_checkout_opts *opts = data->checkout_opts;
+	bool do_checkout = false, do_notify = false;
+
+	GIT_UNUSED(progress);
+	data->stats->processed++;
+
+	if (delta->status == GIT_DELTA_MODIFIED ||
+		delta->status == GIT_DELTA_TYPECHANGE)
+	{
+		if ((opts->checkout_strategy & GIT_CHECKOUT_OVERWRITE_MODIFIED) != 0)
+			do_checkout = true;
+		else if (opts->skipped_notify_cb != NULL)
+			do_notify = !data->create_submodules;
+	}
+	else if (delta->status == GIT_DELTA_DELETED &&
+			 (opts->checkout_strategy & GIT_CHECKOUT_CREATE_MISSING) != 0)
+		do_checkout = true;
+
+	if (do_notify) {
+		if (opts->skipped_notify_cb(
+			delta->old_file.path, &delta->old_file.oid,
+			delta->old_file.mode, opts->notify_payload))
+		{
+			giterr_clear();
+			error = GIT_EUSER;
+		}
 	}
 
-	git_buf_truncate(data->path, data->workdir_len);
-
-	if ((error = git_buf_joinpath(
-			data->path, git_buf_cstr(data->path), delta->new_file.path)) < 0)
-		goto cleanup;
+	if (do_checkout) {
+		bool is_submodule = S_ISGITLINK(delta->old_file.mode);
 
-	if (do_delete &&
-		(error = git_futils_rmdir_r(
-			git_buf_cstr(data->path), GIT_DIRREMOVAL_FILES_AND_DIRS)) < 0)
-		goto cleanup;
+		if (!is_submodule && !data->create_submodules)
+			error = checkout_blob(data, &delta->old_file);
 
-	if (do_checkout_blob)
-		error = checkout_blob(
-			data->owner,
-			&delta->old_file.oid,
-			git_buf_cstr(data->path),
-			delta->old_file.mode,
-			data->can_symlink,
-			opts);
+		else if (is_submodule && data->create_submodules)
+			error = checkout_submodule(data, &delta->old_file);
+	}
 
-cleanup:
 	if (error)
-		data->error = error; /* preserve real error */
+		data->error = error;
 
 	return error;
 }
@@ -278,7 +303,6 @@ int git_checkout_index(
 	git_checkout_opts *opts,
 	git_indexer_stats *stats)
 {
-	git_index *index = NULL;
 	git_diff_list *diff = NULL;
 	git_indexer_stats dummy_stats;
 
@@ -292,11 +316,13 @@ int git_checkout_index(
 
 	assert(repo);
 
-	if ((git_repository__ensure_not_bare(repo, "checkout")) < 0)
-		return GIT_EBAREREPO;
+	if ((error = git_repository__ensure_not_bare(repo, "checkout")) < 0)
+		return error;
 
-	diff_opts.flags = GIT_DIFF_INCLUDE_UNTRACKED |
-		GIT_DIFF_DONT_SPLIT_TYPECHANGE;
+	diff_opts.flags =
+		GIT_DIFF_INCLUDE_UNTRACKED |
+		GIT_DIFF_INCLUDE_TYPECHANGE |
+		GIT_DIFF_SKIP_BINARY_CHECK;
 
 	if (opts && opts->paths.count > 0)
 		diff_opts.pathspec = opts->paths;
@@ -313,11 +339,7 @@ int git_checkout_index(
 		stats = &dummy_stats;
 
 	stats->processed = 0;
-
-	if ((git_repository_index(&index, repo)) < 0)
-		goto cleanup;
-
-	stats->total = git_index_entrycount(index);
+	stats->total = (unsigned int)git_diff_num_deltas(diff) * 3 /* # passes */;
 
 	memset(&data, 0, sizeof(data));
 
@@ -330,15 +352,33 @@ int git_checkout_index(
 	if ((error = retrieve_symlink_capabilities(repo, &data.can_symlink)) < 0)
 		goto cleanup;
 
-	error = git_diff_foreach(diff, &data, checkout_diff_fn, NULL, NULL);
+	/* Checkout is best performed with three passes through the diff.
+	 *
+	 * 1. First do removes, because we iterate in alphabetical order, thus
+	 *    a new untracked directory will end up sorted *after* a blob that
+	 *    should be checked out with the same name.
+	 * 2. Then checkout all blobs.
+	 * 3. Then checkout all submodules in case a new .gitmodules blob was
+	 *    checked out during pass #2.
+	 */
 
+	if (!(error = git_diff_foreach(
+			diff, &data, checkout_remove_the_old, NULL, NULL)) &&
+		!(error = git_diff_foreach(
+			diff, &data, checkout_create_the_new, NULL, NULL)))
+	{
+		data.create_submodules = true;
+		error = git_diff_foreach(
+			diff, &data, checkout_create_the_new, NULL, NULL);
+	}
+
+cleanup:
 	if (error == GIT_EUSER)
 		error = (data.error != 0) ? data.error : -1;
 
-cleanup:
-	git_index_free(index);
 	git_diff_list_free(diff);
 	git_buf_free(&workdir);
+
 	return error;
 }
 
diff --git a/src/clone.c b/src/clone.c
index 00e39d3b5..d16d09843 100644
--- a/src/clone.c
+++ b/src/clone.c
@@ -314,7 +314,7 @@ static int clone_internal(
 		if ((retcode = setup_remotes_and_fetch(repo, origin_url, fetch_stats)) < 0) {
 			/* Failed to fetch; clean up */
 			git_repository_free(repo);
-			git_futils_rmdir_r(path, GIT_DIRREMOVAL_FILES_AND_DIRS);
+			git_futils_rmdir_r(path, NULL, GIT_DIRREMOVAL_FILES_AND_DIRS);
 		} else {
 			*out = repo;
 			retcode = 0;
diff --git a/src/diff.c b/src/diff.c
index f88bda4aa..7f500b8e8 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -291,6 +291,36 @@ static int diff_delta__from_two(
 	return 0;
 }
 
+static git_diff_delta *diff_delta__last_for_item(
+	git_diff_list *diff,
+	const git_index_entry *item)
+{
+	git_diff_delta *delta = git_vector_last(&diff->deltas);
+	if (!delta)
+		return NULL;
+
+	switch (delta->status) {
+	case GIT_DELTA_UNMODIFIED:
+	case GIT_DELTA_DELETED:
+		if (git_oid_cmp(&delta->old_file.oid, &item->oid) == 0)
+			return delta;
+		break;
+	case GIT_DELTA_ADDED:
+		if (git_oid_cmp(&delta->new_file.oid, &item->oid) == 0)
+			return delta;
+		break;
+	case GIT_DELTA_MODIFIED:
+		if (git_oid_cmp(&delta->old_file.oid, &item->oid) == 0 ||
+			git_oid_cmp(&delta->new_file.oid, &item->oid) == 0)
+			return delta;
+		break;
+	default:
+		break;
+	}
+
+	return NULL;
+}
+
 static char *diff_strdup_prefix(git_pool *pool, const char *prefix)
 {
 	size_t len = strlen(prefix);
@@ -368,6 +398,10 @@ static git_diff_list *git_diff_list_alloc(
 		diff->opts.new_prefix = swap;
 	}
 
+	/* INCLUDE_TYPECHANGE_TREES implies INCLUDE_TYPECHANGE */
+	if (diff->opts.flags & GIT_DIFF_INCLUDE_TYPECHANGE_TREES)
+		diff->opts.flags |= GIT_DIFF_INCLUDE_TYPECHANGE;
+
 	/* only copy pathspec if it is "interesting" so we can test
 	 * diff->pathspec.length > 0 to know if it is worth calling
 	 * fnmatch as we iterate.
@@ -537,7 +571,7 @@ static int maybe_modified(
 
 	/* if basic type of file changed, then split into delete and add */
 	else if (GIT_MODE_TYPE(omode) != GIT_MODE_TYPE(nmode)) {
-		if ((diff->opts.flags & GIT_DIFF_DONT_SPLIT_TYPECHANGE) != 0)
+		if ((diff->opts.flags & GIT_DIFF_INCLUDE_TYPECHANGE) != 0)
 			status = GIT_DELTA_TYPECHANGE;
 		else {
 			if (diff_delta__from_one(diff, GIT_DELTA_DELETED, oitem) < 0 ||
@@ -590,7 +624,7 @@ static int maybe_modified(
 				/* grab OID while we are here */
 				if (git_oid_iszero(&nitem->oid)) {
 					const git_oid *sm_oid = git_submodule_wd_oid(sub);
-					if (sub != NULL) {
+					if (sm_oid != NULL) {
 						git_oid_cpy(&noid, sm_oid);
 						use_noid = &noid;
 					}
@@ -632,6 +666,24 @@ static int git_index_entry_cmp_icase(const void *a, const void *b)
 	return strcasecmp(entry_a->path, entry_b->path);
 }
 
+static bool entry_is_prefixed(
+	const git_index_entry *item,
+	git_iterator *prefix_iterator,
+	const git_index_entry *prefix_item)
+{
+	size_t pathlen;
+
+	if (!prefix_item ||
+		ITERATOR_PREFIXCMP(*prefix_iterator, prefix_item->path, item->path))
+		return false;
+
+	pathlen = strlen(item->path);
+
+	return (item->path[pathlen - 1] == '/' ||
+			prefix_item->path[pathlen] == '\0' ||
+			prefix_item->path[pathlen] == '/');
+}
+
 static int diff_from_iterators(
 	git_repository *repo,
 	const git_diff_options *opts, /**< can be NULL for defaults */
@@ -681,8 +733,24 @@ static int diff_from_iterators(
 
 		/* create DELETED records for old items not matched in new */
 		if (oitem && (!nitem || entry_compare(oitem, nitem) < 0)) {
-			if (diff_delta__from_one(diff, GIT_DELTA_DELETED, oitem) < 0 ||
-				git_iterator_advance(old_iter, &oitem) < 0)
+			if (diff_delta__from_one(diff, GIT_DELTA_DELETED, oitem) < 0)
+				goto fail;
+
+			/* if we are generating TYPECHANGE records then check for that
+			 * instead of just generating a DELETE record
+			 */
+			if ((diff->opts.flags & GIT_DIFF_INCLUDE_TYPECHANGE_TREES) != 0 &&
+				entry_is_prefixed(oitem, new_iter, nitem))
+			{
+				/* this entry has become a tree! convert to TYPECHANGE */
+				git_diff_delta *last = diff_delta__last_for_item(diff, oitem);
+				if (last) {
+					last->status = GIT_DELTA_TYPECHANGE;
+					last->new_file.mode = GIT_FILEMODE_TREE;
+				}
+			}
+
+			if (git_iterator_advance(old_iter, &oitem) < 0)
 				goto fail;
 		}
 
@@ -704,8 +772,7 @@ static int diff_from_iterators(
 				 * directories and it is not under an ignored directory.
 				 */
 				bool contains_tracked =
-					(oitem &&
-					 !ITERATOR_PREFIXCMP(*old_iter, oitem->path, nitem->path));
+					entry_is_prefixed(nitem, old_iter, oitem);
 				bool recurse_untracked =
 					(delta_type == GIT_DELTA_UNTRACKED &&
 					 (diff->opts.flags & GIT_DIFF_RECURSE_UNTRACKED_DIRS) != 0);
@@ -761,8 +828,25 @@ static int diff_from_iterators(
 			else if (new_iter->type != GIT_ITERATOR_WORKDIR)
 				delta_type = GIT_DELTA_ADDED;
 
-			if (diff_delta__from_one(diff, delta_type, nitem) < 0 ||
-				git_iterator_advance(new_iter, &nitem) < 0)
+			if (diff_delta__from_one(diff, delta_type, nitem) < 0)
+				goto fail;
+
+			/* if we are generating TYPECHANGE records then check for that
+			 * instead of just generating an ADD/UNTRACKED record
+			 */
+			if (delta_type != GIT_DELTA_IGNORED &&
+				(diff->opts.flags & GIT_DIFF_INCLUDE_TYPECHANGE_TREES) != 0 &&
+				entry_is_prefixed(nitem, old_iter, oitem))
+			{
+				/* this entry was a tree! convert to TYPECHANGE */
+				git_diff_delta *last = diff_delta__last_for_item(diff, oitem);
+				if (last) {
+					last->status = GIT_DELTA_TYPECHANGE;
+					last->old_file.mode = GIT_FILEMODE_TREE;
+				}
+			}
+
+			if (git_iterator_advance(new_iter, &nitem) < 0)
 				goto fail;
 		}
 
diff --git a/src/diff_output.c b/src/diff_output.c
index 10fbd391c..5f0d13c64 100644
--- a/src/diff_output.c
+++ b/src/diff_output.c
@@ -533,6 +533,11 @@ static int diff_patch_load(
 	if (delta->binary == 1)
 		goto cleanup;
 
+	if (!ctxt->hunk_cb &&
+		!ctxt->data_cb &&
+		(ctxt->opts->flags & GIT_DIFF_SKIP_BINARY_CHECK) != 0)
+		goto cleanup;
+
 	switch (delta->status) {
 	case GIT_DELTA_ADDED:
 		delta->old_file.flags |= GIT_DIFF_FILE_NO_DATA;
@@ -698,8 +703,10 @@ static int diff_patch_generate(
 	if ((patch->flags & GIT_DIFF_PATCH_DIFFABLE) == 0)
 		return 0;
 
-	if (ctxt)
-		patch->ctxt = ctxt;
+	if (!ctxt->file_cb && !ctxt->hunk_cb)
+		return 0;
+
+	patch->ctxt = ctxt;
 
 	memset(&xdiff_callback, 0, sizeof(xdiff_callback));
 	xdiff_callback.outf = diff_patch_cb;
@@ -1360,7 +1367,9 @@ int git_diff_get_patch(
 	if (delta_ptr)
 		*delta_ptr = delta;
 
-	if (!patch_ptr && delta->binary != -1)
+	if (!patch_ptr &&
+		(delta->binary != -1 ||
+		 (diff->opts.flags & GIT_DIFF_SKIP_BINARY_CHECK) != 0))
 		return 0;
 
 	diff_context_init(
diff --git a/src/fileops.c b/src/fileops.c
index 8ccf063d5..23bfa8e55 100644
--- a/src/fileops.c
+++ b/src/fileops.c
@@ -323,10 +323,6 @@ static int _rmdir_recurs_foreach(void *opaque, git_buf *path)
 {
 	git_directory_removal_type removal_type = *(git_directory_removal_type *)opaque;
 
-	assert(removal_type == GIT_DIRREMOVAL_EMPTY_HIERARCHY
-		|| removal_type == GIT_DIRREMOVAL_FILES_AND_DIRS
-		|| removal_type == GIT_DIRREMOVAL_ONLY_EMPTY_DIRS);
-
 	if (git_path_isdir(path->ptr) == true) {
 		if (git_path_direach(path, _rmdir_recurs_foreach, opaque) < 0)
 			return -1;
@@ -359,15 +355,24 @@ static int _rmdir_recurs_foreach(void *opaque, git_buf *path)
 	return 0;
 }
 
-int git_futils_rmdir_r(const char *path, git_directory_removal_type removal_type)
+int git_futils_rmdir_r(
+	const char *path, const char *base, git_directory_removal_type removal_type)
 {
 	int error;
-	git_buf p = GIT_BUF_INIT;
+	git_buf fullpath = GIT_BUF_INIT;
+
+	assert(removal_type == GIT_DIRREMOVAL_EMPTY_HIERARCHY
+		|| removal_type == GIT_DIRREMOVAL_FILES_AND_DIRS
+		|| removal_type == GIT_DIRREMOVAL_ONLY_EMPTY_DIRS);
+
+	/* build path and find "root" where we should start calling mkdir */
+	if (git_path_join_unrooted(&fullpath, path, base, NULL) < 0)
+		return -1;
+
+	error = _rmdir_recurs_foreach(&removal_type, &fullpath);
+
+	git_buf_free(&fullpath);
 
-	error = git_buf_sets(&p, path);
-	if (!error)
-		error = _rmdir_recurs_foreach(&removal_type, &p);
-	git_buf_free(&p);
 	return error;
 }
 
diff --git a/src/fileops.h b/src/fileops.h
index d2944f460..19f7ffd54 100644
--- a/src/fileops.h
+++ b/src/fileops.h
@@ -107,15 +107,17 @@ typedef enum {
  * Remove path and any files and directories beneath it.
  *
  * @param path Path to to top level directory to process.
- *
+ * @param base Root for relative path.
  * @param removal_type GIT_DIRREMOVAL_EMPTY_HIERARCHY to remove a hierarchy
- * of empty directories (will fail if any file is found), GIT_DIRREMOVAL_FILES_AND_DIRS
- * to remove a hierarchy of files and folders, GIT_DIRREMOVAL_ONLY_EMPTY_DIRS to only remove
- * empty directories (no failure on file encounter).
+ *                     of empty directories (will fail if any file is found),
+ *                     GIT_DIRREMOVAL_FILES_AND_DIRS to remove a hierarchy of
+ *                     files and folders,
+ *                     GIT_DIRREMOVAL_ONLY_EMPTY_DIRS to only remove empty
+ *                     directories (no failure on file encounter).
  *
  * @return 0 on success; -1 on error.
  */
-extern int git_futils_rmdir_r(const char *path, git_directory_removal_type removal_type);
+extern int git_futils_rmdir_r(const char *path, const char *base, git_directory_removal_type removal_type);
 
 /**
  * Create and open a temporary file with a `_git2_` suffix.
diff --git a/src/iterator.c b/src/iterator.c
index 267687e01..df6da9a87 100644
--- a/src/iterator.c
+++ b/src/iterator.c
@@ -82,7 +82,7 @@ int git_iterator_for_nothing(git_iterator **iter)
 
 typedef struct tree_iterator_frame tree_iterator_frame;
 struct tree_iterator_frame {
-	tree_iterator_frame *next;
+	tree_iterator_frame *next, *prev;
 	git_tree *tree;
 	char *start;
 	unsigned int index;
@@ -91,7 +91,7 @@ struct tree_iterator_frame {
 typedef struct {
 	git_iterator base;
 	git_repository *repo;
-	tree_iterator_frame *stack;
+	tree_iterator_frame *stack, *tail;
 	git_index_entry entry;
 	git_buf path;
 	bool path_has_filename;
@@ -119,8 +119,10 @@ static void tree_iterator__pop_frame(tree_iterator *ti)
 {
 	tree_iterator_frame *tf = ti->stack;
 	ti->stack = tf->next;
-	if (ti->stack != NULL) /* don't free the initial tree */
-		git_tree_free(tf->tree);
+	if (ti->stack != NULL) {
+		git_tree_free(tf->tree); /* don't free the initial tree */
+		ti->stack->prev = NULL;  /* disconnect prev */
+	}
 	git__free(tf);
 }
 
@@ -221,6 +223,7 @@ static int tree_iterator__expand_tree(tree_iterator *ti)
 
 		tf->next  = ti->stack;
 		ti->stack = tf;
+		tf->next->prev = tf;
 
 		te = tree_iterator__tree_entry(ti);
 	}
@@ -312,7 +315,7 @@ int git_iterator_for_tree_range(
 	ITERATOR_BASE_INIT(ti, tree, TREE);
 
 	ti->repo  = repo;
-	ti->stack = tree_iterator__alloc_frame(tree, ti->base.start);
+	ti->stack = ti->tail = tree_iterator__alloc_frame(tree, ti->base.start);
 
 	if ((error = tree_iterator__expand_tree(ti)) < 0)
 		git_iterator_free((git_iterator *)ti);
@@ -864,6 +867,45 @@ int git_iterator_current_tree_entry(
 	return 0;
 }
 
+int git_iterator_current_parent_tree(
+	git_iterator *iter,
+	const char *parent_path,
+	const git_tree **tree_ptr)
+{
+	tree_iterator *ti = (tree_iterator *)iter;
+	tree_iterator_frame *tf;
+	const char *scan = parent_path;
+
+	if (iter->type != GIT_ITERATOR_TREE || ti->stack == NULL)
+		goto notfound;
+
+	for (tf = ti->tail; tf != NULL; tf = tf->prev) {
+		const git_tree_entry *te;
+
+		if (!*scan) {
+			*tree_ptr = tf->tree;
+			return 0;
+		}
+
+		te = git_tree_entry_byindex(tf->tree, tf->index);
+
+		if (strncmp(scan, te->filename, te->filename_len) != 0)
+			goto notfound;
+
+		scan += te->filename_len;
+
+		if (*scan) {
+			if (*scan != '/')
+				goto notfound;
+			scan++;
+		}
+	}
+
+notfound:
+	*tree_ptr = NULL;
+	return 0;
+}
+
 int git_iterator_current_is_ignored(git_iterator *iter)
 {
 	return (iter->type != GIT_ITERATOR_WORKDIR) ? 0 :
diff --git a/src/iterator.h b/src/iterator.h
index 29c8985d4..d7df50137 100644
--- a/src/iterator.h
+++ b/src/iterator.h
@@ -142,6 +142,9 @@ GIT_INLINE(git_iterator_type_t) git_iterator_type(git_iterator *iter)
 extern int git_iterator_current_tree_entry(
 	git_iterator *iter, const git_tree_entry **tree_entry);
 
+extern int git_iterator_current_parent_tree(
+	git_iterator *iter, const char *parent_path, const git_tree **tree_ptr);
+
 extern int git_iterator_current_is_ignored(git_iterator *iter);
 
 /**
diff --git a/src/reflog.c b/src/reflog.c
index 80e40b960..a1ea7a27d 100644
--- a/src/reflog.c
+++ b/src/reflog.c
@@ -372,7 +372,7 @@ int git_reflog_rename(git_reference *ref, const char *new_name)
 		goto cleanup;
 
 	if (git_path_isdir(git_buf_cstr(&new_path)) && 
-		(git_futils_rmdir_r(git_buf_cstr(&new_path), GIT_DIRREMOVAL_ONLY_EMPTY_DIRS) < 0))
+		(git_futils_rmdir_r(git_buf_cstr(&new_path), NULL, GIT_DIRREMOVAL_ONLY_EMPTY_DIRS) < 0))
 		goto cleanup;
 
 	if (git_futils_mkpath2file(git_buf_cstr(&new_path), GIT_REFLOG_DIR_MODE) < 0)
diff --git a/src/refs.c b/src/refs.c
index 9dc422e1b..9249391e1 100644
--- a/src/refs.c
+++ b/src/refs.c
@@ -262,14 +262,15 @@ static int loose_write(git_reference *ref)
 	if (git_buf_joinpath(&ref_path, ref->owner->path_repository, ref->name) < 0)
 		return -1;
 
-	/* Remove a possibly existing empty directory hierarchy 
+	/* Remove a possibly existing empty directory hierarchy
 	 * which name would collide with the reference name
 	 */
-	if (git_path_isdir(git_buf_cstr(&ref_path)) && 
-		(git_futils_rmdir_r(git_buf_cstr(&ref_path), GIT_DIRREMOVAL_ONLY_EMPTY_DIRS) < 0)) {
-			git_buf_free(&ref_path);
-			return -1;
-		}
+	if (git_path_isdir(git_buf_cstr(&ref_path)) &&
+		git_futils_rmdir_r(git_buf_cstr(&ref_path), NULL,
+			GIT_DIRREMOVAL_ONLY_EMPTY_DIRS) < 0) {
+		git_buf_free(&ref_path);
+		return -1;
+	}
 
 	if (git_filebuf_open(&file, ref_path.ptr, GIT_FILEBUF_FORCE) < 0) {
 		git_buf_free(&ref_path);
diff --git a/src/status.c b/src/status.c
index e39006e93..50ac19d3b 100644
--- a/src/status.c
+++ b/src/status.c
@@ -100,7 +100,7 @@ int git_status_foreach_ext(
 	memset(&diffopt, 0, sizeof(diffopt));
 	memcpy(&diffopt.pathspec, &opts->pathspec, sizeof(diffopt.pathspec));
 
-	diffopt.flags = GIT_DIFF_DONT_SPLIT_TYPECHANGE;
+	diffopt.flags = GIT_DIFF_INCLUDE_TYPECHANGE;
 
 	if ((opts->flags & GIT_STATUS_OPT_INCLUDE_UNTRACKED) != 0)
 		diffopt.flags = diffopt.flags | GIT_DIFF_INCLUDE_UNTRACKED;
diff --git a/src/tree.c b/src/tree.c
index 83aa303d4..8d3f2665c 100644
--- a/src/tree.c
+++ b/src/tree.c
@@ -180,9 +180,9 @@ void git_tree__free(git_tree *tree)
 	git__free(tree);
 }
 
-const git_oid *git_tree_id(git_tree *c)
+const git_oid *git_tree_id(const git_tree *c)
 {
-	return git_object_id((git_object *)c);
+	return git_object_id((const git_object *)c);
 }
 
 git_filemode_t git_tree_entry_filemode(const git_tree_entry *entry)
@@ -286,7 +286,7 @@ int git_tree__prefix_position(git_tree *tree, const char *path)
 	return at_pos;
 }
 
-unsigned int git_tree_entrycount(git_tree *tree)
+unsigned int git_tree_entrycount(const git_tree *tree)
 {
 	assert(tree);
 	return (unsigned int)tree->entries.length;