Fix git_checkout_tree() to do index filemodes correctly on Windows.

git_checkout_tree() has some fallback behaviors for file systems
which don't have full support of filemodes.  Generally works fine,
but if a given file had a change of type from a 0644 to 0755 (i.e.,
you add executable permissions), the fallback behavior incorrectly
triggers when writing hte updated index.

This would cause a git_checkout_tree() command, even with the
GIT_CHECKOUT_FORCE option set, to leave a dirty index on Windows.

Also added checks to an existing test to catch this case.

2 files changed, 37 insertions, 2 deletions

diff --git a/src/checkout.c b/src/checkout.c
index 0b6e298a0..93343cccc 100644
--- a/src/checkout.c
+++ b/src/checkout.c
@@ -1794,6 +1794,9 @@ static int checkout_create_the_new(
 	int error = 0;
 	git_diff_delta *delta;
 	size_t i;
+	int caps = git_index_caps(data->index);
+
+	git_index_set_caps(data->index, caps & !GIT_INDEXCAP_NO_FILEMODE);
 
 	git_vector_foreach(&data->diff->deltas, i, delta) {
 		if (actions[i] & CHECKOUT_ACTION__DEFER_REMOVE) {
@@ -1815,6 +1818,8 @@ static int checkout_create_the_new(
 		}
 	}
 
+	git_index_set_caps(data->index, caps);
+
 	return 0;
 }
 
@@ -2543,7 +2548,12 @@ int git_checkout_iterator(
 cleanup:
 	if (!error && data.index != NULL &&
 		(data.strategy & CHECKOUT_INDEX_DONT_WRITE_MASK) == 0)
+	{
+		int caps = git_index_caps(data.index);
+		git_index_set_caps(data.index, caps & !GIT_INDEXCAP_NO_FILEMODE);
 		error = git_index_write(data.index);
+		git_index_set_caps(data.index, caps);
+	}
 
 	git_diff_free(data.diff);
 	git_iterator_free(workdir);
diff --git a/tests/checkout/tree.c b/tests/checkout/tree.c
index 7d4c784a1..3973d9320 100644
--- a/tests/checkout/tree.c
+++ b/tests/checkout/tree.c
@@ -925,18 +925,43 @@ void test_checkout_tree__filemode_preserved_in_index(void)
 	git_index *index;
 	const git_index_entry *entry;
 
+	opts.checkout_strategy = GIT_CHECKOUT_FORCE;
+
 	cl_git_pass(git_repository_index(&index, g_repo));
 
+	/* test a freshly added executable */
 	cl_git_pass(git_oid_fromstr(&executable_oid, "afe4393b2b2a965f06acf2ca9658eaa01e0cd6b6"));
 	cl_git_pass(git_commit_lookup(&commit, g_repo, &executable_oid));
 
-	opts.checkout_strategy = GIT_CHECKOUT_FORCE;
-
 	cl_git_pass(git_checkout_tree(g_repo, (const git_object *)commit, &opts));
 	cl_assert(entry = git_index_get_bypath(index, "executable.txt", 0));
 	cl_assert_equal_i(0100755, entry->mode);
 
 	git_commit_free(commit);
+
+
+	/* Now start with a commit which has a text file */
+	cl_git_pass(git_oid_fromstr(&executable_oid, "cf80f8de9f1185bf3a05f993f6121880dd0cfbc9"));
+	cl_git_pass(git_commit_lookup(&commit, g_repo, &executable_oid));
+
+	cl_git_pass(git_checkout_tree(g_repo, (const git_object *)commit, &opts));
+	cl_assert(entry = git_index_get_bypath(index, "a/b.txt", 0));
+	cl_assert_equal_i(0100644, entry->mode);
+
+	git_commit_free(commit);
+
+
+	/* And then check out to a commit which converts the text file to an executable */
+	cl_git_pass(git_oid_fromstr(&executable_oid, "144344043ba4d4a405da03de3844aa829ae8be0e"));
+	cl_git_pass(git_commit_lookup(&commit, g_repo, &executable_oid));
+
+	cl_git_pass(git_checkout_tree(g_repo, (const git_object *)commit, &opts));
+	cl_assert(entry = git_index_get_bypath(index, "a/b.txt", 0));
+	cl_assert_equal_i(0100755, entry->mode);
+
+	git_commit_free(commit);
+
+
 	git_index_free(index);
 }