Merge pull request #2704

12 files changed, 39 insertions, 3 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 96bd9a16e..e1c02f965 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,9 @@ v0.22 + 1
 
 ### Changes or improvements
 
+* Updated binary identification in CRLF filtering to avoid false positives in
+  UTF-8 files.
+
 * Rename and copy detection is enabled for small files.
 
 ### API additions
diff --git a/src/buf_text.c b/src/buf_text.c
index cead599f4..cb3661edb 100644
--- a/src/buf_text.c
+++ b/src/buf_text.c
@@ -191,7 +191,10 @@ bool git_buf_text_is_binary(const git_buf *buf)
 	while (scan < end) {
 		unsigned char c = *scan++;
 
-		if (c > 0x1F && c < 0x7F)
+		/* Printable characters are those above SPACE (0x1F) excluding DEL,
+		 * and including BS, ESC and FF.
+		 */
+		if ((c > 0x1F && c != 127) || c == '\b' || c == '\033' || c == '\014')
 			printable++;
 		else if (c == '\0')
 			return true;
diff --git a/tests/checkout/crlf.c b/tests/checkout/crlf.c
index 496f83d5d..b6d4e949a 100644
--- a/tests/checkout/crlf.c
+++ b/tests/checkout/crlf.c
@@ -106,6 +106,31 @@ void test_checkout_crlf__all_crlf_autocrlf_true(void)
 	check_file_contents("./crlf/all-crlf", ALL_CRLF_TEXT_RAW);
 }
 
+void test_checkout_crlf__detect_crlf_autocrlf_true_utf8(void)
+{
+	git_checkout_options opts = GIT_CHECKOUT_OPTIONS_INIT;
+	opts.checkout_strategy = GIT_CHECKOUT_SAFE_CREATE;
+
+	cl_repo_set_bool(g_repo, "core.autocrlf", true);
+
+	git_repository_set_head(g_repo, "refs/heads/utf8", NULL, NULL);
+	git_checkout_head(g_repo, &opts);
+
+	if (GIT_EOL_NATIVE == GIT_EOL_LF)
+	{
+		check_file_contents("./crlf/few-utf8-chars-lf.txt", FEW_UTF8_LF_RAW);
+		check_file_contents("./crlf/many-utf8-chars-lf.txt", MANY_UTF8_LF_RAW);
+	}
+	else
+	{
+		check_file_contents("./crlf/few-utf8-chars-lf.txt", FEW_UTF8_CRLF_RAW);
+		check_file_contents("./crlf/many-utf8-chars-lf.txt", MANY_UTF8_CRLF_RAW);
+	}
+
+	check_file_contents("./crlf/few-utf8-chars-crlf.txt", FEW_UTF8_CRLF_RAW);
+	check_file_contents("./crlf/many-utf8-chars-crlf.txt", MANY_UTF8_CRLF_RAW);
+}
+
 void test_checkout_crlf__autocrlf_true_index_size_is_filtered_size(void)
 {
 	git_index *index;
diff --git a/tests/core/buffer.c b/tests/core/buffer.c
index 87dec4607..d28aa218f 100644
--- a/tests/core/buffer.c
+++ b/tests/core/buffer.c
@@ -830,7 +830,7 @@ void test_core_buffer__classify_with_utf8(void)
 	cl_assert(!git_buf_text_contains_nul(&b));
 
 	b.ptr = data1; b.size = b.asize = data1len;
-	cl_assert(git_buf_text_is_binary(&b));
+	cl_assert(!git_buf_text_is_binary(&b));
 	cl_assert(!git_buf_text_contains_nul(&b));
 
 	b.ptr = data2; b.size = b.asize = data2len;
diff --git a/tests/filter/crlf.h b/tests/filter/crlf.h
index 9cb98ad4c..786edfc96 100644
--- a/tests/filter/crlf.h
+++ b/tests/filter/crlf.h
@@ -22,4 +22,9 @@
 #define MORE_CRLF_TEXT_AS_LF	"crlf\ncrlf\nlf\ncrlf\ncrlf\n"
 #define MORE_LF_TEXT_AS_LF		"lf\nlf\ncrlf\nlf\nlf\n"
 
+#define FEW_UTF8_CRLF_RAW		"\xe2\x9a\xbdThe rest is ASCII01.\r\nThe rest is ASCII02.\r\nThe rest is ASCII03.\r\nThe rest is ASCII04.\r\nThe rest is ASCII05.\r\nThe rest is ASCII06.\r\nThe rest is ASCII07.\r\nThe rest is ASCII08.\r\nThe rest is ASCII09.\r\nThe rest is ASCII10.\r\nThe rest is ASCII11.\r\nThe rest is ASCII12.\r\nThe rest is ASCII13.\r\nThe rest is ASCII14.\r\nThe rest is ASCII15.\r\nThe rest is ASCII16.\r\nThe rest is ASCII17.\r\nThe rest is ASCII18.\r\nThe rest is ASCII19.\r\nThe rest is ASCII20.\r\nThe rest is ASCII21.\r\nThe rest is ASCII22.\r\n"
+#define FEW_UTF8_LF_RAW			"\xe2\x9a\xbdThe rest is ASCII01.\nThe rest is ASCII02.\nThe rest is ASCII03.\nThe rest is ASCII04.\nThe rest is ASCII05.\nThe rest is ASCII06.\nThe rest is ASCII07.\nThe rest is ASCII08.\nThe rest is ASCII09.\nThe rest is ASCII10.\nThe rest is ASCII11.\nThe rest is ASCII12.\nThe rest is ASCII13.\nThe rest is ASCII14.\nThe rest is ASCII15.\nThe rest is ASCII16.\nThe rest is ASCII17.\nThe rest is ASCII18.\nThe rest is ASCII19.\nThe rest is ASCII20.\nThe rest is ASCII21.\nThe rest is ASCII22.\n"
+#define MANY_UTF8_CRLF_RAW		"Lets sing!\r\n\xe2\x99\xab\xe2\x99\xaa\xe2\x99\xac\xe2\x99\xa9\r\nEat food\r\n\xf0\x9f\x8d\x85\xf0\x9f\x8d\x95\r\n"
+#define MANY_UTF8_LF_RAW		"Lets sing!\n\xe2\x99\xab\xe2\x99\xaa\xe2\x99\xac\xe2\x99\xa9\nEat food\n\xf0\x9f\x8d\x85\xf0\x9f\x8d\x95\n"
+
 #endif
diff --git a/tests/resources/crlf/.gitted/objects/0e/052888828a954ca17e5882638e3c6a083e75c0 b/tests/resources/crlf/.gitted/objects/0e/052888828a954ca17e5882638e3c6a083e75c0
new file mode 100644
index 000000000..746143f85
--- /dev/null
+++ b/tests/resources/crlf/.gitted/objects/0e/052888828a954ca17e5882638e3c6a083e75c0
diff --git a/tests/resources/crlf/.gitted/objects/9a/6c3533fef19abd6eec8e61206b5c51982b80d9 b/tests/resources/crlf/.gitted/objects/9a/6c3533fef19abd6eec8e61206b5c51982b80d9
new file mode 100644
index 000000000..78fc8aeb7
--- /dev/null
+++ b/tests/resources/crlf/.gitted/objects/9a/6c3533fef19abd6eec8e61206b5c51982b80d9
diff --git a/tests/resources/crlf/.gitted/objects/a2/34455d62297f1856c4603686150c59fcb0aafe b/tests/resources/crlf/.gitted/objects/a2/34455d62297f1856c4603686150c59fcb0aafe
new file mode 100644
index 000000000..7d204f4c8
--- /dev/null
+++ b/tests/resources/crlf/.gitted/objects/a2/34455d62297f1856c4603686150c59fcb0aafe
diff --git a/tests/resources/crlf/.gitted/objects/c3/e11722855ff260bd27418988ac1467c4e9e73a b/tests/resources/crlf/.gitted/objects/c3/e11722855ff260bd27418988ac1467c4e9e73a
new file mode 100644
index 000000000..5f96dc76c
--- /dev/null
+++ b/tests/resources/crlf/.gitted/objects/c3/e11722855ff260bd27418988ac1467c4e9e73a
diff --git a/tests/resources/crlf/.gitted/objects/cd/574f5a2baa4c79504f8837b730fa0b11defe99 b/tests/resources/crlf/.gitted/objects/cd/574f5a2baa4c79504f8837b730fa0b11defe99
new file mode 100644
index 000000000..e8d020246
--- /dev/null
+++ b/tests/resources/crlf/.gitted/objects/cd/574f5a2baa4c79504f8837b730fa0b11defe99
diff --git a/tests/resources/crlf/.gitted/objects/f4/d25b796d86387205a5498175d66e91d1e5006a b/tests/resources/crlf/.gitted/objects/f4/d25b796d86387205a5498175d66e91d1e5006a
new file mode 100644
index 000000000..792b1659d
--- /dev/null
+++ b/tests/resources/crlf/.gitted/objects/f4/d25b796d86387205a5498175d66e91d1e5006a
diff --git a/tests/resources/crlf/.gitted/refs/heads/utf8 b/tests/resources/crlf/.gitted/refs/heads/utf8
index f8e6cf51c..30ab61cd3 100644
--- a/tests/resources/crlf/.gitted/refs/heads/utf8
+++ b/tests/resources/crlf/.gitted/refs/heads/utf8
@@ -1 +1 @@
-2b55b4b94f655c857635b6a9005c056aa7de3532
+a234455d62297f1856c4603686150c59fcb0aafe