Implement filters for status/diff blobs

This adds support to diff and status for running filters (a la crlf)
on blobs in the workdir before computing SHAs and before generating
text diffs.  This ended up being a bit more code change than I had
thought since I had to reorganize some of the diff logic to minimize
peak memory use when filtering blobs in a diff.

This also adds a cap on the maximum size of data that will be loaded
to diff.  I set it at 512Mb which should match core git.  Right now
it is a #define in src/diff.h but it could be moved into the public
API if desired.

1 files changed, 31 insertions, 2 deletions

diff --git a/src/odb.c b/src/odb.c
index 34033d15c..83c7a80fc 100644
--- a/src/odb.c
+++ b/src/odb.c
@@ -12,6 +12,7 @@
 #include "hash.h"
 #include "odb.h"
 #include "delta-apply.h"
+#include "filter.h"
 
 #include "git2/odb_backend.h"
 #include "git2/oid.h"
@@ -118,11 +119,12 @@ int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
 	hdr_len = format_object_header(hdr, sizeof(hdr), size, type);
 
 	ctx = git_hash_new_ctx();
+	GITERR_CHECK_ALLOC(ctx);
 
 	git_hash_update(ctx, hdr, hdr_len);
 
 	while (size > 0) {
-		ssize_t read_len = read(fd, buffer, sizeof(buffer));
+		ssize_t read_len = p_read(fd, buffer, sizeof(buffer));
 
 		if (read_len < 0) {
 			git_hash_free_ctx(ctx);
@@ -140,6 +142,33 @@ int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
 	return 0;
 }
 
+int git_odb__hashfd_filtered(
+	git_oid *out, git_file fd, size_t size, git_otype type, git_vector *filters)
+{
+	int error;
+	git_buf raw = GIT_BUF_INIT;
+	git_buf filtered = GIT_BUF_INIT;
+
+	if (!filters || !filters->length)
+		return git_odb__hashfd(out, fd, size, type);
+
+	/* size of data is used in header, so we have to read the whole file
+	 * into memory to apply filters before beginning to calculate the hash
+	 */
+
+	if (!(error = git_futils_readbuffer_fd(&raw, fd, size)))
+		error = git_filters_apply(&filtered, &raw, filters);
+
+	git_buf_free(&raw);
+
+	if (!error)
+		error = git_odb_hash(out, filtered.ptr, filtered.size, type);
+
+	git_buf_free(&filtered);
+
+	return error;
+}
+
 int git_odb__hashlink(git_oid *out, const char *path)
 {
 	struct stat st;
@@ -171,7 +200,7 @@ int git_odb__hashlink(git_oid *out, const char *path)
 
 		result = git_odb_hash(out, link_data, (size_t)size, GIT_OBJ_BLOB);
 		git__free(link_data);
-	} else { 
+	} else {
 		int fd = git_futils_open_ro(path);
 		if (fd < 0)
 			return -1;