Merge pull request #3177 from ethomson/binary_diff

Binary diffs: store deltas in the diff structure, include binary data in diff callbacks

7 files changed, 452 insertions, 209 deletions

diff --git a/src/blame.c b/src/blame.c
index 4a12cb85b..a52f9402b 100644
--- a/src/blame.c
+++ b/src/blame.c
@@ -496,8 +496,8 @@ int git_blame_buffer(
 
 	/* Diff to the reference blob */
 	git_diff_blob_to_buffer(reference->final_blob, blame->path,
-			buffer, buffer_len, blame->path,
-			&diffopts, NULL, buffer_hunk_cb, buffer_line_cb, blame);
+		buffer, buffer_len, blame->path, &diffopts,
+		NULL, NULL, buffer_hunk_cb, buffer_line_cb, blame);
 
 	*out = blame;
 	return 0;
diff --git a/src/diff_file.c b/src/diff_file.c
index f7061ae83..cef4bc169 100644
--- a/src/diff_file.c
+++ b/src/diff_file.c
@@ -89,10 +89,9 @@ static int diff_file_content_init_common(
 int git_diff_file_content__init_from_diff(
 	git_diff_file_content *fc,
 	git_diff *diff,
-	size_t delta_index,
+	git_diff_delta *delta,
 	bool use_old)
 {
-	git_diff_delta *delta = git_vector_get(&diff->deltas, delta_index);
 	bool has_data = true;
 
 	memset(fc, 0, sizeof(*fc));
@@ -218,7 +217,9 @@ static int diff_file_content_commit_to_str(
 	return 0;
 }
 
-static int diff_file_content_load_blob(git_diff_file_content *fc)
+static int diff_file_content_load_blob(
+	git_diff_file_content *fc,
+	git_diff_options *opts)
 {
 	int error = 0;
 	git_odb_object *odb_obj = NULL;
@@ -236,7 +237,8 @@ static int diff_file_content_load_blob(git_diff_file_content *fc)
 			return error;
 	}
 
-	if (diff_file_content_binary_by_size(fc))
+	if ((opts->flags & GIT_DIFF_SHOW_BINARY) == 0 &&
+		diff_file_content_binary_by_size(fc))
 		return 0;
 
 	if (odb_obj != NULL) {
@@ -283,7 +285,9 @@ static int diff_file_content_load_workdir_symlink(
 }
 
 static int diff_file_content_load_workdir_file(
-	git_diff_file_content *fc, git_buf *path)
+	git_diff_file_content *fc,
+	git_buf *path,
+	git_diff_options *diff_opts)
 {
 	int error = 0;
 	git_filter_list *fl = NULL;
@@ -297,7 +301,8 @@ static int diff_file_content_load_workdir_file(
 		!(fc->file->size = git_futils_filesize(fd)))
 		goto cleanup;
 
-	if (diff_file_content_binary_by_size(fc))
+	if ((diff_opts->flags & GIT_DIFF_SHOW_BINARY) == 0 &&
+		diff_file_content_binary_by_size(fc))
 		goto cleanup;
 
 	if ((error = git_filter_list_load(
@@ -339,7 +344,9 @@ cleanup:
 	return error;
 }
 
-static int diff_file_content_load_workdir(git_diff_file_content *fc)
+static int diff_file_content_load_workdir(
+	git_diff_file_content *fc,
+	git_diff_options *diff_opts)
 {
 	int error = 0;
 	git_buf path = GIT_BUF_INIT;
@@ -357,7 +364,7 @@ static int diff_file_content_load_workdir(git_diff_file_content *fc)
 	if (S_ISLNK(fc->file->mode))
 		error = diff_file_content_load_workdir_symlink(fc, &path);
 	else
-		error = diff_file_content_load_workdir_file(fc, &path);
+		error = diff_file_content_load_workdir_file(fc, &path, diff_opts);
 
 	/* once data is loaded, update OID if we didn't have it previously */
 	if (!error && (fc->file->flags & GIT_DIFF_FLAG_VALID_ID) == 0) {
@@ -370,20 +377,23 @@ static int diff_file_content_load_workdir(git_diff_file_content *fc)
 	return error;
 }
 
-int git_diff_file_content__load(git_diff_file_content *fc)
+int git_diff_file_content__load(
+	git_diff_file_content *fc,
+	git_diff_options *diff_opts)
 {
 	int error = 0;
 
 	if ((fc->flags & GIT_DIFF_FLAG__LOADED) != 0)
 		return 0;
 
-	if ((fc->file->flags & GIT_DIFF_FLAG_BINARY) != 0)
+	if ((fc->file->flags & GIT_DIFF_FLAG_BINARY) != 0 &&
+		(diff_opts->flags & GIT_DIFF_SHOW_BINARY) == 0)
 		return 0;
 
 	if (fc->src == GIT_ITERATOR_TYPE_WORKDIR)
-		error = diff_file_content_load_workdir(fc);
+		error = diff_file_content_load_workdir(fc, diff_opts);
 	else
-		error = diff_file_content_load_blob(fc);
+		error = diff_file_content_load_blob(fc, diff_opts);
 	if (error)
 		return error;
 
diff --git a/src/diff_file.h b/src/diff_file.h
index 4d290ad43..0d54b6d33 100644
--- a/src/diff_file.h
+++ b/src/diff_file.h
@@ -28,7 +28,7 @@ typedef struct {
 extern int git_diff_file_content__init_from_diff(
 	git_diff_file_content *fc,
 	git_diff *diff,
-	size_t delta_index,
+	git_diff_delta *delta,
 	bool use_old);
 
 typedef struct {
@@ -49,7 +49,9 @@ extern int git_diff_file_content__init_from_src(
 	git_diff_file *as_file);
 
 /* this loads the blob/file-on-disk as needed */
-extern int git_diff_file_content__load(git_diff_file_content *fc);
+extern int git_diff_file_content__load(
+	git_diff_file_content *fc,
+	git_diff_options *diff_opts);
 
 /* this releases the blob/file-in-memory */
 extern void git_diff_file_content__unload(git_diff_file_content *fc);
diff --git a/src/diff_patch.c b/src/diff_patch.c
index 1c4c0e8b8..63253c2c5 100644
--- a/src/diff_patch.c
+++ b/src/diff_patch.c
@@ -11,41 +11,13 @@
 #include "diff_driver.h"
 #include "diff_patch.h"
 #include "diff_xdiff.h"
+#include "delta.h"
+#include "zstream.h"
 #include "fileops.h"
 
-/* cached information about a hunk in a diff */
-typedef struct diff_patch_hunk {
-	git_diff_hunk hunk;
-	size_t line_start;
-	size_t line_count;
-} diff_patch_hunk;
-
-struct git_patch {
-	git_refcount rc;
-	git_diff *diff; /* for refcount purposes, maybe NULL for blob diffs */
-	git_diff_delta *delta;
-	size_t delta_index;
-	git_diff_file_content ofile;
-	git_diff_file_content nfile;
-	uint32_t flags;
-	git_array_t(diff_patch_hunk) hunks;
-	git_array_t(git_diff_line)   lines;
-	size_t content_size, context_size, header_size;
-	git_pool flattened;
-};
-
-enum {
-	GIT_DIFF_PATCH_ALLOCATED   = (1 << 0),
-	GIT_DIFF_PATCH_INITIALIZED = (1 << 1),
-	GIT_DIFF_PATCH_LOADED      = (1 << 2),
-	GIT_DIFF_PATCH_DIFFABLE    = (1 << 3),
-	GIT_DIFF_PATCH_DIFFED      = (1 << 4),
-	GIT_DIFF_PATCH_FLATTENED   = (1 << 5),
-};
-
 static void diff_output_init(
-	git_diff_output*, const git_diff_options*,
-	git_diff_file_cb, git_diff_hunk_cb, git_diff_line_cb, void*);
+	git_diff_output*, const git_diff_options*, git_diff_file_cb,
+	git_diff_binary_cb, git_diff_hunk_cb, git_diff_line_cb, void*);
 
 static void diff_output_to_patch(git_diff_output *, git_patch *);
 
@@ -67,15 +39,38 @@ static void diff_patch_init_common(git_patch *patch)
 {
 	diff_patch_update_binary(patch);
 
-	if ((patch->delta->flags & GIT_DIFF_FLAG_BINARY) != 0)
-		patch->flags |= GIT_DIFF_PATCH_LOADED; /* LOADED but not DIFFABLE */
-
 	patch->flags |= GIT_DIFF_PATCH_INITIALIZED;
 
 	if (patch->diff)
 		git_diff_addref(patch->diff);
 }
 
+static int diff_patch_normalize_options(
+	git_diff_options *out,
+	const git_diff_options *opts)
+{
+	if (opts) {
+		GITERR_CHECK_VERSION(opts, GIT_DIFF_OPTIONS_VERSION, "git_diff_options");
+		memcpy(out, opts, sizeof(git_diff_options));
+	} else {
+		git_diff_options default_opts = GIT_DIFF_OPTIONS_INIT;
+		memcpy(out, &default_opts, sizeof(git_diff_options));
+	}
+
+	out->old_prefix = opts && opts->old_prefix ?
+		git__strdup(opts->old_prefix) :
+		git__strdup(DIFF_OLD_PREFIX_DEFAULT);
+
+	out->new_prefix = opts && opts->new_prefix ?
+		git__strdup(opts->new_prefix) :
+		git__strdup(DIFF_NEW_PREFIX_DEFAULT);
+
+	GITERR_CHECK_ALLOC(out->old_prefix);
+	GITERR_CHECK_ALLOC(out->new_prefix);
+
+	return 0;
+}
+
 static int diff_patch_init_from_diff(
 	git_patch *patch, git_diff *diff, size_t delta_index)
 {
@@ -86,10 +81,12 @@ static int diff_patch_init_from_diff(
 	patch->delta = git_vector_get(&diff->deltas, delta_index);
 	patch->delta_index = delta_index;
 
-	if ((error = git_diff_file_content__init_from_diff(
-			&patch->ofile, diff, delta_index, true)) < 0 ||
+	if ((error = diff_patch_normalize_options(
+			&patch->diff_opts, &diff->opts)) < 0 ||
 		(error = git_diff_file_content__init_from_diff(
-			&patch->nfile, diff, delta_index, false)) < 0)
+			&patch->ofile, diff, patch->delta, true)) < 0 ||
+		(error = git_diff_file_content__init_from_diff(
+			&patch->nfile, diff, patch->delta, false)) < 0)
 		return error;
 
 	diff_patch_init_common(patch);
@@ -116,6 +113,14 @@ static int diff_patch_alloc_from_diff(
 	return error;
 }
 
+GIT_INLINE(bool) should_skip_binary(git_patch *patch, git_diff_file *file)
+{
+	if ((patch->diff_opts.flags & GIT_DIFF_SHOW_BINARY) != 0)
+		return false;
+
+	return (file->flags & GIT_DIFF_FLAG_BINARY) != 0;
+}
+
 static int diff_patch_load(git_patch *patch, git_diff_output *output)
 {
 	int error = 0;
@@ -128,7 +133,7 @@ static int diff_patch_load(git_patch *patch, git_diff_output *output)
 	 * binary, then there is no need to actually load the data
 	 */
 	if ((patch->ofile.opts_flags & GIT_DIFF_SKIP_BINARY_CHECK) != 0 &&
-		output && !output->hunk_cb && !output->data_cb)
+		output && !output->binary_cb && !output->hunk_cb && !output->data_cb)
 		return 0;
 
 	incomplete_data =
@@ -141,25 +146,29 @@ static int diff_patch_load(git_patch *patch, git_diff_output *output)
 	 * need 2x data size and this minimizes peak memory footprint
 	 */
 	if (patch->ofile.src == GIT_ITERATOR_TYPE_WORKDIR) {
-		if ((error = git_diff_file_content__load(&patch->ofile)) < 0 ||
-			(patch->ofile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
+		if ((error = git_diff_file_content__load(
+				&patch->ofile, &patch->diff_opts)) < 0 ||
+			should_skip_binary(patch, patch->ofile.file))
 			goto cleanup;
 	}
 	if (patch->nfile.src == GIT_ITERATOR_TYPE_WORKDIR) {
-		if ((error = git_diff_file_content__load(&patch->nfile)) < 0 ||
-			(patch->nfile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
+		if ((error = git_diff_file_content__load(
+				&patch->nfile, &patch->diff_opts)) < 0 ||
+			should_skip_binary(patch, patch->nfile.file))
 			goto cleanup;
 	}
 
 	/* once workdir has been tried, load other data as needed */
 	if (patch->ofile.src != GIT_ITERATOR_TYPE_WORKDIR) {
-		if ((error = git_diff_file_content__load(&patch->ofile)) < 0 ||
-			(patch->ofile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
+		if ((error = git_diff_file_content__load(
+				&patch->ofile, &patch->diff_opts)) < 0 ||
+			should_skip_binary(patch, patch->ofile.file))
 			goto cleanup;
 	}
 	if (patch->nfile.src != GIT_ITERATOR_TYPE_WORKDIR) {
-		if ((error = git_diff_file_content__load(&patch->nfile)) < 0 ||
-			(patch->nfile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
+		if ((error = git_diff_file_content__load(
+				&patch->nfile, &patch->diff_opts)) < 0 ||
+			should_skip_binary(patch, patch->nfile.file))
 			goto cleanup;
 	}
 
@@ -177,10 +186,14 @@ cleanup:
 	diff_patch_update_binary(patch);
 
 	if (!error) {
+		bool skip_binary =
+			(patch->delta->flags & GIT_DIFF_FLAG_BINARY) != 0 &&
+			(patch->diff_opts.flags & GIT_DIFF_SHOW_BINARY) == 0;
+
 		/* patch is diffable only for non-binary, modified files where
-		 * at least one side has data and the data actually changed
-		 */
-		if ((patch->delta->flags & GIT_DIFF_FLAG_BINARY) == 0 &&
+		* at least one side has data and the data actually changed
+		*/
+		if (!skip_binary &&
 			patch->delta->status != GIT_DELTA_UNMODIFIED &&
 			(patch->ofile.map.len || patch->nfile.map.len) &&
 			(patch->ofile.map.len != patch->nfile.map.len ||
@@ -207,6 +220,98 @@ static int diff_patch_invoke_file_callback(
 		"git_patch");
 }
 
+static int create_binary(
+	git_diff_binary_t *out_type,
+	char **out_data,
+	size_t *out_datalen,
+	size_t *out_inflatedlen,
+	const char *a_data,
+	size_t a_datalen,
+	const char *b_data,
+	size_t b_datalen)
+{
+	git_buf deflate = GIT_BUF_INIT, delta = GIT_BUF_INIT;
+	unsigned long delta_data_len;
+	int error;
+
+	/* The git_delta function accepts unsigned long only */
+	if (!git__is_ulong(a_datalen) || !git__is_ulong(b_datalen))
+		return GIT_EBUFS;
+
+	if ((error = git_zstream_deflatebuf(&deflate, b_data, b_datalen)) < 0)
+		goto done;
+
+	/* The git_delta function accepts unsigned long only */
+	if (!git__is_ulong(deflate.size)) {
+		error = GIT_EBUFS;
+		goto done;
+	}
+
+	if (a_datalen && b_datalen) {
+		void *delta_data = git_delta(
+			a_data, (unsigned long)a_datalen,
+			b_data, (unsigned long)b_datalen,
+			&delta_data_len, (unsigned long)deflate.size);
+
+		if (delta_data) {
+			error = git_zstream_deflatebuf(
+				&delta, delta_data, (size_t)delta_data_len);
+
+			git__free(delta_data);
+
+			if (error < 0)
+				goto done;
+		}
+	}
+
+	if (delta.size && delta.size < deflate.size) {
+		*out_type = GIT_DIFF_BINARY_DELTA;
+		*out_datalen = delta.size;
+		*out_data = git_buf_detach(&delta);
+		*out_inflatedlen = delta_data_len;
+	} else {
+		*out_type = GIT_DIFF_BINARY_LITERAL;
+		*out_datalen = deflate.size;
+		*out_data = git_buf_detach(&deflate);
+		*out_inflatedlen = b_datalen;
+	}
+
+done:
+	git_buf_free(&deflate);
+	git_buf_free(&delta);
+
+	return error;
+}
+
+static int diff_binary(git_diff_output *output, git_patch *patch)
+{
+	git_diff_binary binary = { 0 };
+	const char *old_data = patch->ofile.map.data;
+	const char *new_data = patch->nfile.map.data;
+	size_t old_len = patch->ofile.map.len,
+		new_len = patch->nfile.map.len;
+	int error;
+
+	/* Create the old->new delta (as the "new" side of the patch),
+	 * and the new->old delta (as the "old" side)
+	 */
+	if ((error = create_binary(&binary.old_file.type,
+			(char **)&binary.old_file.data,
+			&binary.old_file.datalen,
+			&binary.old_file.inflatedlen,
+			new_data, new_len, old_data, old_len)) < 0 ||
+		(error = create_binary(&binary.new_file.type,
+			(char **)&binary.new_file.data,
+			&binary.new_file.datalen,
+			&binary.new_file.inflatedlen,
+			old_data, old_len, new_data, new_len)) < 0)
+		return error;
+
+	return giterr_set_after_callback_function(
+		output->binary_cb(patch->delta, &binary, output->payload),
+		"git_patch");
+}
+
 static int diff_patch_generate(git_patch *patch, git_diff_output *output)
 {
 	int error = 0;
@@ -214,8 +319,8 @@ static int diff_patch_generate(git_patch *patch, git_diff_output *output)
 	if ((patch->flags & GIT_DIFF_PATCH_DIFFED) != 0)
 		return 0;
 
-	/* if we are not looking at the hunks and lines, don't do the diff */
-	if (!output->hunk_cb && !output->data_cb)
+	/* if we are not looking at the binary or text data, don't do the diff */
+	if (!output->binary_cb && !output->hunk_cb && !output->data_cb)
 		return 0;
 
 	if ((patch->flags & GIT_DIFF_PATCH_LOADED) == 0 &&
@@ -225,10 +330,16 @@ static int diff_patch_generate(git_patch *patch, git_diff_output *output)
 	if ((patch->flags & GIT_DIFF_PATCH_DIFFABLE) == 0)
 		return 0;
 
-	if (output->diff_cb != NULL &&
-		(error = output->diff_cb(output, patch)) < 0)
-		patch->flags |= GIT_DIFF_PATCH_DIFFED;
+	if ((patch->delta->flags & GIT_DIFF_FLAG_BINARY) != 0) {
+		if (output->binary_cb)
+			error = diff_binary(output, patch);
+	}
+	else {
+		if (output->diff_cb)
+			error = output->diff_cb(output, patch);
+	}
 
+	patch->flags |= GIT_DIFF_PATCH_DIFFED;
 	return error;
 }
 
@@ -245,6 +356,9 @@ static void diff_patch_free(git_patch *patch)
 
 	git_pool_clear(&patch->flattened);
 
+	git__free((char *)patch->diff_opts.old_prefix);
+	git__free((char *)patch->diff_opts.new_prefix);
+
 	if (patch->flags & GIT_DIFF_PATCH_ALLOCATED)
 		git__free(patch);
 }
@@ -260,6 +374,7 @@ static int diff_required(git_diff *diff, const char *action)
 int git_diff_foreach(
 	git_diff *diff,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb data_cb,
 	void *payload)
@@ -275,7 +390,7 @@ int git_diff_foreach(
 	memset(&xo, 0, sizeof(xo));
 	memset(&patch, 0, sizeof(patch));
 	diff_output_init(
-		&xo.output, &diff->opts, file_cb, hunk_cb, data_cb, payload);
+		&xo.output, &diff->opts, file_cb, binary_cb, hunk_cb, data_cb, payload);
 	git_xdiff_init(&xo, &diff->opts);
 
 	git_vector_foreach(&diff->deltas, idx, patch.delta) {
@@ -284,11 +399,15 @@ int git_diff_foreach(
 		if (git_diff_delta__should_skip(&diff->opts, patch.delta))
 			continue;
 
+		if (binary_cb || hunk_cb || data_cb) {
+			if ((error = diff_patch_init_from_diff(&patch, diff, idx)) != 0 ||
+				(error = diff_patch_load(&patch, &xo.output)) != 0)
+				return error;
+		}
+
 		if ((error = diff_patch_invoke_file_callback(&patch, &xo.output)) == 0) {
-			if (hunk_cb || data_cb) {
-				if ((error = diff_patch_init_from_diff(&patch, diff, idx)) == 0)
+			if (binary_cb || hunk_cb || data_cb)
 					error = diff_patch_generate(&patch, &xo.output);
-			}
 		}
 
 		git_patch_free(&patch);
@@ -350,7 +469,8 @@ static int diff_patch_from_sources(
 	git_diff_file *lfile = &pd->delta.old_file, *rfile = &pd->delta.new_file;
 	git_diff_file_content *ldata = &pd->patch.ofile, *rdata = &pd->patch.nfile;
 
-	GITERR_CHECK_VERSION(opts, GIT_DIFF_OPTIONS_VERSION, "git_diff_options");
+	if ((error = diff_patch_normalize_options(&pd->patch.diff_opts, opts)) < 0)
+		return error;
 
 	if (opts && (opts->flags & GIT_DIFF_REVERSE) != 0) {
 		void *tmp = lfile; lfile = rfile; rfile = tmp;
@@ -419,6 +539,7 @@ static int diff_from_sources(
 	git_diff_file_content_src *newsrc,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb data_cb,
 	void *payload)
@@ -429,7 +550,7 @@ static int diff_from_sources(
 
 	memset(&xo, 0, sizeof(xo));
 	diff_output_init(
-		&xo.output, opts, file_cb, hunk_cb, data_cb, payload);
+		&xo.output, opts, file_cb, binary_cb, hunk_cb, data_cb, payload);
 	git_xdiff_init(&xo, opts);
 
 	memset(&pd, 0, sizeof(pd));
@@ -477,6 +598,7 @@ int git_diff_blobs(
 	const char *new_path,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb data_cb,
 	void *payload)
@@ -486,7 +608,7 @@ int git_diff_blobs(
 	git_diff_file_content_src nsrc =
 		GIT_DIFF_FILE_CONTENT_SRC__BLOB(new_blob, new_path);
 	return diff_from_sources(
-		&osrc, &nsrc, opts, file_cb, hunk_cb, data_cb, payload);
+		&osrc, &nsrc, opts, file_cb, binary_cb, hunk_cb, data_cb, payload);
 }
 
 int git_patch_from_blobs(
@@ -512,6 +634,7 @@ int git_diff_blob_to_buffer(
 	const char *buf_path,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb data_cb,
 	void *payload)
@@ -521,7 +644,7 @@ int git_diff_blob_to_buffer(
 	git_diff_file_content_src nsrc =
 		GIT_DIFF_FILE_CONTENT_SRC__BUF(buf, buflen, buf_path);
 	return diff_from_sources(
-		&osrc, &nsrc, opts, file_cb, hunk_cb, data_cb, payload);
+		&osrc, &nsrc, opts, file_cb, binary_cb, hunk_cb, data_cb, payload);
 }
 
 int git_patch_from_blob_and_buffer(
@@ -549,6 +672,7 @@ int git_diff_buffers(
 	const char *new_path,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb data_cb,
 	void *payload)
@@ -558,7 +682,7 @@ int git_diff_buffers(
 	git_diff_file_content_src nsrc =
 		GIT_DIFF_FILE_CONTENT_SRC__BUF(new_buf, new_len, new_path);
 	return diff_from_sources(
-		&osrc, &nsrc, opts, file_cb, hunk_cb, data_cb, payload);
+		&osrc, &nsrc, opts, file_cb, binary_cb, hunk_cb, data_cb, payload);
 }
 
 int git_patch_from_buffers(
@@ -812,6 +936,7 @@ void git_patch__new_data(
 int git_patch__invoke_callbacks(
 	git_patch *patch,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb line_cb,
 	void *payload)
@@ -822,6 +947,13 @@ int git_patch__invoke_callbacks(
 	if (file_cb)
 		error = file_cb(patch->delta, 0, payload);
 
+	if ((patch->delta->flags & GIT_DIFF_FLAG_BINARY) != 0) {
+		if (binary_cb)
+			error = binary_cb(patch->delta, &patch->binary, payload);
+
+		return error;
+	}
+
 	if (!hunk_cb && !line_cb)
 		return error;
 
@@ -855,6 +987,36 @@ static int diff_patch_file_cb(
 	return 0;
 }
 
+static int diff_patch_binary_cb(
+	const git_diff_delta *delta,
+	const git_diff_binary *binary,
+	void *payload)
+{
+	git_patch *patch = payload;
+
+	GIT_UNUSED(delta);
+
+	memcpy(&patch->binary, binary, sizeof(git_diff_binary));
+
+	if (binary->old_file.data) {
+		patch->binary.old_file.data = git__malloc(binary->old_file.datalen);
+		GITERR_CHECK_ALLOC(patch->binary.old_file.data);
+
+		memcpy((char *)patch->binary.old_file.data,
+			binary->old_file.data, binary->old_file.datalen);
+	}
+
+	if (binary->new_file.data) {
+		patch->binary.new_file.data = git__malloc(binary->new_file.datalen);
+		GITERR_CHECK_ALLOC(patch->binary.new_file.data);
+
+		memcpy((char *)patch->binary.new_file.data,
+			binary->new_file.data, binary->new_file.datalen);
+	}
+
+	return 0;
+}
+
 static int diff_patch_hunk_cb(
 	const git_diff_delta *delta,
 	const git_diff_hunk *hunk_,
@@ -921,6 +1083,7 @@ static void diff_output_init(
 	git_diff_output *out,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb data_cb,
 	void *payload)
@@ -930,6 +1093,7 @@ static void diff_output_init(
 	memset(out, 0, sizeof(*out));
 
 	out->file_cb = file_cb;
+	out->binary_cb = binary_cb;
 	out->hunk_cb = hunk_cb;
 	out->data_cb = data_cb;
 	out->payload = payload;
@@ -938,6 +1102,11 @@ static void diff_output_init(
 static void diff_output_to_patch(git_diff_output *out, git_patch *patch)
 {
 	diff_output_init(
-		out, NULL,
-		diff_patch_file_cb, diff_patch_hunk_cb, diff_patch_line_cb, patch);
+		out,
+		NULL,
+		diff_patch_file_cb,
+		diff_patch_binary_cb,
+		diff_patch_hunk_cb,
+		diff_patch_line_cb,
+		patch);
 }
diff --git a/src/diff_patch.h b/src/diff_patch.h
index df2ba4c31..f6ce57ddd 100644
--- a/src/diff_patch.h
+++ b/src/diff_patch.h
@@ -13,6 +13,38 @@
 #include "array.h"
 #include "git2/patch.h"
 
+ /* cached information about a hunk in a diff */
+typedef struct diff_patch_hunk {
+	git_diff_hunk hunk;
+	size_t line_start;
+	size_t line_count;
+} diff_patch_hunk;
+
+enum {
+	GIT_DIFF_PATCH_ALLOCATED = (1 << 0),
+	GIT_DIFF_PATCH_INITIALIZED = (1 << 1),
+	GIT_DIFF_PATCH_LOADED = (1 << 2),
+	GIT_DIFF_PATCH_DIFFABLE = (1 << 3),
+	GIT_DIFF_PATCH_DIFFED = (1 << 4),
+	GIT_DIFF_PATCH_FLATTENED = (1 << 5),
+};
+
+struct git_patch {
+	git_refcount rc;
+	git_diff *diff; /* for refcount purposes, maybe NULL for blob diffs */
+	git_diff_options diff_opts;
+	git_diff_delta *delta;
+	size_t delta_index;
+	git_diff_file_content ofile;
+	git_diff_file_content nfile;
+	uint32_t flags;
+	git_diff_binary binary;
+	git_array_t(diff_patch_hunk) hunks;
+	git_array_t(git_diff_line)   lines;
+	size_t content_size, context_size, header_size;
+	git_pool flattened;
+};
+
 extern git_diff *git_patch__diff(git_patch *);
 
 extern git_diff_driver *git_patch__driver(git_patch *);
@@ -23,6 +55,7 @@ extern void git_patch__new_data(char **, size_t *, git_patch *);
 extern int git_patch__invoke_callbacks(
 	git_patch *patch,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb line_cb,
 	void *payload);
@@ -31,6 +64,7 @@ typedef struct git_diff_output git_diff_output;
 struct git_diff_output {
 	/* these callbacks are issued with the diff data */
 	git_diff_file_cb file_cb;
+	git_diff_binary_cb binary_cb;
 	git_diff_hunk_cb hunk_cb;
 	git_diff_line_cb data_cb;
 	void *payload;
diff --git a/src/diff_print.c b/src/diff_print.c
index 43a90b3d8..ebae4ea75 100644
--- a/src/diff_print.c
+++ b/src/diff_print.c
@@ -22,50 +22,92 @@ typedef struct {
 	uint32_t flags;
 	int oid_strlen;
 	git_diff_line line;
+	unsigned int
+		content_loaded : 1,
+		content_allocated : 1;
+	git_diff_file_content *ofile;
+	git_diff_file_content *nfile;
 } diff_print_info;
 
-static int diff_print_info_init(
+static int diff_print_info_init__common(
 	diff_print_info *pi,
 	git_buf *out,
-	git_diff *diff,
+	git_repository *repo,
 	git_diff_format_t format,
 	git_diff_line_cb cb,
 	void *payload)
 {
-	pi->diff     = diff;
-	pi->format   = format;
+	pi->format = format;
 	pi->print_cb = cb;
-	pi->payload  = payload;
-	pi->buf      = out;
-
-	if (diff)
-		pi->flags = diff->opts.flags;
-	else
-		pi->flags = 0;
-
-	if (diff && diff->opts.id_abbrev != 0)
-		pi->oid_strlen = diff->opts.id_abbrev;
-	else if (!diff || !diff->repo)
-		pi->oid_strlen = GIT_ABBREV_DEFAULT;
-	else if (git_repository__cvar(
-		&pi->oid_strlen, diff->repo, GIT_CVAR_ABBREV) < 0)
-		return -1;
+	pi->payload = payload;
+	pi->buf = out;
+
+	if (!pi->oid_strlen) {
+		if (!repo)
+			pi->oid_strlen = GIT_ABBREV_DEFAULT;
+		else if (git_repository__cvar(&pi->oid_strlen, repo, GIT_CVAR_ABBREV) < 0)
+			return -1;
+	}
 
 	pi->oid_strlen += 1; /* for NUL byte */
 
-	if (pi->oid_strlen < 2)
-		pi->oid_strlen = 2;
-	else if (pi->oid_strlen > GIT_OID_HEXSZ + 1)
+	if (pi->oid_strlen > GIT_OID_HEXSZ + 1)
 		pi->oid_strlen = GIT_OID_HEXSZ + 1;
 
 	memset(&pi->line, 0, sizeof(pi->line));
 	pi->line.old_lineno = -1;
 	pi->line.new_lineno = -1;
-	pi->line.num_lines  = 1;
+	pi->line.num_lines = 1;
 
 	return 0;
 }
 
+static int diff_print_info_init_fromdiff(
+	diff_print_info *pi,
+	git_buf *out,
+	git_diff *diff,
+	git_diff_format_t format,
+	git_diff_line_cb cb,
+	void *payload)
+{
+	git_repository *repo = diff ? diff->repo : NULL;
+
+	memset(pi, 0, sizeof(diff_print_info));
+
+	pi->diff = diff;
+
+	if (diff) {
+		pi->flags = diff->opts.flags;
+		pi->oid_strlen = diff->opts.id_abbrev;
+	}
+
+	return diff_print_info_init__common(pi, out, repo, format, cb, payload);
+}
+
+static int diff_print_info_init_frompatch(
+	diff_print_info *pi,
+	git_buf *out,
+	git_patch *patch,
+	git_diff_format_t format,
+	git_diff_line_cb cb,
+	void *payload)
+{
+	git_repository *repo = patch && patch->diff ? patch->diff->repo : NULL;
+
+	memset(pi, 0, sizeof(diff_print_info));
+
+	pi->diff = patch->diff;
+
+	pi->flags = patch->diff_opts.flags;
+	pi->oid_strlen = patch->diff_opts.id_abbrev;
+
+	pi->content_loaded = 1;
+	pi->ofile = &patch->ofile;
+	pi->nfile = &patch->nfile;
+
+	return diff_print_info_init__common(pi, out, repo, format, cb, payload);
+}
+
 static char diff_pick_suffix(int mode)
 {
 	if (S_ISDIR(mode))
@@ -283,66 +325,22 @@ int git_diff_delta__format_file_header(
 	return git_buf_oom(out) ? -1 : 0;
 }
 
-static int print_binary_hunk(diff_print_info *pi, git_blob *old, git_blob *new)
+static int format_binary(
+	diff_print_info *pi,
+	git_diff_binary_t type,
+	const char *data,
+	size_t datalen,
+	size_t inflatedlen)
 {
-	git_buf deflate = GIT_BUF_INIT, delta = GIT_BUF_INIT, *out = NULL;
-	const void *old_data, *new_data;
-	git_off_t old_data_len, new_data_len;
-	unsigned long delta_data_len, inflated_len;
-	const char *out_type = "literal";
-	char *scan, *end;
-	int error;
-
-	old_data = old ? git_blob_rawcontent(old) : NULL;
-	new_data = new ? git_blob_rawcontent(new) : NULL;
-
-	old_data_len = old ? git_blob_rawsize(old) : 0;
-	new_data_len = new ? git_blob_rawsize(new) : 0;
-
-	/* The git_delta function accepts unsigned long only */
-	if (!git__is_ulong(old_data_len) || !git__is_ulong(new_data_len))
-		return GIT_EBUFS;
-
-	out = &deflate;
-	inflated_len = (unsigned long)new_data_len;
-
-	if ((error = git_zstream_deflatebuf(
-			out, new_data, (size_t)new_data_len)) < 0)
-		goto done;
-
-	/* The git_delta function accepts unsigned long only */
-	if (!git__is_ulong((git_off_t)deflate.size)) {
-		error = GIT_EBUFS;
-		goto done;
-	}
+	const char *typename = type == GIT_DIFF_BINARY_DELTA ?
+		"delta" : "literal";
+	const char *scan, *end;
+	int error = 0;
 
-	if (old && new) {
-		void *delta_data = git_delta(
-			old_data, (unsigned long)old_data_len,
-			new_data, (unsigned long)new_data_len,
-			&delta_data_len, (unsigned long)deflate.size);
-
-		if (delta_data) {
-			error = git_zstream_deflatebuf(
-				&delta, delta_data, (size_t)delta_data_len);
-
-			git__free(delta_data);
-
-			if (error < 0)
-				goto done;
-
-			if (delta.size < deflate.size) {
-				out = &delta;
-				out_type = "delta";
-				inflated_len = delta_data_len;
-			}
-		}
-	}
-
-	git_buf_printf(pi->buf, "%s %lu\n", out_type, inflated_len);
+	git_buf_printf(pi->buf, "%s %lu\n", typename, inflatedlen);
 	pi->line.num_lines++;
 
-	for (scan = out->ptr, end = out->ptr + out->size; scan < end; ) {
+	for (scan = data, end = data + datalen; scan < end; ) {
 		size_t chunk_len = end - scan;
 		if (chunk_len > 52)
 			chunk_len = 52;
@@ -355,51 +353,74 @@ static int print_binary_hunk(diff_print_info *pi, git_blob *old, git_blob *new)
 		git_buf_encode_base85(pi->buf, scan, chunk_len);
 		git_buf_putc(pi->buf, '\n');
 
-		if (git_buf_oom(pi->buf)) {
-			error = -1;
-			goto done;
-		}
+		if (git_buf_oom(pi->buf))
+			return -1;
 
 		scan += chunk_len;
 		pi->line.num_lines++;
 	}
 
-done:
-	git_buf_free(&deflate);
-	git_buf_free(&delta);
+	return 0;
+}
 
-	return error;
+static int diff_print_load_content(
+	diff_print_info *pi,
+	git_diff_delta *delta)
+{
+	git_diff_file_content *ofile, *nfile;
+	int error;
+
+	assert(pi->diff);
+
+	ofile = git__calloc(1, sizeof(git_diff_file_content));
+	nfile = git__calloc(1, sizeof(git_diff_file_content));
+
+	GITERR_CHECK_ALLOC(ofile);
+	GITERR_CHECK_ALLOC(nfile);
+
+	if ((error = git_diff_file_content__init_from_diff(
+			ofile, pi->diff, delta, true)) < 0 ||
+		(error = git_diff_file_content__init_from_diff(
+			nfile, pi->diff, delta, true)) < 0) {
+
+		git__free(ofile);
+		git__free(nfile);
+		return error;
+	}
+
+	pi->content_loaded = 1;
+	pi->content_allocated = 1;
+	pi->ofile = ofile;
+	pi->nfile = nfile;
+
+	return 0;
 }
 
-/* git diff --binary 8d7523f~2 8d7523f~1 */
 static int diff_print_patch_file_binary(
-	diff_print_info *pi, const git_diff_delta *delta,
-	const char *oldpfx, const char *newpfx)
+	diff_print_info *pi, git_diff_delta *delta,
+	const char *old_pfx, const char *new_pfx,
+	const git_diff_binary *binary)
 {
-	git_blob *old = NULL, *new = NULL;
-	const git_oid *old_id, *new_id;
-	int error;
 	size_t pre_binary_size;
+	int error;
 
 	if ((pi->flags & GIT_DIFF_SHOW_BINARY) == 0)
 		goto noshow;
 
+	if (!pi->content_loaded &&
+		(error = diff_print_load_content(pi, delta)) < 0)
+		return error;
+
 	pre_binary_size = pi->buf->size;
 	git_buf_printf(pi->buf, "GIT binary patch\n");
 	pi->line.num_lines++;
 
-	old_id = (delta->status != GIT_DELTA_ADDED) ? &delta->old_file.id : NULL;
-	new_id = (delta->status != GIT_DELTA_DELETED) ? &delta->new_file.id : NULL;
-
-	if (old_id && (error = git_blob_lookup(&old, pi->diff->repo, old_id)) < 0)
-		goto done;
-	if (new_id && (error = git_blob_lookup(&new, pi->diff->repo,new_id)) < 0)
-		goto done;
-
-	if ((error = print_binary_hunk(pi, old, new)) < 0 ||
+	if ((error = format_binary(pi, binary->new_file.type, binary->new_file.data,
+		binary->new_file.datalen, binary->new_file.inflatedlen)) < 0 ||
 		(error = git_buf_putc(pi->buf, '\n')) < 0 ||
-		(error = print_binary_hunk(pi, new, old)) < 0)
-	{
+		(error = format_binary(pi, binary->old_file.type, binary->old_file.data,
+			binary->old_file.datalen, binary->old_file.inflatedlen)) < 0) {
+
 		if (error == GIT_EBUFS) {
 			giterr_clear();
 			git_buf_truncate(pi->buf, pre_binary_size);
@@ -408,17 +429,12 @@ static int diff_print_patch_file_binary(
 	}
 
 	pi->line.num_lines++;
-
-done:
-	git_blob_free(old);
-	git_blob_free(new);
-
 	return error;
 
 noshow:
 	pi->line.num_lines = 1;
 	return diff_delta_format_with_paths(
-		pi->buf, delta, oldpfx, newpfx,
+		pi->buf, delta, old_pfx, new_pfx,
 		"Binary files %s%s and %s%s differ\n");
 }
 
@@ -432,7 +448,8 @@ static int diff_print_patch_file(
 	const char *newpfx =
 		pi->diff ? pi->diff->opts.new_prefix : DIFF_NEW_PREFIX_DEFAULT;
 
-	bool binary = !!(delta->flags & GIT_DIFF_FLAG_BINARY);
+	bool binary = (delta->flags & GIT_DIFF_FLAG_BINARY) ||
+		(pi->flags & GIT_DIFF_FORCE_BINARY);
 	bool show_binary = !!(pi->flags & GIT_DIFF_SHOW_BINARY);
 	int oid_strlen = binary && show_binary ?
 		GIT_OID_HEXSZ + 1 : pi->oid_strlen;
@@ -455,19 +472,29 @@ static int diff_print_patch_file(
 	pi->line.content     = git_buf_cstr(pi->buf);
 	pi->line.content_len = git_buf_len(pi->buf);
 
-	if ((error = pi->print_cb(delta, NULL, &pi->line, pi->payload)) != 0)
-		return error;
+	return pi->print_cb(delta, NULL, &pi->line, pi->payload);
+}
 
-	if (!binary)
-		return 0;
+static int diff_print_patch_binary(
+	const git_diff_delta *delta,
+	const git_diff_binary *binary,
+	void *data)
+{
+	diff_print_info *pi = data;
+	const char *old_pfx =
+		pi->diff ? pi->diff->opts.old_prefix : DIFF_OLD_PREFIX_DEFAULT;
+	const char *new_pfx =
+		pi->diff ? pi->diff->opts.new_prefix : DIFF_NEW_PREFIX_DEFAULT;
+	int error;
 
 	git_buf_clear(pi->buf);
 
-	if ((error = diff_print_patch_file_binary(pi, delta, oldpfx, newpfx)) < 0)
+	if ((error = diff_print_patch_file_binary(
+		pi, (git_diff_delta *)delta, old_pfx, new_pfx, binary)) < 0)
 		return error;
 
-	pi->line.origin      = GIT_DIFF_LINE_BINARY;
-	pi->line.content     = git_buf_cstr(pi->buf);
+	pi->line.origin = GIT_DIFF_LINE_BINARY;
+	pi->line.content = git_buf_cstr(pi->buf);
 	pi->line.content_len = git_buf_len(pi->buf);
 
 	return pi->print_cb(delta, NULL, &pi->line, pi->payload);
@@ -515,12 +542,14 @@ int git_diff_print(
 	git_buf buf = GIT_BUF_INIT;
 	diff_print_info pi;
 	git_diff_file_cb print_file = NULL;
+	git_diff_binary_cb print_binary = NULL;
 	git_diff_hunk_cb print_hunk = NULL;
 	git_diff_line_cb print_line = NULL;
 
 	switch (format) {
 	case GIT_DIFF_FORMAT_PATCH:
 		print_file = diff_print_patch_file;
+		print_binary = diff_print_patch_binary;
 		print_hunk = diff_print_patch_hunk;
 		print_line = diff_print_patch_line;
 		break;
@@ -541,11 +570,10 @@ int git_diff_print(
 		return -1;
 	}
 
-	if (!(error = diff_print_info_init(
-			&pi, &buf, diff, format, print_cb, payload)))
-	{
+	if (!(error = diff_print_info_init_fromdiff(
+			&pi, &buf, diff, format, print_cb, payload))) {
 		error = git_diff_foreach(
-			diff, print_file, print_hunk, print_line, &pi);
+			diff, print_file, print_binary, print_hunk, print_line, &pi);
 
 		if (error) /* make sure error message is set */
 			giterr_set_after_callback_function(error, "git_diff_print");
@@ -568,13 +596,13 @@ int git_patch_print(
 
 	assert(patch && print_cb);
 
-	if (!(error = diff_print_info_init(
-			&pi, &temp, git_patch__diff(patch),
+	if (!(error = diff_print_info_init_frompatch(
+			&pi, &temp, patch,
 			GIT_DIFF_FORMAT_PATCH, print_cb, payload)))
 	{
 		error = git_patch__invoke_callbacks(
-			patch, diff_print_patch_file, diff_print_patch_hunk,
-			diff_print_patch_line, &pi);
+			patch, diff_print_patch_file, diff_print_patch_binary,
+			diff_print_patch_hunk, diff_print_patch_line, &pi);
 
 		if (error) /* make sure error message is set */
 			giterr_set_after_callback_function(error, "git_patch_print");
diff --git a/src/index.c b/src/index.c
index 0422ff1d4..14d8d3684 100644
--- a/src/index.c
+++ b/src/index.c
@@ -2727,7 +2727,7 @@ static int index_apply_to_wd_diff(git_index *index, int action, const git_strarr
 		goto cleanup;
 
 	data.pathspec = &ps;
-	error = git_diff_foreach(diff, apply_each_file, NULL, NULL, &data);
+	error = git_diff_foreach(diff, apply_each_file, NULL, NULL, NULL, &data);
 	git_diff_free(diff);
 
 	if (error) /* make sure error is set if callback stopped iteration */