diff: introduce binary diff callbacks

Introduce a new binary diff callback to provide the actual binary
delta contents to callers.  Create this data from the diff contents
(instead of directly from the ODB) to support binary diffs including
the workdir, not just things coming out of the ODB.

7 files changed, 450 insertions, 208 deletions

diff --git a/src/blame.c b/src/blame.c
index 4a12cb85b..a52f9402b 100644
--- a/src/blame.c
+++ b/src/blame.c
@@ -496,8 +496,8 @@ int git_blame_buffer(
 
 	/* Diff to the reference blob */
 	git_diff_blob_to_buffer(reference->final_blob, blame->path,
-			buffer, buffer_len, blame->path,
-			&diffopts, NULL, buffer_hunk_cb, buffer_line_cb, blame);
+		buffer, buffer_len, blame->path, &diffopts,
+		NULL, NULL, buffer_hunk_cb, buffer_line_cb, blame);
 
 	*out = blame;
 	return 0;
diff --git a/src/diff_file.c b/src/diff_file.c
index f7061ae83..cef4bc169 100644
--- a/src/diff_file.c
+++ b/src/diff_file.c
@@ -89,10 +89,9 @@ static int diff_file_content_init_common(
 int git_diff_file_content__init_from_diff(
 	git_diff_file_content *fc,
 	git_diff *diff,
-	size_t delta_index,
+	git_diff_delta *delta,
 	bool use_old)
 {
-	git_diff_delta *delta = git_vector_get(&diff->deltas, delta_index);
 	bool has_data = true;
 
 	memset(fc, 0, sizeof(*fc));
@@ -218,7 +217,9 @@ static int diff_file_content_commit_to_str(
 	return 0;
 }
 
-static int diff_file_content_load_blob(git_diff_file_content *fc)
+static int diff_file_content_load_blob(
+	git_diff_file_content *fc,
+	git_diff_options *opts)
 {
 	int error = 0;
 	git_odb_object *odb_obj = NULL;
@@ -236,7 +237,8 @@ static int diff_file_content_load_blob(git_diff_file_content *fc)
 			return error;
 	}
 
-	if (diff_file_content_binary_by_size(fc))
+	if ((opts->flags & GIT_DIFF_SHOW_BINARY) == 0 &&
+		diff_file_content_binary_by_size(fc))
 		return 0;
 
 	if (odb_obj != NULL) {
@@ -283,7 +285,9 @@ static int diff_file_content_load_workdir_symlink(
 }
 
 static int diff_file_content_load_workdir_file(
-	git_diff_file_content *fc, git_buf *path)
+	git_diff_file_content *fc,
+	git_buf *path,
+	git_diff_options *diff_opts)
 {
 	int error = 0;
 	git_filter_list *fl = NULL;
@@ -297,7 +301,8 @@ static int diff_file_content_load_workdir_file(
 		!(fc->file->size = git_futils_filesize(fd)))
 		goto cleanup;
 
-	if (diff_file_content_binary_by_size(fc))
+	if ((diff_opts->flags & GIT_DIFF_SHOW_BINARY) == 0 &&
+		diff_file_content_binary_by_size(fc))
 		goto cleanup;
 
 	if ((error = git_filter_list_load(
@@ -339,7 +344,9 @@ cleanup:
 	return error;
 }
 
-static int diff_file_content_load_workdir(git_diff_file_content *fc)
+static int diff_file_content_load_workdir(
+	git_diff_file_content *fc,
+	git_diff_options *diff_opts)
 {
 	int error = 0;
 	git_buf path = GIT_BUF_INIT;
@@ -357,7 +364,7 @@ static int diff_file_content_load_workdir(git_diff_file_content *fc)
 	if (S_ISLNK(fc->file->mode))
 		error = diff_file_content_load_workdir_symlink(fc, &path);
 	else
-		error = diff_file_content_load_workdir_file(fc, &path);
+		error = diff_file_content_load_workdir_file(fc, &path, diff_opts);
 
 	/* once data is loaded, update OID if we didn't have it previously */
 	if (!error && (fc->file->flags & GIT_DIFF_FLAG_VALID_ID) == 0) {
@@ -370,20 +377,23 @@ static int diff_file_content_load_workdir(git_diff_file_content *fc)
 	return error;
 }
 
-int git_diff_file_content__load(git_diff_file_content *fc)
+int git_diff_file_content__load(
+	git_diff_file_content *fc,
+	git_diff_options *diff_opts)
 {
 	int error = 0;
 
 	if ((fc->flags & GIT_DIFF_FLAG__LOADED) != 0)
 		return 0;
 
-	if ((fc->file->flags & GIT_DIFF_FLAG_BINARY) != 0)
+	if ((fc->file->flags & GIT_DIFF_FLAG_BINARY) != 0 &&
+		(diff_opts->flags & GIT_DIFF_SHOW_BINARY) == 0)
 		return 0;
 
 	if (fc->src == GIT_ITERATOR_TYPE_WORKDIR)
-		error = diff_file_content_load_workdir(fc);
+		error = diff_file_content_load_workdir(fc, diff_opts);
 	else
-		error = diff_file_content_load_blob(fc);
+		error = diff_file_content_load_blob(fc, diff_opts);
 	if (error)
 		return error;
 
diff --git a/src/diff_file.h b/src/diff_file.h
index 4d290ad43..0d54b6d33 100644
--- a/src/diff_file.h
+++ b/src/diff_file.h
@@ -28,7 +28,7 @@ typedef struct {
 extern int git_diff_file_content__init_from_diff(
 	git_diff_file_content *fc,
 	git_diff *diff,
-	size_t delta_index,
+	git_diff_delta *delta,
 	bool use_old);
 
 typedef struct {
@@ -49,7 +49,9 @@ extern int git_diff_file_content__init_from_src(
 	git_diff_file *as_file);
 
 /* this loads the blob/file-on-disk as needed */
-extern int git_diff_file_content__load(git_diff_file_content *fc);
+extern int git_diff_file_content__load(
+	git_diff_file_content *fc,
+	git_diff_options *diff_opts);
 
 /* this releases the blob/file-in-memory */
 extern void git_diff_file_content__unload(git_diff_file_content *fc);
diff --git a/src/diff_patch.c b/src/diff_patch.c
index 1c4c0e8b8..81b93e93e 100644
--- a/src/diff_patch.c
+++ b/src/diff_patch.c
@@ -11,41 +11,13 @@
 #include "diff_driver.h"
 #include "diff_patch.h"
 #include "diff_xdiff.h"
+#include "delta.h"
+#include "zstream.h"
 #include "fileops.h"
 
-/* cached information about a hunk in a diff */
-typedef struct diff_patch_hunk {
-	git_diff_hunk hunk;
-	size_t line_start;
-	size_t line_count;
-} diff_patch_hunk;
-
-struct git_patch {
-	git_refcount rc;
-	git_diff *diff; /* for refcount purposes, maybe NULL for blob diffs */
-	git_diff_delta *delta;
-	size_t delta_index;
-	git_diff_file_content ofile;
-	git_diff_file_content nfile;
-	uint32_t flags;
-	git_array_t(diff_patch_hunk) hunks;
-	git_array_t(git_diff_line)   lines;
-	size_t content_size, context_size, header_size;
-	git_pool flattened;
-};
-
-enum {
-	GIT_DIFF_PATCH_ALLOCATED   = (1 << 0),
-	GIT_DIFF_PATCH_INITIALIZED = (1 << 1),
-	GIT_DIFF_PATCH_LOADED      = (1 << 2),
-	GIT_DIFF_PATCH_DIFFABLE    = (1 << 3),
-	GIT_DIFF_PATCH_DIFFED      = (1 << 4),
-	GIT_DIFF_PATCH_FLATTENED   = (1 << 5),
-};
-
 static void diff_output_init(
-	git_diff_output*, const git_diff_options*,
-	git_diff_file_cb, git_diff_hunk_cb, git_diff_line_cb, void*);
+	git_diff_output*, const git_diff_options*, git_diff_file_cb,
+	git_diff_binary_cb, git_diff_hunk_cb, git_diff_line_cb, void*);
 
 static void diff_output_to_patch(git_diff_output *, git_patch *);
 
@@ -67,15 +39,38 @@ static void diff_patch_init_common(git_patch *patch)
 {
 	diff_patch_update_binary(patch);
 
-	if ((patch->delta->flags & GIT_DIFF_FLAG_BINARY) != 0)
-		patch->flags |= GIT_DIFF_PATCH_LOADED; /* LOADED but not DIFFABLE */
-
 	patch->flags |= GIT_DIFF_PATCH_INITIALIZED;
 
 	if (patch->diff)
 		git_diff_addref(patch->diff);
 }
 
+static int diff_patch_normalize_options(
+	git_diff_options *out,
+	const git_diff_options *opts)
+{
+	if (opts) {
+		GITERR_CHECK_VERSION(opts, GIT_DIFF_OPTIONS_VERSION, "git_diff_options");
+		memcpy(out, opts, sizeof(git_diff_options));
+	} else {
+		git_diff_options default_opts = GIT_DIFF_OPTIONS_INIT;
+		memcpy(out, &default_opts, sizeof(git_diff_options));
+	}
+
+	out->old_prefix = opts && opts->old_prefix ?
+		git__strdup(opts->old_prefix) :
+		git__strdup(DIFF_OLD_PREFIX_DEFAULT);
+
+	out->new_prefix = opts && opts->new_prefix ?
+		git__strdup(opts->new_prefix) :
+		git__strdup(DIFF_NEW_PREFIX_DEFAULT);
+
+	GITERR_CHECK_ALLOC(out->old_prefix);
+	GITERR_CHECK_ALLOC(out->new_prefix);
+
+	return 0;
+}
+
 static int diff_patch_init_from_diff(
 	git_patch *patch, git_diff *diff, size_t delta_index)
 {
@@ -86,10 +81,12 @@ static int diff_patch_init_from_diff(
 	patch->delta = git_vector_get(&diff->deltas, delta_index);
 	patch->delta_index = delta_index;
 
-	if ((error = git_diff_file_content__init_from_diff(
-			&patch->ofile, diff, delta_index, true)) < 0 ||
+	if ((error = diff_patch_normalize_options(
+			&patch->diff_opts, &diff->opts)) < 0 ||
 		(error = git_diff_file_content__init_from_diff(
-			&patch->nfile, diff, delta_index, false)) < 0)
+			&patch->ofile, diff, patch->delta, true)) < 0 ||
+		(error = git_diff_file_content__init_from_diff(
+			&patch->nfile, diff, patch->delta, false)) < 0)
 		return error;
 
 	diff_patch_init_common(patch);
@@ -116,6 +113,14 @@ static int diff_patch_alloc_from_diff(
 	return error;
 }
 
+GIT_INLINE(bool) should_skip_binary(git_patch *patch, git_diff_file *file)
+{
+	if ((patch->diff_opts.flags & GIT_DIFF_SHOW_BINARY) != 0)
+		return false;
+
+	return (file->flags & GIT_DIFF_FLAG_BINARY) != 0;
+}
+
 static int diff_patch_load(git_patch *patch, git_diff_output *output)
 {
 	int error = 0;
@@ -128,7 +133,7 @@ static int diff_patch_load(git_patch *patch, git_diff_output *output)
 	 * binary, then there is no need to actually load the data
 	 */
 	if ((patch->ofile.opts_flags & GIT_DIFF_SKIP_BINARY_CHECK) != 0 &&
-		output && !output->hunk_cb && !output->data_cb)
+		output && !output->binary_cb && !output->hunk_cb && !output->data_cb)
 		return 0;
 
 	incomplete_data =
@@ -141,25 +146,29 @@ static int diff_patch_load(git_patch *patch, git_diff_output *output)
 	 * need 2x data size and this minimizes peak memory footprint
 	 */
 	if (patch->ofile.src == GIT_ITERATOR_TYPE_WORKDIR) {
-		if ((error = git_diff_file_content__load(&patch->ofile)) < 0 ||
-			(patch->ofile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
+		if ((error = git_diff_file_content__load(
+				&patch->ofile, &patch->diff_opts)) < 0 ||
+			should_skip_binary(patch, patch->ofile.file))
 			goto cleanup;
 	}
 	if (patch->nfile.src == GIT_ITERATOR_TYPE_WORKDIR) {
-		if ((error = git_diff_file_content__load(&patch->nfile)) < 0 ||
-			(patch->nfile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
+		if ((error = git_diff_file_content__load(
+				&patch->nfile, &patch->diff_opts)) < 0 ||
+			should_skip_binary(patch, patch->nfile.file))
 			goto cleanup;
 	}
 
 	/* once workdir has been tried, load other data as needed */
 	if (patch->ofile.src != GIT_ITERATOR_TYPE_WORKDIR) {
-		if ((error = git_diff_file_content__load(&patch->ofile)) < 0 ||
-			(patch->ofile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
+		if ((error = git_diff_file_content__load(
+				&patch->ofile, &patch->diff_opts)) < 0 ||
+			should_skip_binary(patch, patch->ofile.file))
 			goto cleanup;
 	}
 	if (patch->nfile.src != GIT_ITERATOR_TYPE_WORKDIR) {
-		if ((error = git_diff_file_content__load(&patch->nfile)) < 0 ||
-			(patch->nfile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
+		if ((error = git_diff_file_content__load(
+				&patch->nfile, &patch->diff_opts)) < 0 ||
+			should_skip_binary(patch, patch->nfile.file))
 			goto cleanup;
 	}
 
@@ -177,10 +186,14 @@ cleanup:
 	diff_patch_update_binary(patch);
 
 	if (!error) {
+		bool skip_binary =
+			(patch->delta->flags & GIT_DIFF_FLAG_BINARY) != 0 &&
+			(patch->diff_opts.flags & GIT_DIFF_SHOW_BINARY) == 0;
+
 		/* patch is diffable only for non-binary, modified files where
-		 * at least one side has data and the data actually changed
-		 */
-		if ((patch->delta->flags & GIT_DIFF_FLAG_BINARY) == 0 &&
+		* at least one side has data and the data actually changed
+		*/
+		if (!skip_binary &&
 			patch->delta->status != GIT_DELTA_UNMODIFIED &&
 			(patch->ofile.map.len || patch->nfile.map.len) &&
 			(patch->ofile.map.len != patch->nfile.map.len ||
@@ -207,6 +220,98 @@ static int diff_patch_invoke_file_callback(
 		"git_patch");
 }
 
+static int create_binary(
+	git_diff_binary_t *out_type,
+	char **out_data,
+	size_t *out_datalen,
+	size_t *out_inflatedlen,
+	const char *a_data,
+	size_t a_datalen,
+	const char *b_data,
+	size_t b_datalen)
+{
+	git_buf deflate = GIT_BUF_INIT, delta = GIT_BUF_INIT;
+	unsigned long delta_data_len;
+	int error;
+
+	/* The git_delta function accepts unsigned long only */
+	if (!git__is_ulong(a_datalen) || !git__is_ulong(b_datalen))
+		return GIT_EBUFS;
+
+	if ((error = git_zstream_deflatebuf(&deflate, b_data, b_datalen)) < 0)
+		goto done;
+
+	/* The git_delta function accepts unsigned long only */
+	if (!git__is_ulong(deflate.size)) {
+		error = GIT_EBUFS;
+		goto done;
+	}
+
+	if (a_datalen && b_datalen) {
+		void *delta_data = git_delta(
+			a_data, (unsigned long)a_datalen,
+			b_data, (unsigned long)b_datalen,
+			&delta_data_len, (unsigned long)deflate.size);
+
+		if (delta_data) {
+			error = git_zstream_deflatebuf(
+				&delta, delta_data, (size_t)delta_data_len);
+
+			git__free(delta_data);
+
+			if (error < 0)
+				goto done;
+		}
+	}
+
+	if (delta.size && delta.size < deflate.size) {
+		*out_type = GIT_DIFF_BINARY_DELTA;
+		*out_datalen = delta.size;
+		*out_data = git_buf_detach(&delta);
+		*out_inflatedlen = delta_data_len;
+	} else {
+		*out_type = GIT_DIFF_BINARY_LITERAL;
+		*out_datalen = deflate.size;
+		*out_data = git_buf_detach(&deflate);
+		*out_inflatedlen = b_datalen;
+	}
+
+done:
+	git_buf_free(&deflate);
+	git_buf_free(&delta);
+
+	return error;
+}
+
+static int diff_binary(git_diff_output *output, git_patch *patch)
+{
+	git_diff_binary binary = { 0 };
+	const char *old_data = patch->ofile.map.data;
+	const char *new_data = patch->nfile.map.data;
+	size_t old_len = patch->ofile.map.len,
+		new_len = patch->nfile.map.len;
+	int error;
+
+	/* Create the old->new delta (as the "new" side of the patch),
+	 * and the new->old delta (as the "old" side)
+	 */
+	if ((error = create_binary(&binary.old_file.type,
+			(char **)&binary.old_file.data,
+			&binary.old_file.datalen,
+			&binary.old_file.inflatedlen,
+			new_data, new_len, old_data, old_len)) < 0 ||
+		(error = create_binary(&binary.new_file.type,
+			(char **)&binary.new_file.data,
+			&binary.new_file.datalen,
+			&binary.new_file.inflatedlen,
+			old_data, old_len, new_data, new_len)) < 0)
+		return error;
+
+	return giterr_set_after_callback_function(
+		output->binary_cb(patch->delta, &binary, output->payload),
+		"git_patch");
+}
+
 static int diff_patch_generate(git_patch *patch, git_diff_output *output)
 {
 	int error = 0;
@@ -214,8 +319,8 @@ static int diff_patch_generate(git_patch *patch, git_diff_output *output)
 	if ((patch->flags & GIT_DIFF_PATCH_DIFFED) != 0)
 		return 0;
 
-	/* if we are not looking at the hunks and lines, don't do the diff */
-	if (!output->hunk_cb && !output->data_cb)
+	/* if we are not looking at the binary or text data, don't do the diff */
+	if (!output->binary_cb && !output->hunk_cb && !output->data_cb)
 		return 0;
 
 	if ((patch->flags & GIT_DIFF_PATCH_LOADED) == 0 &&
@@ -225,10 +330,16 @@ static int diff_patch_generate(git_patch *patch, git_diff_output *output)
 	if ((patch->flags & GIT_DIFF_PATCH_DIFFABLE) == 0)
 		return 0;
 
-	if (output->diff_cb != NULL &&
-		(error = output->diff_cb(output, patch)) < 0)
-		patch->flags |= GIT_DIFF_PATCH_DIFFED;
+	if ((patch->delta->flags & GIT_DIFF_FLAG_BINARY) != 0) {
+		if (output->binary_cb)
+			error = diff_binary(output, patch);
+	}
+	else {
+		if (output->diff_cb)
+			error = output->diff_cb(output, patch);
+	}
 
+	patch->flags |= GIT_DIFF_PATCH_DIFFED;
 	return error;
 }
 
@@ -245,6 +356,9 @@ static void diff_patch_free(git_patch *patch)
 
 	git_pool_clear(&patch->flattened);
 
+	git__free((char *)patch->diff_opts.old_prefix);
+	git__free((char *)patch->diff_opts.new_prefix);
+
 	if (patch->flags & GIT_DIFF_PATCH_ALLOCATED)
 		git__free(patch);
 }
@@ -260,6 +374,7 @@ static int diff_required(git_diff *diff, const char *action)
 int git_diff_foreach(
 	git_diff *diff,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb data_cb,
 	void *payload)
@@ -275,7 +390,7 @@ int git_diff_foreach(
 	memset(&xo, 0, sizeof(xo));
 	memset(&patch, 0, sizeof(patch));
 	diff_output_init(
-		&xo.output, &diff->opts, file_cb, hunk_cb, data_cb, payload);
+		&xo.output, &diff->opts, file_cb, binary_cb, hunk_cb, data_cb, payload);
 	git_xdiff_init(&xo, &diff->opts);
 
 	git_vector_foreach(&diff->deltas, idx, patch.delta) {
@@ -284,11 +399,15 @@ int git_diff_foreach(
 		if (git_diff_delta__should_skip(&diff->opts, patch.delta))
 			continue;
 
+		if (binary_cb || hunk_cb || data_cb) {
+			if ((error = diff_patch_init_from_diff(&patch, diff, idx)) != 0 ||
+				(error = diff_patch_load(&patch, &xo.output)) != 0)
+				return error;
+		}
+
 		if ((error = diff_patch_invoke_file_callback(&patch, &xo.output)) == 0) {
-			if (hunk_cb || data_cb) {
-				if ((error = diff_patch_init_from_diff(&patch, diff, idx)) == 0)
+			if (binary_cb || hunk_cb || data_cb)
 					error = diff_patch_generate(&patch, &xo.output);
-			}
 		}
 
 		git_patch_free(&patch);
@@ -419,6 +538,7 @@ static int diff_from_sources(
 	git_diff_file_content_src *newsrc,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb data_cb,
 	void *payload)
@@ -429,7 +549,7 @@ static int diff_from_sources(
 
 	memset(&xo, 0, sizeof(xo));
 	diff_output_init(
-		&xo.output, opts, file_cb, hunk_cb, data_cb, payload);
+		&xo.output, opts, file_cb, binary_cb, hunk_cb, data_cb, payload);
 	git_xdiff_init(&xo, opts);
 
 	memset(&pd, 0, sizeof(pd));
@@ -477,6 +597,7 @@ int git_diff_blobs(
 	const char *new_path,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb data_cb,
 	void *payload)
@@ -486,7 +607,7 @@ int git_diff_blobs(
 	git_diff_file_content_src nsrc =
 		GIT_DIFF_FILE_CONTENT_SRC__BLOB(new_blob, new_path);
 	return diff_from_sources(
-		&osrc, &nsrc, opts, file_cb, hunk_cb, data_cb, payload);
+		&osrc, &nsrc, opts, file_cb, binary_cb, hunk_cb, data_cb, payload);
 }
 
 int git_patch_from_blobs(
@@ -512,6 +633,7 @@ int git_diff_blob_to_buffer(
 	const char *buf_path,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb data_cb,
 	void *payload)
@@ -521,7 +643,7 @@ int git_diff_blob_to_buffer(
 	git_diff_file_content_src nsrc =
 		GIT_DIFF_FILE_CONTENT_SRC__BUF(buf, buflen, buf_path);
 	return diff_from_sources(
-		&osrc, &nsrc, opts, file_cb, hunk_cb, data_cb, payload);
+		&osrc, &nsrc, opts, file_cb, binary_cb, hunk_cb, data_cb, payload);
 }
 
 int git_patch_from_blob_and_buffer(
@@ -549,6 +671,7 @@ int git_diff_buffers(
 	const char *new_path,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb data_cb,
 	void *payload)
@@ -558,7 +681,7 @@ int git_diff_buffers(
 	git_diff_file_content_src nsrc =
 		GIT_DIFF_FILE_CONTENT_SRC__BUF(new_buf, new_len, new_path);
 	return diff_from_sources(
-		&osrc, &nsrc, opts, file_cb, hunk_cb, data_cb, payload);
+		&osrc, &nsrc, opts, file_cb, binary_cb, hunk_cb, data_cb, payload);
 }
 
 int git_patch_from_buffers(
@@ -812,6 +935,7 @@ void git_patch__new_data(
 int git_patch__invoke_callbacks(
 	git_patch *patch,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb line_cb,
 	void *payload)
@@ -822,6 +946,13 @@ int git_patch__invoke_callbacks(
 	if (file_cb)
 		error = file_cb(patch->delta, 0, payload);
 
+	if ((patch->delta->flags & GIT_DIFF_FLAG_BINARY) != 0) {
+		if (binary_cb)
+			error = binary_cb(patch->delta, &patch->binary, payload);
+
+		return error;
+	}
+
 	if (!hunk_cb && !line_cb)
 		return error;
 
@@ -855,6 +986,36 @@ static int diff_patch_file_cb(
 	return 0;
 }
 
+static int diff_patch_binary_cb(
+	const git_diff_delta *delta,
+	const git_diff_binary *binary,
+	void *payload)
+{
+	git_patch *patch = payload;
+
+	GIT_UNUSED(delta);
+
+	memcpy(&patch->binary, binary, sizeof(git_diff_binary));
+
+	if (binary->old_file.data) {
+		patch->binary.old_file.data = git__malloc(binary->old_file.datalen);
+		GITERR_CHECK_ALLOC(patch->binary.old_file.data);
+
+		memcpy((char *)patch->binary.old_file.data,
+			binary->old_file.data, binary->old_file.datalen);
+	}
+
+	if (binary->new_file.data) {
+		patch->binary.new_file.data = git__malloc(binary->new_file.datalen);
+		GITERR_CHECK_ALLOC(patch->binary.new_file.data);
+
+		memcpy((char *)patch->binary.new_file.data,
+			binary->new_file.data, binary->new_file.datalen);
+	}
+
+	return 0;
+}
+
 static int diff_patch_hunk_cb(
 	const git_diff_delta *delta,
 	const git_diff_hunk *hunk_,
@@ -921,6 +1082,7 @@ static void diff_output_init(
 	git_diff_output *out,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb data_cb,
 	void *payload)
@@ -930,6 +1092,7 @@ static void diff_output_init(
 	memset(out, 0, sizeof(*out));
 
 	out->file_cb = file_cb;
+	out->binary_cb = binary_cb;
 	out->hunk_cb = hunk_cb;
 	out->data_cb = data_cb;
 	out->payload = payload;
@@ -938,6 +1101,11 @@ static void diff_output_init(
 static void diff_output_to_patch(git_diff_output *out, git_patch *patch)
 {
 	diff_output_init(
-		out, NULL,
-		diff_patch_file_cb, diff_patch_hunk_cb, diff_patch_line_cb, patch);
+		out,
+		NULL,
+		diff_patch_file_cb,
+		diff_patch_binary_cb,
+		diff_patch_hunk_cb,
+		diff_patch_line_cb,
+		patch);
 }
diff --git a/src/diff_patch.h b/src/diff_patch.h
index df2ba4c31..f6ce57ddd 100644
--- a/src/diff_patch.h
+++ b/src/diff_patch.h
@@ -13,6 +13,38 @@
 #include "array.h"
 #include "git2/patch.h"
 
+ /* cached information about a hunk in a diff */
+typedef struct diff_patch_hunk {
+	git_diff_hunk hunk;
+	size_t line_start;
+	size_t line_count;
+} diff_patch_hunk;
+
+enum {
+	GIT_DIFF_PATCH_ALLOCATED = (1 << 0),
+	GIT_DIFF_PATCH_INITIALIZED = (1 << 1),
+	GIT_DIFF_PATCH_LOADED = (1 << 2),
+	GIT_DIFF_PATCH_DIFFABLE = (1 << 3),
+	GIT_DIFF_PATCH_DIFFED = (1 << 4),
+	GIT_DIFF_PATCH_FLATTENED = (1 << 5),
+};
+
+struct git_patch {
+	git_refcount rc;
+	git_diff *diff; /* for refcount purposes, maybe NULL for blob diffs */
+	git_diff_options diff_opts;
+	git_diff_delta *delta;
+	size_t delta_index;
+	git_diff_file_content ofile;
+	git_diff_file_content nfile;
+	uint32_t flags;
+	git_diff_binary binary;
+	git_array_t(diff_patch_hunk) hunks;
+	git_array_t(git_diff_line)   lines;
+	size_t content_size, context_size, header_size;
+	git_pool flattened;
+};
+
 extern git_diff *git_patch__diff(git_patch *);
 
 extern git_diff_driver *git_patch__driver(git_patch *);
@@ -23,6 +55,7 @@ extern void git_patch__new_data(char **, size_t *, git_patch *);
 extern int git_patch__invoke_callbacks(
 	git_patch *patch,
 	git_diff_file_cb file_cb,
+	git_diff_binary_cb binary_cb,
 	git_diff_hunk_cb hunk_cb,
 	git_diff_line_cb line_cb,
 	void *payload);
@@ -31,6 +64,7 @@ typedef struct git_diff_output git_diff_output;
 struct git_diff_output {
 	/* these callbacks are issued with the diff data */
 	git_diff_file_cb file_cb;
+	git_diff_binary_cb binary_cb;
 	git_diff_hunk_cb hunk_cb;
 	git_diff_line_cb data_cb;
 	void *payload;
diff --git a/src/diff_print.c b/src/diff_print.c
index 43a90b3d8..ebae4ea75 100644
--- a/src/diff_print.c
+++ b/src/diff_print.c
@@ -22,50 +22,92 @@ typedef struct {
 	uint32_t flags;
 	int oid_strlen;
 	git_diff_line line;
+	unsigned int
+		content_loaded : 1,
+		content_allocated : 1;
+	git_diff_file_content *ofile;
+	git_diff_file_content *nfile;
 } diff_print_info;
 
-static int diff_print_info_init(
+static int diff_print_info_init__common(
 	diff_print_info *pi,
 	git_buf *out,
-	git_diff *diff,
+	git_repository *repo,
 	git_diff_format_t format,
 	git_diff_line_cb cb,
 	void *payload)
 {
-	pi->diff     = diff;
-	pi->format   = format;
+	pi->format = format;
 	pi->print_cb = cb;
-	pi->payload  = payload;
-	pi->buf      = out;
-
-	if (diff)
-		pi->flags = diff->opts.flags;
-	else
-		pi->flags = 0;
-
-	if (diff && diff->opts.id_abbrev != 0)
-		pi->oid_strlen = diff->opts.id_abbrev;
-	else if (!diff || !diff->repo)
-		pi->oid_strlen = GIT_ABBREV_DEFAULT;
-	else if (git_repository__cvar(
-		&pi->oid_strlen, diff->repo, GIT_CVAR_ABBREV) < 0)
-		return -1;
+	pi->payload = payload;
+	pi->buf = out;
+
+	if (!pi->oid_strlen) {
+		if (!repo)
+			pi->oid_strlen = GIT_ABBREV_DEFAULT;
+		else if (git_repository__cvar(&pi->oid_strlen, repo, GIT_CVAR_ABBREV) < 0)
+			return -1;
+	}
 
 	pi->oid_strlen += 1; /* for NUL byte */
 
-	if (pi->oid_strlen < 2)
-		pi->oid_strlen = 2;
-	else if (pi->oid_strlen > GIT_OID_HEXSZ + 1)
+	if (pi->oid_strlen > GIT_OID_HEXSZ + 1)
 		pi->oid_strlen = GIT_OID_HEXSZ + 1;
 
 	memset(&pi->line, 0, sizeof(pi->line));
 	pi->line.old_lineno = -1;
 	pi->line.new_lineno = -1;
-	pi->line.num_lines  = 1;
+	pi->line.num_lines = 1;
 
 	return 0;
 }
 
+static int diff_print_info_init_fromdiff(
+	diff_print_info *pi,
+	git_buf *out,
+	git_diff *diff,
+	git_diff_format_t format,
+	git_diff_line_cb cb,
+	void *payload)
+{
+	git_repository *repo = diff ? diff->repo : NULL;
+
+	memset(pi, 0, sizeof(diff_print_info));
+
+	pi->diff = diff;
+
+	if (diff) {
+		pi->flags = diff->opts.flags;
+		pi->oid_strlen = diff->opts.id_abbrev;
+	}
+
+	return diff_print_info_init__common(pi, out, repo, format, cb, payload);
+}
+
+static int diff_print_info_init_frompatch(
+	diff_print_info *pi,
+	git_buf *out,
+	git_patch *patch,
+	git_diff_format_t format,
+	git_diff_line_cb cb,
+	void *payload)
+{
+	git_repository *repo = patch && patch->diff ? patch->diff->repo : NULL;
+
+	memset(pi, 0, sizeof(diff_print_info));
+
+	pi->diff = patch->diff;
+
+	pi->flags = patch->diff_opts.flags;
+	pi->oid_strlen = patch->diff_opts.id_abbrev;
+
+	pi->content_loaded = 1;
+	pi->ofile = &patch->ofile;
+	pi->nfile = &patch->nfile;
+
+	return diff_print_info_init__common(pi, out, repo, format, cb, payload);
+}
+
 static char diff_pick_suffix(int mode)
 {
 	if (S_ISDIR(mode))
@@ -283,66 +325,22 @@ int git_diff_delta__format_file_header(
 	return git_buf_oom(out) ? -1 : 0;
 }
 
-static int print_binary_hunk(diff_print_info *pi, git_blob *old, git_blob *new)
+static int format_binary(
+	diff_print_info *pi,
+	git_diff_binary_t type,
+	const char *data,
+	size_t datalen,
+	size_t inflatedlen)
 {
-	git_buf deflate = GIT_BUF_INIT, delta = GIT_BUF_INIT, *out = NULL;
-	const void *old_data, *new_data;
-	git_off_t old_data_len, new_data_len;
-	unsigned long delta_data_len, inflated_len;
-	const char *out_type = "literal";
-	char *scan, *end;
-	int error;
-
-	old_data = old ? git_blob_rawcontent(old) : NULL;
-	new_data = new ? git_blob_rawcontent(new) : NULL;
-
-	old_data_len = old ? git_blob_rawsize(old) : 0;
-	new_data_len = new ? git_blob_rawsize(new) : 0;
-
-	/* The git_delta function accepts unsigned long only */
-	if (!git__is_ulong(old_data_len) || !git__is_ulong(new_data_len))
-		return GIT_EBUFS;
-
-	out = &deflate;
-	inflated_len = (unsigned long)new_data_len;
-
-	if ((error = git_zstream_deflatebuf(
-			out, new_data, (size_t)new_data_len)) < 0)
-		goto done;
-
-	/* The git_delta function accepts unsigned long only */
-	if (!git__is_ulong((git_off_t)deflate.size)) {
-		error = GIT_EBUFS;
-		goto done;
-	}
+	const char *typename = type == GIT_DIFF_BINARY_DELTA ?
+		"delta" : "literal";
+	const char *scan, *end;
+	int error = 0;
 
-	if (old && new) {
-		void *delta_data = git_delta(
-			old_data, (unsigned long)old_data_len,
-			new_data, (unsigned long)new_data_len,
-			&delta_data_len, (unsigned long)deflate.size);
-
-		if (delta_data) {
-			error = git_zstream_deflatebuf(
-				&delta, delta_data, (size_t)delta_data_len);
-
-			git__free(delta_data);
-
-			if (error < 0)
-				goto done;
-
-			if (delta.size < deflate.size) {
-				out = &delta;
-				out_type = "delta";
-				inflated_len = delta_data_len;
-			}
-		}
-	}
-
-	git_buf_printf(pi->buf, "%s %lu\n", out_type, inflated_len);
+	git_buf_printf(pi->buf, "%s %lu\n", typename, inflatedlen);
 	pi->line.num_lines++;
 
-	for (scan = out->ptr, end = out->ptr + out->size; scan < end; ) {
+	for (scan = data, end = data + datalen; scan < end; ) {
 		size_t chunk_len = end - scan;
 		if (chunk_len > 52)
 			chunk_len = 52;
@@ -355,51 +353,74 @@ static int print_binary_hunk(diff_print_info *pi, git_blob *old, git_blob *new)
 		git_buf_encode_base85(pi->buf, scan, chunk_len);
 		git_buf_putc(pi->buf, '\n');
 
-		if (git_buf_oom(pi->buf)) {
-			error = -1;
-			goto done;
-		}
+		if (git_buf_oom(pi->buf))
+			return -1;
 
 		scan += chunk_len;
 		pi->line.num_lines++;
 	}
 
-done:
-	git_buf_free(&deflate);
-	git_buf_free(&delta);
+	return 0;
+}
 
-	return error;
+static int diff_print_load_content(
+	diff_print_info *pi,
+	git_diff_delta *delta)
+{
+	git_diff_file_content *ofile, *nfile;
+	int error;
+
+	assert(pi->diff);
+
+	ofile = git__calloc(1, sizeof(git_diff_file_content));
+	nfile = git__calloc(1, sizeof(git_diff_file_content));
+
+	GITERR_CHECK_ALLOC(ofile);
+	GITERR_CHECK_ALLOC(nfile);
+
+	if ((error = git_diff_file_content__init_from_diff(
+			ofile, pi->diff, delta, true)) < 0 ||
+		(error = git_diff_file_content__init_from_diff(
+			nfile, pi->diff, delta, true)) < 0) {
+
+		git__free(ofile);
+		git__free(nfile);
+		return error;
+	}
+
+	pi->content_loaded = 1;
+	pi->content_allocated = 1;
+	pi->ofile = ofile;
+	pi->nfile = nfile;
+
+	return 0;
 }
 
-/* git diff --binary 8d7523f~2 8d7523f~1 */
 static int diff_print_patch_file_binary(
-	diff_print_info *pi, const git_diff_delta *delta,
-	const char *oldpfx, const char *newpfx)
+	diff_print_info *pi, git_diff_delta *delta,
+	const char *old_pfx, const char *new_pfx,
+	const git_diff_binary *binary)
 {
-	git_blob *old = NULL, *new = NULL;
-	const git_oid *old_id, *new_id;
-	int error;
 	size_t pre_binary_size;
+	int error;
 
 	if ((pi->flags & GIT_DIFF_SHOW_BINARY) == 0)
 		goto noshow;
 
+	if (!pi->content_loaded &&
+		(error = diff_print_load_content(pi, delta)) < 0)
+		return error;
+
 	pre_binary_size = pi->buf->size;
 	git_buf_printf(pi->buf, "GIT binary patch\n");
 	pi->line.num_lines++;
 
-	old_id = (delta->status != GIT_DELTA_ADDED) ? &delta->old_file.id : NULL;
-	new_id = (delta->status != GIT_DELTA_DELETED) ? &delta->new_file.id : NULL;
-
-	if (old_id && (error = git_blob_lookup(&old, pi->diff->repo, old_id)) < 0)
-		goto done;
-	if (new_id && (error = git_blob_lookup(&new, pi->diff->repo,new_id)) < 0)
-		goto done;
-
-	if ((error = print_binary_hunk(pi, old, new)) < 0 ||
+	if ((error = format_binary(pi, binary->new_file.type, binary->new_file.data,
+		binary->new_file.datalen, binary->new_file.inflatedlen)) < 0 ||
 		(error = git_buf_putc(pi->buf, '\n')) < 0 ||
-		(error = print_binary_hunk(pi, new, old)) < 0)
-	{
+		(error = format_binary(pi, binary->old_file.type, binary->old_file.data,
+			binary->old_file.datalen, binary->old_file.inflatedlen)) < 0) {
+
 		if (error == GIT_EBUFS) {
 			giterr_clear();
 			git_buf_truncate(pi->buf, pre_binary_size);
@@ -408,17 +429,12 @@ static int diff_print_patch_file_binary(
 	}
 
 	pi->line.num_lines++;
-
-done:
-	git_blob_free(old);
-	git_blob_free(new);
-
 	return error;
 
 noshow:
 	pi->line.num_lines = 1;
 	return diff_delta_format_with_paths(
-		pi->buf, delta, oldpfx, newpfx,
+		pi->buf, delta, old_pfx, new_pfx,
 		"Binary files %s%s and %s%s differ\n");
 }
 
@@ -432,7 +448,8 @@ static int diff_print_patch_file(
 	const char *newpfx =
 		pi->diff ? pi->diff->opts.new_prefix : DIFF_NEW_PREFIX_DEFAULT;
 
-	bool binary = !!(delta->flags & GIT_DIFF_FLAG_BINARY);
+	bool binary = (delta->flags & GIT_DIFF_FLAG_BINARY) ||
+		(pi->flags & GIT_DIFF_FORCE_BINARY);
 	bool show_binary = !!(pi->flags & GIT_DIFF_SHOW_BINARY);
 	int oid_strlen = binary && show_binary ?
 		GIT_OID_HEXSZ + 1 : pi->oid_strlen;
@@ -455,19 +472,29 @@ static int diff_print_patch_file(
 	pi->line.content     = git_buf_cstr(pi->buf);
 	pi->line.content_len = git_buf_len(pi->buf);
 
-	if ((error = pi->print_cb(delta, NULL, &pi->line, pi->payload)) != 0)
-		return error;
+	return pi->print_cb(delta, NULL, &pi->line, pi->payload);
+}
 
-	if (!binary)
-		return 0;
+static int diff_print_patch_binary(
+	const git_diff_delta *delta,
+	const git_diff_binary *binary,
+	void *data)
+{
+	diff_print_info *pi = data;
+	const char *old_pfx =
+		pi->diff ? pi->diff->opts.old_prefix : DIFF_OLD_PREFIX_DEFAULT;
+	const char *new_pfx =
+		pi->diff ? pi->diff->opts.new_prefix : DIFF_NEW_PREFIX_DEFAULT;
+	int error;
 
 	git_buf_clear(pi->buf);
 
-	if ((error = diff_print_patch_file_binary(pi, delta, oldpfx, newpfx)) < 0)
+	if ((error = diff_print_patch_file_binary(
+		pi, (git_diff_delta *)delta, old_pfx, new_pfx, binary)) < 0)
 		return error;
 
-	pi->line.origin      = GIT_DIFF_LINE_BINARY;
-	pi->line.content     = git_buf_cstr(pi->buf);
+	pi->line.origin = GIT_DIFF_LINE_BINARY;
+	pi->line.content = git_buf_cstr(pi->buf);
 	pi->line.content_len = git_buf_len(pi->buf);
 
 	return pi->print_cb(delta, NULL, &pi->line, pi->payload);
@@ -515,12 +542,14 @@ int git_diff_print(
 	git_buf buf = GIT_BUF_INIT;
 	diff_print_info pi;
 	git_diff_file_cb print_file = NULL;
+	git_diff_binary_cb print_binary = NULL;
 	git_diff_hunk_cb print_hunk = NULL;
 	git_diff_line_cb print_line = NULL;
 
 	switch (format) {
 	case GIT_DIFF_FORMAT_PATCH:
 		print_file = diff_print_patch_file;
+		print_binary = diff_print_patch_binary;
 		print_hunk = diff_print_patch_hunk;
 		print_line = diff_print_patch_line;
 		break;
@@ -541,11 +570,10 @@ int git_diff_print(
 		return -1;
 	}
 
-	if (!(error = diff_print_info_init(
-			&pi, &buf, diff, format, print_cb, payload)))
-	{
+	if (!(error = diff_print_info_init_fromdiff(
+			&pi, &buf, diff, format, print_cb, payload))) {
 		error = git_diff_foreach(
-			diff, print_file, print_hunk, print_line, &pi);
+			diff, print_file, print_binary, print_hunk, print_line, &pi);
 
 		if (error) /* make sure error message is set */
 			giterr_set_after_callback_function(error, "git_diff_print");
@@ -568,13 +596,13 @@ int git_patch_print(
 
 	assert(patch && print_cb);
 
-	if (!(error = diff_print_info_init(
-			&pi, &temp, git_patch__diff(patch),
+	if (!(error = diff_print_info_init_frompatch(
+			&pi, &temp, patch,
 			GIT_DIFF_FORMAT_PATCH, print_cb, payload)))
 	{
 		error = git_patch__invoke_callbacks(
-			patch, diff_print_patch_file, diff_print_patch_hunk,
-			diff_print_patch_line, &pi);
+			patch, diff_print_patch_file, diff_print_patch_binary,
+			diff_print_patch_hunk, diff_print_patch_line, &pi);
 
 		if (error) /* make sure error message is set */
 			giterr_set_after_callback_function(error, "git_patch_print");
diff --git a/src/index.c b/src/index.c
index 0422ff1d4..14d8d3684 100644
--- a/src/index.c
+++ b/src/index.c
@@ -2727,7 +2727,7 @@ static int index_apply_to_wd_diff(git_index *index, int action, const git_strarr
 		goto cleanup;
 
 	data.pathspec = &ps;
-	error = git_diff_foreach(diff, apply_each_file, NULL, NULL, &data);
+	error = git_diff_foreach(diff, apply_each_file, NULL, NULL, NULL, &data);
 	git_diff_free(diff);
 
 	if (error) /* make sure error is set if callback stopped iteration */