Extend public filter api with filter lists

This moves the git_filter_list into the public API so that users
can create, apply, and dispose of filter lists.  This allows more
granular application of filters to user data outside of libgit2
internals.

This also converts all the internal usage of filters to the public
APIs along with a few small tweaks to make it easier to use the
public git_buffer stuff alongside the internal git_buf.

9 files changed, 171 insertions, 205 deletions

diff --git a/src/blob.c b/src/blob.c
index 3581ee9d1..e6bba033a 100644
--- a/src/blob.c
+++ b/src/blob.c
@@ -111,26 +111,18 @@ static int write_file_filtered(
 	git_filter_list *fl)
 {
 	int error;
-	git_buf source = GIT_BUF_INIT;
-	git_buf dest = GIT_BUF_INIT;
+	git_buffer tgt = GIT_BUFFER_INIT;
 
-	if ((error = git_futils_readbuffer(&source, full_path)) < 0)
-		return error;
-
-	error = git_filter_list_apply(&dest, &source, fl);
-
-	/* Free the source as soon as possible. This can be big in memory,
-	 * and we don't want to ODB write to choke */
-	git_buf_free(&source);
+	error = git_filter_list_apply_to_file(&tgt, fl, NULL, full_path);
 
 	/* Write the file to disk if it was properly filtered */
 	if (!error) {
-		*size = dest.size;
+		*size = tgt.size;
 
-		error = git_odb_write(oid, odb, dest.ptr, dest.size, GIT_OBJ_BLOB);
+		error = git_odb_write(oid, odb, tgt.ptr, tgt.size, GIT_OBJ_BLOB);
 	}
 
-	git_buf_free(&dest);
+	git_buffer_free(&tgt);
 	return error;
 }
 
@@ -329,8 +321,9 @@ int git_blob_is_binary(git_blob *blob)
 
 	assert(blob);
 
-	content.ptr = blob->odb_object->buffer;
-	content.size = min(blob->odb_object->cached.size, 4000);
+	content.ptr   = blob->odb_object->buffer;
+	content.size  = min(blob->odb_object->cached.size, 4000);
+	content.asize = 0;
 
 	return git_buf_text_is_binary(&content);
 }
@@ -342,46 +335,20 @@ int git_blob_filtered_content(
 	int check_for_binary_data)
 {
 	int error = 0;
-	git_buf filtered = GIT_BUF_INIT, unfiltered = GIT_BUF_INIT;
 	git_filter_list *fl = NULL;
 
 	assert(blob && as_path && out);
 
-	/* Create a fake git_buf from the blob raw data... */
-	filtered.ptr   = (void *)git_blob_rawcontent(blob);
-	filtered.size  = (size_t)git_blob_rawsize(blob);
-	filtered.asize = 0;
-
-	if (check_for_binary_data && git_buf_text_is_binary(&filtered))
+	if (check_for_binary_data && git_blob_is_binary(blob))
 		return 0;
 
-	error = git_filter_list_load(
-		&fl, git_blob_owner(blob), as_path, GIT_FILTER_TO_WORKTREE);
-	if (error < 0)
-		return error;
+	if (!(error = git_filter_list_load(
+			&fl, git_blob_owner(blob), as_path, GIT_FILTER_TO_WORKTREE))) {
 
-	if (fl != NULL) {
-		if (out->ptr && out->available) {
-			filtered.ptr   = out->ptr;
-			filtered.size  = out->size;
-			filtered.asize = out->available;
-		} else {
-			git_buf_init(&filtered, filtered.size + 1);
-		}
-
-		if (!(error = git_blob__getbuf(&unfiltered, blob)))
-			error = git_filter_list_apply(&filtered, &unfiltered, fl);
+		error = git_filter_list_apply_to_blob(out, fl, blob);
 
 		git_filter_list_free(fl);
-		git_buf_free(&unfiltered);
-	}
-
-	if (!error) {
-		out->ptr  = filtered.ptr;
-		out->size = filtered.size;
-		out->available = filtered.asize;
 	}
 
 	return error;
 }
-
diff --git a/src/buffer.c b/src/buffer.c
index a92133674..aaebac776 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -501,7 +501,8 @@ void git_buffer_free(git_buffer *buffer)
 	git__memzero(buffer, sizeof(*buffer));
 }
 
-int git_buffer_resize(git_buffer *buffer, size_t want_size)
+static int git_buffer__resize(
+	git_buffer *buffer, size_t want_size, int preserve_data)
 {
 	int non_allocated_buffer = 0;
 	char *new_ptr;
@@ -514,7 +515,7 @@ int git_buffer_resize(git_buffer *buffer, size_t want_size)
 	if (non_allocated_buffer && !want_size)
 		want_size = buffer->size;
 
-	if (buffer->available <= want_size)
+	if (buffer->available >= want_size)
 		return 0;
 
 	if (non_allocated_buffer) {
@@ -530,7 +531,7 @@ int git_buffer_resize(git_buffer *buffer, size_t want_size)
 	new_ptr = git__realloc(new_ptr, want_size);
 	GITERR_CHECK_ALLOC(new_ptr);
 
-	if (non_allocated_buffer)
+	if (non_allocated_buffer && preserve_data)
 		memcpy(new_ptr, buffer->ptr, buffer->size);
 
 	buffer->ptr       = new_ptr;
@@ -538,3 +539,19 @@ int git_buffer_resize(git_buffer *buffer, size_t want_size)
 
 	return 0;
 }
+
+int git_buffer_resize(git_buffer *buffer, size_t want_size)
+{
+	return git_buffer__resize(buffer, want_size, true);
+}
+
+int git_buffer_copy(
+	git_buffer *buffer, const void *data, size_t datalen)
+{
+	if (git_buffer__resize(buffer, datalen, false) < 0)
+		return -1;
+	memcpy(buffer->ptr, data, datalen);
+	buffer->size = datalen;
+	return 0;
+}
+
diff --git a/src/buffer.h b/src/buffer.h
index b1cb5d06a..e07f29131 100644
--- a/src/buffer.h
+++ b/src/buffer.h
@@ -176,6 +176,11 @@ int git_buf_splice(
 	size_t nb_to_insert);
 
 
+GIT_INLINE(bool) git_buffer_is_allocated(const git_buffer *buffer)
+{
+	return (buffer->ptr != NULL && buffer->available > 0);
+}
+
 #define GIT_BUF_FROM_BUFFER(buffer) \
 	{ (buffer)->ptr, (buffer)->available, (buffer)->size }
 
diff --git a/src/checkout.c b/src/checkout.c
index 5ce4a19c5..1def58b0a 100644
--- a/src/checkout.c
+++ b/src/checkout.c
@@ -678,19 +678,20 @@ fail:
 
 static int buffer_to_file(
 	struct stat *st,
-	git_buf *buffer,
+	git_buffer *buffer,
 	const char *path,
 	mode_t dir_mode,
 	int file_open_flags,
 	mode_t file_mode)
 {
 	int error;
+	git_buf buf = GIT_BUF_FROM_BUFFER(buffer);
 
 	if ((error = git_futils_mkpath2file(path, dir_mode)) < 0)
 		return error;
 
 	if ((error = git_futils_writebuffer(
-			buffer, path, file_open_flags, file_mode)) < 0)
+			&buf, path, file_open_flags, file_mode)) < 0)
 		return error;
 
 	if (st != NULL && (error = p_stat(path, st)) < 0)
@@ -712,39 +713,26 @@ static int blob_content_to_file(
 {
 	int error = 0;
 	mode_t file_mode = opts->file_mode ? opts->file_mode : entry_filemode;
-	git_buf unfiltered = GIT_BUF_INIT, filtered = GIT_BUF_INIT;
+	git_buffer out = GIT_BUFFER_INIT;
 	git_filter_list *fl = NULL;
 
-	/* Create a fake git_buf from the blob raw data... */
-	filtered.ptr  = (void *)git_blob_rawcontent(blob);
-	filtered.size = (size_t)git_blob_rawsize(blob);
-
-	if (!opts->disable_filters && !git_buf_text_is_binary(&filtered)) {
+	if (!opts->disable_filters && !git_blob_is_binary(blob))
 		error = git_filter_list_load(
 			&fl, git_blob_owner(blob), path, GIT_FILTER_TO_WORKTREE);
-	}
 
-	if (fl != NULL) {
-		/* reset 'filtered' so it can be a filter target */
-		git_buf_init(&filtered, 0);
+	if (!error)
+		error = git_filter_list_apply_to_blob(&out, fl, blob);
 
-		if (!(error = git_blob__getbuf(&unfiltered, blob))) {
-			error = git_filter_list_apply(&filtered, &unfiltered, fl);
-
-			git_buf_free(&unfiltered);
-		}
+	git_filter_list_free(fl);
 
-		git_filter_list_free(fl);
-	}
+	if (!error) {
+		error = buffer_to_file(
+			st, &out, path, opts->dir_mode, opts->file_open_flags, file_mode);
 
-	if (!error &&
-		!(error = buffer_to_file(
-			st, &filtered, path, opts->dir_mode,
-			opts->file_open_flags, file_mode)))
 		st->st_mode = entry_filemode;
 
-	if (filtered.asize != 0)
-		git_buf_free(&filtered);
+		git_buffer_free(&out);
+	}
 
 	return error;
 }
diff --git a/src/crlf.c b/src/crlf.c
index 1242450d8..cc256fc70 100644
--- a/src/crlf.c
+++ b/src/crlf.c
@@ -8,6 +8,7 @@
 #include "git2/attr.h"
 #include "git2/blob.h"
 #include "git2/index.h"
+#include "git2/sys/filter.h"
 
 #include "common.h"
 #include "fileops.h"
@@ -249,7 +250,6 @@ static int crlf_apply_to_workdir(
 static int crlf_check(
 	git_filter        *self,
 	void              **payload, /* points to NULL ptr on entry, may be set */
-	git_filter_mode_t mode,
 	const git_filter_source *src,
 	const char **attr_values)
 {
@@ -257,7 +257,6 @@ static int crlf_check(
 	struct crlf_attrs ca;
 
 	GIT_UNUSED(self);
-	GIT_UNUSED(mode);
 
 	if (!attr_values) {
 		ca.crlf_action = GIT_CRLF_GUESS;
@@ -299,14 +298,13 @@ static int crlf_check(
 static int crlf_apply(
 	git_filter        *self,
 	void              **payload, /* may be read and/or set */
-	git_filter_mode_t mode,
 	git_buffer        *to,
 	const git_buffer  *from,
 	const git_filter_source *src)
 {
 	GIT_UNUSED(self);
 
-	if (mode == GIT_FILTER_SMUDGE)
+	if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE)
 		return crlf_apply_to_workdir(*payload, to, from);
 	else
 		return crlf_apply_to_odb(*payload, to, from, src);
@@ -331,5 +329,6 @@ git_filter *git_crlf_filter_new(void)
 	f->f.check   = crlf_check;
 	f->f.apply   = crlf_apply;
 	f->f.cleanup = crlf_cleanup;
+
 	return (git_filter *)f;
 }
diff --git a/src/diff_file.c b/src/diff_file.c
index 7602591cf..e0e244b65 100644
--- a/src/diff_file.c
+++ b/src/diff_file.c
@@ -297,8 +297,8 @@ static int diff_file_content_load_workdir_file(
 {
 	int error = 0;
 	git_filter_list *fl = NULL;
-	git_buf raw = GIT_BUF_INIT, filtered = GIT_BUF_INIT;
 	git_file fd = git_futils_open_ro(git_buf_cstr(path));
+	git_buf raw = GIT_BUF_INIT;
 
 	if (fd < 0)
 		return fd;
@@ -326,16 +326,19 @@ static int diff_file_content_load_workdir_file(
 		giterr_clear();
 	}
 
-	if (!(error = git_futils_readbuffer_fd(&raw, fd, (size_t)fc->file->size)) &&
-		!(error = git_filter_list_apply(&filtered, &raw, fl)))
-	{
-		fc->map.len  = git_buf_len(&filtered);
-		fc->map.data = git_buf_detach(&filtered);
-		fc->flags |= GIT_DIFF_FLAG__FREE_DATA;
-	}
+	if (!(error = git_futils_readbuffer_fd(&raw, fd, (size_t)fc->file->size))) {
+		git_buffer in = GIT_BUFFER_FROM_BUF(&raw), out = GIT_BUFFER_INIT;
+
+		error = git_filter_list_apply_to_data(&out, fl, &in);
+
+		git_buffer_free(&in);
 
-	git_buf_free(&raw);
-	git_buf_free(&filtered);
+		if (!error) {
+			fc->map.len  = out.size;
+			fc->map.data = out.ptr;
+			fc->flags |= GIT_DIFF_FLAG__FREE_DATA;
+		}
+	}
 
 cleanup:
 	git_filter_list_free(fl);
diff --git a/src/filter.c b/src/filter.c
index 7fbc20a41..de0d490aa 100644
--- a/src/filter.c
+++ b/src/filter.c
@@ -10,15 +10,18 @@
 #include "hash.h"
 #include "filter.h"
 #include "repository.h"
+#include "git2/sys/filter.h"
 #include "git2/config.h"
 #include "blob.h"
 #include "attr_file.h"
+#include "array.h"
 
 struct git_filter_source {
 	git_repository *repo;
 	const char     *path;
 	git_oid         oid;  /* zero if unknown (which is likely) */
 	uint16_t        filemode; /* zero if unknown */
+	git_filter_mode_t mode;
 };
 
 typedef struct {
@@ -28,7 +31,6 @@ typedef struct {
 
 struct git_filter_list {
 	git_array_t(git_filter_entry) filters;
-	git_filter_mode_t mode;
 	git_filter_source source;
 	char path[GIT_FLEX_ARRAY];
 };
@@ -238,8 +240,13 @@ const git_oid *git_filter_source_id(const git_filter_source *src)
 	return git_oid_iszero(&src->oid) ? NULL : &src->oid;
 }
 
+git_filter_mode_t git_filter_source_mode(const git_filter_source *src)
+{
+	return src->mode;
+}
+
 static int git_filter_list_new(
-	git_filter_list **out, git_filter_mode_t mode, const git_filter_source *src)
+	git_filter_list **out, const git_filter_source *src)
 {
 	git_filter_list *fl = NULL;
 	size_t pathlen = src->path ? strlen(src->path) : 0;
@@ -247,11 +254,11 @@ static int git_filter_list_new(
 	fl = git__calloc(1, sizeof(git_filter_list) + pathlen + 1);
 	GITERR_CHECK_ALLOC(fl);
 
-	fl->mode = mode;
 	if (src->path)
 		memcpy(fl->path, src->path, pathlen);
 	fl->source.repo = src->repo;
 	fl->source.path = fl->path;
+	fl->source.mode = src->mode;
 
 	*out = fl;
 	return 0;
@@ -316,6 +323,7 @@ int git_filter_list_load(
 
 	src.repo = repo;
 	src.path = path;
+	src.mode = mode;
 
 	git_vector_foreach(&git__filter_registry, idx, fdef) {
 		const char **values = NULL;
@@ -335,7 +343,7 @@ int git_filter_list_load(
 
 		if (fdef->filter->check)
 			error = fdef->filter->check(
-				fdef->filter, &payload, mode, &src, values);
+				fdef->filter, &payload, &src, values);
 
 		git__free(values);
 
@@ -344,7 +352,7 @@ int git_filter_list_load(
 		else if (error < 0)
 			break;
 		else {
-			if (!fl && (error = git_filter_list_new(&fl, mode, &src)) < 0)
+			if (!fl && (error = git_filter_list_new(&fl, &src)) < 0)
 				return error;
 
 			fe = git_array_alloc(fl->filters);
@@ -381,40 +389,46 @@ void git_filter_list_free(git_filter_list *fl)
 	git__free(fl);
 }
 
-int git_filter_list_apply(
-	git_buf *dest,
-	git_buf *source,
-	git_filter_list *fl)
+static int filter_list_out_buffer_from_raw(
+	git_buffer *out, const void *ptr, size_t size)
+{
+	if (git_buffer_is_allocated(out))
+		git_buffer_free(out);
+
+	out->ptr  = (char *)ptr;
+	out->size = size;
+	out->available = 0;
+	return 0;
+}
+
+int git_filter_list_apply_to_data(
+	git_buffer *tgt, git_filter_list *fl, git_buffer *src)
 {
 	int error = 0;
 	uint32_t i;
-	unsigned int src;
-	git_buf *dbuffer[2];
-	git_filter_entry *fe;
+	git_buffer *dbuffer[2], local = GIT_BUFFER_INIT;
+	unsigned int si = 0;
 
-	if (!fl) {
-		git_buf_swap(dest, source);
-		return 0;
-	}
-
-	dbuffer[0] = source;
-	dbuffer[1] = dest;
+	if (!fl)
+		return filter_list_out_buffer_from_raw(tgt, src->ptr, src->size);
 
-	src = 0;
+	dbuffer[0] = src;
+	dbuffer[1] = tgt;
 
-	/* Pre-grow the destination buffer to more or less the size
-	 * we expect it to have */
-	if (git_buf_grow(dest, git_buf_len(source)) < 0)
-		return -1;
+	/* if `src` buffer is reallocable, then use it, otherwise copy it */
+	if (!git_buffer_is_allocated(src)) {
+		if (git_buffer_copy(&local, src->ptr, src->size) < 0)
+			return -1;
+		dbuffer[0] = &local;
+	}
 
 	for (i = 0; i < git_array_size(fl->filters); ++i) {
-		unsigned int dst = 1 - src;
-
-		git_buf_clear(dbuffer[dst]);
+		unsigned int di = 1 - si;
+		uint32_t fidx = (fl->source.mode == GIT_FILTER_TO_ODB) ?
+			i : git_array_size(fl->filters) - 1 - i;
+		git_filter_entry *fe = git_array_get(fl->filters, fidx);
 
-		fe = git_array_get(
-			fl->filters, (fl->mode == GIT_FILTER_TO_ODB) ?
-			i : git_array_size(fl->filters) - 1 - i);
+		dbuffer[di]->size = 0;
 
 		/* Apply the filter from dbuffer[src] to the other buffer;
 		 * if the filtering is canceled by the user mid-filter,
@@ -422,33 +436,64 @@ int git_filter_list_apply(
 		 * of the double buffering (so that the text goes through
 		 * cleanly).
 		 */
-		{
-			git_buffer srcb = GIT_BUFFER_FROM_BUF(dbuffer[src]);
-			git_buffer dstb = GIT_BUFFER_FROM_BUF(dbuffer[dst]);
 
-			error = fe->filter->apply(
-				fe->filter, &fe->payload, fl->mode, &dstb, &srcb, &fl->source);
+		error = fe->filter->apply(
+			fe->filter, &fe->payload, dbuffer[di], dbuffer[si], &fl->source);
 
-			if (error == GIT_ENOTFOUND)
-				error = 0;
-			else if (error < 0) {
-				git_buf_clear(dest);
-				return error;
-			}
-			else {
-				git_buf_from_buffer(dbuffer[src], &srcb);
-				git_buf_from_buffer(dbuffer[dst], &dstb);
-				src = dst;
-			}
+		if (error == GIT_ENOTFOUND)
+			error = 0;
+		else if (!error)
+			si = di; /* swap buffers */
+		else {
+			tgt->size = 0;
+			return error;
 		}
-
-		if (git_buf_oom(dbuffer[dst]))
-			return -1;
 	}
 
 	/* Ensure that the output ends up in dbuffer[1] (i.e. the dest) */
-	if (src != 1)
-		git_buf_swap(dest, source);
+	if (si != 1) {
+		git_buffer sw = *dbuffer[1];
+		*dbuffer[1] = *dbuffer[0];
+		*dbuffer[0] = sw;
+	}
+
+	git_buffer_free(&local); /* don't leak if we allocated locally */
 
 	return 0;
 }
+
+int git_filter_list_apply_to_file(
+	git_buffer *out,
+	git_filter_list *filters,
+	git_repository *repo,
+	const char *path)
+{
+	int error;
+	const char *base = repo ? git_repository_workdir(repo) : NULL;
+	git_buf abspath = GIT_BUF_INIT, raw = GIT_BUF_INIT;
+
+	if (!(error = git_path_join_unrooted(&abspath, path, base, NULL)) &&
+		!(error = git_futils_readbuffer(&raw, abspath.ptr)))
+	{
+		git_buffer in = GIT_BUFFER_FROM_BUF(&raw);
+
+		error = git_filter_list_apply_to_data(out, filters, &in);
+
+		git_buffer_free(&in);
+	}
+
+	git_buf_free(&abspath);
+	return error;
+}
+
+int git_filter_list_apply_to_blob(
+	git_buffer *out,
+	git_filter_list *filters,
+	git_blob *blob)
+{
+	git_buffer in = {
+		(char *)git_blob_rawcontent(blob), git_blob_rawsize(blob), 0
+	};
+
+	return git_filter_list_apply_to_data(out, filters, &in);
+}
diff --git a/src/filter.h b/src/filter.h
index a4ee2172d..1bde1e306 100644
--- a/src/filter.h
+++ b/src/filter.h
@@ -8,12 +8,7 @@
 #define INCLUDE_filter_h__
 
 #include "common.h"
-#include "buffer.h"
-#include "array.h"
-#include "git2/odb.h"
-#include "git2/repository.h"
 #include "git2/filter.h"
-#include "git2/sys/filter.h"
 
 typedef enum {
 	GIT_CRLF_GUESS = -1,
@@ -24,61 +19,6 @@ typedef enum {
 	GIT_CRLF_AUTO,
 } git_crlf_t;
 
-typedef struct git_filter_list git_filter_list;
-
-/*
- * FILTER API
- */
-
-/*
- * For any given path in the working directory, create a `git_filter_list`
- * with the relevant filters that need to be applied.
- *
- * This will return 0 (success) but set the output git_filter_list to NULL
- * if no filters are requested for the given file.
- *
- * @param filters Output newly created git_filter_list (or NULL)
- * @param repo Repository object that contains `path`
- * @param path Relative path of the file to be filtered
- * @param mode Filtering direction (WT->ODB or ODB->WT)
- * @return 0 on success (which could still return NULL if no filters are
- *         needed for the requested file), <0 on error
- */
-extern int git_filter_list_load(
-	git_filter_list **filters,
-	git_repository *repo,
-	const char *path,
-	git_filter_mode_t mode);
-
-/*
- * Apply one or more filters to a data buffer.
- *
- * The source data must have been loaded as a `git_buf` object. Both the
- * `source` and `dest` buffers are owned by the caller and must be freed
- * once they are no longer needed.
- *
- * NOTE: Because of the double-buffering schema, the `source` buffer that
- * contains the original file may be tampered once the filtering is
- * complete. Regardless, the `dest` buffer will always contain the final
- * result of the filtering
- *
- * @param dest Buffer to store the result of the filtering
- * @param source Buffer containing the document to filter
- * @param filters An already loaded git_filter_list
- * @return 0 on success, an error code otherwise
- */
-extern int git_filter_list_apply(
-	git_buf *dest,
-	git_buf *source,
-	git_filter_list *filters);
-
-/*
- * Free the git_filter_list
- *
- * @param filters A git_filter_list created by `git_filter_list_load`
- */
-extern void git_filter_list_free(git_filter_list *filters);
-
 /*
  * Available filters
  */
diff --git a/src/odb.c b/src/odb.c
index d9310a9d7..b71b038bf 100644
--- a/src/odb.c
+++ b/src/odb.c
@@ -183,7 +183,6 @@ int git_odb__hashfd_filtered(
 {
 	int error;
 	git_buf raw = GIT_BUF_INIT;
-	git_buf filtered = GIT_BUF_INIT;
 
 	if (!fl)
 		return git_odb__hashfd(out, fd, size, type);
@@ -192,15 +191,18 @@ int git_odb__hashfd_filtered(
 	 * into memory to apply filters before beginning to calculate the hash
 	 */
 
-	if (!(error = git_futils_readbuffer_fd(&raw, fd, size)))
-		error = git_filter_list_apply(&filtered, &raw, fl);
+	if (!(error = git_futils_readbuffer_fd(&raw, fd, size))) {
+		git_buffer pre = GIT_BUFFER_FROM_BUF(&raw), post = GIT_BUFFER_INIT;
 
-	git_buf_free(&raw);
+		error = git_filter_list_apply_to_data(&post, fl, &pre);
 
-	if (!error)
-		error = git_odb_hash(out, filtered.ptr, filtered.size, type);
+		git_buffer_free(&pre);
 
-	git_buf_free(&filtered);
+		if (!error)
+			error = git_odb_hash(out, post.ptr, post.size, type);
+
+		git_buffer_free(&post);
+	}
 
 	return error;
 }