From a9a7bfb506ee4db156abf1ba55cda9de86c18fdb Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 12 May 2021 22:13:12 +0100 Subject: filter: add docs for `git_filter_stream_fn` --- include/git2/sys/filter.h | 69 ++++++++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/include/git2/sys/filter.h b/include/git2/sys/filter.h index e43fde55c..098f8f7d3 100644 --- a/include/git2/sys/filter.h +++ b/include/git2/sys/filter.h @@ -167,16 +167,16 @@ typedef void GIT_CALLBACK(git_filter_shutdown_fn)(git_filter *self); * * The `payload` will be a pointer to a reference payload for the filter. * This will start as NULL, but `check` can assign to this pointer for - * later use by the `apply` callback. Note that the value should be heap - * allocated (not stack), so that it doesn't go away before the `apply` + * later use by the `stream` callback. Note that the value should be heap + * allocated (not stack), so that it doesn't go away before the `stream` * callback can use it. If a filter allocates and assigns a value to the * `payload`, it will need a `cleanup` callback to free the payload. */ typedef int GIT_CALLBACK(git_filter_check_fn)( - git_filter *self, - void **payload, /* points to NULL ptr on entry, may be set */ + git_filter *self, + void **payload, /* NULL on entry, may be set */ const git_filter_source *src, - const char **attr_values); + const char **attr_values); /** * Callback to actually perform the data filtering @@ -191,30 +191,40 @@ typedef int GIT_CALLBACK(git_filter_check_fn)( * `check` callback. It may be read from or written to as needed. */ typedef int GIT_CALLBACK(git_filter_apply_fn)( - git_filter *self, - void **payload, /* may be read and/or set */ - git_buf *to, - const git_buf *from, + git_filter *self, + void **payload, /* may be read and/or set */ + git_buf *to, + const git_buf *from, const git_filter_source *src); +/** + * Callback to perform the data filtering. + * + * Specified as `filter.stream`, this is a callback that filters data + * in a streaming manner. This function will provide a + * `git_writestream` that will the original data will be written to; + * with that data, the `git_writestream` will then perform the filter + * translation and stream the filtered data out to the `next` location. + */ typedef int GIT_CALLBACK(git_filter_stream_fn)( - git_writestream **out, - git_filter *self, - void **payload, + git_writestream **out, + git_filter *self, + void **payload, const git_filter_source *src, - git_writestream *next); + git_writestream *next); /** * Callback to clean up after filtering has been applied * * Specified as `filter.cleanup`, this is an optional callback invoked - * after the filter has been applied. If the `check` or `apply` callbacks - * allocated a `payload` to keep per-source filter state, use this - * callback to free that payload and release resources as required. + * after the filter has been applied. If the `check`, `apply`, or + * `stream` callbacks allocated a `payload` to keep per-source filter + * state, use this callback to free that payload and release resources + * as required. */ typedef void GIT_CALLBACK(git_filter_cleanup_fn)( - git_filter *self, - void *payload); + git_filter *self, + void *payload); /** * Filter structure used to register custom filters. @@ -248,21 +258,24 @@ struct git_filter { /** * Called to determine whether the filter should be invoked for a * given file. If this function returns `GIT_PASSTHROUGH` then the - * `apply` function will not be invoked and the contents will be passed - * through unmodified. + * `stream` or `apply` functions will not be invoked and the + * contents will be passed through unmodified. */ git_filter_check_fn check; /** - * Called to actually apply the filter to file contents. If this - * function returns `GIT_PASSTHROUGH` then the contents will be passed - * through unmodified. + * Provided for backward compatibility; this will apply the + * filter to the given contents in a `git_buf`. Callers should + * provide a `stream` function instead. */ git_filter_apply_fn apply; /** - * Called to apply the filter in a streaming manner. If this is not - * specified then the system will call `apply` with the whole buffer. + * Called to apply the filter, this function will provide a + * `git_writestream` that will the original data will be + * written to; with that data, the `git_writestream` will then + * perform the filter translation and stream the filtered data + * out to the `next` location. */ git_filter_stream_fn stream; @@ -289,9 +302,9 @@ GIT_EXTERN(int) git_filter_init(git_filter *filter, unsigned int version); * As mentioned elsewhere, the initialize callback will not be invoked * immediately. It is deferred until the filter is used in some way. * - * A filter's attribute checks and `check` and `apply` callbacks will be - * issued in order of `priority` on smudge (to workdir), and in reverse - * order of `priority` on clean (to odb). + * A filter's attribute checks and `check` and `stream` (or `apply`) + * callbacks will be issued in order of `priority` on smudge (to + * workdir), and in reverse order of `priority` on clean (to odb). * * Two filters are preregistered with libgit2: * - GIT_FILTER_CRLF with priority 0 -- cgit v1.2.1 From a8943c04dcb8079595c075daf1b7ba76d4bb455e Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 27 Aug 2021 15:59:01 -0400 Subject: filter: proxy_stream is now git_filter_buffered_stream The filter's proxy_stream is used to adapt filters that only provided an `apply` function into a `stream` function. Make this internal to the library instead of private to the filter file. This will allow the filters to use it directly, instead of relying on the filter functionality to do the proxying. --- src/filter.c | 89 +++++++++++++++++++++++++++++++----------------------------- src/filter.h | 10 +++++++ 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/src/filter.c b/src/filter.c index eed175e88..21ae27167 100644 --- a/src/filter.c +++ b/src/filter.c @@ -839,9 +839,10 @@ int git_filter_list_apply_to_blob( return error; } -struct proxy_stream { +struct buffered_stream { git_writestream parent; git_filter *filter; + int (*write_fn)(git_filter *, void **, git_buf *, const git_buf *, const git_filter_source *); const git_filter_source *source; void **payload; git_buf input; @@ -850,89 +851,91 @@ struct proxy_stream { git_writestream *target; }; -static int proxy_stream_write( +static int buffered_stream_write( git_writestream *s, const char *buffer, size_t len) { - struct proxy_stream *proxy_stream = (struct proxy_stream *)s; - GIT_ASSERT_ARG(proxy_stream); + struct buffered_stream *buffered_stream = (struct buffered_stream *)s; + GIT_ASSERT_ARG(buffered_stream); - return git_buf_put(&proxy_stream->input, buffer, len); + return git_buf_put(&buffered_stream->input, buffer, len); } -static int proxy_stream_close(git_writestream *s) +static int buffered_stream_close(git_writestream *s) { - struct proxy_stream *proxy_stream = (struct proxy_stream *)s; + struct buffered_stream *buffered_stream = (struct buffered_stream *)s; git_buf *writebuf; git_error_state error_state = {0}; int error; - GIT_ASSERT_ARG(proxy_stream); + GIT_ASSERT_ARG(buffered_stream); - error = proxy_stream->filter->apply( - proxy_stream->filter, - proxy_stream->payload, - proxy_stream->output, - &proxy_stream->input, - proxy_stream->source); + error = buffered_stream->write_fn( + buffered_stream->filter, + buffered_stream->payload, + buffered_stream->output, + &buffered_stream->input, + buffered_stream->source); if (error == GIT_PASSTHROUGH) { - writebuf = &proxy_stream->input; + writebuf = &buffered_stream->input; } else if (error == 0) { - if ((error = git_buf_sanitize(proxy_stream->output)) < 0) + if ((error = git_buf_sanitize(buffered_stream->output)) < 0) return error; - writebuf = proxy_stream->output; + writebuf = buffered_stream->output; } else { /* close stream before erroring out taking care * to preserve the original error */ git_error_state_capture(&error_state, error); - proxy_stream->target->close(proxy_stream->target); + buffered_stream->target->close(buffered_stream->target); git_error_state_restore(&error_state); return error; } - if ((error = proxy_stream->target->write( - proxy_stream->target, writebuf->ptr, writebuf->size)) == 0) - error = proxy_stream->target->close(proxy_stream->target); + if ((error = buffered_stream->target->write( + buffered_stream->target, writebuf->ptr, writebuf->size)) == 0) + error = buffered_stream->target->close(buffered_stream->target); return error; } -static void proxy_stream_free(git_writestream *s) +static void buffered_stream_free(git_writestream *s) { - struct proxy_stream *proxy_stream = (struct proxy_stream *)s; + struct buffered_stream *buffered_stream = (struct buffered_stream *)s; - if (proxy_stream) { - git_buf_dispose(&proxy_stream->input); - git_buf_dispose(&proxy_stream->temp_buf); - git__free(proxy_stream); + if (buffered_stream) { + git_buf_dispose(&buffered_stream->input); + git_buf_dispose(&buffered_stream->temp_buf); + git__free(buffered_stream); } } -static int proxy_stream_init( +int git_filter_buffered_stream_new( git_writestream **out, git_filter *filter, + int (*write_fn)(git_filter *, void **, git_buf *, const git_buf *, const git_filter_source *), git_buf *temp_buf, void **payload, const git_filter_source *source, git_writestream *target) { - struct proxy_stream *proxy_stream = git__calloc(1, sizeof(struct proxy_stream)); - GIT_ERROR_CHECK_ALLOC(proxy_stream); - - proxy_stream->parent.write = proxy_stream_write; - proxy_stream->parent.close = proxy_stream_close; - proxy_stream->parent.free = proxy_stream_free; - proxy_stream->filter = filter; - proxy_stream->payload = payload; - proxy_stream->source = source; - proxy_stream->target = target; - proxy_stream->output = temp_buf ? temp_buf : &proxy_stream->temp_buf; + struct buffered_stream *buffered_stream = git__calloc(1, sizeof(struct buffered_stream)); + GIT_ERROR_CHECK_ALLOC(buffered_stream); + + buffered_stream->parent.write = buffered_stream_write; + buffered_stream->parent.close = buffered_stream_close; + buffered_stream->parent.free = buffered_stream_free; + buffered_stream->filter = filter; + buffered_stream->write_fn = write_fn; + buffered_stream->output = temp_buf ? temp_buf : &buffered_stream->temp_buf; + buffered_stream->payload = payload; + buffered_stream->source = source; + buffered_stream->target = target; if (temp_buf) git_buf_clear(temp_buf); - *out = (git_writestream *)proxy_stream; + *out = (git_writestream *)buffered_stream; return 0; } @@ -970,9 +973,9 @@ static int stream_list_init( &fe->payload, &filters->source, last_stream); else /* Create a stream that proxies the one-shot apply */ - error = proxy_stream_init(&filter_stream, fe->filter, - filters->temp_buf, &fe->payload, &filters->source, - last_stream); + error = git_filter_buffered_stream_new(&filter_stream, + fe->filter, fe->filter->apply, filters->temp_buf, + &fe->payload, &filters->source, last_stream); if (error < 0) goto out; diff --git a/src/filter.h b/src/filter.h index 55ed50e97..241791276 100644 --- a/src/filter.h +++ b/src/filter.h @@ -11,6 +11,7 @@ #include "attr_file.h" #include "git2/filter.h" +#include "git2/sys/filter.h" /* Amount of file to examine for NUL byte when checking binary-ness */ #define GIT_FILTER_BYTES_TO_CHECK_NUL 8000 @@ -51,4 +52,13 @@ extern int git_filter_list__convert_buf( extern git_filter *git_crlf_filter_new(void); extern git_filter *git_ident_filter_new(void); +extern int git_filter_buffered_stream_new( + git_writestream **out, + git_filter *filter, + int (*write_fn)(git_filter *, void **, git_buf *, const git_buf *, const git_filter_source *), + git_buf *temp_buf, + void **payload, + const git_filter_source *source, + git_writestream *target); + #endif -- cgit v1.2.1 From c1f4f45ef84d11d1baf41469eac28ec1b90dabf3 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 27 Aug 2021 16:43:00 -0400 Subject: crlf: use streaming filters --- src/crlf.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/crlf.c b/src/crlf.c index 1de9d8c3b..406f7140f 100644 --- a/src/crlf.c +++ b/src/crlf.c @@ -386,6 +386,17 @@ static int crlf_apply( return crlf_apply_to_odb(*payload, to, from, src); } +static int crlf_stream( + git_writestream **out, + git_filter *self, + void **payload, + const git_filter_source *src, + git_writestream *next) +{ + return git_filter_buffered_stream_new(out, + self, crlf_apply, NULL, payload, src, next); +} + static void crlf_cleanup( git_filter *self, void *payload) @@ -405,7 +416,7 @@ git_filter *git_crlf_filter_new(void) f->f.initialize = NULL; f->f.shutdown = git_filter_free; f->f.check = crlf_check; - f->f.apply = crlf_apply; + f->f.stream = crlf_stream; f->f.cleanup = crlf_cleanup; return (git_filter *)f; -- cgit v1.2.1 From f593fa979319d628295cf7a4d57051e68caa4823 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 27 Aug 2021 16:46:21 -0400 Subject: ident: use streaming filters --- src/ident.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ident.c b/src/ident.c index ae3ef1b45..e5aab80ed 100644 --- a/src/ident.c +++ b/src/ident.c @@ -113,6 +113,17 @@ static int ident_apply( return ident_remove_id(to, from); } +static int ident_stream( + git_writestream **out, + git_filter *self, + void **payload, + const git_filter_source *src, + git_writestream *next) +{ + return git_filter_buffered_stream_new(out, + self, ident_apply, NULL, payload, src, next); +} + git_filter *git_ident_filter_new(void) { git_filter *f = git__calloc(1, sizeof(git_filter)); @@ -122,7 +133,7 @@ git_filter *git_ident_filter_new(void) f->version = GIT_FILTER_VERSION; f->attributes = "+ident"; /* apply to files with ident attribute set */ f->shutdown = git_filter_free; - f->apply = ident_apply; + f->stream = ident_stream; return f; } -- cgit v1.2.1 From c089d5ac6145e4498167c9a3157835e416084d18 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 27 Aug 2021 16:51:38 -0400 Subject: filter: use streaming filters in tests --- tests/filter/custom_helpers.c | 26 ++++++++++++++++++++++++-- tests/filter/wildcard.c | 13 ++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/tests/filter/custom_helpers.c b/tests/filter/custom_helpers.c index 233ba3219..ee3b6353b 100644 --- a/tests/filter/custom_helpers.c +++ b/tests/filter/custom_helpers.c @@ -37,6 +37,17 @@ int bitflip_filter_apply( return 0; } +static int bitflip_filter_stream( + git_writestream **out, + git_filter *self, + void **payload, + const git_filter_source *src, + git_writestream *next) +{ + return git_filter_buffered_stream_new(out, + self, bitflip_filter_apply, NULL, payload, src, next); +} + static void bitflip_filter_free(git_filter *f) { git__free(f); @@ -50,7 +61,7 @@ git_filter *create_bitflip_filter(void) filter->version = GIT_FILTER_VERSION; filter->attributes = "+bitflip"; filter->shutdown = bitflip_filter_free; - filter->apply = bitflip_filter_apply; + filter->stream = bitflip_filter_stream; return filter; } @@ -88,6 +99,17 @@ int reverse_filter_apply( return 0; } +static int reverse_filter_stream( + git_writestream **out, + git_filter *self, + void **payload, + const git_filter_source *src, + git_writestream *next) +{ + return git_filter_buffered_stream_new(out, + self, reverse_filter_apply, NULL, payload, src, next); +} + static void reverse_filter_free(git_filter *f) { git__free(f); @@ -101,7 +123,7 @@ git_filter *create_reverse_filter(const char *attrs) filter->version = GIT_FILTER_VERSION; filter->attributes = attrs; filter->shutdown = reverse_filter_free; - filter->apply = reverse_filter_apply; + filter->stream = reverse_filter_stream; return filter; } diff --git a/tests/filter/wildcard.c b/tests/filter/wildcard.c index ca1ba1a28..0c9c13b1e 100644 --- a/tests/filter/wildcard.c +++ b/tests/filter/wildcard.c @@ -93,6 +93,17 @@ static int wildcard_filter_apply( return GIT_PASSTHROUGH; } +static int wildcard_filter_stream( + git_writestream **out, + git_filter *self, + void **payload, + const git_filter_source *src, + git_writestream *next) +{ + return git_filter_buffered_stream_new(out, + self, wildcard_filter_apply, NULL, payload, src, next); +} + static void wildcard_filter_cleanup(git_filter *self, void *payload) { GIT_UNUSED(self); @@ -112,7 +123,7 @@ static git_filter *create_wildcard_filter(void) filter->version = GIT_FILTER_VERSION; filter->attributes = "filter=*"; filter->check = wildcard_filter_check; - filter->apply = wildcard_filter_apply; + filter->stream = wildcard_filter_stream; filter->cleanup = wildcard_filter_cleanup; filter->shutdown = wildcard_filter_free; -- cgit v1.2.1 From 5bcef522f382042c68d6a7577ec67732687305f2 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 27 Aug 2021 17:06:50 -0400 Subject: filter: deprecate apply function --- include/git2/sys/filter.h | 8 ++++++++ src/filter.c | 41 ++++++++++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/include/git2/sys/filter.h b/include/git2/sys/filter.h index 098f8f7d3..b3759416a 100644 --- a/include/git2/sys/filter.h +++ b/include/git2/sys/filter.h @@ -178,6 +178,7 @@ typedef int GIT_CALLBACK(git_filter_check_fn)( const git_filter_source *src, const char **attr_values); +#ifndef GIT_DEPRECATE_HARD /** * Callback to actually perform the data filtering * @@ -189,6 +190,8 @@ typedef int GIT_CALLBACK(git_filter_check_fn)( * * The `payload` value will refer to any payload that was set by the * `check` callback. It may be read from or written to as needed. + * + * @deprecated use git_filter_stream_fn */ typedef int GIT_CALLBACK(git_filter_apply_fn)( git_filter *self, @@ -196,6 +199,7 @@ typedef int GIT_CALLBACK(git_filter_apply_fn)( git_buf *to, const git_buf *from, const git_filter_source *src); +#endif /** * Callback to perform the data filtering. @@ -263,12 +267,16 @@ struct git_filter { */ git_filter_check_fn check; +#ifdef GIT_DEPRECATE_HARD + void *reserved; +#else /** * Provided for backward compatibility; this will apply the * filter to the given contents in a `git_buf`. Callers should * provide a `stream` function instead. */ git_filter_apply_fn apply; +#endif /** * Called to apply the filter, this function will provide a diff --git a/src/filter.c b/src/filter.c index 21ae27167..f1d961406 100644 --- a/src/filter.c +++ b/src/filter.c @@ -939,6 +939,32 @@ int git_filter_buffered_stream_new( return 0; } +static int setup_stream( + git_writestream **out, + git_filter_entry *fe, + git_filter_list *filters, + git_writestream *last_stream) +{ +#ifndef GIT_DEPRECATE_HARD + GIT_ASSERT(fe->filter->stream || fe->filter->apply); + + /* + * If necessary, create a stream that proxies the traditional + * application. + */ + if (!fe->filter->stream) { + /* Create a stream that proxies the one-shot apply */ + return git_filter_buffered_stream_new(out, + fe->filter, fe->filter->apply, filters->temp_buf, + &fe->payload, &filters->source, last_stream); + } +#endif + + GIT_ASSERT(fe->filter->stream); + return fe->filter->stream(out, fe->filter, + &fe->payload, &filters->source, last_stream); +} + static int stream_list_init( git_writestream **out, git_vector *streams, @@ -960,22 +986,11 @@ static int stream_list_init( for (i = 0; i < git_array_size(filters->filters); ++i) { size_t filter_idx = (filters->source.mode == GIT_FILTER_TO_WORKTREE) ? git_array_size(filters->filters) - 1 - i : i; + git_filter_entry *fe = git_array_get(filters->filters, filter_idx); git_writestream *filter_stream; - GIT_ASSERT(fe->filter->stream || fe->filter->apply); - - /* If necessary, create a stream that proxies the traditional - * application. - */ - if (fe->filter->stream) - error = fe->filter->stream(&filter_stream, fe->filter, - &fe->payload, &filters->source, last_stream); - else - /* Create a stream that proxies the one-shot apply */ - error = git_filter_buffered_stream_new(&filter_stream, - fe->filter, fe->filter->apply, filters->temp_buf, - &fe->payload, &filters->source, last_stream); + error = setup_stream(&filter_stream, fe, filters, last_stream); if (error < 0) goto out; -- cgit v1.2.1