From 8c2dfb38dbc782a6b964bfcfd43e4f46bb71526e Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 17 Feb 2015 13:27:06 -0500 Subject: filter: test a large file through the stream Test pushing a file on-disk into a streaming filter that compresses it into the ODB, and inflates it back into the working directory. --- tests/filter/stream.c | 221 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 tests/filter/stream.c diff --git a/tests/filter/stream.c b/tests/filter/stream.c new file mode 100644 index 000000000..ff5361a37 --- /dev/null +++ b/tests/filter/stream.c @@ -0,0 +1,221 @@ +#include "clar_libgit2.h" +#include "posix.h" +#include "blob.h" +#include "filter.h" +#include "buf_text.h" +#include "git2/sys/filter.h" +#include "git2/sys/repository.h" + +static git_repository *g_repo = NULL; + +static git_filter *create_compress_filter(void); +static git_filter *compress_filter; + +void test_filter_stream__initialize(void) +{ + compress_filter = create_compress_filter(); + + cl_git_pass(git_filter_register("compress", compress_filter, 50)); + g_repo = cl_git_sandbox_init("empty_standard_repo"); +} + +void test_filter_stream__cleanup(void) +{ + cl_git_sandbox_cleanup(); + g_repo = NULL; + + git_filter_unregister("compress"); +} + +#define CHUNKSIZE 10240 + +struct compress_stream { + git_filter_stream base; + git_filter_stream *next; + git_filter_mode_t mode; + char current; + size_t current_chunk; +}; + +static int compress_stream_write__deflated(struct compress_stream *stream, const char *buffer, size_t len) +{ + size_t idx = 0; + + while (len > 0) { + size_t chunkremain, chunksize; + + if (stream->current_chunk == 0) + stream->current = buffer[idx]; + + chunkremain = CHUNKSIZE - stream->current_chunk; + chunksize = min(chunkremain, len); + + stream->current_chunk += chunksize; + len -= chunksize; + idx += chunksize; + + if (stream->current_chunk == CHUNKSIZE) { + cl_git_pass(stream->next->write(stream->next, &stream->current, 1)); + stream->current_chunk = 0; + } + } + + return 0; +} + +static int compress_stream_write__inflated(struct compress_stream *stream, const char *buffer, size_t len) +{ + char inflated[CHUNKSIZE]; + size_t i, j; + + for (i = 0; i < len; i++) { + for (j = 0; j < CHUNKSIZE; j++) + inflated[j] = buffer[i]; + + cl_git_pass(stream->next->write(stream->next, inflated, CHUNKSIZE)); + } + + return 0; +} + +static int compress_stream_write(git_filter_stream *s, const char *buffer, size_t len) +{ + struct compress_stream *stream = (struct compress_stream *)s; + + return (stream->mode == GIT_FILTER_TO_ODB) ? + compress_stream_write__deflated(stream, buffer, len) : + compress_stream_write__inflated(stream, buffer, len); +} + +static int compress_stream_close(git_filter_stream *s) +{ + struct compress_stream *stream = (struct compress_stream *)s; + cl_assert_equal_i(0, stream->current_chunk); + stream->next->close(stream->next); + return 0; +} + +static void compress_stream_free(git_filter_stream *stream) +{ + git__free(stream); +} + +static int compress_filter_stream_init( + git_filter_stream **out, + git_filter *self, + void **payload, + const git_filter_source *src, + git_filter_stream *next) +{ + struct compress_stream *stream = git__calloc(1, sizeof(struct compress_stream)); + cl_assert(stream); + + GIT_UNUSED(self); + GIT_UNUSED(payload); + + stream->base.write = compress_stream_write; + stream->base.close = compress_stream_close; + stream->base.free = compress_stream_free; + stream->next = next; + stream->mode = git_filter_source_mode(src); + + *out = (git_filter_stream *)stream; + return 0; +} + +static void compress_filter_free(git_filter *f) +{ + git__free(f); +} + +git_filter *create_compress_filter(void) +{ + git_filter *filter = git__calloc(1, sizeof(git_filter)); + cl_assert(filter); + + filter->version = GIT_FILTER_VERSION; + filter->attributes = "+compress"; + filter->stream = compress_filter_stream_init; + filter->shutdown = compress_filter_free; + + return filter; +} + +static void writefile(const char *filename, size_t numchunks) +{ + git_buf path = GIT_BUF_INIT; + char buf[CHUNKSIZE]; + size_t i = 0, j = 0; + int fd; + + cl_git_pass(git_buf_joinpath(&path, "empty_standard_repo", filename)); + + fd = p_open(path.ptr, O_RDWR|O_CREAT, 0666); + cl_assert(fd >= 0); + + for (i = 0; i < numchunks; i++) { + for (j = 0; j < CHUNKSIZE; j++) { + buf[j] = i % 256; + } + + cl_git_pass(p_write(fd, buf, CHUNKSIZE)); + } + p_close(fd); + + git_buf_free(&path); +} + +static void test_stream(size_t numchunks) +{ + git_index *index; + const git_index_entry *entry; + git_blob *blob; + struct stat st; + git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT; + + checkout_opts.checkout_strategy = GIT_CHECKOUT_FORCE; + + cl_git_mkfile( + "empty_standard_repo/.gitattributes", + "* compress\n"); + + /* write a file to disk */ + writefile("streamed_file", numchunks); + + /* place it in the index */ + cl_git_pass(git_repository_index(&index, g_repo)); + cl_git_pass(git_index_add_bypath(index, "streamed_file")); + cl_git_pass(git_index_write(index)); + + /* ensure it was appropriately compressed */ + cl_assert(entry = git_index_get_bypath(index, "streamed_file", 0)); + + cl_git_pass(git_blob_lookup(&blob, g_repo, &entry->id)); + cl_assert_equal_i(numchunks, git_blob_rawsize(blob)); + + /* check the file back out */ + cl_must_pass(p_unlink("empty_standard_repo/streamed_file")); + cl_git_pass(git_checkout_index(g_repo, index, &checkout_opts)); + + /* ensure it was decompressed */ + cl_must_pass(p_stat("empty_standard_repo/streamed_file", &st)); + cl_assert_equal_sz((numchunks * CHUNKSIZE), st.st_size); + + git_index_free(index); + git_blob_free(blob); +} + +/* write a 50KB file through the "compression" stream */ +void test_filter_stream__smallfile(void) +{ + test_stream(5); +} + +/* optionally write a 500 MB file through the compression stream */ +void test_filter_stream__bigfile(void) +{ + if (!cl_getenv("GITTEST_INVASIVE_FILESYSTEM")) + cl_skip(); + + test_stream(51200); +} -- cgit v1.2.1