diff options
-rw-r--r-- | Makefile.am | 2 | ||||
-rw-r--r-- | include/erasurecode/erasurecode_helpers.h | 2 | ||||
-rw-r--r-- | include/erasurecode/erasurecode_postprocessing.h | 36 | ||||
-rw-r--r-- | src/Makefile.am | 3 | ||||
-rw-r--r-- | src/backends/jerasure/jerasure_rs_vand.c | 18 | ||||
-rw-r--r-- | src/erasurecode.c | 9 | ||||
-rw-r--r-- | src/erasurecode_helpers.c | 32 | ||||
-rw-r--r-- | src/erasurecode_postprocessing.c | 69 | ||||
-rw-r--r-- | src/erasurecode_preprocessing.c | 19 | ||||
-rw-r--r-- | test/liberasurecode_test.c | 40 |
10 files changed, 188 insertions, 42 deletions
diff --git a/Makefile.am b/Makefile.am index f0c0f06..660a0e9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -19,6 +19,8 @@ include_HEADERS = \ include/erasurecode/erasurecode.h \ include/erasurecode/erasurecode_backend.h \ include/erasurecode/erasurecode_helpers.h \ + include/erasurecode/erasurecode_preprocessing.h \ + include/erasurecode/erasurecode_postprocessing.h \ include/erasurecode/erasurecode_stdinc.h \ include/erasurecode/erasurecode_version.h \ include/erasurecode/list.h \ diff --git a/include/erasurecode/erasurecode_helpers.h b/include/erasurecode/erasurecode_helpers.h index 8596da7..0f72c16 100644 --- a/include/erasurecode/erasurecode_helpers.h +++ b/include/erasurecode/erasurecode_helpers.h @@ -155,6 +155,8 @@ int get_fragment_payload_size(char *buf); int set_orig_data_size(char *buf, int orig_data_size); int get_orig_data_size(char *buf); int validate_fragment(char *buf); +int set_checksum(char *buf, int chksum); +int get_checksum(char *buf); /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ diff --git a/include/erasurecode/erasurecode_postprocessing.h b/include/erasurecode/erasurecode_postprocessing.h new file mode 100644 index 0000000..b6c7b2f --- /dev/null +++ b/include/erasurecode/erasurecode_postprocessing.h @@ -0,0 +1,36 @@ +/* + * <Copyright> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY + * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * liberasurecode API helpers header + * + * vi: set noai tw=79 ts=4 sw=4: + */ + +#ifndef _ERASURECODE_POSTPROCESSING_H_ +#define _ERASURECODE_POSTPROCESSING_H_ + +int finalize_fragments_after_encode(ec_backend_t instance, + int k, int m, int blocksize, + char **encoded_data, char **encoded_parity); + +#endif diff --git a/src/Makefile.am b/src/Makefile.am index 805fbec..3d13226 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -10,7 +10,8 @@ INCLUDES = \ liberasurecode_la_SOURCES = \ erasurecode.c \ erasurecode_helpers.c \ - erasurecode_preprocessing.c \ + erasurecode_preprocessing.c \ + erasurecode_postprocessing.c \ utils/chksum/crc32.c \ utils/chksum/alg_sig.c \ utils/chksum/galois.c \ diff --git a/src/backends/jerasure/jerasure_rs_vand.c b/src/backends/jerasure/jerasure_rs_vand.c index 0df892a..6558fa7 100644 --- a/src/backends/jerasure/jerasure_rs_vand.c +++ b/src/backends/jerasure/jerasure_rs_vand.c @@ -31,6 +31,7 @@ #include "erasurecode.h" #include "erasurecode_backend.h" +#include "erasurecode_helpers.h" /* Forward declarations */ struct ec_backend_op_stubs jerasure_rs_vand_ops; @@ -58,23 +59,6 @@ struct jerasure_rs_vand_descriptor { int w; }; -/** - * ToDo (KMG): Move this to a util package, or replace with calloc - */ -static -void * alloc_zeroed_buffer(int size) -{ - void * buf = NULL; /* buffer to allocate and return */ - - /* Allocate and zero the buffer, or set the appropriate error */ - buf = malloc((size_t) size); - if (buf) { - buf = memset(buf, 0, (size_t) size); - } - - return buf; -} - static int jerasure_rs_vand_encode(void *desc, char **data, char **parity, int blocksize) { diff --git a/src/erasurecode.c b/src/erasurecode.c index ca05fd4..d70227e 100644 --- a/src/erasurecode.c +++ b/src/erasurecode.c @@ -356,9 +356,18 @@ int liberasurecode_encode(int desc, /* call the backend encode function passing it desc instance */ ret = instance->common.ops->encode(instance->desc.backend_desc, *encoded_data, *encoded_parity, blocksize); + if (ret < 0) { + goto out; + } + + ret = finalize_fragments_after_encode(instance, k, m, blocksize, + *encoded_data, *encoded_parity); *fragment_len = get_fragment_size((*encoded_data)[0]); out: + /* FIXME add cleanup API to call when encode() has an error */ + if (ret) + log_error("Error in liberasurecode_encode %d", ret); return ret; } diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c index 839d101..6c0d158 100644 --- a/src/erasurecode_helpers.c +++ b/src/erasurecode_helpers.c @@ -57,9 +57,7 @@ void *get_aligned_buffer16(int size) } /** - * Allocate a zero-ed buffer of a specific size. This method allocates from - * the Python stack in order to comply with the recommendations of the - * Python\C API. On error, return NULL and call PyErr_NoMemory. + * Allocate a zero-ed buffer of a specific size. * * @param size integer size in bytes of buffer to allocate * @return pointer to start of allocated buffer or NULL on error @@ -307,3 +305,31 @@ int validate_fragment(char *buf) /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ +inline int set_chksum(char *buf, int chksum) +{ + fragment_header_t* header = (fragment_header_t*) buf; + + if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) { + log_error("Invalid fragment header (set chksum)!\n"); + return -1; + } + + header->chksum = chksum; + + return 0; +} + +inline int get_chksum(char *buf) +{ + fragment_header_t* header = (fragment_header_t*) buf; + + if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) { + log_error("Invalid fragment header (get chksum)!"); + return -1; + } + + return header->chksum; +} + +/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ + diff --git a/src/erasurecode_postprocessing.c b/src/erasurecode_postprocessing.c new file mode 100644 index 0000000..e086f2c --- /dev/null +++ b/src/erasurecode_postprocessing.c @@ -0,0 +1,69 @@ +/* + * <Copyright> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY + * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * liberasurecode postprocessing helpers implementation + * + * vi: set noai tw=79 ts=4 sw=4: + */ + +#include "erasurecode_backend.h" +#include "erasurecode_helpers.h" +#include "erasurecode_stdinc.h" + +int finalize_fragments_after_encode(ec_backend_t instance, + int k, int m, int blocksize, + char **encoded_data, char **encoded_parity) +{ + int i; + // int add_checksum = instance->args.uargs.inline_chksum; + int add_checksum = 1; + char *fragment = get_fragment_ptr_from_data(encoded_parity[0]); +printf ("fragment = %p\n", fragment); + fragment = get_fragment_ptr_from_data(encoded_parity[1]); +printf ("fragment = %p\n", fragment); + + /* finalize data fragments */ + for (i = 0; i < k; i++) { + char *fragment = get_fragment_ptr_from_data(encoded_data[i]); + set_fragment_idx(fragment, i); + if (add_checksum) { + int chksum = crc32(0, encoded_data[i], blocksize); + set_chksum(fragment, chksum); + } + encoded_data[i] = fragment; + } + + /* finalize parity fragments */ + for (i = 0; i < m; i++) { + char *fragment = get_fragment_ptr_from_data(encoded_parity[i]); + set_fragment_idx(fragment, i + k); + if (add_checksum) { + int chksum = crc32(0, encoded_parity[i], blocksize); + set_chksum(fragment, chksum); + } + encoded_parity[i] = fragment; + } + + return 0; +} + diff --git a/src/erasurecode_preprocessing.c b/src/erasurecode_preprocessing.c index 082243f..6591180 100644 --- a/src/erasurecode_preprocessing.c +++ b/src/erasurecode_preprocessing.c @@ -40,15 +40,16 @@ int prepare_fragments_for_encode(ec_backend_t instance, int i, ret = 0; int data_len; /* data len to write to fragment headers */ int aligned_data_len; /* EC algorithm compatible data length */ + int bsize = 0; /* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/ data_len = orig_data_size; aligned_data_len = get_aligned_data_size(instance, orig_data_size); - *blocksize = aligned_data_len / k; + *blocksize = bsize = (aligned_data_len / k); for (i = 0; i < k; i++) { - int payload_size = data_len > *blocksize ? *blocksize : data_len; - char *fragment = alloc_fragment_buffer(*blocksize); + int payload_size = data_len > bsize ? bsize : data_len; + char *fragment = (char *) alloc_fragment_buffer(bsize); if (NULL == fragment) { ret = -ENOMEM; goto out_error; @@ -56,32 +57,38 @@ int prepare_fragments_for_encode(ec_backend_t instance, /* Copy existing data into clean, zero'd out buffer */ encoded_data[i] = get_data_ptr_from_fragment(fragment); + if (data_len > 0) { memcpy(encoded_data[i], orig_data, payload_size); } /* Fragment size will always be the same * (may be able to get rid of this) */ - set_fragment_payload_size(fragment, *blocksize); + set_fragment_payload_size(fragment, bsize); + + /* Original data length */ + set_orig_data_size(fragment, orig_data_size); orig_data += payload_size; data_len -= payload_size; } for (i = 0; i < m; i++) { - char *fragment = alloc_fragment_buffer(*blocksize); + char *fragment = (char *) alloc_fragment_buffer(bsize); if (NULL == fragment) { ret = -ENOMEM; goto out_error; } + encoded_parity[i] = get_data_ptr_from_fragment(fragment); - set_fragment_payload_size(fragment, *blocksize); + set_fragment_payload_size(fragment, bsize); } out: return ret; out_error: + printf ("ERROR in encode\n"); if (encoded_data) { for (i = 0; i < k; i++) { if (encoded_data[i]) diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c index a6255f6..93e7d72 100644 --- a/test/liberasurecode_test.c +++ b/test/liberasurecode_test.c @@ -42,10 +42,10 @@ struct testcase { }; //TODO Make this a but more useful -char *create_buffer(int size) +char *create_buffer(int size, int fill) { char *buf = malloc(size); - memset(buf, 'x', size); + memset(buf, fill, size); return buf; } @@ -67,32 +67,42 @@ static int test_create_and_destroy_backend( return 0; } -static int test_simple_encode( +static int test_simple_encode_decode( const char *backend, struct ec_args *args) { int rc = 0; int desc = -1; - int size = 1024 * 1024; - char **encoded_data = NULL, **parity = NULL; - char *data = NULL; + + int orig_data_size = 1024 * 1024; + char *orig_data = NULL; + char **encoded_data = NULL, **encoded_parity = NULL; + uint64_t encoded_fragment_len = 0; + + uint64_t decoded_data_len = 0; + char *decoded_data = NULL; desc = liberasurecode_instance_create(backend, args); assert(desc > 0 || EBACKENDNOTAVAIL == desc); - data = create_buffer(size); - if (NULL == data) { + orig_data = create_buffer(orig_data_size, 'x'); + if (NULL == orig_data) { rc = -ENOMEM; goto out; } - rc = liberasurecode_encode(desc, data, size, encoded_data, parity); + rc = liberasurecode_encode(desc, orig_data, orig_data_size, + &encoded_data, &encoded_parity, &encoded_fragment_len); + assert(0 == rc); + + rc = liberasurecode_decode(desc, encoded_data, + 10, -1, decoded_data, &decoded_data_len); assert(0 == rc); if (desc) liberasurecode_instance_destroy(desc); - free(data); + free(orig_data); out: return rc; @@ -100,7 +110,7 @@ out: struct ec_args flat_xor_hd_args = { .k = 10, - .m = 4, + .m = 6, .priv_args1.flat_xor_hd_args.hd = 3, }; @@ -118,19 +128,19 @@ struct testcase testcases[] = { {"create_and_destroy_backend", test_create_and_destroy_backend, "flat_xor_hd", &flat_xor_hd_args, - .skip = true}, + .skip = false}, {"create_and_destroy_backend", test_create_and_destroy_backend, "jerasure_rs_vand", &jerasure_rs_vand_args, .skip = false}, {"simple_encode_flat_xor_hd", - test_simple_encode, + test_simple_encode_decode, "flat_xor_hd", &flat_xor_hd_args, .skip = true}, {"simple_encode_jerasure_rs_vand", - test_simple_encode, + test_simple_encode_decode, "jerasure_rs_vand", &jerasure_rs_vand_args, - .skip = false}, + .skip = true}, { NULL, NULL, NULL, NULL, false }, }; |