Add encode postprocessing, checksum helpers

Signed-off-by: Tushar Gohad <tushar.gohad@intel.com>

10 files changed, 188 insertions, 42 deletions

diff --git a/Makefile.am b/Makefile.am
index f0c0f06..660a0e9 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -19,6 +19,8 @@ include_HEADERS = \
 	include/erasurecode/erasurecode.h \
 	include/erasurecode/erasurecode_backend.h \
 	include/erasurecode/erasurecode_helpers.h \
+	include/erasurecode/erasurecode_preprocessing.h \
+	include/erasurecode/erasurecode_postprocessing.h \
 	include/erasurecode/erasurecode_stdinc.h \
 	include/erasurecode/erasurecode_version.h \
 	include/erasurecode/list.h \
diff --git a/include/erasurecode/erasurecode_helpers.h b/include/erasurecode/erasurecode_helpers.h
index 8596da7..0f72c16 100644
--- a/include/erasurecode/erasurecode_helpers.h
+++ b/include/erasurecode/erasurecode_helpers.h
@@ -155,6 +155,8 @@ int get_fragment_payload_size(char *buf);
 int set_orig_data_size(char *buf, int orig_data_size);
 int get_orig_data_size(char *buf);
 int validate_fragment(char *buf);
+int set_checksum(char *buf, int chksum);
+int get_checksum(char *buf);
 
 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
 
diff --git a/include/erasurecode/erasurecode_postprocessing.h b/include/erasurecode/erasurecode_postprocessing.h
new file mode 100644
index 0000000..b6c7b2f
--- /dev/null
+++ b/include/erasurecode/erasurecode_postprocessing.h
@@ -0,0 +1,36 @@
+/*
+ * <Copyright>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice, this
+ * list of conditions and the following disclaimer in the documentation and/or
+ * other materials provided with the distribution.  THIS SOFTWARE IS PROVIDED BY
+ * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * liberasurecode API helpers header
+ *
+ * vi: set noai tw=79 ts=4 sw=4:
+ */
+
+#ifndef _ERASURECODE_POSTPROCESSING_H_
+#define _ERASURECODE_POSTPROCESSING_H_
+
+int finalize_fragments_after_encode(ec_backend_t instance,
+        int k, int m, int blocksize,
+        char **encoded_data, char **encoded_parity);
+
+#endif
diff --git a/src/Makefile.am b/src/Makefile.am
index 805fbec..3d13226 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -10,7 +10,8 @@ INCLUDES = \
 liberasurecode_la_SOURCES = \
 		erasurecode.c \
 		erasurecode_helpers.c \
-        erasurecode_preprocessing.c \
+		erasurecode_preprocessing.c \
+		erasurecode_postprocessing.c \
 		utils/chksum/crc32.c \
 		utils/chksum/alg_sig.c \
 		utils/chksum/galois.c \
diff --git a/src/backends/jerasure/jerasure_rs_vand.c b/src/backends/jerasure/jerasure_rs_vand.c
index 0df892a..6558fa7 100644
--- a/src/backends/jerasure/jerasure_rs_vand.c
+++ b/src/backends/jerasure/jerasure_rs_vand.c
@@ -31,6 +31,7 @@
 
 #include "erasurecode.h"
 #include "erasurecode_backend.h"
+#include "erasurecode_helpers.h"
 
 /* Forward declarations */
 struct ec_backend_op_stubs jerasure_rs_vand_ops;
@@ -58,23 +59,6 @@ struct jerasure_rs_vand_descriptor {
     int w;
 };
 
-/**
- * ToDo (KMG): Move this to a util package, or replace with calloc
- */
-static
-void * alloc_zeroed_buffer(int size)
-{
-    void * buf = NULL;  /* buffer to allocate and return */
-  
-    /* Allocate and zero the buffer, or set the appropriate error */
-    buf = malloc((size_t) size);
-    if (buf) {
-      buf = memset(buf, 0, (size_t) size);
-    } 
-
-    return buf;
-}
-
 static int jerasure_rs_vand_encode(void *desc, char **data, char **parity,
         int blocksize)
 {
diff --git a/src/erasurecode.c b/src/erasurecode.c
index ca05fd4..d70227e 100644
--- a/src/erasurecode.c
+++ b/src/erasurecode.c
@@ -356,9 +356,18 @@ int liberasurecode_encode(int desc,
     /* call the backend encode function passing it desc instance */
     ret = instance->common.ops->encode(instance->desc.backend_desc,
                                        *encoded_data, *encoded_parity, blocksize);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = finalize_fragments_after_encode(instance, k, m, blocksize,
+                                          *encoded_data, *encoded_parity);
 
     *fragment_len = get_fragment_size((*encoded_data)[0]);
 out:
+    /* FIXME add cleanup API to call when encode() has an error */
+    if (ret)
+        log_error("Error in liberasurecode_encode %d", ret);
     return ret;
 }
 
diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c
index 839d101..6c0d158 100644
--- a/src/erasurecode_helpers.c
+++ b/src/erasurecode_helpers.c
@@ -57,9 +57,7 @@ void *get_aligned_buffer16(int size)
 }
 
 /**
- * Allocate a zero-ed buffer of a specific size.  This method allocates from
- * the Python stack in order to comply with the recommendations of the
- * Python\C API.  On error, return NULL and call PyErr_NoMemory.
+ * Allocate a zero-ed buffer of a specific size.
  *
  * @param size integer size in bytes of buffer to allocate
  * @return pointer to start of allocated buffer or NULL on error
@@ -307,3 +305,31 @@ int validate_fragment(char *buf)
 
 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
 
+inline int set_chksum(char *buf, int chksum)
+{
+    fragment_header_t* header = (fragment_header_t*) buf;
+
+    if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
+        log_error("Invalid fragment header (set chksum)!\n");
+        return -1; 
+    }
+
+    header->chksum = chksum;
+    
+    return 0;
+}
+
+inline int get_chksum(char *buf)
+{
+    fragment_header_t* header = (fragment_header_t*) buf;
+
+    if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
+        log_error("Invalid fragment header (get chksum)!");
+        return -1;
+    }
+
+    return header->chksum;
+}
+
+/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
+
diff --git a/src/erasurecode_postprocessing.c b/src/erasurecode_postprocessing.c
new file mode 100644
index 0000000..e086f2c
--- /dev/null
+++ b/src/erasurecode_postprocessing.c
@@ -0,0 +1,69 @@
+/*
+ * <Copyright>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice, this
+ * list of conditions and the following disclaimer in the documentation and/or
+ * other materials provided with the distribution.  THIS SOFTWARE IS PROVIDED BY
+ * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * liberasurecode postprocessing helpers implementation
+ *
+ * vi: set noai tw=79 ts=4 sw=4:
+ */
+
+#include "erasurecode_backend.h"
+#include "erasurecode_helpers.h"
+#include "erasurecode_stdinc.h"
+
+int finalize_fragments_after_encode(ec_backend_t instance,
+        int k, int m, int blocksize,
+        char **encoded_data, char **encoded_parity)
+{
+    int i;
+    // int add_checksum = instance->args.uargs.inline_chksum;
+    int add_checksum = 1;
+    char *fragment = get_fragment_ptr_from_data(encoded_parity[0]);
+printf ("fragment = %p\n", fragment);
+    fragment = get_fragment_ptr_from_data(encoded_parity[1]);
+printf ("fragment = %p\n", fragment);
+
+    /* finalize data fragments */
+    for (i = 0; i < k; i++) {
+        char *fragment = get_fragment_ptr_from_data(encoded_data[i]);
+        set_fragment_idx(fragment, i);
+        if (add_checksum) {
+            int chksum = crc32(0, encoded_data[i], blocksize);
+            set_chksum(fragment, chksum);
+        }
+        encoded_data[i] = fragment;
+    }
+
+    /* finalize parity fragments */
+    for (i = 0; i < m; i++) {
+        char *fragment = get_fragment_ptr_from_data(encoded_parity[i]);
+        set_fragment_idx(fragment, i + k);
+        if (add_checksum) {
+            int chksum = crc32(0, encoded_parity[i], blocksize);
+            set_chksum(fragment, chksum);
+        }
+        encoded_parity[i] = fragment;
+    }
+
+    return 0;
+}
+
diff --git a/src/erasurecode_preprocessing.c b/src/erasurecode_preprocessing.c
index 082243f..6591180 100644
--- a/src/erasurecode_preprocessing.c
+++ b/src/erasurecode_preprocessing.c
@@ -40,15 +40,16 @@ int prepare_fragments_for_encode(ec_backend_t instance,
     int i, ret = 0;
     int data_len;           /* data len to write to fragment headers */
     int aligned_data_len;   /* EC algorithm compatible data length */
+    int bsize = 0;
 
     /* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/
     data_len = orig_data_size;
     aligned_data_len = get_aligned_data_size(instance, orig_data_size);
-    *blocksize = aligned_data_len / k;
+    *blocksize = bsize = (aligned_data_len / k);
 
     for (i = 0; i < k; i++) {
-        int payload_size = data_len > *blocksize ? *blocksize : data_len;
-        char *fragment = alloc_fragment_buffer(*blocksize);
+        int payload_size = data_len > bsize ? bsize : data_len;
+        char *fragment = (char *) alloc_fragment_buffer(bsize);
         if (NULL == fragment) {
             ret = -ENOMEM;
             goto out_error;
@@ -56,32 +57,38 @@ int prepare_fragments_for_encode(ec_backend_t instance,
       
         /* Copy existing data into clean, zero'd out buffer */
         encoded_data[i] = get_data_ptr_from_fragment(fragment);
+      
         if (data_len > 0) {
             memcpy(encoded_data[i], orig_data, payload_size);
         }
 
         /* Fragment size will always be the same
          * (may be able to get rid of this) */
-        set_fragment_payload_size(fragment, *blocksize);
+        set_fragment_payload_size(fragment, bsize);
+
+        /* Original data length */
+        set_orig_data_size(fragment, orig_data_size);
 
         orig_data += payload_size;
         data_len -= payload_size;
     }
 
     for (i = 0; i < m; i++) {
-        char *fragment = alloc_fragment_buffer(*blocksize);
+        char *fragment = (char *) alloc_fragment_buffer(bsize);
         if (NULL == fragment) {
             ret = -ENOMEM;
             goto out_error;
         }
+
         encoded_parity[i] = get_data_ptr_from_fragment(fragment);
-        set_fragment_payload_size(fragment, *blocksize);
+        set_fragment_payload_size(fragment, bsize);
     }
 
 out:
     return ret;
 
 out_error:
+    printf ("ERROR in encode\n");
     if (encoded_data) {
         for (i = 0; i < k; i++) {
             if (encoded_data[i])
diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c
index a6255f6..93e7d72 100644
--- a/test/liberasurecode_test.c
+++ b/test/liberasurecode_test.c
@@ -42,10 +42,10 @@ struct testcase {
 };
 
 //TODO Make this a but more useful
-char *create_buffer(int size)
+char *create_buffer(int size, int fill)
 {
     char *buf = malloc(size);
-    memset(buf, 'x', size);
+    memset(buf, fill, size);
     return buf;
 }
 
@@ -67,32 +67,42 @@ static int test_create_and_destroy_backend(
     return 0;
 }
 
-static int test_simple_encode(
+static int test_simple_encode_decode(
         const char *backend,
         struct ec_args *args)
 {
     int rc = 0;
     int desc = -1;
-    int size = 1024 * 1024;
-    char **encoded_data = NULL, **parity = NULL;
-    char *data = NULL;
+
+    int orig_data_size = 1024 * 1024;
+    char *orig_data = NULL;
+    char **encoded_data = NULL, **encoded_parity = NULL;
+    uint64_t encoded_fragment_len = 0;
+
+    uint64_t decoded_data_len = 0;
+    char *decoded_data = NULL;
         
     desc = liberasurecode_instance_create(backend, args);
     assert(desc > 0 || EBACKENDNOTAVAIL == desc);
 
-    data = create_buffer(size);
-    if (NULL == data) {
+    orig_data = create_buffer(orig_data_size, 'x');
+    if (NULL == orig_data) {
         rc = -ENOMEM;
         goto out;
     }
 
-    rc = liberasurecode_encode(desc, data, size, encoded_data, parity);
+    rc = liberasurecode_encode(desc, orig_data, orig_data_size,
+            &encoded_data, &encoded_parity, &encoded_fragment_len);
+    assert(0 == rc);
+
+    rc = liberasurecode_decode(desc, encoded_data,
+            10, -1, decoded_data, &decoded_data_len);
     assert(0 == rc);
 
     if (desc)
         liberasurecode_instance_destroy(desc);
 
-    free(data);
+    free(orig_data);
 
 out:
     return rc;
@@ -100,7 +110,7 @@ out:
 
 struct ec_args flat_xor_hd_args = {
     .k = 10,
-    .m = 4,
+    .m = 6,
     .priv_args1.flat_xor_hd_args.hd = 3,
 };
 
@@ -118,19 +128,19 @@ struct testcase testcases[] = {
     {"create_and_destroy_backend",
         test_create_and_destroy_backend,
         "flat_xor_hd", &flat_xor_hd_args,
-        .skip = true},
+        .skip = false},
     {"create_and_destroy_backend",
         test_create_and_destroy_backend,
         "jerasure_rs_vand", &jerasure_rs_vand_args,
         .skip = false},
     {"simple_encode_flat_xor_hd",
-        test_simple_encode,
+        test_simple_encode_decode,
         "flat_xor_hd", &flat_xor_hd_args,
         .skip = true},
     {"simple_encode_jerasure_rs_vand",
-        test_simple_encode,
+        test_simple_encode_decode,
         "jerasure_rs_vand", &jerasure_rs_vand_args,
-        .skip = false},
+        .skip = true},
     { NULL, NULL, NULL, NULL, false },
 };