Split metadata handling into own routine, add crc32 support

Signed-off-by: Tushar Gohad <tushar.gohad@intel.com>

8 files changed, 258 insertions, 108 deletions

diff --git a/include/erasurecode/erasurecode.h b/include/erasurecode/erasurecode.h
index f6bf457..12031ba 100644
--- a/include/erasurecode/erasurecode.h
+++ b/include/erasurecode/erasurecode.h
@@ -40,10 +40,10 @@ extern "C" {
 /* =~=*=~==~=*=~==~=*=~= Supported EC backends =~=*=~==~=*=~==~=*=~==~=*=~== */
 
 typedef enum {
-    EC_BACKEND_NULL                     = 0,
-    EC_BACKEND_JERASURE_RS_VAND         = 1,
-    EC_BACKEND_JERASURE_RS_CAUCHY       = 2,
-    EC_BACKEND_FLAT_XOR_HD              = 3,
+    EC_BACKEND_NULL                 = 0,
+    EC_BACKEND_JERASURE_RS_VAND     = 1,
+    EC_BACKEND_JERASURE_RS_CAUCHY   = 2,
+    EC_BACKEND_FLAT_XOR_HD          = 3,
     EC_BACKENDS_MAX,
 } ec_backend_id_t;
 
@@ -58,6 +58,26 @@ static const char *ec_backend_names[EC_BACKENDS_MAX] = {
 };
 #endif // EC_BACKENDS_SUPPORTED
 
+/* ~=*=~==~=*=~= EC Fragment metadata - supported checksum types ~=*=~==~=*=~ */
+
+/* Checksum types supported for fragment metadata stored in each fragment */
+typedef enum {
+    CHKSUM_NONE                     = 0,
+    CHKSUM_CRC32                    = 1,
+    CHKSUM_MD5                      = 2,
+    CHKSUM_TYPES_MAX,
+} ec_checksum_type_t;
+
+#ifndef EC_CHKSUM_TYPES_SUPPORTED
+#define EC_CHKSUM_TYPES_SUPPORTED
+/* Supported EC backends */
+static const char *ec_chksum_types[CHKSUM_TYPES_MAX] = {
+    "none",
+    "crc32",
+    "md5",
+};
+#endif // EC_CHKSUM_TYPES_SUPPORTED
+
 /* =~=*=~==~=*=~== EC Arguments - Common and backend-specific =~=*=~==~=*=~== */
 
 /**
@@ -68,7 +88,7 @@ struct ec_args {
     int k;                  /* number of data fragments */
     int m;                  /* number of parity fragments */
     int w;                  /* word size, in bits (optional) */
-    int hd;                 /* Hamming distance (=m for Reed-Solomon) */
+    int hd;                 /* hamming distance (=m for Reed-Solomon) */
 
     union {
         struct {
@@ -82,9 +102,7 @@ struct ec_args {
 
     void *priv_args2;       /** flexible placeholder for
                               * future backend args */
-
-    int inline_chksum;      /* embedded fragment checksums (yes/no), type */
-    int algsig_chksum;      /* use algorithmic signature checksums */
+    ec_checksum_type_t ct;  /* fragment checksum type */
 };
 
 /* =~=*=~==~=*=~== liberasurecode frontend API functions =~=*=~==~=~=*=~==~= */
@@ -99,11 +117,21 @@ struct ec_args {
  *
  * @param num_backends - pointer to return number of backends in
  *
- * @returns 
+ * @returns list of EC backends implemented
  */
 const char ** liberasurecode_supported_backends(int *num_backends);
 
 /**
+ * Returns a list of checksum types supported for fragment data, stored in
+ * individual fragment headers as part of fragment metadata
+ *
+ * @param num_checksum_types - pointer to return number of checksum types in
+ *
+ * @returns list of checksum types supported for fragment data
+ */
+const char ** liberasurecode_supported_checksum_types(void);
+
+/**
  * Create a liberasurecode instance and return a descriptor 
  * for use with EC operations (encode, decode, reconstruct)
  *
@@ -113,12 +141,12 @@ const char ** liberasurecode_supported_backends(int *num_backends);
  *        arguments common to all backends
  *          k - number of data fragments
  *          m - number of parity fragments
- *          inline_checksum - 
- *          algsig_checksum -
+ *          w - word size, in bits
+ *          hd - hamming distance (=m for Reed-Solomon)
+ *          ct - fragment checksum type (stored with the fragment metadata)
  *        backend-specific arguments
  *          null_args - arguments for the null backend
- *          flat_xor_hd_args - arguments for the xor_hd backend
- *          jerasure_args - arguments for the Jerasure backend
+ *          flat_xor_hd, jerasure do not require any special args
  *      
  * @returns liberasurecode instance descriptor (int > 0)
  */
@@ -173,16 +201,16 @@ int liberasurecode_encode(int desc,
 int liberasurecode_decode(int desc,
         char **available_fragments,                     /* input */
         int num_fragments, uint64_t fragment_len,       /* input */
-        char **out_data, uint64_t *out_data_len);        /* output */
+        char **out_data, uint64_t *out_data_len);       /* output */
 
 /**
  * Reconstruct a missing fragment from a subset of available fragments
  *
  * @param desc - liberasurecode descriptor/handle
  *        from liberasurecode_instance_create()
- * @param fragment_len - size in bytes of the fragments
  * @param available_fragments - erasure encoded fragments
  * @param num_fragments - number of fragments being passed in
+ * @param fragment_len - size in bytes of the fragments
  * @param destination_idx - missing idx to reconstruct
  * @param out_fragment - output of reconstruct
  *
@@ -202,20 +230,54 @@ int liberasurecode_reconstruct_fragment(int desc,
 int liberasurecode_fragments_needed(int desc,
         int *missing_idxs, int *fragments_needed);
 
+
+/* ==~=*=~==~=*=~== liberasurecode fragment metadata routines ==~*==~=*=~==~ */
+
+#define LIBERASURECODE_MAX_CHECKSUM_LEN 8
+typedef struct __attribute__((__packed__))
+fragment_metadata
+{
+    uint32_t    idx;                /* 4 */
+    uint32_t    size;               /* 4 */
+    uint64_t    orig_data_size;     /* 8 */
+    uint8_t     chksum_type;        /* 1 */
+    uint32_t    chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 32 */
+    uint8_t     chksum_mismatch;    /* 1 */
+} fragment_metadata_t;
+
+#define FRAGSIZE_2_BLOCKSIZE(fragment_size) \
+    (fragment_size - sizeof(fragment_header_t))
+
 /**
  * Get opaque metadata for a fragment.  The metadata is opaque to the
  * client, but meaningful to the underlying library.  It is used to verify
  * stripes in verify_stripe_metadata().
+ *
+ * @param desc - liberasurecode descriptor/handle
+ *        from liberasurecode_instance_create()
+ * @param fragment - fragment data pointer
+ *
+ * @param fragment_metadata - pointer to output fragment metadata struct
+ *          (reference passed by the user)
  */
-int liberasurecode_get_fragment_metadata(char *fragment, 
-        char **fragment_metadata, int *fragment_metadata_len);
-
+int liberasurecode_get_fragment_metadata(int desc,
+        char *fragment, fragment_metadata_t *fragment_metadata);
 
 /**
  * Verify a subset of fragments generated by encode()
+ *
+ * @param desc - liberasurecode descriptor/handle
+ *        from liberasurecode_instance_create()
+ * @param fragments - fragments part of the EC stripe to verify
+ * @num_fragments - number of fragments part of the EC stripe
+ *
+ * @ returns 1 if stripe checksum verification is successful
+ *           0 otherwise
  */
-int liberasurecode_verify_stripe_metadata(char **fragment_metadata, 
-        int num_fragments, int fragment_metadata_len);
+int liberasurecode_verify_stripe_metadata(int desc,
+        char **fragments, int num_fragments);
+
+/* ==~=*=~===~=*=~==~=*=~== liberasurecode Helpers ==~*==~=*=~==~=~=*=~==~= */
 
 /**
  * This computes the aligned size of a buffer passed into 
@@ -227,12 +289,11 @@ int liberasurecode_verify_stripe_metadata(char **fragment_metadata,
 int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len);
  
 /**
- * This will return the minumum encode size, which is the minimum
+ * This will return the minimum encode size, which is the minimum
  * buffer size that can be encoded.
  */
 int liberasurecode_get_minimum_encode_size(int desc);
 
-
 /* ==~=*=~===~=*=~==~=*=~== liberasurecode Error codes =~=*=~==~=~=*=~==~== */
 
 /* Error codes */
diff --git a/include/erasurecode/erasurecode_backend.h b/include/erasurecode/erasurecode_backend.h
index c5ead2a..2d30371 100644
--- a/include/erasurecode/erasurecode_backend.h
+++ b/include/erasurecode/erasurecode_backend.h
@@ -20,6 +20,10 @@
  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * liberasurecode backend API definition
+ *
+ * vi: set noai tw=79 ts=4 sw=4:
  */
 
 #ifndef _ERASURECODE_BACKEND_H_
diff --git a/include/erasurecode/erasurecode_helpers.h b/include/erasurecode/erasurecode_helpers.h
index 27684c3..428460f 100644
--- a/include/erasurecode/erasurecode_helpers.h
+++ b/include/erasurecode/erasurecode_helpers.h
@@ -39,21 +39,19 @@
  *
  * Prevent the compiler from padding this by using the __packed__ keyword
  */
+
+#define LIBERASURECODE_FRAG_HEADER_MAGIC    0xb0c5ecc
+#define LIBERASURECODE_MAX_CHECKSUM_LEN     8   /* quad words */
+
 typedef struct __attribute__((__packed__)) fragment_header_s
 {
-    uint32_t magic;
-    uint32_t idx;
-    uint32_t size;
-    uint32_t orig_data_size;
-    // FIXME - reserve 16-bytes for md5
-    uint32_t chksum;
+    fragment_metadata_t meta;   /* 50 bytes */
+    uint32_t            magic;  /*  4 bytes */
     // We must be aligned to 16-byte boundaries
     // So, size this array accordingly
-    uint32_t aligned_padding[3];
+    uint8_t             aligned_padding[10];
 } fragment_header_t;
 
-#define LIBERASURECODE_FRAG_HEADER_MAGIC  0xb0c5ecc
-
 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
 
 #define talloc(type, num)   (type *) malloc(sizeof(type) * (num))
@@ -125,7 +123,8 @@ int free_fragment_buffer(char *buf);
 void *get_aligned_buffer16(int size);
 int get_aligned_data_size(ec_backend_t instance, int data_len);
 char *get_data_ptr_from_fragment(char *buf);
-int get_data_ptr_array_from_fragments(char **data_array, char **fragments, int num_fragments);
+int get_data_ptr_array_from_fragments(char **data_array, char **fragments,
+        int num_fragments);
 char *get_fragment_ptr_from_data_novalidate(char *buf);
 char *get_fragment_ptr_from_data(char *buf);
 uint64_t get_fragment_size(char *buf);
diff --git a/src/erasurecode.c b/src/erasurecode.c
index 21dff35..cc81cbc 100644
--- a/src/erasurecode.c
+++ b/src/erasurecode.c
@@ -227,7 +227,14 @@ liberasurecode_exit(void) {
 /* =~=*=~==~=*=~= liberasurecode frontend API implementation =~=*=~==~=*=~== */
 
 /**
- * Returns a list of EC backends implemented/enabled
+ * Returns a list of EC backends implemented/enabled - the user
+ * should always rely on the return from this function as this
+ * set of backends can be different from the names listed in
+ * ec_backend_names above.
+ *
+ * @param num_backends - pointer to return number of backends in
+ *
+ * @returns list of EC backends implemented
  */
 const char ** liberasurecode_supported_backends(int *num_backends)
 {
@@ -236,6 +243,19 @@ const char ** liberasurecode_supported_backends(int *num_backends)
 }
 
 /**
+ * Returns a list of checksum types supported for fragment data, stored in
+ * individual fragment headers as part of fragment metadata
+ *
+ * @param num_checksum_types - pointer to return number of checksum types in
+ *
+ * @returns list of checksum types supported for fragment data
+ */
+const char ** liberasurecode_supported_checksum_types(void)
+{
+    return (const char **) ec_chksum_types;
+}
+
+/**
  * Create a liberasurecode instance and return a descriptor 
  * for use with EC operations (encode, decode, reconstruct)
  *
@@ -245,13 +265,13 @@ const char ** liberasurecode_supported_backends(int *num_backends)
  *        arguments common to all backends
  *          k - number of data fragments
  *          m - number of parity fragments
- *          inline_checksum - 
- *          algsig_checksum -
+ *          w - word size, in bits
+ *          hd - hamming distance (=m for Reed-Solomon)
+ *          ct - fragment checksum type (stored with the fragment metadata)
  *        backend-specific arguments
  *          null_args - arguments for the null backend
- *          flat_xor_hd_args - arguments for the xor_hd backend
- *          jerasure_args - arguments for the Jerasure backend
- *      
+ *          flat_xor_hd, jerasure do not require any special args
+ *
  * @returns liberasurecode instance descriptor (int > 0)
  */
 int liberasurecode_instance_create(const char *backend_name,
@@ -391,7 +411,7 @@ int liberasurecode_encode(int desc,
         goto out;
     }
 
-    ret = finalize_fragments_after_encode(instance, k, m, blocksize,
+    ret = finalize_fragments_after_encode(instance, k, m, blocksize, orig_data_size,
                                           *encoded_data, *encoded_parity);
 
     *fragment_len = get_fragment_size((*encoded_data)[0]);
@@ -525,14 +545,14 @@ int liberasurecode_decode(int desc,
      */
     j = 0;
     while (missing_idxs[j] >= 0) { 
+        int set_chksum = 1;
         int missing_idx = missing_idxs[j]; 
         if (missing_idx < k) {
             /* Generate headers */
             char *fragment_ptr = data[missing_idx];
             init_fragment_header(fragment_ptr);
-            set_fragment_idx(fragment_ptr, missing_idx);
-            set_orig_data_size(fragment_ptr, orig_data_size);
-            set_fragment_payload_size(fragment_ptr, blocksize);
+            add_fragment_metadata(fragment_ptr, missing_idx,
+                    orig_data_size, blocksize, !set_chksum);
         }
         j++;
     }
@@ -598,8 +618,6 @@ int liberasurecode_reconstruct_fragment(int desc,
     int m;
     int i;
     int j;
-    int checksum = 0;
-    int add_checksum = 1;
     uint64_t realloc_bm = 0;
     char **data_segments = NULL;
     char **parity_segments = NULL;
@@ -677,16 +695,13 @@ int liberasurecode_reconstruct_fragment(int desc,
      */
     if (destination_idx < k) {
         fragment_ptr = data[destination_idx];
-        checksum = crc32(0, data_segments[destination_idx], blocksize);
+        // checksum = crc32(0, data_segments[destination_idx], blocksize);
     } else {
         fragment_ptr = parity[destination_idx - k];
-        checksum = crc32(0, parity_segments[destination_idx - k], blocksize);
+        // checksum = crc32(0, parity_segments[destination_idx - k], blocksize);
     }
     init_fragment_header(fragment_ptr);
-    set_fragment_idx(fragment_ptr, destination_idx);
-    set_orig_data_size(fragment_ptr, orig_data_size);
-    set_fragment_payload_size(fragment_ptr, blocksize);
-    set_chksum(fragment_ptr, checksum);
+    add_fragment_metadata(fragment_ptr, destination_idx, orig_data_size, blocksize);
 
     /*
      * Copy the reconstructed fragment to the output buffer
@@ -753,6 +768,81 @@ out_error:
     return ret;
 }
 
+/* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */
+
+/**
+ * Get opaque metadata for a fragment.  The metadata is opaque to the
+ * client, but meaningful to the underlying library.  It is used to verify
+ * stripes in verify_stripe_metadata().
+ *
+ * @param desc - liberasurecode descriptor/handle
+ *        from liberasurecode_instance_create()
+ * @param fragment - fragment pointer
+ *
+ * @param fragment_metadata - pointer to output fragment metadata struct
+ *          (reference passed by the user)
+ */
+int liberasurecode_get_fragment_metadata(int desc,
+        char *fragment, fragment_metadata_t *fragment_metadata)
+{
+    int ret = 0;
+    fragment_header_t *fragment_hdr = NULL;
+
+    if (NULL == fragment) {
+        log_error("Need valid fragment object to get metadata for");
+        ret = -1;
+        goto out;
+    }
+
+    if (NULL == fragment_metadata) {
+        log_error("Need valid fragment_metadata object for return value");
+        ret = -2;
+        goto out;
+    }
+
+    memcpy(fragment_metadata, fragment, sizeof(struct fragment_metadata));
+    fragment_hdr = (fragment_header_t *) fragment;
+    switch(fragment_hdr->meta.chksum_type) {
+        case CHKSUM_CRC32: {
+            uint32_t computed_chksum = 0;
+            uint32_t stored_chksum = fragment_hdr->meta.chksum[0];
+            char *fragment_data = get_data_ptr_from_fragment(fragment);
+            uint64_t fragment_size = get_fragment_size(fragment);
+            computed_chksum = crc32(0, fragment_data, fragment_size);
+            if (stored_chksum != computed_chksum) {
+                fragment_metadata->chksum_mismatch = 1;
+            }
+            break;
+        }
+        case CHKSUM_MD5:
+            break;
+        case CHKSUM_NONE:
+        default:
+            break;
+    }
+
+out:
+    return ret;
+}
+
+/**
+ * Verify a subset of fragments generated by encode()
+ *
+ * @param desc - liberasurecode descriptor/handle
+ *        from liberasurecode_instance_create()
+ * @param fragments - fragments part of the EC stripe to verify
+ * @num_fragments - number of fragments part of the EC stripe
+ *
+ * @ returns 0 if stripe checksum verification is successful
+ *           -1 otherwise
+ */
+int liberasurecode_verify_stripe_metadata(int desc,
+        char **fragments, int num_fragments)
+{
+    return 0;
+}
+/* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */
+
 /**
  * This computes the aligned size of a buffer passed into 
  * the encode function.  The encode function must pad fragments
@@ -796,29 +886,6 @@ int liberasurecode_get_minimum_encode_size(int desc)
     return liberasurecode_get_aligned_data_size(desc, 1);
 }
 
-/**
- * Get opaque metadata for a fragment.  The metadata is opaque to the
- * client, but meaningful to the underlying library.  It is used to verify
- * stripes in verify_stripe_metadata().
- */
-int liberasurecode_get_fragment_metadata(char *fragment, char **fragment_metadata, 
-        int *fragment_metadata_len)
-{
-  int ret = 0;
-  return ret;
-}
-
-
-/**
- * Verify a subset of fragments generated by encode()
- */
-int liberasurecode_verify_stripe_metadata(char **fragments, int num_fragments,
-        int fragment_metadata_len)
-{
-  int ret = 0;
-  return ret;
-}
- 
 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=* misc *=~==~=*=~==~=*=~==~=*=~==~=*=~== */
 
 #if 0
diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c
index af44837..a8d26f3 100644
--- a/src/erasurecode_helpers.c
+++ b/src/erasurecode_helpers.c
@@ -155,7 +155,7 @@ uint64_t get_fragment_size(char *buf)
         return -1;
 
     header = (fragment_header_t *) buf;
-    return (header->size + sizeof(fragment_header_t));
+    return (header->meta.size + sizeof(fragment_header_t));
  }
 
 /**
@@ -251,7 +251,7 @@ int set_fragment_idx(char *buf, int idx)
         return -1;
     }
 
-    header->idx = idx;
+    header->meta.idx = idx;
 
     return 0;
 }
@@ -266,7 +266,7 @@ int get_fragment_idx(char *buf)
         return -1;
     }
 
-    return header->idx;
+    return header->meta.idx;
 }
 
 int set_fragment_payload_size(char *buf, int size)
@@ -279,7 +279,7 @@ int set_fragment_payload_size(char *buf, int size)
         return -1;
     }
 
-    header->size = size;
+    header->meta.size = size;
 
     return 0;
 }
@@ -294,7 +294,7 @@ int get_fragment_payload_size(char *buf)
         return -1;
     }
 
-    return header->size;
+    return header->meta.size;
 }
 
 int set_orig_data_size(char *buf, int orig_data_size)
@@ -307,7 +307,7 @@ int set_orig_data_size(char *buf, int orig_data_size)
         return -1;
     }
 
-    header->orig_data_size = orig_data_size;
+    header->meta.orig_data_size = orig_data_size;
 
     return 0;
 }
@@ -322,7 +322,7 @@ int get_orig_data_size(char *buf)
         return -1;
     }
 
-    return header->orig_data_size;
+    return header->meta.orig_data_size;
 }
 
 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
@@ -341,9 +341,10 @@ int validate_fragment(char *buf)
 
 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
 
-inline int set_chksum(char *buf, int chksum)
+inline int set_chksum(char *buf, int blocksize)
 {
     fragment_header_t* header = (fragment_header_t*) buf;
+    char *data = get_data_ptr_from_fragment(buf);
 
     assert(NULL != header);
     if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
@@ -351,22 +352,31 @@ inline int set_chksum(char *buf, int chksum)
         return -1; 
     }
 
-    header->chksum = chksum;
+    switch(header->meta.chksum_type) {
+        case CHKSUM_CRC32:
+            header->meta.chksum[0] = crc32(0, data, blocksize);
+            break;
+        case CHKSUM_MD5:
+            break;
+        case CHKSUM_NONE:
+        default:
+            break;
+    }
     
     return 0;
 }
 
-inline int get_chksum(char *buf)
+inline uint32_t* get_chksum(char *buf)
 {
     fragment_header_t* header = (fragment_header_t*) buf;
 
     assert(NULL != header);
     if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
         log_error("Invalid fragment header (get chksum)!");
-        return -1;
+        return NULL;
     }
 
-    return header->chksum;
+    return (uint32_t *) header->meta.chksum;
 }
 
 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
diff --git a/src/erasurecode_postprocessing.c b/src/erasurecode_postprocessing.c
index 4a4798e..2ee62d4 100644
--- a/src/erasurecode_postprocessing.c
+++ b/src/erasurecode_postprocessing.c
@@ -30,33 +30,37 @@
 #include "erasurecode_helpers.h"
 #include "erasurecode_stdinc.h"
 
+void add_fragment_metadata(char *fragment,
+        int idx, uint64_t orig_data_size, int blocksize,
+        int add_chksum)
+{
+    set_fragment_idx(fragment, idx);
+    set_orig_data_size(fragment, orig_data_size);
+    set_fragment_payload_size(fragment, blocksize);
+    if (add_chksum) {
+        set_chksum(fragment, blocksize);
+    }
+}
+
 int finalize_fragments_after_encode(ec_backend_t instance,
-        int k, int m, int blocksize,
+        int k, int m, int blocksize, uint64_t orig_data_size,
         char **encoded_data, char **encoded_parity)
 {
-    int i;
-    // int add_checksum = instance->args.uargs.inline_chksum;
-    int add_checksum = 1;
+    int i, set_chksum = 1;
 
     /* finalize data fragments */
     for (i = 0; i < k; i++) {
         char *fragment = get_fragment_ptr_from_data(encoded_data[i]);
-        set_fragment_idx(fragment, i);
-        if (add_checksum) {
-            int chksum = crc32(0, encoded_data[i], blocksize);
-            set_chksum(fragment, chksum);
-        }
+        add_fragment_metadata(fragment, i, orig_data_size,
+                blocksize, set_chksum);
         encoded_data[i] = fragment;
     }
 
     /* finalize parity fragments */
     for (i = 0; i < m; i++) {
         char *fragment = get_fragment_ptr_from_data(encoded_parity[i]);
-        set_fragment_idx(fragment, i + k);
-        if (add_checksum) {
-            int chksum = crc32(0, encoded_parity[i], blocksize);
-            set_chksum(fragment, chksum);
-        }
+        add_fragment_metadata(fragment, i + k, orig_data_size,
+                blocksize, set_chksum);
         encoded_parity[i] = fragment;
     }
 
diff --git a/src/erasurecode_preprocessing.c b/src/erasurecode_preprocessing.c
index a45cd65..02ffe51 100644
--- a/src/erasurecode_preprocessing.c
+++ b/src/erasurecode_preprocessing.c
@@ -62,13 +62,6 @@ int prepare_fragments_for_encode(ec_backend_t instance,
             memcpy(encoded_data[i], orig_data, payload_size);
         }
 
-        /* Fragment size will always be the same
-         * (may be able to get rid of this) */
-        set_fragment_payload_size(fragment, bsize);
-
-        /* Original data length */
-        set_orig_data_size(fragment, orig_data_size);
-
         orig_data += payload_size;
         data_len -= payload_size;
     }
@@ -81,8 +74,6 @@ int prepare_fragments_for_encode(ec_backend_t instance,
         }
 
         encoded_parity[i] = get_data_ptr_from_fragment(fragment);
-        set_orig_data_size(fragment, orig_data_size);
-        set_fragment_payload_size(fragment, bsize);
     }
 
 out:
diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c
index 8ae088c..c0695ce 100644
--- a/test/liberasurecode_test.c
+++ b/test/liberasurecode_test.c
@@ -113,6 +113,16 @@ static void test_liberasurecode_supported_backends()
         printf("%s\n", supported_ec_backends[i]);
 }
 
+static void test_liberasurecode_supported_checksum_types()
+{
+    int i;
+    const char **supported_checksum_types =
+        liberasurecode_supported_checksum_types();
+
+    for (i = 0; i < CHKSUM_TYPES_MAX; i++)
+        printf("%s\n", supported_checksum_types[i]);
+}
+
 static void test_create_and_destroy_backend(
         const char *backend,
         struct ec_args *args)
@@ -369,6 +379,10 @@ struct testcase testcases[] = {
         test_liberasurecode_supported_backends,
         NULL, NULL,
         .skip = false},
+    {"test_liberasurecode_supported_checksum_types",
+        test_liberasurecode_supported_checksum_types,
+        NULL, NULL,
+        .skip = false},
     {"create_and_destroy_backend",
         test_create_and_destroy_backend,
         "null", &null_args,