Merged in metadata_adder_new_patches2 (pull request #8)

New (fixed) patch set of metadata_adder discussion

11 files changed, 242 insertions, 52 deletions

diff --git a/AUTHORS b/AUTHORS
index 8bb9907..031a748 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -6,3 +6,9 @@ Original Authors
  * Eric Lambert
  * Mark W Storer
 
+Contributors
+----------------
+
+ * Kota Tsuyuzaki
+ * Ryuta Kon
+
diff --git a/include/erasurecode/erasurecode.h b/include/erasurecode/erasurecode.h
index 4bd2a59..7dc7ba7 100644
--- a/include/erasurecode/erasurecode.h
+++ b/include/erasurecode/erasurecode.h
@@ -247,6 +247,7 @@ fragment_metadata
 {
     uint32_t    idx;                /* 4 */
     uint32_t    size;               /* 4 */
+    uint32_t    frag_adder_size;    /* 4 */
     uint64_t    orig_data_size;     /* 8 */
     uint8_t     chksum_type;        /* 1 */
     uint32_t    chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 32 */
@@ -327,6 +328,18 @@ int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len);
  */
 int liberasurecode_get_minimum_encode_size(int desc);
 
+/**
+ * This will return the fragment size, which is each fragment data
+ * length the backend will allocate when encoding.
+ *
+ * @param desc - liberasurecode descriptor/handle
+ *        from liberasurecode_instance_create()
+ * @param data_len - original data length in bytes
+ *
+ * @return fragment size
+ */
+int liberasurecode_get_fragment_size(int desc, int data_len);
+
 /* ==~=*=~===~=*=~==~=*=~== liberasurecode Error codes =~=*=~==~=~=*=~==~== */
 
 /* Error codes */
diff --git a/include/erasurecode/erasurecode_backend.h b/include/erasurecode/erasurecode_backend.h
index 66e858b..921a9a1 100644
--- a/include/erasurecode/erasurecode_backend.h
+++ b/include/erasurecode/erasurecode_backend.h
@@ -109,7 +109,7 @@ struct ec_backend_common {
     char                        soversion[MAX_LEN]; /* EC backend shared library version */
 
     struct ec_backend_op_stubs  *ops;               /* EC backend stubs */
-    int                         metadata_adder;     /* EC backend custom metadata adder -
+    size_t                      metadata_adder;     /* EC backend custom metadata adder -
                                                      * metadata_adder bytes are added to
                                                      * the fragment size when allocating
                                                      * data/parity fragment buffers */
diff --git a/include/erasurecode/erasurecode_helpers.h b/include/erasurecode/erasurecode_helpers.h
index 93a8b6f..f7eefd7 100644
--- a/include/erasurecode/erasurecode_helpers.h
+++ b/include/erasurecode/erasurecode_helpers.h
@@ -45,12 +45,12 @@
 
 typedef struct __attribute__((__packed__)) fragment_header_s
 {
-    fragment_metadata_t meta;   /* 55 bytes */
+    fragment_metadata_t meta;   /* 59 bytes */
     uint32_t            magic;  /*  4 bytes */
     uint32_t            libec_version; /* 4 bytes */
     // We must be aligned to 16-byte boundaries
     // So, size this array accordingly
-    uint8_t             aligned_padding[1];
+    uint8_t             aligned_padding[13];
 } fragment_header_t;
 
 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
@@ -126,6 +126,8 @@ int get_aligned_data_size(ec_backend_t instance, int data_len);
 char *get_data_ptr_from_fragment(char *buf);
 int get_data_ptr_array_from_fragments(char **data_array, char **fragments,
         int num_fragments);
+int get_fragment_ptr_array_from_data(char **frag_array, char **data,
+        int num_data);
 char *get_fragment_ptr_from_data_novalidate(char *buf);
 char *get_fragment_ptr_from_data(char *buf);
 uint64_t get_fragment_size(char *buf);
@@ -133,6 +135,9 @@ int set_fragment_idx(char *buf, int idx);
 int get_fragment_idx(char *buf);
 int set_fragment_payload_size(char *buf, int size);
 int get_fragment_payload_size(char *buf);
+int set_fragment_adder_size(char *buf, int size);
+int get_fragment_adder_size(char *buf);
+int get_fragment_buffer_size(char *buf);
 int set_orig_data_size(char *buf, int orig_data_size);
 int get_orig_data_size(char *buf);
 int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize);
diff --git a/src/backends/isa-l/isa_l_rs_vand.c b/src/backends/isa-l/isa_l_rs_vand.c
index d701a2f..98a9226 100644
--- a/src/backends/isa-l/isa_l_rs_vand.c
+++ b/src/backends/isa-l/isa_l_rs_vand.c
@@ -480,7 +480,9 @@ static void * isa_l_rs_vand_init(struct ec_backend_args *args,
 
     desc->k = args->uargs.k;
     desc->m = args->uargs.m;
-    desc->w = ISA_L_W;
+    if (args->uargs.w <= 0)
+        args->uargs.w = ISA_L_W;
+    desc->w = args->uargs.w;
 
     /* validate EC arguments */
     {
diff --git a/src/backends/shss/shss.c b/src/backends/shss/shss.c
index f8ba53a..3553456 100644
--- a/src/backends/shss/shss.c
+++ b/src/backends/shss/shss.c
@@ -31,7 +31,6 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <alloca.h>
 
 #include "erasurecode.h"
 #include "erasurecode_helpers.h"
@@ -40,6 +39,7 @@
 /* Forward declarations */
 struct ec_backend shss;
 struct ec_backend_op_stubs shss_ops;
+struct ec_backend_common backend_shss;
 
 typedef int (*shss_encode_func)(char **, size_t, int, int, int, int, long long *);
 typedef int (*shss_decode_func)(char **, size_t, int *, int, int, int, int, int, long long *);
@@ -67,18 +67,7 @@ struct shss_descriptor {
 #define SHSS_SO_NAME "libshss.so"
 #endif
 #define DEFAULT_W 128
-
-/* TODO:
-   metadata_adder is still in discussion. shss needs to a fixed value to allocate extra bytes
-   for *each* fragment. However, current liberasurecode calculates the extra bytes as
-   "(alined_data_size + metadata_adder) / k" so that shss has to define the METADATA as a big value
-   to alloc enough memory for the maximum number of k even if k is smaller than the maximum value.
-
-   i.e. (shss specification is)
-   Enough Extra Bytes (for *each* fragment): 32
-   The Maximum Number: 20 (k=20)
-*/
-#define METADATA 32 * 20
+#define METADATA 32
 
 static int shss_encode(void *desc, char **data, char **parity,
         int blocksize)
@@ -224,18 +213,29 @@ static int shss_element_size(void* desc)
 
 static void * shss_init(struct ec_backend_args *args, void *backend_sohandle)
 {
-    static struct shss_descriptor xdesc;
+    struct shss_descriptor *desc = NULL;
 
-    xdesc.k = args->uargs.k;
-    xdesc.m = args->uargs.m;
-    xdesc.n = args->uargs.k + args->uargs.m;
-    xdesc.w = DEFAULT_W;
+    desc = (struct shss_descriptor *)
+           malloc(sizeof(struct shss_descriptor));
+    if (NULL == desc) {
+        return NULL;
+    }
+
+    desc->k = args->uargs.k;
+    desc->m = args->uargs.m;
+    desc->n = args->uargs.k + args->uargs.m;
+    desc->w = DEFAULT_W;
     args->uargs.w = DEFAULT_W;
 
     /* Sample on how to pass extra args to the backend */
     // TODO: Need discussion how to pass extra args.
+    // tentatively we could pass with priv_args2 as the bit_length
     int *priv = (int *)args->uargs.priv_args2;
-    xdesc.aes_bit_length = priv[0]; // AES bit number
+    if(priv != NULL){
+        desc->aes_bit_length = priv[0]; // AES bit number
+    }else{
+        desc->aes_bit_length = 128;
+    }
 
     union {
         shss_encode_func encodep;
@@ -246,36 +246,45 @@ static void * shss_init(struct ec_backend_args *args, void *backend_sohandle)
 
     func_handle.vptr = NULL;
     func_handle.vptr = dlsym(backend_sohandle, "ssencode");
-    xdesc.ssencode = func_handle.encodep;
-    if (NULL == xdesc.ssencode) {
+    desc->ssencode = func_handle.encodep;
+    if (NULL == desc->ssencode) {
         goto error;
     }
 
     func_handle.vptr = NULL;
     func_handle.vptr = dlsym(backend_sohandle, "ssdecode");
-    xdesc.ssdecode = func_handle.decodep;
-    if (NULL == xdesc.ssdecode) {
+    desc->ssdecode = func_handle.decodep;
+    if (NULL == desc->ssdecode) {
         goto error;
     }
 
     func_handle.vptr = NULL;
     func_handle.vptr = dlsym(backend_sohandle, "ssreconst");
-    xdesc.ssreconst = func_handle.reconp;
-    if (NULL == xdesc.ssreconst) {
+    desc->ssreconst = func_handle.reconp;
+    if (NULL == desc->ssreconst) {
         goto error;
     }
 
-    return (void *)&xdesc;
+    return desc;
 
 error:
+    free(desc);
+
     return NULL;
 }
 
 static int shss_exit(void *desc)
 {
+    if (desc != NULL) {
+        free(desc);
+    }
     return 0;
 }
 
+static bool shss_is_compatible_with(uint32_t version) {
+    return version == backend_shss.ec_backend_version;
+}
+
 struct ec_backend_op_stubs shss_op_stubs = {
     .INIT                       = shss_init,
     .EXIT                       = shss_exit,
@@ -284,6 +293,7 @@ struct ec_backend_op_stubs shss_op_stubs = {
     .FRAGSNEEDED                = shss_fragments_needed,
     .RECONSTRUCT                = shss_reconstruct,
     .ELEMENTSIZE                = shss_element_size,
+    .ISCOMPATIBLEWITH           = shss_is_compatible_with,
 };
 
 struct ec_backend_common backend_shss = {
diff --git a/src/erasurecode.c b/src/erasurecode.c
index c9a32f1..66a7e2f 100644
--- a/src/erasurecode.c
+++ b/src/erasurecode.c
@@ -333,6 +333,7 @@ int liberasurecode_encode_cleanup(int desc,
                                   char **encoded_parity)
 {
     int i, k, m;
+
     ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
     if (NULL == instance) {
         return -EBACKENDNOTAVAIL;
@@ -353,7 +354,6 @@ int liberasurecode_encode_cleanup(int desc,
         for (i = 0; i < m; i++) {
             free(encoded_parity[i]);
         }
-
         free(encoded_parity);
     }
 
@@ -441,6 +441,9 @@ int liberasurecode_encode(int desc,
     ret = prepare_fragments_for_encode(instance, k, m, orig_data, orig_data_size,
                                        *encoded_data, *encoded_parity, &blocksize);
     if (ret < 0) {
+        // ensure encoded_data/parity point the head of fragment_ptr
+        get_fragment_ptr_array_from_data(*encoded_data, *encoded_data, k);
+        get_fragment_ptr_array_from_data(*encoded_parity, *encoded_parity, m);
         goto out;
     }
 
@@ -448,6 +451,9 @@ int liberasurecode_encode(int desc,
     ret = instance->common.ops->encode(instance->desc.backend_desc,
                                        *encoded_data, *encoded_parity, blocksize);
     if (ret < 0) {
+        // ensure encoded_data/parity point the head of fragment_ptr
+        get_fragment_ptr_array_from_data(*encoded_data, *encoded_data, k);
+        get_fragment_ptr_array_from_data(*encoded_parity, *encoded_parity, m);
         goto out;
     }
 
@@ -455,6 +461,7 @@ int liberasurecode_encode(int desc,
                                           *encoded_data, *encoded_parity);
 
     *fragment_len = get_fragment_size((*encoded_data)[0]);
+
 out:
     if (ret) {
         /* Cleanup the allocations we have done */
@@ -625,7 +632,7 @@ int liberasurecode_decode(int desc,
      *
      */
     ret = prepare_fragments_for_decode(k, m,
-                                       data, parity, missing_idxs,
+                                       data, parity, missing_idxs, 
                                        &orig_data_size, &blocksize,
                                        fragment_len, &realloc_bm);
     if (ret < 0) {
@@ -1111,6 +1118,16 @@ int liberasurecode_get_minimum_encode_size(int desc)
     return liberasurecode_get_aligned_data_size(desc, 1);
 }
 
+int liberasurecode_get_fragment_size(int desc, int data_len)
+{
+    ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
+    // TODO: Create a common function to calculate fragment size also for preprocessing
+    int aligned_data_len = get_aligned_data_size(instance, data_len);
+    int size = (aligned_data_len / instance->args.uargs.k) + instance->common.metadata_adder;
+
+    return size;
+}
+
 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=* misc *=~==~=*=~==~=*=~==~=*=~==~=*=~== */
 
 #if 0
diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c
index 94aa308..05bc838 100644
--- a/src/erasurecode_helpers.c
+++ b/src/erasurecode_helpers.c
@@ -163,13 +163,11 @@ int free_fragment_buffer(char *buf)
  */
 uint64_t get_fragment_size(char *buf)
 {
-    fragment_header_t *header = NULL;
 
     if (NULL == buf)
         return -1;
 
-    header = (fragment_header_t *) buf;
-    return (header->meta.size + sizeof(fragment_header_t));
+    return get_fragment_buffer_size(buf) + sizeof(fragment_header_t);
  }
 
 /**
@@ -188,9 +186,6 @@ int get_aligned_data_size(ec_backend_t instance, int data_len)
     int alignment_multiple;
     int aligned_size = 0;
 
-    /* Account for any custom metadata the backend wants to add in data_len */
-    data_len += instance->common.metadata_adder;
-
     /*
      * For Cauchy reed-solomon align to k*word_size*packet_size
      * For Vandermonde reed-solomon and flat-XOR, align to k*word_size
@@ -232,6 +227,22 @@ int get_data_ptr_array_from_fragments(char **data_array, char **fragments,
     return num;
 }
 
+int get_fragment_ptr_array_from_data(char **frag_array, char **data,
+                                     int num_data)
+{
+    int i = 0, num = 0;
+    for (i = 0; i < num_data; i++) {
+        char *data_ptr = frag_array[i];
+        if (data_ptr == NULL) {
+            data[i] = NULL;
+            continue;
+        }
+        data[i] = get_fragment_ptr_from_data(data_ptr);
+        num++;
+    }
+    return num;
+}
+
 char *get_fragment_ptr_from_data_novalidate(char *buf)
 {
     buf -= sizeof(fragment_header_t);
@@ -313,6 +324,47 @@ int get_fragment_payload_size(char *buf)
     return header->meta.size;
 }
 
+int set_fragment_adder_size(char *buf, int size)
+{
+    fragment_header_t *header = (fragment_header_t *) buf;
+
+    assert(NULL != header);
+    if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
+        log_error("Invalid fragment header (set adder size)!");
+        return -1;
+    }
+
+    header->meta.frag_adder_size = size;
+
+    return 0;
+}
+
+int get_fragment_adder_size(char *buf)
+{
+    fragment_header_t *header = (fragment_header_t *) buf;
+
+    assert(NULL != header);
+    if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
+        log_error("Invalid fragment header (get adder size)!");
+        return -1;
+    }
+
+    return header->meta.frag_adder_size;
+}
+
+int get_fragment_buffer_size(char *buf)
+{
+    fragment_header_t *header = (fragment_header_t *) buf;
+
+    assert(NULL != header);
+    if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
+        log_error("Invalid fragment header (get size)!");
+        return -1;
+    }
+
+    return header->meta.size + header->meta.frag_adder_size;
+}
+
 int set_orig_data_size(char *buf, int orig_data_size)
 {
     fragment_header_t *header = (fragment_header_t *) buf;
diff --git a/src/erasurecode_postprocessing.c b/src/erasurecode_postprocessing.c
index 9ec68c7..435325e 100644
--- a/src/erasurecode_postprocessing.c
+++ b/src/erasurecode_postprocessing.c
@@ -41,7 +41,8 @@ void add_fragment_metadata(ec_backend_t be, char *fragment,
     set_fragment_payload_size(fragment, blocksize);
     set_backend_id(fragment, be->common.id);
     set_backend_version(fragment, be->common.ec_backend_version);
-    
+    set_fragment_adder_size(fragment, be->common.metadata_adder);
+
     if (add_chksum) {
         set_checksum(ct, fragment, blocksize);
     }
diff --git a/src/erasurecode_preprocessing.c b/src/erasurecode_preprocessing.c
index 62134ba..9666b9d 100644
--- a/src/erasurecode_preprocessing.c
+++ b/src/erasurecode_preprocessing.c
@@ -40,16 +40,17 @@ int prepare_fragments_for_encode(ec_backend_t instance,
     int i, ret = 0;
     int data_len;           /* data len to write to fragment headers */
     int aligned_data_len;   /* EC algorithm compatible data length */
-    int bsize = 0;
+    int buffer_size, payload_size = 0;
 
     /* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/
     data_len = orig_data_size;
     aligned_data_len = get_aligned_data_size(instance, orig_data_size);
-    *blocksize = bsize = (aligned_data_len / k);
+    *blocksize = payload_size = (aligned_data_len / k);
+    buffer_size = payload_size + instance->common.metadata_adder;
 
     for (i = 0; i < k; i++) {
-        int payload_size = data_len > bsize ? bsize : data_len;
-        char *fragment = (char *) alloc_fragment_buffer(bsize);
+        int copy_size = data_len > payload_size ? payload_size : data_len;
+        char *fragment = (char *) alloc_fragment_buffer(buffer_size);
         if (NULL == fragment) {
             ret = -ENOMEM;
             goto out_error;
@@ -59,15 +60,15 @@ int prepare_fragments_for_encode(ec_backend_t instance,
         encoded_data[i] = get_data_ptr_from_fragment(fragment);
       
         if (data_len > 0) {
-            memcpy(encoded_data[i], orig_data, payload_size);
+            memcpy(encoded_data[i], orig_data, copy_size);
         }
 
-        orig_data += payload_size;
-        data_len -= payload_size;
+        orig_data += copy_size;
+        data_len -= copy_size;
     }
 
     for (i = 0; i < m; i++) {
-        char *fragment = (char *) alloc_fragment_buffer(bsize);
+        char *fragment = (char *) alloc_fragment_buffer(buffer_size);
         if (NULL == fragment) {
             ret = -ENOMEM;
             goto out_error;
@@ -245,8 +246,9 @@ int get_fragment_partition(
             num_missing++;
         }
     }
-
-    return (num_missing > m) ? 1 : 0;
+    // TODO: In general, it is possible to reconstruct one or more fragments
+    // when more than m fragments are missing (e.g. flat XOR codes)
+    return (num_missing > m) ? -1 : 0;
 }
 
 int fragments_to_string(int k, int m,
diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c
index 18a008f..7fb9557 100644
--- a/test/liberasurecode_test.c
+++ b/test/liberasurecode_test.c
@@ -30,6 +30,7 @@
 #include <stdbool.h>
 #include "erasurecode.h"
 #include "erasurecode_helpers.h"
+#include "erasurecode_preprocessing.h"
 #include "erasurecode_backend.h"
 #include "alg_sig.h"
 #define NULL_BACKEND "null"
@@ -212,6 +213,12 @@ out:
     return num_frags;
 }
 
+static int encode_failure_stub(void *desc, char **data,
+                               char **parity, int blocksize)
+{
+    return -1;
+}
+
 static void validate_fragment_checksum(struct ec_args *args,
     fragment_metadata_t *metadata, char *fragment_data)
 {
@@ -279,6 +286,8 @@ static void test_encode_invalid_args()
     char *orig_data = create_buffer(orig_data_size, 'x');
     char **encoded_data = NULL, **encoded_parity = NULL;
     uint64_t encoded_fragment_len = 0;
+    ec_backend_t instance = NULL;
+    int (*orig_encode_func)(void *, char **, char **, int);
 
     assert(orig_data != NULL);
     rc = liberasurecode_encode(desc, orig_data, orig_data_size,
@@ -307,6 +316,15 @@ static void test_encode_invalid_args()
     rc = liberasurecode_encode(desc, orig_data, orig_data_size,
             &encoded_data, &encoded_parity, NULL);
     assert(rc < 0);
+
+    instance = liberasurecode_backend_instance_get_by_desc(desc);
+    orig_encode_func = instance->common.ops->encode;
+    instance->common.ops->encode = encode_failure_stub;
+    rc = liberasurecode_encode(desc, orig_data, orig_data_size,
+            &encoded_data, &encoded_parity, &encoded_fragment_len);
+    assert(rc < 0);
+    instance->common.ops->encode = orig_encode_func;
+
     free(orig_data);
 }
 
@@ -501,6 +519,61 @@ static void test_verify_stripe_metadata_invalid_args() {
 
 }
 
+static void test_get_fragment_partition()
+{
+    int i;
+    int rc = 0;
+    int desc = -1;
+    int orig_data_size = 1024 * 1024;
+    char *orig_data = create_buffer(orig_data_size, 'x');
+    char **encoded_data = NULL, **encoded_parity = NULL;
+    uint64_t encoded_fragment_len = 0;
+    int num_avail_frags = -1;
+    char **avail_frags = NULL;
+    int *skips = create_skips_array(&null_args, -1);
+    int *missing;
+
+    desc = liberasurecode_instance_create(EC_BACKEND_NULL, &null_args);
+    assert(desc > 0);
+    rc = liberasurecode_encode(desc, orig_data, orig_data_size,
+            &encoded_data, &encoded_parity, &encoded_fragment_len);
+    assert(0 == rc);
+
+    missing = alloc_and_set_buffer(sizeof(char*) * null_args.k, -1);
+
+    for(i = 0; i < null_args.m; i++) skips[i] = 1;
+    num_avail_frags = create_frags_array(&avail_frags, encoded_data,
+                                         encoded_parity, &null_args, skips);
+
+    rc = get_fragment_partition(null_args.k, null_args.m, avail_frags, num_avail_frags,
+                                encoded_data, encoded_parity, missing);
+    assert(0 == rc);
+
+    for(i = 0; i < null_args.m; i++) assert(missing[i] == i);
+    assert(missing[++i] == -1);
+
+    free(missing);
+
+    for(i = 0; i < null_args.m + 1; i++) skips[i] = 1;
+    num_avail_frags = create_frags_array(&avail_frags, encoded_data,
+                                         encoded_parity, &null_args, skips);
+
+    missing = alloc_and_set_buffer(sizeof(char*) * null_args.k, -1);
+    rc = get_fragment_partition(null_args.k, null_args.m, avail_frags, num_avail_frags,
+                                encoded_data, encoded_parity, missing);
+
+    for(i = 0; i < null_args.m + 1; i++) assert(missing[i] == i);
+    assert(missing[++i] == -1);
+
+    assert(rc < 0);
+
+    free(missing);
+    free(skips);
+    liberasurecode_encode_cleanup(desc, encoded_data, encoded_parity);
+    free(avail_frags);
+    free(orig_data);
+}
+
 static void encode_decode_test_impl(const ec_backend_id_t be_id,
                                    struct ec_args *args,
                                    int *skip)
@@ -519,8 +592,11 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
     int num_avail_frags = 0;
     char *orig_data_ptr = NULL;
     int remaining = 0;
+    ec_backend_t be = NULL;
 
     desc = liberasurecode_instance_create(be_id, args);
+    be = liberasurecode_backend_instance_get_by_desc(desc);
+
     if (-EBACKENDNOTAVAIL == desc) {
         fprintf (stderr, "Backend library not available!\n");
         return;
@@ -541,11 +617,14 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
         assert(header != NULL);
         fragment_metadata_t metadata = header->meta;
         assert(metadata.idx == i);
-        assert(metadata.size == encoded_fragment_len - frag_header_size);
+        assert(metadata.size == encoded_fragment_len - frag_header_size - be->common.metadata_adder);
         assert(metadata.orig_data_size == orig_data_size);
         char *data_ptr = frag + frag_header_size;
         int cmp_size = remaining >= metadata.size ? metadata.size : remaining;
-        assert(memcmp(data_ptr, orig_data_ptr, cmp_size) == 0);
+        // shss doesn't keep original data on data fragments
+        if (be_id != 5) {
+            assert(memcmp(data_ptr, orig_data_ptr, cmp_size) == 0);
+        }
         remaining -= cmp_size;
         orig_data_ptr += metadata.size;
     }
@@ -553,7 +632,6 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
     num_avail_frags = create_frags_array(&avail_frags, encoded_data,
                                          encoded_parity, args, skip);
     assert(num_avail_frags != -1);
-
     rc = liberasurecode_decode(desc, avail_frags, num_avail_frags,
                                encoded_fragment_len, 1,
                                &decoded_data, &decoded_data_len);
@@ -1109,6 +1187,10 @@ struct testcase testcases[] = {
         test_fragments_needed_invalid_args,
         EC_BACKENDS_MAX, CHKSUM_TYPES_MAX,
         .skip = false},
+    {"test_get_fragment_partition",
+        test_get_fragment_partition,
+        EC_BACKENDS_MAX, CHKSUM_TYPES_MAX,
+        .skip = false},
     // NULL backend test
     {"create_and_destroy_backend",
         test_create_and_destroy_backend,