Merge branch 'newapi' of https://bitbucket.org/tsg-/liberasurecode into newapi

Conflicts:
	test/liberasurecode_test.c

3 files changed, 129 insertions, 93 deletions

diff --git a/include/erasurecode/erasurecode.h b/include/erasurecode/erasurecode.h
index 1b26b73..9db76da 100644
--- a/include/erasurecode/erasurecode.h
+++ b/include/erasurecode/erasurecode.h
@@ -117,7 +117,7 @@ struct ec_args {
  *
  * @param num_backends - pointer to int, size of list returned
  *
- * @returns list of EC backends implemented
+ * @return list of EC backends implemented
  */
 const char ** liberasurecode_supported_backends(int *num_backends);
 
@@ -127,7 +127,7 @@ const char ** liberasurecode_supported_backends(int *num_backends);
  *
  * @param num_checksum_types - pointer to int, size of list returned
  *
- * @returns list of checksum types supported for fragment data
+ * @return list of checksum types supported for fragment data
  */
 const char ** liberasurecode_supported_checksum_types(int *num_checksum_types);
 
@@ -148,7 +148,7 @@ const char ** liberasurecode_supported_checksum_types(int *num_checksum_types);
  *          null_args - arguments for the null backend
  *          flat_xor_hd, jerasure do not require any special args
  *      
- * @returns liberasurecode instance descriptor (int > 0)
+ * @return liberasurecode instance descriptor (int > 0)
  */
 int liberasurecode_instance_create(const char *backend_name,
                                    struct ec_args *args);
@@ -156,7 +156,9 @@ int liberasurecode_instance_create(const char *backend_name,
 /**
  * Close a liberasurecode instance
  *
- * @param liberasurecode descriptor to close
+ * @param desc - liberasurecode descriptor to close
+ *
+ * @return 0 on success, otherwise non-zero error code
  */
 int liberasurecode_instance_destroy(int desc);
 
@@ -195,6 +197,7 @@ int liberasurecode_encode(int desc,
  *        fragments (char *), allocated by liberasurecode_encode
  * @param encoded_parity - (char **) array of m parity
  *        fragments (char *), allocated by liberasurecode_encode
+ *
  * @return 0 in success; -error otherwise
  */
 int liberasurecode_encode_cleanup(int desc, char **encoded_data, char **encoded_parity);
@@ -229,16 +232,16 @@ int liberasurecode_decode(int desc,
  *
  * @param desc - liberasurecode descriptor/handle
  *        from liberasurecode_instance_create()
- * @param data - (char *) buffer of data decoded by
- *        librasurecode_decode
- * @return 0 in success; -error otherwise
+ * @param data - (char *) buffer of data decoded by librasurecode_decode
+ *
+ * @return 0 on success; -error otherwise
  */
 int liberasurecode_decode_cleanup(int desc, char *data);
 
 /**
  * Reconstruct a missing fragment from a subset of available fragments
  *
- * @param desc - liberasurecode descriptor/handle
+ * @param desc - liberasurecode descriptor/handle 
  *        from liberasurecode_instance_create()
  * @param available_fragments - erasure encoded fragments
  * @param num_fragments - number of fragments being passed in
@@ -255,12 +258,23 @@ int liberasurecode_reconstruct_fragment(int desc,
         char* out_fragment);                            /* output */
 
 /**
- * Determine which fragments are needed to reconstruct some subset
- * of missing fragments.  Returns a list of lists (as bitmaps)
- * of fragments required to reconstruct missing indexes.
+ * Return a list of lists with valid rebuild indexes given
+ * a list of missing indexes.
+ *
+ * @desc: liberasurecode instance descriptor (obtained with
+ *        liberasurecode_instance_create)
+ * @fragments_to_reconstruct list of indexes to reconstruct
+ * @fragments_to_exclude list of indexes to exclude from 
+ *        reconstruction equation
+ * @fragments_needed list of fragments needed to reconstruct
+ *        fragments in fragments_to_reconstruct
+ *
+ * @return 0 on success, non-zero on error
  */
 int liberasurecode_fragments_needed(int desc,
-        int *missing_idxs, int *fragments_needed);
+        int *fragments_to_reconstruct, 
+        int *fragments_to_exclude,
+        int *fragments_needed);
 
 
 /* ==~=*=~==~=*=~== liberasurecode fragment metadata routines ==~*==~=*=~==~ */
@@ -288,9 +302,10 @@ fragment_metadata
  * @param desc - liberasurecode descriptor/handle
  *        from liberasurecode_instance_create()
  * @param fragment - fragment data pointer
+ * @param fragment_metadata - pointer to allocated buffer of size at least
+ *        sizeof(struct fragment_metadata) to hold fragment metadata struct
  *
- * @param fragment_metadata - pointer to output fragment metadata struct
- *          (reference passed by the user)
+ * @return 0 on success, non-zero on error
  */
 int liberasurecode_get_fragment_metadata(int desc,
         char *fragment, fragment_metadata_t *fragment_metadata);
@@ -301,10 +316,9 @@ int liberasurecode_get_fragment_metadata(int desc,
  * @param desc - liberasurecode descriptor/handle
  *        from liberasurecode_instance_create()
  * @param fragments - fragments part of the EC stripe to verify
- * @num_fragments - number of fragments part of the EC stripe
+ * @param num_fragments - number of fragments part of the EC stripe
  *
- * @ returns 1 if stripe checksum verification is successful
- *           0 otherwise
+ * @return 1 if stripe checksum verification is successful, 0 otherwise
  */
 int liberasurecode_verify_stripe_metadata(int desc,
         char **fragments, int num_fragments);
@@ -317,12 +331,23 @@ int liberasurecode_verify_stripe_metadata(int desc,
  * to be algined with the word size (w) and the last fragment also
  * needs to be aligned.  This computes the sum of the algined fragment
  * sizes for a given buffer to encode.
+ *
+ * @param desc - liberasurecode descriptor/handle
+ *        from liberasurecode_instance_create()
+ * @param data_len - original data length in bytes
+ *
+ * @return aligned length, or -error code on error
  */
 int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len);
  
 /**
  * This will return the minimum encode size, which is the minimum
  * buffer size that can be encoded.
+ * 
+ * @param desc - liberasurecode descriptor/handle
+ *        from liberasurecode_instance_create()
+ *
+ * @return minimum data length length, or -error code on error
  */
 int liberasurecode_get_minimum_encode_size(int desc);
 
diff --git a/src/erasurecode.c b/src/erasurecode.c
index 2a5bd9b..c9bf0ad 100644
--- a/src/erasurecode.c
+++ b/src/erasurecode.c
@@ -93,7 +93,7 @@ int next_backend_desc = 0;
 /**
  * Look up a backend instance by descriptor
  *
- * Returns pointer to a registered liberasurecode instance
+ * @returns pointer to a registered liberasurecode instance
  * The caller must hold active_instances_rwlock
  */
 ec_backend_t liberasurecode_backend_instance_get_by_desc(int desc)
@@ -192,7 +192,8 @@ int liberasurecode_backend_open(ec_backend_t instance)
         return 0;
 
     /* Use RTLD_LOCAL to avoid symbol collisions */
-    instance->desc.backend_sohandle = dlopen(instance->common.soname, RTLD_LAZY | RTLD_LOCAL);
+    instance->desc.backend_sohandle = dlopen(instance->common.soname,
+                                             RTLD_LAZY | RTLD_LOCAL);
     if (NULL == instance->desc.backend_sohandle) {
         print_dlerror(__func__);
         return -EBACKENDNOTAVAIL;
@@ -322,8 +323,8 @@ int liberasurecode_instance_create(const char *backend_name,
     }
 
     /* Call private init() for the backend */
-    instance->desc.backend_desc = instance->common.ops->init(&instance->args,
-            instance->desc.backend_sohandle);
+    instance->desc.backend_desc = instance->common.ops->init(
+            &instance->args, instance->desc.backend_sohandle);
     if (NULL == instance->desc.backend_desc) {
         free (instance);
         return -EBACKENDINITERR;
@@ -342,10 +343,12 @@ int liberasurecode_instance_create(const char *backend_name,
  */
 int liberasurecode_instance_destroy(int desc)
 {
-    ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
+    ec_backend_t instance = NULL;  /* instance to destroy */
+    int rc = 0;                    /* return code */
 
+    instance = liberasurecode_backend_instance_get_by_desc(desc);
     if (NULL == instance)
-        return 0;
+        return EBACKENDNOTAVAIL;
 
     /* Call private exit() for the backend */
     instance->common.ops->exit(instance->desc.backend_desc);
@@ -354,12 +357,12 @@ int liberasurecode_instance_destroy(int desc)
     liberasurecode_backend_close(instance);
 
     /* Remove instace from registry */
-    liberasurecode_backend_instance_unregister(instance);
-
-    /* Cleanup */
-    free(instance);
+    rc = liberasurecode_backend_instance_unregister(instance);
+    if (rc == 0) {
+        free(instance);
+    }
 
-    return 0;
+    return rc;
 }
 
 /**
@@ -377,7 +380,9 @@ int liberasurecode_instance_destroy(int desc)
  *        fragments (char *), allocated by liberasurecode_encode
  * @return 0 in success; -error otherwise
  */
-int liberasurecode_encode_cleanup(int desc, char **encoded_data, char **encoded_parity)
+int liberasurecode_encode_cleanup(int desc,
+                                  char **encoded_data,
+                                  char **encoded_parity)
 {
     int i, k, m;
     ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
@@ -742,7 +747,8 @@ int liberasurecode_reconstruct_fragment(int desc,
      * Separate the fragments into data and parity.  Also determine which
      * pieces are missing.
      */
-    ret = get_fragment_partition(k, m, available_fragments, num_fragments, data, parity, missing_idxs);
+    ret = get_fragment_partition(k, m, available_fragments, num_fragments,
+                                 data, parity, missing_idxs);
 
     if (ret < 0) {
         log_error("Could not properly partition the fragments!");
@@ -750,12 +756,14 @@ int liberasurecode_reconstruct_fragment(int desc,
     }
 
     /*
-     * Preparing the fragments for reconstruction.  This will alloc aligned buffers when unaligned buffers
-     * were passed in available_fragments.  It passes back a bitmap telling us which buffers need to
-     * be freed by us (realloc_bm).
-     *
+     * Preparing the fragments for reconstruction.  This will alloc aligned
+     * buffers when unaligned buffers were passed in available_fragments.
+     * It passes back a bitmap telling us which buffers need to be freed by
+     * us (realloc_bm).
      */
-    ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs, &orig_data_size, &blocksize, fragment_len, &realloc_bm);
+    ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs,
+                                       &orig_data_size, &blocksize,
+                                       fragment_len, &realloc_bm);
     if (ret < 0) {
         log_error("Could not prepare fragments for reconstruction!");
         goto out;
@@ -785,7 +793,8 @@ int liberasurecode_reconstruct_fragment(int desc,
         fragment_ptr = parity[destination_idx - k];
     }
     init_fragment_header(fragment_ptr);
-    add_fragment_metadata(fragment_ptr, destination_idx, orig_data_size, blocksize);
+    add_fragment_metadata(fragment_ptr, destination_idx, orig_data_size,
+                          blocksize);
 
     /*
      * Copy the reconstructed fragment to the output buffer
@@ -825,12 +834,18 @@ out:
  *
  * @desc: liberasurecode instance descriptor (obtained with
  *        liberasurecode_instance_create)
- * @missing_idx_list: list of indexes of missing elements
+ * @fragments_to_reconstruct list of indexes to reconstruct
+ * @fragments_to_exclude list of indexes to exclude from 
+          reconstruction equation
+ * @fragments_needed list of fragments needed to reconstruct
+          fragments in fragments_to_reconstruct
  *
  * @return a list of lists (bitmaps) of indexes to rebuild data
  *        from (in 'fragments_needed')
  */
-int liberasurecode_fragments_needed(int desc, int *missing_idxs,
+int liberasurecode_fragments_needed(int desc,
+                                    int *fragments_to_reconstruct, 
+                                    int *fragments_to_exclude,
                                     int *fragments_needed)
 {
     int ret = 0;
@@ -843,10 +858,12 @@ int liberasurecode_fragments_needed(int desc, int *missing_idxs,
 
     /* FIXME preprocessing */
 
+    /* FIXME use fragments_to_exclude */
+
     /* call the backend fragments_needed function passing it desc instance */
     ret = instance->common.ops->fragments_needed(
-            instance->desc.backend_desc, missing_idxs, 
-            fragments_needed);
+            instance->desc.backend_desc,
+            fragments_to_reconstruct, fragments_needed);
 
 out_error:
     return ret;
diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c
index 0e656b0..e0eb6a1 100644
--- a/test/liberasurecode_test.c
+++ b/test/liberasurecode_test.c
@@ -55,13 +55,11 @@ int *create_skips_array(struct ec_args *args, int skip)
     int num = args->k + args->m;
     size_t array_size = sizeof(int) * num;
     int *buf = malloc(array_size);
-    if (buf == NULL)
-    {
+    if (buf == NULL) {
         return NULL;
     }
     memset(buf, 0, array_size);
-    if (skip >= 0 && skip < num)
-    {
+    if (skip >= 0 && skip < num) {
         buf[skip] = 1;
     }
     return buf;
@@ -82,8 +80,7 @@ static int create_frags_array(char ***array,
     //add data frags
     int i = 0;
     char **ptr = *array;
-    for (i = 0; i < args->k; i++)
-    {
+    for (i = 0; i < args->k; i++) {
         if (data[i] == NULL || skips[i] == 1)
         {
             continue;
@@ -92,10 +89,8 @@ static int create_frags_array(char ***array,
         num_frags++;
     }
     //add parity frags
-    for (i = 0; i < args->m; i++)
-    {
-        if (parity[i] == NULL || skips[i + args->k] == 1)
-        {
+    for (i = 0; i < args->m; i++) {
+        if (parity[i] == NULL || skips[i + args->k] == 1) {
             continue;
         }
         *ptr++ = parity[i]; 
@@ -111,8 +106,9 @@ static void test_liberasurecode_supported_backends()
     const char **supported_ec_backends =
         liberasurecode_supported_backends(&num_backends);
 
-    for (i = 0; i < num_backends; i++)
+    for (i = 0; i < num_backends; i++) {
         printf("%s\n", supported_ec_backends[i]);
+    }
 }
 
 static void test_liberasurecode_supported_checksum_types()
@@ -123,8 +119,9 @@ static void test_liberasurecode_supported_checksum_types()
         liberasurecode_supported_checksum_types(&num_checksum_types);
 
     assert(num_checksum_types == CHKSUM_TYPES_MAX);
-    for (i = 0; i < CHKSUM_TYPES_MAX; i++)
+    for (i = 0; i < CHKSUM_TYPES_MAX; i++) {
         printf("%s\n", supported_checksum_types[i]);
+    }
 }
 
 static void test_create_and_destroy_backend(
@@ -137,7 +134,7 @@ static void test_create_and_destroy_backend(
         return;
     }
     assert(desc > 0);
-    liberasurecode_instance_destroy(desc);
+    assert(0 == liberasurecode_instance_destroy(desc));
 }
 
 static void encode_decode_test_impl(const char *backend,
@@ -155,6 +152,8 @@ static void encode_decode_test_impl(const char *backend,
     uint64_t decoded_data_len = 0;
     char *decoded_data = NULL;
     size_t frag_header_size =  sizeof(fragment_header_t);
+    char **avail_frags = NULL;
+    int num_avail_frags = 0;
         
     desc = liberasurecode_instance_create(backend, args);
     if (-EBACKENDNOTAVAIL == desc) {
@@ -186,25 +185,25 @@ static void encode_decode_test_impl(const char *backend,
         orig_data_ptr += metadata.size;
     }
 
-    char **avail_frags = NULL;
-    int num_avail_frags = create_frags_array(&avail_frags, encoded_data,
-                                             encoded_parity, args, skip);
+    num_avail_frags = create_frags_array(&avail_frags, encoded_data,
+                                         encoded_parity, args, skip);
     assert(num_avail_frags != -1);
 
-    rc = liberasurecode_decode(desc, avail_frags,
-            num_avail_frags, encoded_fragment_len, &decoded_data, &decoded_data_len);
+    rc = liberasurecode_decode(desc, avail_frags, num_avail_frags,
+                               encoded_fragment_len, &decoded_data,
+                               &decoded_data_len);
     assert(0 == rc);
     assert(decoded_data_len == orig_data_size);
     assert(memcmp(decoded_data, orig_data, orig_data_size) == 0);
 
-    if (desc)
-        liberasurecode_instance_destroy(desc);
+    if (desc) {
+        assert(0 == liberasurecode_instance_destroy(desc));
+    }
+
     free(orig_data);
-    if (avail_frags != NULL)
-    {
+    if (avail_frags != NULL) {
         int idx = 0;
-        for (idx = 0; idx < num_avail_frags; idx++) 
-        {
+        for (idx = 0; idx < num_avail_frags; idx++) {
             free(avail_frags[idx]);
         }
         free(avail_frags);
@@ -225,6 +224,12 @@ static void reconstruct_test_impl(const char *backend,
     uint64_t decoded_data_len = 0;
     char *decoded_data = NULL;
 
+    char **avail_frags = NULL;
+    int num_avail_frags = 0;
+
+    int i = 0;
+    char *out = NULL;
+
     desc = liberasurecode_instance_create(backend, args);
     if (-EBACKENDNOTAVAIL == desc) {
         fprintf (stderr, "Backend library not available!\n");
@@ -237,24 +242,19 @@ static void reconstruct_test_impl(const char *backend,
     rc = liberasurecode_encode(desc, orig_data, orig_data_size,
             &encoded_data, &encoded_parity, &encoded_fragment_len);
     assert(rc == 0);
-    char **avail_frags = NULL;
-    int num_avail_frags = create_frags_array(&avail_frags, encoded_data,
+    num_avail_frags = create_frags_array(&avail_frags, encoded_data,
                                              encoded_parity, args, skip);
-    int i = 0;
-    char *out = malloc(encoded_fragment_len);
+    out = malloc(encoded_fragment_len);
     assert(out != NULL);
     for (i = 0; i < num_fragments; i++) {
-        if (skip[i] == 0)
-        {
+        if (skip[i] == 0) {
             continue;
         }
         char *cmp = NULL;
-        if (i < args->k)
-        {
+        if (i < args->k) {
             cmp = encoded_data[i];
         }
-        else
-        {
+        else {
             cmp = encoded_parity[i - args->k];
         }
         memset(out, 0, encoded_fragment_len);
@@ -299,12 +299,10 @@ static void test_decode_with_missing_multi_data(const char *backend,
 {
     int max_num_missing = args->hd;
     int i,j;
-    for (i = 0; i < args->k - max_num_missing + 1; i++)
-    {
+    for (i = 0; i < args->k - max_num_missing + 1; i++) {
         int *skip = create_skips_array(args,-1);
         assert(skip != NULL);
-        for (j = i; j < i + max_num_missing; j++)
-        {
+        for (j = i; j < i + max_num_missing; j++) {
             skip[j]=1;
         }
         encode_decode_test_impl(backend, args, skip);
@@ -317,12 +315,10 @@ static void test_decode_with_missing_multi_parity(const char *backend,
 {
     int i,j;
     int max_num_missing = args->hd;
-    for (i = args->k; i < args->k + args->m - max_num_missing + 1; i++)
-    {
+    for (i = args->k; i < args->k + args->m - max_num_missing + 1; i++) {
         int *skip = create_skips_array(args,-1);
         assert(skip != NULL);
-        for (j = i; j < i + max_num_missing; j++)
-        {
+        for (j = i; j < i + max_num_missing; j++) {
             skip[j]=1;
         }
         encode_decode_test_impl(backend, args, skip);
@@ -336,12 +332,10 @@ static void test_decode_with_missing_multi_data_parity(const char *backend,
     int i,j;
     int max_num_missing = args->hd;
     int start = args->k - max_num_missing + 1;
-    for (i = start; i < start + max_num_missing -1; i++)
-    {
+    for (i = start; i < start + max_num_missing -1; i++) {
         int *skip = create_skips_array(args,-1);
         assert(skip != NULL);
-        for (j = i; j < i + max_num_missing; j++)
-        {
+        for (j = i; j < i + max_num_missing; j++) {
             skip[j]=1;
         }
         encode_decode_test_impl(backend, args, skip);
@@ -362,8 +356,7 @@ static void test_simple_reconstruct(const char *backend,
                                      struct ec_args *args)
 {
     int i = 0;
-    for (i = 0; i < args->k + args->m; i++)
-    {
+    for (i = 0; i < args->k + args->m; i++) {
         int *skip = create_skips_array(args,i);
         assert(skip != NULL);
         reconstruct_test_impl(backend, args, skip);
@@ -422,11 +415,12 @@ struct testcase testcases[] = {
         test_create_and_destroy_backend,
         "jerasure_rs_cauchy", &jerasure_rs_cauchy_args,
         .skip = false},
-    // Flat XOR tests
-    {"simple_encode_flat_xor_hd",
+    // NULL backend tests
+    {"simple_encode_null",
         test_simple_encode_decode,
         "null", &null_args,
-        .skip = true},
+        .skip = false},
+    // Flat XOR backend tests
     {"simple_encode_flat_xor_hd",
         test_simple_encode_decode,
         "flat_xor_hd", &flat_xor_hd_args,
@@ -455,7 +449,7 @@ struct testcase testcases[] = {
         test_simple_reconstruct,
         "flat_xor_hd", &flat_xor_hd_args,
         .skip = false},
-    // Jerasure RS Vand tests
+    // Jerasure RS Vand backend tests
     {"simple_encode_jerasure_rs_vand",
         test_simple_encode_decode,
         "jerasure_rs_vand", &jerasure_rs_vand_args,
@@ -480,7 +474,7 @@ struct testcase testcases[] = {
         test_simple_reconstruct,
         "jerasure_rs_vand", &jerasure_rs_vand_args,
         .skip = false},
-    // Jerasure RS Cauchy tests
+    // Jerasure RS Cauchy backend tests
     {"simple_encode_jerasureFlat XOR tests_rs_cauchy",
         test_simple_encode_decode,
         "jerasure_rs_cauchy", &jerasure_rs_cauchy_args,