add fragment validation

5 files changed, 316 insertions, 26 deletions

diff --git a/include/erasurecode/erasurecode.h b/include/erasurecode/erasurecode.h
index a8b1e43..e8ad056 100644
--- a/include/erasurecode/erasurecode.h
+++ b/include/erasurecode/erasurecode.h
@@ -266,10 +266,24 @@ fragment_metadata
  *
  * @return 0 on success, non-zero on error
  */
+//EDL: This needs to be implemented
 int liberasurecode_get_fragment_metadata(char *fragment,
         fragment_metadata_t *fragment_metadata);
 
 /**
+* Verify that the specified pointer points to a well formed fragment that can
+* be processed by both this instance of liberasurecode and the specified
+* backend.
+*
+* @param desc - liberasurecode descriptor/handle
+*        from liberasurecode_instance_create()
+* @param fragment - fragment to verify
+*
+* @return 0 if fragment validation is successful, 1 otherwise.
+*/
+int is_valid_fragment(int desc, char *fragment);
+
+/**
  * Verify a subset of fragments generated by encode()
  *
  * @param desc - liberasurecode descriptor/handle
diff --git a/include/erasurecode/erasurecode_helpers.h b/include/erasurecode/erasurecode_helpers.h
index 5bef344..93a8b6f 100644
--- a/include/erasurecode/erasurecode_helpers.h
+++ b/include/erasurecode/erasurecode_helpers.h
@@ -135,9 +135,8 @@ int set_fragment_payload_size(char *buf, int size);
 int get_fragment_payload_size(char *buf);
 int set_orig_data_size(char *buf, int orig_data_size);
 int get_orig_data_size(char *buf);
-int validate_fragment(char *buf);
 int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize);
-int get_checksum(char *buf);
+int get_checksum(char *buf); //TODO implement this
 int set_libec_version(char *fragment);
 int get_libec_version(char *fragment, uint32_t *ver);
 int set_backend_id(char *buf, ec_backend_id_t id);
diff --git a/src/erasurecode.c b/src/erasurecode.c
index 2071702..ad29673 100644
--- a/src/erasurecode.c
+++ b/src/erasurecode.c
@@ -33,7 +33,7 @@
 #include "erasurecode_preprocessing.h"
 #include "erasurecode_postprocessing.h"
 #include "erasurecode_stdinc.h"
-#include "erasurecode/alg_sig.h"
+#include "alg_sig.h"
 
 /* =~=*=~==~=*=~==~=*=~= Supported EC backends =~=*=~==~=*=~==~=*=~==~=*=~== */
 
@@ -951,22 +951,70 @@ out:
     return ret;
 }
 
-/**
- * Verify a subset of fragments generated by encode()
- *
- * @param desc - liberasurecode descriptor/handle
- *        from liberasurecode_instance_create()
- * @param fragments - fragments part of the EC stripe to verify
- * @num_fragments - number of fragments part of the EC stripe
- *
- * @ returns 0 if stripe checksum verification is successful
- *           -1 otherwise
- */
+int liberasurecode_verify_fragment_metadata(ec_backend_t be,
+                                            fragment_metadata_t *md)
+{
+    if (md->backend_id != be->common.id) {
+        return 1;
+    }
+    if (!be->common.ops->is_compatible_with(md->backend_version))  {
+        return 1;
+    }
+    return 0;
+}
+
+int is_valid_fragment(int desc, char *fragment)
+{
+    uint32_t ver = 0;
+    fragment_metadata_t fragment_metadata;
+    ec_backend_t be = liberasurecode_backend_instance_get_by_desc(desc);
+    if (!be) {
+        log_error("Unable to verify stripe metadata: invalid backend id %d.",
+                desc);
+        return 1;
+    }
+    if (!fragment) {
+        log_error("Unable to verify fragment validity: fragments missing.");
+        return 1;
+    }
+    if (get_libec_version(fragment, &ver) != 0 ||
+            ver != LIBERASURECODE_VERSION) {
+        return 1;
+    }
+    if (liberasurecode_get_fragment_metadata(fragment,
+            &fragment_metadata) != 0) {
+        return 1;
+    }
+    if (liberasurecode_verify_fragment_metadata(be,
+            &fragment_metadata) != 0) {
+        return 1;
+    }
+    return 0;
+}
+
 int liberasurecode_verify_stripe_metadata(int desc,
         char **fragments, int num_fragments)
 {
+    int i = 0;
+    if (!fragments) {
+        log_error("Unable to verify stripe metadata: fragments missing.");
+        return 1;
+    }
+    if (num_fragments <= 0) {
+        log_error("Unable to verify stripe metadata: "
+                "number of fragments must be greater than 0.");
+        return 1;
+    }
+
+    for (i = 0; i < num_fragments; i++) {
+        if (is_valid_fragment(desc, fragments[i]) == 1) {
+            return 1;
+        }
+    }
+
     return 0;
 }
+
 /* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */
 
 /**
diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c
index 9f07647..94aa308 100644
--- a/src/erasurecode_helpers.c
+++ b/src/erasurecode_helpers.c
@@ -32,11 +32,11 @@
 #include "erasurecode_helpers.h"
 #include "erasurecode_stdinc.h"
 #include "erasurecode_version.h"
-#include "erasurecode/alg_sig.h"
+#include "alg_sig.h"
 
 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
 
-static bool is_valid_fragment(char *buf)
+static bool is_fragment(char *buf)
 {
     fragment_header_t *header = (fragment_header_t *) buf;
 
@@ -343,7 +343,7 @@ int get_orig_data_size(char *buf)
 
 int set_libec_version(char *buf)
 {
-    if (!is_valid_fragment(buf)) {
+    if (!is_fragment(buf)) {
             return -1;
     }
     fragment_header_t *header = (fragment_header_t *) buf;
@@ -353,7 +353,7 @@ int set_libec_version(char *buf)
 
 int get_libec_version(char *buf, uint32_t *ver)
 {
-    if (!is_valid_fragment(buf)) {
+    if (!is_fragment(buf)) {
             return -1;
     }
     fragment_header_t *header = (fragment_header_t *) buf;
@@ -363,7 +363,7 @@ int get_libec_version(char *buf, uint32_t *ver)
 
 int set_backend_id(char *buf, ec_backend_id_t id)
 {
-    if (!is_valid_fragment(buf)) {
+    if (!is_fragment(buf)) {
             return -1;
     }
     fragment_header_t *header = (fragment_header_t *) buf;
@@ -373,7 +373,7 @@ int set_backend_id(char *buf, ec_backend_id_t id)
 
 int get_backend_id(char *buf, ec_backend_id_t *id) 
 {
-    if (!is_valid_fragment(buf)) {
+    if (!is_fragment(buf)) {
             return -1;
     }
     fragment_header_t *header = (fragment_header_t *) buf;
@@ -383,7 +383,7 @@ int get_backend_id(char *buf, ec_backend_id_t *id)
 
 int set_backend_version(char *buf, uint32_t version) 
 {
-    if (!is_valid_fragment(buf)) {
+    if (!is_fragment(buf)) {
             return -1;
     }
     fragment_header_t *header = (fragment_header_t *) buf;
@@ -393,7 +393,7 @@ int set_backend_version(char *buf, uint32_t version)
 
 int get_backend_version(char *buf, uint32_t *version) 
 {
-    if (!is_valid_fragment(buf)) {
+    if (!is_fragment(buf)) {
             return -1;
     }
     fragment_header_t *header = (fragment_header_t *) buf;
diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c
index c0d9199..61492d1 100644
--- a/test/liberasurecode_test.c
+++ b/test/liberasurecode_test.c
@@ -85,6 +85,13 @@ struct ec_args isa_l_args = {
     .hd = 5,
 };
 
+typedef enum {
+    LIBEC_VERSION_MISMATCH,
+    MAGIC_MISMATCH,
+    BACKEND_ID_MISMATCH,
+    BACKEND_VERSION_MISMATCH,
+} fragment_mismatch_scenario_t;
+
 char * get_name_from_backend_id(ec_backend_id_t be) {
     switch(be) {
         case EC_BACKEND_NULL:
@@ -463,16 +470,20 @@ static void test_verify_stripe_metadata_invalid_args() {
     char **frags = malloc(sizeof(char *) * num_frags);
 
     rc = liberasurecode_verify_stripe_metadata(desc, frags, num_frags);
-    assert(rc < 0);
+    assert(rc == 1);
 
     desc = liberasurecode_instance_create(EC_BACKEND_NULL, &null_args);
     assert(desc > 0);
 
     rc = liberasurecode_verify_stripe_metadata(desc, NULL, num_frags);
-    assert(rc < 0);
+    assert(rc == 1);
 
     rc = liberasurecode_verify_stripe_metadata(desc, frags, -1);
-    assert(rc < 0);
+    assert(rc == 1);
+
+    rc = liberasurecode_verify_stripe_metadata(desc, frags, 0);
+    assert(rc == 1);
+
 }
 
 static void encode_decode_test_impl(const ec_backend_id_t be_id,
@@ -905,6 +916,144 @@ static void test_fragments_needed(const ec_backend_id_t be_id,
     test_fragments_needed_impl(be_id, args);
 }
 
+static void test_verify_stripe_metadata(const ec_backend_id_t be_id, 
+                                        struct ec_args *args)
+{
+    int orig_data_size = 1024;
+    char **encoded_data = NULL, **encoded_parity = NULL;
+    char **avail_frags = NULL;
+    uint64_t encoded_fragment_len = 0;
+    int rc = -1;
+    int num_avail_frags = -1;
+    int *skip = create_skips_array(args,-1);
+    char *orig_data = create_buffer(orig_data_size, 'x');
+    int desc = liberasurecode_instance_create(be_id, args);
+
+    if (-EBACKENDNOTAVAIL == desc) {
+        fprintf (stderr, "Backend library not available!\n");
+        return;
+    }
+    assert(desc > 0);
+
+    assert(orig_data != NULL);
+    rc = liberasurecode_encode(desc, orig_data, orig_data_size,
+            &encoded_data, &encoded_parity, &encoded_fragment_len);
+    assert(0 == rc);
+
+    num_avail_frags = create_frags_array(&avail_frags, encoded_data,
+            encoded_parity, args, skip);
+
+    rc = liberasurecode_verify_stripe_metadata(desc, avail_frags,
+            num_avail_frags);
+    assert(0 == rc);
+
+    liberasurecode_encode_cleanup(desc, encoded_data, encoded_parity);
+    free(orig_data);
+    free(skip);
+}
+
+static void verify_stripe_metadata_mismatch_impl(const ec_backend_id_t be_id, struct ec_args *args,
+        fragment_mismatch_scenario_t scenario)
+{
+    int orig_data_size = 1024;
+    char **encoded_data = NULL, **encoded_parity = NULL;
+    char **avail_frags = NULL;
+    uint64_t encoded_fragment_len = 0;
+    int rc = -1;
+    int num_avail_frags = -1;
+    int i = 0;
+    uint32_t orig_libec_ver = 0;
+    uint32_t orig_be_ver = 0;
+    uint8_t orig_be_id = 0;
+    int *skip = create_skips_array(args,-1);
+    char *orig_data = create_buffer(orig_data_size, 'x');
+    int desc = liberasurecode_instance_create(be_id, args);
+
+    if (-EBACKENDNOTAVAIL == desc) {
+        fprintf (stderr, "Backend library not available!\n");
+        return;
+    }
+    assert(desc > 0);
+    assert(orig_data != NULL);
+    rc = liberasurecode_encode(desc, orig_data, orig_data_size,
+            &encoded_data, &encoded_parity, &encoded_fragment_len);
+    assert(0 == rc);
+    num_avail_frags = create_frags_array(&avail_frags, encoded_data,
+            encoded_parity, args, skip);
+    for (i = 0; i < num_avail_frags; i++) {
+        char * cur_frag = avail_frags[i];
+        //corrupt fragment
+        switch (scenario) {
+            case LIBEC_VERSION_MISMATCH:
+                orig_libec_ver = ((fragment_header_t*)cur_frag)->libec_version;
+                ((fragment_header_t*)cur_frag)->libec_version = orig_libec_ver + 1;
+                break;
+            case MAGIC_MISMATCH:
+                ((fragment_header_t*)cur_frag)->magic = 0;
+                break;
+            case BACKEND_ID_MISMATCH:
+                orig_be_id = ((fragment_header_t*)cur_frag)->meta.backend_id;
+                ((fragment_header_t*)cur_frag)->meta.backend_id = orig_be_id + 1;
+                break;
+            case BACKEND_VERSION_MISMATCH:
+                orig_be_ver = ((fragment_header_t*)cur_frag)->meta.backend_version;
+                ((fragment_header_t*)cur_frag)->meta.backend_version = orig_be_ver + 1;
+                break;
+            default:
+                assert(false);
+        }
+        rc = liberasurecode_verify_stripe_metadata(desc, avail_frags,
+                num_avail_frags);
+        assert(rc == 1);
+        //heal fragment
+        switch (scenario) {
+            case LIBEC_VERSION_MISMATCH:
+                ((fragment_header_t*)cur_frag)->libec_version = orig_libec_ver;
+                break;
+            case MAGIC_MISMATCH:
+                ((fragment_header_t*)cur_frag)->magic = LIBERASURECODE_FRAG_HEADER_MAGIC;
+                break;
+            case BACKEND_ID_MISMATCH:
+                ((fragment_header_t*)cur_frag)->meta.backend_id = orig_be_id;
+                break;
+            case BACKEND_VERSION_MISMATCH:
+                ((fragment_header_t*)cur_frag)->meta.backend_version = orig_be_ver;
+                break;
+            default:
+                break;
+        }
+    }
+    liberasurecode_encode_cleanup(desc, encoded_data, encoded_parity);
+    free(orig_data);
+    free(skip);
+}
+static void test_verify_stripe_metadata_libec_mismatch(
+        const ec_backend_id_t be_id, struct ec_args *args)
+{
+    verify_stripe_metadata_mismatch_impl(be_id, args, LIBEC_VERSION_MISMATCH);
+}
+
+static void test_verify_stripe_metadata_magic_mismatch(
+        const ec_backend_id_t be_id, struct ec_args *args)
+{
+    verify_stripe_metadata_mismatch_impl(be_id, args, MAGIC_MISMATCH);
+}
+
+static void test_verify_stripe_metadata_be_id_mismatch(
+        const ec_backend_id_t be_id, struct ec_args *args)
+{
+    verify_stripe_metadata_mismatch_impl(be_id, args, BACKEND_ID_MISMATCH);
+}
+
+static void test_verify_stripe_metadata_be_ver_mismatch(
+        const ec_backend_id_t be_id, struct ec_args *args)
+{
+    verify_stripe_metadata_mismatch_impl(be_id, args, BACKEND_VERSION_MISMATCH);
+}
+
+
+//static void test_verify_str
+
 struct testcase testcases[] = {
     {"test_create_backend_invalid_args",
         test_create_backend_invalid_args,
@@ -941,7 +1090,7 @@ struct testcase testcases[] = {
     {"test_verify_stripe_metadata_invalid_args",
         test_verify_stripe_metadata_invalid_args,
         EC_BACKENDS_MAX, CHKSUM_TYPES_MAX,
-        .skip = true}, //EDL, liberasurecode_verify_stripe_metadata is not implemented at the moment
+        .skip = false},
     {"test_fragments_needed_invalid_args",
         test_fragments_needed_invalid_args,
         EC_BACKENDS_MAX, CHKSUM_TYPES_MAX,
@@ -1004,6 +1153,26 @@ struct testcase testcases[] = {
         test_get_fragment_metadata,
         EC_BACKEND_FLAT_XOR_HD, CHKSUM_CRC32,
         .skip = false},
+    {"test_verify_stripe_metadata",
+        test_verify_stripe_metadata,
+        EC_BACKEND_FLAT_XOR_HD, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_libec_mismatch",
+        test_verify_stripe_metadata_libec_mismatch,
+        EC_BACKEND_FLAT_XOR_HD, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_magic_mismatch",
+        test_verify_stripe_metadata_magic_mismatch,
+        EC_BACKEND_FLAT_XOR_HD, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_be_id_mismatch",
+        test_verify_stripe_metadata_be_id_mismatch,
+        EC_BACKEND_FLAT_XOR_HD, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_be_ver_mismatch",
+        test_verify_stripe_metadata_be_ver_mismatch,
+        EC_BACKEND_FLAT_XOR_HD, CHKSUM_CRC32,
+        .skip = false},
     // Jerasure RS Vand backend tests
     {"create_and_destroy_backend",
         test_create_and_destroy_backend,
@@ -1045,6 +1214,26 @@ struct testcase testcases[] = {
         test_get_fragment_metadata,
         EC_BACKEND_JERASURE_RS_VAND, CHKSUM_CRC32,
         .skip = false},
+    {"test_verify_stripe_metadata",
+        test_verify_stripe_metadata,
+        EC_BACKEND_JERASURE_RS_VAND, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_libec_mismatch",
+         test_verify_stripe_metadata_libec_mismatch,
+         EC_BACKEND_JERASURE_RS_VAND, CHKSUM_CRC32,
+         .skip = false},
+    {"test_verify_stripe_metadata_magic_mismatch",
+          test_verify_stripe_metadata_magic_mismatch,
+          EC_BACKEND_JERASURE_RS_VAND, CHKSUM_CRC32,
+         .skip = false},
+    {"test_verify_stripe_metadata_be_id_mismatch",
+         test_verify_stripe_metadata_be_id_mismatch,
+         EC_BACKEND_JERASURE_RS_VAND, CHKSUM_CRC32,
+         .skip = false},
+    {"test_verify_stripe_metadata_be_ver_mismatch",
+        test_verify_stripe_metadata_be_ver_mismatch,
+        EC_BACKEND_JERASURE_RS_VAND, CHKSUM_CRC32,
+        .skip = false},
     // Jerasure RS Cauchy backend tests
     {"create_and_destroy_backend",
         test_create_and_destroy_backend,
@@ -1086,6 +1275,26 @@ struct testcase testcases[] = {
         test_get_fragment_metadata,
         EC_BACKEND_JERASURE_RS_CAUCHY, CHKSUM_CRC32,
         .skip = false},
+    {"test_verify_stripe_metadata",
+        test_verify_stripe_metadata,
+        EC_BACKEND_JERASURE_RS_CAUCHY, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_libec_mismatch",
+         test_verify_stripe_metadata_libec_mismatch,
+         EC_BACKEND_JERASURE_RS_CAUCHY, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_magic_mismatch",
+         test_verify_stripe_metadata_magic_mismatch,
+         EC_BACKEND_JERASURE_RS_CAUCHY, CHKSUM_CRC32,
+         .skip = false},
+    {"test_verify_stripe_metadata_be_id_mismatch",
+         test_verify_stripe_metadata_be_id_mismatch,
+         EC_BACKEND_JERASURE_RS_CAUCHY, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_be_ver_mismatch",
+        test_verify_stripe_metadata_be_ver_mismatch,
+        EC_BACKEND_JERASURE_RS_CAUCHY, CHKSUM_CRC32,
+        .skip = false},
     // ISA-L tests
     {"create_and_destroy_backend",
         test_create_and_destroy_backend,
@@ -1123,6 +1332,26 @@ struct testcase testcases[] = {
         test_get_fragment_metadata,
         EC_BACKEND_ISA_L_RS_VAND, CHKSUM_NONE,
         .skip = false},
+    {"test_verify_stripe_metadata",
+        test_verify_stripe_metadata,
+        EC_BACKEND_ISA_L_RS_VAND, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_libec_mismatch",
+        test_verify_stripe_metadata_libec_mismatch,
+        EC_BACKEND_ISA_L_RS_VAND, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_magic_mismatch",
+        test_verify_stripe_metadata_magic_mismatch,
+        EC_BACKEND_ISA_L_RS_VAND, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_be_id_mismatch",
+        test_verify_stripe_metadata_be_id_mismatch,
+        EC_BACKEND_ISA_L_RS_VAND, CHKSUM_CRC32,
+        .skip = false},
+    {"test_verify_stripe_metadata_be_ver_mismatch",
+        test_verify_stripe_metadata_be_ver_mismatch,
+        EC_BACKEND_ISA_L_RS_VAND, CHKSUM_CRC32,
+        .skip = false},
     { NULL, NULL, 0, 0, false },
 };