From 5626cd5791bd35279594e067c10581e89ae66abc Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 1 Jul 2020 21:59:45 -0700 Subject: Be willing to write fragments with legacy crc ...if users *really* want to. They opt-in at run time by setting LIBERASURECODE_WRITE_LEGACY_CRC=1 in the environment; leaving it unset, set to an empty string, or set to the string "0" continues to write zlib crcs. UpgradeImpact ============= This option is intended to allow a smooth upgrade from liberasurecode 1.5.0 and earlier in a system with multiple readers and writers: * Before upgrade, ensure the environment variable is set on all nodes. This will be ignored by earlier versions. * Upgrade liberasurecode on each node in the system, restarting any services that use it. Every node continues writing CRCs that are still usable by nodes that have not yet upgraded. * Now that every node is capable of reading zlib CRCs, remove the environment variable from each node to start writing new CRCs. If you are already using 1.6.0 or later, just upgrade normally. Closes-Bug: #1886088 Closes-Bug: #1867937 Related-Bug: #1666320 Needed-By: https://review.opendev.org/#/c/739164/ Change-Id: I9adfbe631a2dddc592fd08f8a325f3e8331b92f1 --- src/erasurecode_helpers.c | 8 +++++++- src/erasurecode_postprocessing.c | 11 +++++++++-- test/liberasurecode_test.c | 14 +++++++++++++- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c index 4be2634..84b114c 100644 --- a/src/erasurecode_helpers.c +++ b/src/erasurecode_helpers.c @@ -463,6 +463,7 @@ inline int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize) { fragment_header_t* header = (fragment_header_t*) buf; char *data = get_data_ptr_from_fragment(buf); + char *flag; assert(NULL != header); if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) { @@ -475,7 +476,12 @@ inline int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize) switch(header->meta.chksum_type) { case CHKSUM_CRC32: - header->meta.chksum[0] = crc32(0, (unsigned char *) data, blocksize); + flag = getenv("LIBERASURECODE_WRITE_LEGACY_CRC"); + if (flag && !(flag[0] == '\0' || (flag[0] == '0' && flag[1] == '\0'))) { + header->meta.chksum[0] = liberasurecode_crc32_alt(0, data, blocksize); + } else { + header->meta.chksum[0] = crc32(0, (unsigned char *) data, blocksize); + } break; case CHKSUM_MD5: break; diff --git a/src/erasurecode_postprocessing.c b/src/erasurecode_postprocessing.c index 07c08f5..f665e60 100644 --- a/src/erasurecode_postprocessing.c +++ b/src/erasurecode_postprocessing.c @@ -32,6 +32,7 @@ #include "erasurecode_helpers_ext.h" #include "erasurecode_log.h" #include "erasurecode_stdinc.h" +#include "alg_sig.h" void add_fragment_metadata(ec_backend_t be, char *fragment, int idx, uint64_t orig_data_size, int blocksize, @@ -59,8 +60,14 @@ void add_fragment_metadata(ec_backend_t be, char *fragment, return; } - header->metadata_chksum = crc32(0, (unsigned char *) &header->meta, - sizeof(fragment_metadata_t)); + char *flag = getenv("LIBERASURECODE_WRITE_LEGACY_CRC"); + if (flag && !(flag[0] == '\0' || (flag[0] == '0' && flag[1] == '\0'))) { + header->metadata_chksum = liberasurecode_crc32_alt( + 0, &header->meta, sizeof(fragment_metadata_t)); + } else { + header->metadata_chksum = crc32(0, (unsigned char *) &header->meta, + sizeof(fragment_metadata_t)); + } } int finalize_fragments_after_encode(ec_backend_t instance, diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c index e60140f..714ee8a 100644 --- a/test/liberasurecode_test.c +++ b/test/liberasurecode_test.c @@ -475,7 +475,11 @@ static void validate_fragment_checksum(struct ec_args *args, assert(false); //currently only have support crc32 break; case CHKSUM_CRC32: - computed = crc32(0, (unsigned char *) fragment_data, size); + if (getenv("LIBERASURECODE_WRITE_LEGACY_CRC")) { + computed = liberasurecode_crc32_alt(0, fragment_data, size); + } else { + computed = crc32(0, (unsigned char *) fragment_data, size); + } break; case CHKSUM_NONE: assert(metadata->chksum_mismatch == 0); @@ -1341,6 +1345,13 @@ static void test_get_fragment_metadata(const ec_backend_id_t be_id, struct ec_ar free(orig_data); } +static void test_write_legacy_fragment_metadata(const ec_backend_id_t be_id, struct ec_args *args) +{ + setenv("LIBERASURECODE_WRITE_LEGACY_CRC", "1", 1); + test_get_fragment_metadata(be_id, args); + unsetenv("LIBERASURECODE_WRITE_LEGACY_CRC"); +} + static void test_decode_with_missing_data(const ec_backend_id_t be_id, struct ec_args *args) { @@ -1855,6 +1866,7 @@ static void test_metadata_crcs_be() TEST(test_fragments_needed, backend, CHKSUM_NONE), \ TEST(test_get_fragment_metadata, backend, CHKSUM_NONE), \ TEST(test_get_fragment_metadata, backend, CHKSUM_CRC32), \ + TEST(test_write_legacy_fragment_metadata, backend, CHKSUM_CRC32), \ TEST(test_verify_stripe_metadata, backend, CHKSUM_CRC32), \ TEST(test_verify_stripe_metadata_libec_mismatch, backend, CHKSUM_CRC32), \ TEST(test_verify_stripe_metadata_magic_mismatch, backend, CHKSUM_CRC32), \ -- cgit v1.2.1