diff options
author | Jenkins <jenkins@review.openstack.org> | 2017-03-10 06:03:56 +0000 |
---|---|---|
committer | Gerrit Code Review <review@openstack.org> | 2017-03-10 06:03:56 +0000 |
commit | b52657606572740359ead8b6892d744a05e9dc2c (patch) | |
tree | e8bd8f9e8d009235cb997af695e940decd0bab08 | |
parent | 6a5ec37f151f63fc6c7f2f84908bd04cdf9ff164 (diff) | |
parent | 69cb73f9d1a1066325b5c7ed9b2c9a2b16d0343b (diff) | |
download | pyeclib-b52657606572740359ead8b6892d744a05e9dc2c.tar.gz |
Merge "Add Phazr.IO libphazr backend support"
-rw-r--r-- | README.rst | 5 | ||||
-rw-r--r-- | pyeclib/ec_iface.py | 4 | ||||
-rw-r--r-- | src/c/pyeclib_c/pyeclib_c.c | 3 | ||||
-rw-r--r-- | test/test_pyeclib_api.py | 28 | ||||
-rw-r--r-- | test/test_pyeclib_c.py | 31 |
5 files changed, 66 insertions, 5 deletions
@@ -51,6 +51,7 @@ Supported ``ec_type`` values: * ``isa_l_rs_vand`` => Intel Storage Acceleration Library (ISA-L) - SIMD accelerated Erasure Coding backends [4] * ``isa_l_rs_cauchy`` => Cauchy Reed-Solomon encoding (ISA-L variant) [4] * ``shss`` => NTT Lab Japan's Erasure Coding Library [5] + * ``libphazr`` => Phazr.IO's erasure code library with built-in privacy [6] The Python API supports the following functions: @@ -240,7 +241,7 @@ Quick Start An example for ubuntu to install dependency packages:: $ sudo apt-get install build-essential python-dev python-pip liberasurecode-dev - $ sudo pip install -U bindep -r test-requirement.txt + $ sudo pip install -U bindep -r test-requirements.txt If you want to confirm all dependency packages installed successfully, try:: @@ -280,3 +281,5 @@ References [4] Intel(R) Storage Acceleration Library (Open Source Version), https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version [5] Kota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>, "NTT SHSS Erasure Coding backend" + + [6] Jim Cheung <support@phazr.io>, "Phazr.IO libphazr erasure code backend with built-in privacy" diff --git a/pyeclib/ec_iface.py b/pyeclib/ec_iface.py index 20d6337..59bf1fb 100644 --- a/pyeclib/ec_iface.py +++ b/pyeclib/ec_iface.py @@ -124,6 +124,7 @@ class PyECLib_EC_Types(PyECLibEnum): shss = 5 liberasurecode_rs_vand = 6 isa_l_rs_cauchy = 7 + libphazr = 8 # Output of Erasure (en)Coding process are data "fragments". Fragment data @@ -175,6 +176,8 @@ class ECDriver(object): elif value == "flat_xor_hd_4": self.hd = 4 value = "flat_xor_hd" + elif value == "libphazr": + self.hd = 1 if PyECLib_EC_Types.has_enum(value): self.ec_type = PyECLib_EC_Types.get_by_name(value) else: @@ -521,6 +524,7 @@ ALL_EC_TYPES = [ 'shss', 'liberasurecode_rs_vand', 'isa_l_rs_cauchy', + 'libphazr', ] diff --git a/src/c/pyeclib_c/pyeclib_c.c b/src/c/pyeclib_c/pyeclib_c.c index 8d7cd06..5d709f3 100644 --- a/src/c/pyeclib_c/pyeclib_c.c +++ b/src/c/pyeclib_c/pyeclib_c.c @@ -971,6 +971,9 @@ static const char* backend_id_to_str(uint8_t backend_id) case 7: backend_id_str = "isa_l_rs_cauchy\0"; break; + case 8: + backend_id_str = "libphazr\0"; + break; default: backend_id_str = "unknown\0"; } diff --git a/test/test_pyeclib_api.py b/test/test_pyeclib_api.py index b3026a0..9585035 100644 --- a/test/test_pyeclib_api.py +++ b/test/test_pyeclib_api.py @@ -134,7 +134,7 @@ class TestPyECLibDriver(unittest.TestCase): "Invalid Argument: m is required") with self.assertRaises(ECDriverError) as err_context: - # m is smaller than 1 + # k is smaller than 1 ECDriver(ec_type=ec_type, k=-100, m=1) self.assertEqual(str(err_context.exception), "Invalid number of data fragments (k)") @@ -151,10 +151,15 @@ class TestPyECLibDriver(unittest.TestCase): for _type in ALL_EC_TYPES: try: if _type is 'shss': + _k = 10 + _m = 4 + elif _type is 'libphazr': + _k = 4 _m = 4 else: + _k = 10 _m = 5 - ECDriver(k=10, m=_m, ec_type=_type, validate=True) + ECDriver(k=_k, m=_m, ec_type=_type, validate=True) available_ec_types.append(_type) except Exception: # ignore any errors, assume backend not available @@ -171,10 +176,12 @@ class TestPyECLibDriver(unittest.TestCase): try: if _type is 'shss': _instance = ECDriver(k=10, m=4, ec_type=_type) + elif _type is 'libphazr': + _instance = ECDriver(k=4, m=4, ec_type=_type) else: _instance = ECDriver(k=10, m=5, ec_type=_type) except ECDriverError: - self.fail("%p: %s algorithm not supported" % _instance, _type) + self.fail("%s algorithm not supported" % _type) self.assertRaises(ECBackendNotSupported, ECDriver, k=10, m=5, ec_type="invalid_algo") @@ -244,6 +251,11 @@ class TestPyECLibDriver(unittest.TestCase): chksum_type=csum)) pyeclib_drivers.append(ECDriver(k=11, m=7, ec_type=_type6, chksum_type=csum)) + + _type7 = 'libphazr' + if _type7 in VALID_EC_TYPES: + pyeclib_drivers.append(ECDriver(k=4, m=4, ec_type=_type7, + chksum_type=csum)) return pyeclib_drivers def test_small_encode(self): @@ -740,7 +752,15 @@ class TestBackendsEnabled(unittest.TestCase): def dummy(self, ec_type=ec_type): if ec_type not in VALID_EC_TYPES: raise unittest.SkipTest - k, m = 10, 4 if ec_type == 'shss' else 5 + if ec_type == 'shss': + k = 10 + m = 4 + elif ec_type == 'libphazr': + k = 4 + m = 4 + else: + k = 10 + m = 5 ECDriver(k=k, m=m, ec_type=ec_type) dummy.__name__ = 'test_%s_available' % ec_type cls_dict[dummy.__name__] = dummy diff --git a/test/test_pyeclib_c.py b/test/test_pyeclib_c.py index 0395e21..f505a54 100644 --- a/test/test_pyeclib_c.py +++ b/test/test_pyeclib_c.py @@ -78,6 +78,7 @@ class TestPyECLib(unittest.TestCase): (PyECLib_EC_Types.shss, 10, 4), (PyECLib_EC_Types.shss, 20, 4), (PyECLib_EC_Types.shss, 11, 7)] + self.libphazr = [(PyECLib_EC_Types.libphazr, 4, 4)] # Input temp files for testing self.sizes = ["101-K", "202-K", "303-K"] @@ -357,6 +358,36 @@ class TestPyECLib(unittest.TestCase): print("Reconstruct (%s): %s" % (size_str, self.get_throughput(avg_time, size_str))) + @require_backend("libphazr") + def test_libphazr(self): + for (ec_type, k, m) in self.libphazr: + print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m))) + + success = self._test_get_required_fragments(k, m, ec_type) + self.assertTrue(success) + + for size_str in self.sizes: + avg_time = self.time_encode(k, m, ec_type.value, 0, + size_str, + self.iterations) + print("Encode (%s): %s" % + (size_str, self.get_throughput(avg_time, size_str))) + + for size_str in self.sizes: + success, avg_time = self.time_decode(k, m, ec_type.value, 0, + size_str, + self.iterations) + self.assertTrue(success) + print("Decode (%s): %s" % + (size_str, self.get_throughput(avg_time, size_str))) + + for size_str in self.sizes: + success, avg_time = self.time_reconstruct( + k, m, ec_type.value, 0, size_str, self.iterations) + self.assertTrue(success) + print("Reconstruct (%s): %s" % + (size_str, self.get_throughput(avg_time, size_str))) + def _test_get_required_fragments(self, num_data, num_parity, ec_type): """ :return boolean, True if all tests passed |