summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2017-03-10 06:03:56 +0000
committerGerrit Code Review <review@openstack.org>2017-03-10 06:03:56 +0000
commitb52657606572740359ead8b6892d744a05e9dc2c (patch)
treee8bd8f9e8d009235cb997af695e940decd0bab08
parent6a5ec37f151f63fc6c7f2f84908bd04cdf9ff164 (diff)
parent69cb73f9d1a1066325b5c7ed9b2c9a2b16d0343b (diff)
downloadpyeclib-b52657606572740359ead8b6892d744a05e9dc2c.tar.gz
Merge "Add Phazr.IO libphazr backend support"
-rw-r--r--README.rst5
-rw-r--r--pyeclib/ec_iface.py4
-rw-r--r--src/c/pyeclib_c/pyeclib_c.c3
-rw-r--r--test/test_pyeclib_api.py28
-rw-r--r--test/test_pyeclib_c.py31
5 files changed, 66 insertions, 5 deletions
diff --git a/README.rst b/README.rst
index 645f48f..584b87d 100644
--- a/README.rst
+++ b/README.rst
@@ -51,6 +51,7 @@ Supported ``ec_type`` values:
* ``isa_l_rs_vand`` => Intel Storage Acceleration Library (ISA-L) - SIMD accelerated Erasure Coding backends [4]
* ``isa_l_rs_cauchy`` => Cauchy Reed-Solomon encoding (ISA-L variant) [4]
* ``shss`` => NTT Lab Japan's Erasure Coding Library [5]
+ * ``libphazr`` => Phazr.IO's erasure code library with built-in privacy [6]
The Python API supports the following functions:
@@ -240,7 +241,7 @@ Quick Start
An example for ubuntu to install dependency packages::
$ sudo apt-get install build-essential python-dev python-pip liberasurecode-dev
- $ sudo pip install -U bindep -r test-requirement.txt
+ $ sudo pip install -U bindep -r test-requirements.txt
If you want to confirm all dependency packages installed successfully, try::
@@ -280,3 +281,5 @@ References
[4] Intel(R) Storage Acceleration Library (Open Source Version), https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
[5] Kota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>, "NTT SHSS Erasure Coding backend"
+
+ [6] Jim Cheung <support@phazr.io>, "Phazr.IO libphazr erasure code backend with built-in privacy"
diff --git a/pyeclib/ec_iface.py b/pyeclib/ec_iface.py
index 20d6337..59bf1fb 100644
--- a/pyeclib/ec_iface.py
+++ b/pyeclib/ec_iface.py
@@ -124,6 +124,7 @@ class PyECLib_EC_Types(PyECLibEnum):
shss = 5
liberasurecode_rs_vand = 6
isa_l_rs_cauchy = 7
+ libphazr = 8
# Output of Erasure (en)Coding process are data "fragments". Fragment data
@@ -175,6 +176,8 @@ class ECDriver(object):
elif value == "flat_xor_hd_4":
self.hd = 4
value = "flat_xor_hd"
+ elif value == "libphazr":
+ self.hd = 1
if PyECLib_EC_Types.has_enum(value):
self.ec_type = PyECLib_EC_Types.get_by_name(value)
else:
@@ -521,6 +524,7 @@ ALL_EC_TYPES = [
'shss',
'liberasurecode_rs_vand',
'isa_l_rs_cauchy',
+ 'libphazr',
]
diff --git a/src/c/pyeclib_c/pyeclib_c.c b/src/c/pyeclib_c/pyeclib_c.c
index 8d7cd06..5d709f3 100644
--- a/src/c/pyeclib_c/pyeclib_c.c
+++ b/src/c/pyeclib_c/pyeclib_c.c
@@ -971,6 +971,9 @@ static const char* backend_id_to_str(uint8_t backend_id)
case 7:
backend_id_str = "isa_l_rs_cauchy\0";
break;
+ case 8:
+ backend_id_str = "libphazr\0";
+ break;
default:
backend_id_str = "unknown\0";
}
diff --git a/test/test_pyeclib_api.py b/test/test_pyeclib_api.py
index b3026a0..9585035 100644
--- a/test/test_pyeclib_api.py
+++ b/test/test_pyeclib_api.py
@@ -134,7 +134,7 @@ class TestPyECLibDriver(unittest.TestCase):
"Invalid Argument: m is required")
with self.assertRaises(ECDriverError) as err_context:
- # m is smaller than 1
+ # k is smaller than 1
ECDriver(ec_type=ec_type, k=-100, m=1)
self.assertEqual(str(err_context.exception),
"Invalid number of data fragments (k)")
@@ -151,10 +151,15 @@ class TestPyECLibDriver(unittest.TestCase):
for _type in ALL_EC_TYPES:
try:
if _type is 'shss':
+ _k = 10
+ _m = 4
+ elif _type is 'libphazr':
+ _k = 4
_m = 4
else:
+ _k = 10
_m = 5
- ECDriver(k=10, m=_m, ec_type=_type, validate=True)
+ ECDriver(k=_k, m=_m, ec_type=_type, validate=True)
available_ec_types.append(_type)
except Exception:
# ignore any errors, assume backend not available
@@ -171,10 +176,12 @@ class TestPyECLibDriver(unittest.TestCase):
try:
if _type is 'shss':
_instance = ECDriver(k=10, m=4, ec_type=_type)
+ elif _type is 'libphazr':
+ _instance = ECDriver(k=4, m=4, ec_type=_type)
else:
_instance = ECDriver(k=10, m=5, ec_type=_type)
except ECDriverError:
- self.fail("%p: %s algorithm not supported" % _instance, _type)
+ self.fail("%s algorithm not supported" % _type)
self.assertRaises(ECBackendNotSupported, ECDriver, k=10, m=5,
ec_type="invalid_algo")
@@ -244,6 +251,11 @@ class TestPyECLibDriver(unittest.TestCase):
chksum_type=csum))
pyeclib_drivers.append(ECDriver(k=11, m=7, ec_type=_type6,
chksum_type=csum))
+
+ _type7 = 'libphazr'
+ if _type7 in VALID_EC_TYPES:
+ pyeclib_drivers.append(ECDriver(k=4, m=4, ec_type=_type7,
+ chksum_type=csum))
return pyeclib_drivers
def test_small_encode(self):
@@ -740,7 +752,15 @@ class TestBackendsEnabled(unittest.TestCase):
def dummy(self, ec_type=ec_type):
if ec_type not in VALID_EC_TYPES:
raise unittest.SkipTest
- k, m = 10, 4 if ec_type == 'shss' else 5
+ if ec_type == 'shss':
+ k = 10
+ m = 4
+ elif ec_type == 'libphazr':
+ k = 4
+ m = 4
+ else:
+ k = 10
+ m = 5
ECDriver(k=k, m=m, ec_type=ec_type)
dummy.__name__ = 'test_%s_available' % ec_type
cls_dict[dummy.__name__] = dummy
diff --git a/test/test_pyeclib_c.py b/test/test_pyeclib_c.py
index 0395e21..f505a54 100644
--- a/test/test_pyeclib_c.py
+++ b/test/test_pyeclib_c.py
@@ -78,6 +78,7 @@ class TestPyECLib(unittest.TestCase):
(PyECLib_EC_Types.shss, 10, 4),
(PyECLib_EC_Types.shss, 20, 4),
(PyECLib_EC_Types.shss, 11, 7)]
+ self.libphazr = [(PyECLib_EC_Types.libphazr, 4, 4)]
# Input temp files for testing
self.sizes = ["101-K", "202-K", "303-K"]
@@ -357,6 +358,36 @@ class TestPyECLib(unittest.TestCase):
print("Reconstruct (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))
+ @require_backend("libphazr")
+ def test_libphazr(self):
+ for (ec_type, k, m) in self.libphazr:
+ print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m)))
+
+ success = self._test_get_required_fragments(k, m, ec_type)
+ self.assertTrue(success)
+
+ for size_str in self.sizes:
+ avg_time = self.time_encode(k, m, ec_type.value, 0,
+ size_str,
+ self.iterations)
+ print("Encode (%s): %s" %
+ (size_str, self.get_throughput(avg_time, size_str)))
+
+ for size_str in self.sizes:
+ success, avg_time = self.time_decode(k, m, ec_type.value, 0,
+ size_str,
+ self.iterations)
+ self.assertTrue(success)
+ print("Decode (%s): %s" %
+ (size_str, self.get_throughput(avg_time, size_str)))
+
+ for size_str in self.sizes:
+ success, avg_time = self.time_reconstruct(
+ k, m, ec_type.value, 0, size_str, self.iterations)
+ self.assertTrue(success)
+ print("Reconstruct (%s): %s" %
+ (size_str, self.get_throughput(avg_time, size_str)))
+
def _test_get_required_fragments(self, num_data, num_parity, ec_type):
"""
:return boolean, True if all tests passed