summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKota Tsuyuzaki <bloodeagle40234@gmail.com>2015-02-27 15:16:58 +0900
committerKota Tsuyuzaki <bloodeagle40234@gmail.com>2015-03-04 20:18:10 +0900
commit298826e0a9e85327100aa0a8eccd740bfc955baa (patch)
tree4d2b67d3fc8ea18efd92c24221ee41bd307f0eff
parente10d5aa330898bce4f1ac8d29059e374a12954c4 (diff)
downloadpyeclib-298826e0a9e85327100aa0a8eccd740bfc955baa.tar.gz
Add shss and its unit tests
-rw-r--r--README1
-rw-r--r--pyeclib/ec_iface.py4
-rw-r--r--test/test_pyeclib_api.py13
-rw-r--r--test/test_pyeclib_c.py37
4 files changed, 51 insertions, 4 deletions
diff --git a/README b/README
index ba6a122..e61e849 100644
--- a/README
+++ b/README
@@ -42,6 +42,7 @@ Supported ``ec_type`` values:
* ``jerasure_rs_cauchy`` => Cauchy Reed-Solomon encoding (Jerasure variant)
* ``flat_xor_hd`` => Flat-XOR based HD combination codes
* ``isa_l_rs_vand`` => SIMD-based Reed-Soloman implementation from ISA-L (Intel(R) Storage Acceleration Library)
+ * ``shss`` => NTT Lab Japan's Erasure Coding Library
A configuration utility is provided to help compare available EC schemes in
terms of performance and redundancy:: tools/pyeclib_conf_tool.py
diff --git a/pyeclib/ec_iface.py b/pyeclib/ec_iface.py
index 1a31277..0665058 100644
--- a/pyeclib/ec_iface.py
+++ b/pyeclib/ec_iface.py
@@ -45,7 +45,8 @@ VALID_EC_TYPES = ['jerasure_rs_vand',
'jerasure_rs_cauchy',
'flat_xor_hd_3',
'flat_xor_hd_4',
- 'isa_l_rs_vand']
+ 'isa_l_rs_vand',
+ 'shss']
@unique
@@ -92,6 +93,7 @@ class PyECLib_EC_Types(PyECLibEnum):
jerasure_rs_cauchy = 2
flat_xor_hd = 3
isa_l_rs_vand = 4
+ shss = 5
# Output of Erasure (en)Coding process are data "fragments". Fragment data
diff --git a/test/test_pyeclib_api.py b/test/test_pyeclib_api.py
index 4cfb3e0..c7c5681 100644
--- a/test/test_pyeclib_api.py
+++ b/test/test_pyeclib_api.py
@@ -27,8 +27,9 @@ import sys
import tempfile
import unittest
-from pyeclib.ec_iface import ECDriver, VALID_EC_TYPES, ECDriverError
-
+from pyeclib.ec_iface import ECDriver, VALID_EC_TYPES, ECDriverError, \
+ PyECLib_EC_Types
+from test_pyeclib_c import _available_backends
if sys.version < '3':
def b2i(b):
@@ -106,7 +107,13 @@ class TestPyECLibDriver(unittest.TestCase):
for _type in VALID_EC_TYPES:
# Check if this algo works
try:
- _instance = ECDriver(k=10, m=5, ec_type=_type)
+ if _type is 'shss':
+ if PyECLib_EC_Types.shss in _available_backends:
+ _instance = ECDriver(k=10, m=4, ec_type=_type)
+ else:
+ continue
+ else:
+ _instance = ECDriver(k=10, m=5, ec_type=_type)
except ECDriverError:
self.fail("%s algorithm not supported" % _type)
diff --git a/test/test_pyeclib_c.py b/test/test_pyeclib_c.py
index a580ee7..4cd122d 100644
--- a/test/test_pyeclib_c.py
+++ b/test/test_pyeclib_c.py
@@ -26,6 +26,7 @@ from string import ascii_letters
import tempfile
import time
import unittest
+from unittest import skipIf
import pyeclib_c
from pyeclib.ec_iface import PyECLib_EC_Types
@@ -80,6 +81,10 @@ class TestPyECLib(unittest.TestCase):
self.xor_types = [(PyECLib_EC_Types.flat_xor_hd, 12, 6, 4),
(PyECLib_EC_Types.flat_xor_hd, 10, 5, 4),
(PyECLib_EC_Types.flat_xor_hd, 10, 5, 3)]
+ self.shss = [(PyECLib_EC_Types.shss, 6, 3),
+ (PyECLib_EC_Types.shss, 10, 4),
+ (PyECLib_EC_Types.shss, 20, 4),
+ (PyECLib_EC_Types.shss, 11, 7)]
# Input temp files for testing
self.sizes = ["101-K", "202-K", "303-K"]
@@ -311,6 +316,38 @@ class TestPyECLib(unittest.TestCase):
print("Reconstruct (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str)))
+ @skipIf(PyECLib_EC_Types.shss not in _available_backends,
+ "shss backend is not available in your enviromnet")
+ def test_shss(self):
+ for (ec_type, k, m) in self.shss:
+ print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m)))
+
+ success = self._test_get_required_fragments(k, m, ec_type)
+ self.assertTrue(success)
+
+ for size_str in self.sizes:
+ avg_time = self.time_encode(k, m, ec_type.value, 0,
+ size_str,
+ self.iterations)
+ print("Encode (%s): %s" %
+ (size_str, self.get_throughput(avg_time, size_str)))
+
+ for size_str in self.sizes:
+ success, avg_time = self.time_decode(k, m, ec_type.value, 0,
+ size_str,
+ self.iterations)
+ self.assertTrue(success)
+ print("Decode (%s): %s" %
+ (size_str, self.get_throughput(avg_time, size_str)))
+
+ for size_str in self.sizes:
+ success, avg_time = self.time_reconstruct(k, m, ec_type.value, 0,
+ size_str,
+ self.iterations)
+ self.assertTrue(success)
+ print("Reconstruct (%s): %s" %
+ (size_str, self.get_throughput(avg_time, size_str)))
+
def _test_get_required_fragments(self, num_data, num_parity, ec_type):
"""
:return boolean, True if all tests passed