diff options
author | Tushar Gohad <tushar.gohad@intel.com> | 2014-11-19 09:34:16 -0700 |
---|---|---|
committer | Tushar Gohad <tushar.gohad@intel.com> | 2014-11-19 09:34:16 -0700 |
commit | 5026e16ab15e0247361e5d8cc2b14ed3fa4d4b63 (patch) | |
tree | 41c6e0f1218987c525325c26861e5665d684c85c | |
parent | 3254a95f0bcde0f0626569d390c0b45675d6cfd2 (diff) | |
download | pyeclib-5026e16ab15e0247361e5d8cc2b14ed3fa4d4b63.tar.gz |
Make library_import_str arg to ECDriver conditional
Signed-off-by: Tushar Gohad <tushar.gohad@intel.com>
-rw-r--r-- | README | 4 | ||||
-rw-r--r-- | src/python/pyeclib/ec_iface.py | 15 | ||||
-rw-r--r-- | test/test_pyeclib_api.py | 99 | ||||
-rw-r--r-- | test/test_pyeclib_c.py | 4 | ||||
-rw-r--r-- | tools/pyeclib_conf_tool.py | 6 | ||||
-rw-r--r-- | tools/pyeclib_decode.py | 8 | ||||
-rw-r--r-- | tools/pyeclib_encode.py | 9 | ||||
-rw-r--r-- | tools/pyeclib_fragments_needed.py | 4 |
8 files changed, 48 insertions, 101 deletions
@@ -32,14 +32,14 @@ Examples of using this library are provided in "tools" directory: PyEClib initialization:: - ec_driver = ECDriver("pyeclib.core.ECPyECLibDriver", - k=<num_encoded_data_fragments>, + ec_driver = ECDriver(k=<num_encoded_data_fragments>, m=<num_encoded_parity_fragments>, ec_type=<ec_scheme>)) Supported ``ec_type`` values: * ``jerasure_rs_vand`` => Vandermonde Reed-Solomon encoding + * ``jerasure_rs_cauchy`` => Cauchy Reed-Solomon encoding (Jerasure variant) * ``flat_xor_hd_3``, ``flat_xor_hd_4`` => Flat-XOR based HD combination codes A configuration utility is provided to help compare available EC schemes in diff --git a/src/python/pyeclib/ec_iface.py b/src/python/pyeclib/ec_iface.py index 2eeeb85..ebc98b9 100644 --- a/src/python/pyeclib/ec_iface.py +++ b/src/python/pyeclib/ec_iface.py @@ -115,12 +115,11 @@ class ECDriverError(Exception): # Main ECDriver class class ECDriver(object): - def __init__(self, library_import_str, *args, **kwargs): + def __init__(self, *args, **kwargs): self.k = -1 self.m = -1 self.ec_type = None self.chksum_type = None - self.library_import_str = None for (key, value) in kwargs.items(): if key == "k": try: @@ -149,17 +148,13 @@ class ECDriver(object): raise ECDriverError( "%s is not a valid checksum type for PyECLib!" % value) - if library_import_str is not None: - self.library_import_str = library_import_str - else: - raise ECDriverError( - "Library import string (library_import_str) was not specified " - "and is a required argument!") + self.library_import_str = kwargs.pop('library_import_str', + 'pyeclib.core.ECPyECLibDriver') # # Instantiate EC backend driver # self.ec_lib_reference = create_instance( - library_import_str, + self.library_import_str, k=self.k, m=self.m, ec_type=self.ec_type, @@ -189,7 +184,7 @@ class ECDriver(object): if len(not_implemented_str) > 0: raise ECDriverError( "The following required methods are not implemented " - "in %s: %s" % (library_import_str, not_implemented_str)) + "in %s: %s" % (self.library_import_str, not_implemented_str)) def encode(self, data_bytes): """ diff --git a/test/test_pyeclib_api.py b/test/test_pyeclib_api.py index 75e4f38..71a5201 100644 --- a/test/test_pyeclib_api.py +++ b/test/test_pyeclib_api.py @@ -41,7 +41,8 @@ else: class TestNullDriver(unittest.TestCase): def setUp(self): - self.null_driver = ECDriver("pyeclib.core.ECNullDriver", k=8, m=2) + self.null_driver = ECDriver( + library_import_str="pyeclib.core.ECNullDriver", k=8, m=2) def tearDown(self): pass @@ -54,7 +55,8 @@ class TestNullDriver(unittest.TestCase): class TestStripeDriver(unittest.TestCase): def setUp(self): - self.stripe_driver = ECDriver("pyeclib.core.ECStripingDriver", k=8, m=0) + self.stripe_driver = ECDriver( + library_import_str="pyeclib.core.ECStripingDriver", k=8, m=0) def tearDown(self): pass @@ -102,12 +104,9 @@ class TestPyECLibDriver(unittest.TestCase): def test_small_encode(self): pyeclib_drivers = [] - pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=2, ec_type="jerasure_rs_vand")) - pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver", - k=11, m=2, ec_type="jerasure_rs_vand")) - pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver", - k=10, m=2, ec_type="jerasure_rs_vand")) + pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type="jerasure_rs_vand")) + pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type="jerasure_rs_vand")) + pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type="jerasure_rs_vand")) encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"] @@ -121,17 +120,13 @@ class TestPyECLibDriver(unittest.TestCase): def disabled_test_verify_fragment_algsig_chksum_fail(self): pyeclib_drivers = [] pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig")) + ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig")) + ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig")) + ECDriver(k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig")) + ECDriver(k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig")) filesize = 1024 * 1024 * 3 file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) @@ -159,17 +154,13 @@ class TestPyECLibDriver(unittest.TestCase): def disabled_test_verify_fragment_inline_succeed(self): pyeclib_drivers = [] pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig")) + ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig")) + ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig")) + ECDriver(k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig")) + ECDriver(k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig")) filesize = 1024 * 1024 * 3 file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) @@ -190,17 +181,13 @@ class TestPyECLibDriver(unittest.TestCase): def disabled_test_verify_fragment_inline_chksum_fail(self): pyeclib_drivers = [] pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) + ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) + ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) + ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32")) + ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32")) filesize = 1024 * 1024 * 3 file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) @@ -233,17 +220,13 @@ class TestPyECLibDriver(unittest.TestCase): def disabled_test_verify_fragment_inline_chksum_succeed(self): pyeclib_drivers = [] pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) + ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) + ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) + ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32")) + ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32")) filesize = 1024 * 1024 * 3 file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) @@ -264,14 +247,11 @@ class TestPyECLibDriver(unittest.TestCase): def test_get_segment_byterange_info(self): pyeclib_drivers = [] pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=2, ec_type="jerasure_rs_vand")) + ECDriver(k=12, m=2, ec_type="jerasure_rs_vand")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=11, m=2, ec_type="jerasure_rs_vand")) + ECDriver(k=11, m=2, ec_type="jerasure_rs_vand")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=10, m=2, ec_type="jerasure_rs_vand")) + ECDriver(k=10, m=2, ec_type="jerasure_rs_vand")) file_size = 1024 * 1024 segment_size = 3 * 1024 @@ -300,14 +280,11 @@ class TestPyECLibDriver(unittest.TestCase): def test_get_segment_info(self): pyeclib_drivers = [] pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=2, ec_type="jerasure_rs_vand")) + ECDriver(k=12, m=2, ec_type="jerasure_rs_vand")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=11, m=2, ec_type="jerasure_rs_vand")) + ECDriver(k=11, m=2, ec_type="jerasure_rs_vand")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=10, m=2, ec_type="jerasure_rs_vand")) + ECDriver(k=10, m=2, ec_type="jerasure_rs_vand")) file_sizes = [ 1024 * 1024, @@ -375,23 +352,17 @@ class TestPyECLibDriver(unittest.TestCase): def test_rs(self): pyeclib_drivers = [] pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=2, ec_type="jerasure_rs_vand")) + ECDriver(k=12, m=2, ec_type="jerasure_rs_vand")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=2, ec_type="jerasure_rs_cauchy")) + ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=3, ec_type="jerasure_rs_vand")) + ECDriver(k=12, m=3, ec_type="jerasure_rs_vand")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=3, ec_type="jerasure_rs_cauchy")) + ECDriver(k=12, m=3, ec_type="jerasure_rs_cauchy")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=12, m=6, ec_type="flat_xor_hd")) + ECDriver(k=12, m=6, ec_type="flat_xor_hd")) pyeclib_drivers.append( - ECDriver("pyeclib.core.ECPyECLibDriver", - k=10, m=5, ec_type="flat_xor_hd")) + ECDriver(k=10, m=5, ec_type="flat_xor_hd")) for pyeclib_driver in pyeclib_drivers: for file_size in self.file_sizes: diff --git a/test/test_pyeclib_c.py b/test/test_pyeclib_c.py index b273a84..eafb1f3 100644 --- a/test/test_pyeclib_c.py +++ b/test/test_pyeclib_c.py @@ -184,7 +184,7 @@ class TestPyECLib(unittest.TestCase): fragments = pyeclib_c.encode(handle, whole_file_bytes) orig_fragments = fragments[:] - + for i in range(iterations): missing_idxs = [] num_missing = hd - 1 @@ -367,7 +367,7 @@ class TestPyECLib(unittest.TestCase): self.num_parities[i], ec_type.value, self.num_parities[i] + 1, size_str, self.iterations) - + self.assertTrue(success) print(("Range Decode (%s): %s" % (size_str, self.get_throughput(avg_time, size_str)))) diff --git a/tools/pyeclib_conf_tool.py b/tools/pyeclib_conf_tool.py index 84cd957..c21495a 100644 --- a/tools/pyeclib_conf_tool.py +++ b/tools/pyeclib_conf_tool.py @@ -58,12 +58,10 @@ # (limit 10) # -import pyeclib from pyeclib.ec_iface import ECDriver import random import string import sys -import os import argparse import time import math @@ -238,9 +236,7 @@ for scheme in schemes: string.ascii_uppercase + string.digits) for x in range(args.s)) try: - ec_driver = ECDriver( - "pyeclib.core.ECPyECLibDriver", - k=scheme.k, m=scheme.m, ec_type=scheme.ec_type) + ec_driver = ECDriver(k=scheme.k, m=scheme.m, ec_type=scheme.ec_type) except Exception as e: print("Scheme %s is not defined (%s)." % (scheme, e)) continue diff --git a/tools/pyeclib_decode.py b/tools/pyeclib_decode.py index 4ec7f52..707a63a 100644 --- a/tools/pyeclib_decode.py +++ b/tools/pyeclib_decode.py @@ -21,12 +21,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pyeclib from pyeclib.ec_iface import ECDriver -import random -import string -import sys -import os import argparse parser = argparse.ArgumentParser(description='Decoder for PyECLib.') @@ -44,8 +39,7 @@ print("ec_type = %s" % args.ec_type) print("fragments = %s" % args.fragments) print("filename = %s" % args.filename) -ec_driver = ECDriver( - "pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, ec_type=args.ec_type) +ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type) fragment_list = [] diff --git a/tools/pyeclib_encode.py b/tools/pyeclib_encode.py index 28563c5..093d1cf 100644 --- a/tools/pyeclib_encode.py +++ b/tools/pyeclib_encode.py @@ -21,12 +21,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pyeclib from pyeclib.ec_iface import ECDriver -import random -import string -import sys -import os import argparse parser = argparse.ArgumentParser(description='Encoder for PyECLib.') @@ -43,9 +38,7 @@ print("k = %d, m = %d" % (args.k, args.m)) print("ec_type = %s" % args.ec_type) print("filename = %s" % args.filename) -ec_driver = ECDriver( - "pyeclib.core.ECPyECLibDriver", - k=args.k, m=args.m, ec_type=args.ec_type) +ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type) # read with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp: diff --git a/tools/pyeclib_fragments_needed.py b/tools/pyeclib_fragments_needed.py index 92fd0b3..48d1bdc 100644 --- a/tools/pyeclib_fragments_needed.py +++ b/tools/pyeclib_fragments_needed.py @@ -40,9 +40,7 @@ parser.add_argument('missing_fragments', type=int, metavar='missing_fragment', args = parser.parse_args() -ec_driver = ECDriver( - "pyeclib.core.ECPyECLibDriver", - k=args.k, m=args.m, ec_type=args.ec_type) +ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type) fragments_needed = ec_driver.fragments_needed(args.missing_fragments) |