From b14ac48319554d1399820268e2d81317e072e552 Mon Sep 17 00:00:00 2001 From: Tushar Gohad Date: Fri, 20 Nov 2015 07:03:07 +0000 Subject: Make VALID_EC_TYPES a runtime/dynamic property ... this is needed for compatibility with older versions liberasurecode where a few EC types/backends were not available --- pyeclib/ec_iface.py | 37 +++++++++++++++++++++++++++++-------- test/test_pyeclib_api.py | 29 +++++++++++++---------------- test/test_pyeclib_c.py | 23 ++++------------------- 3 files changed, 46 insertions(+), 43 deletions(-) diff --git a/pyeclib/ec_iface.py b/pyeclib/ec_iface.py index fc8ff03..2b459d6 100644 --- a/pyeclib/ec_iface.py +++ b/pyeclib/ec_iface.py @@ -41,14 +41,6 @@ PYECLIB_VERSION = PyECLibVersion(PYECLIB_MAJOR, PYECLIB_MINOR, PYECLIB_MAX_DATA = 32 PYECLIB_MAX_PARITY = 32 -VALID_EC_TYPES = ['jerasure_rs_vand', - 'jerasure_rs_cauchy', - 'flat_xor_hd_3', - 'flat_xor_hd_4', - 'isa_l_rs_vand', - 'shss', - 'liberasurecode_rs_vand'] - @unique class PyECLibEnum(Enum): @@ -460,3 +452,32 @@ class ECInsufficientFragments(ECDriverError): # Out of memory class ECOutOfMemory(ECDriverError): pass + + +# PyECLib helper for "available" EC types +ALL_EC_TYPES = [ + 'jerasure_rs_vand', + 'jerasure_rs_cauchy', + 'flat_xor_hd_3', + 'flat_xor_hd_4', + 'isa_l_rs_vand', + 'shss', + 'liberasurecode_rs_vand', +] + + +def _PyECLibValidECTypes(): + available_ec_types = [] + for _type in ALL_EC_TYPES: + try: + if _type is 'shss': + ECDriver(k=10, m=4, ec_type=_type) + else: + ECDriver(k=10, m=5, ec_type=_type) + available_ec_types.append(_type) + except Exception: + pass + return available_ec_types + + +VALID_EC_TYPES = _PyECLibValidECTypes() diff --git a/test/test_pyeclib_api.py b/test/test_pyeclib_api.py index 985d88d..6d11076 100644 --- a/test/test_pyeclib_api.py +++ b/test/test_pyeclib_api.py @@ -27,14 +27,11 @@ import sys import tempfile import unittest from pyeclib.ec_iface import ECBackendNotSupported +from pyeclib.ec_iface import ECDriver from pyeclib.ec_iface import ECDriverError from pyeclib.ec_iface import ECInsufficientFragments - -from pyeclib.ec_iface import ECDriver, PyECLib_EC_Types -if sys.version < '3': - from test_pyeclib_c import _available_backends -else: - from .test_pyeclib_c import _available_backends +from pyeclib.ec_iface import PyECLib_EC_Types +from pyeclib.ec_iface import VALID_EC_TYPES if sys.version < '3': def b2i(b): @@ -114,7 +111,7 @@ class TestPyECLibDriver(unittest.TestCase): print("") for _type in PyECLib_EC_Types.names(): # Check if this algo works - if _type not in _available_backends: + if _type not in VALID_EC_TYPES: print("Skipping test for %s backend" % _type) continue try: @@ -131,7 +128,7 @@ class TestPyECLibDriver(unittest.TestCase): def get_pyeclib_testspec(self, csum="none"): pyeclib_drivers = [] _type1 = 'jerasure_rs_vand' - if _type1 in _available_backends: + if _type1 in VALID_EC_TYPES: pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type=_type1, chksum_type=csum)) pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type=_type1, @@ -141,7 +138,7 @@ class TestPyECLibDriver(unittest.TestCase): pyeclib_drivers.append(ECDriver(k=8, m=4, ec_type=_type1, chksum_type=csum)) _type2 = 'liberasurecode_rs_vand' - if _type2 in _available_backends: + if _type2 in VALID_EC_TYPES: pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type=_type2, chksum_type=csum)) pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type=_type2, @@ -151,19 +148,19 @@ class TestPyECLibDriver(unittest.TestCase): pyeclib_drivers.append(ECDriver(k=8, m=4, ec_type=_type2, chksum_type=csum)) _type3_1 = 'flat_xor_hd' - if _type3_1 in _available_backends: + if _type3_1 in VALID_EC_TYPES: pyeclib_drivers.append(ECDriver(k=12, m=6, ec_type=_type3_1, chksum_type=csum)) pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=_type3_1, chksum_type=csum)) _type3_2 = 'flat_xor_hd_4' - if _type3_2 in _available_backends: + if _type3_2 in VALID_EC_TYPES: pyeclib_drivers.append(ECDriver(k=12, m=6, ec_type=_type3_2, chksum_type=csum)) pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=_type3_2, chksum_type=csum)) _type4 = 'shss' - if _type4 in _available_backends: + if _type4 in VALID_EC_TYPES: pyeclib_drivers.append(ECDriver(k=10, m=4, ec_type=_type4, chksum_type=csum)) pyeclib_drivers.append(ECDriver(k=20, m=4, ec_type=_type4, @@ -247,7 +244,7 @@ class TestPyECLibDriver(unittest.TestCase): def check_metadata_formatted(self, k, m, ec_type, chksum_type): - if ec_type not in _available_backends: + if ec_type not in VALID_EC_TYPES: return filesize = 1024 * 1024 * 3 @@ -529,7 +526,7 @@ class TestPyECLibDriver(unittest.TestCase): if ec_type[:11] == "flat_xor_hd": return ECDriver(k=k, m=m, ec_type="flat_xor_hd", chksum_type=chksum_type) - elif ec_type in _available_backends: + elif ec_type in VALID_EC_TYPES: return ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type) else: return None @@ -540,7 +537,7 @@ class TestPyECLibDriver(unittest.TestCase): tmp_file.seek(0) whole_file_bytes = tmp_file.read() for ec_type in ['flat_xor_hd_3', 'liberasurecode_rs_vand']: - if ec_type in _available_backends: + if ec_type in VALID_EC_TYPES: pyeclib_driver = self.get_available_backend( k=10, m=5, ec_type=ec_type) fragments = pyeclib_driver.encode(whole_file_bytes) @@ -551,7 +548,7 @@ class TestPyECLibDriver(unittest.TestCase): def test_min_parity_fragments_needed(self): pyeclib_drivers = [] for ec_type in ['flat_xor_hd_3', 'liberasurecode_rs_vand']: - if ec_type in _available_backends: + if ec_type in VALID_EC_TYPES: pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=ec_type)) self.assertTrue( pyeclib_drivers[0].min_parity_fragments_needed() == 1) diff --git a/test/test_pyeclib_c.py b/test/test_pyeclib_c.py index 26831b1..4975840 100644 --- a/test/test_pyeclib_c.py +++ b/test/test_pyeclib_c.py @@ -30,24 +30,9 @@ import unittest import pyeclib_c from pyeclib.ec_iface import PyECLib_EC_Types +from pyeclib.ec_iface import VALID_EC_TYPES -def collect_available_backends(): - available_backends = [] - for ec_type in PyECLib_EC_Types: - try: - if ec_type == PyECLib_EC_Types.flat_xor_hd: - handle = pyeclib_c.init(10, 5, ec_type.value, 3) - else: - handle = pyeclib_c.init(10, 4, ec_type.value) - available_backends.append(ec_type.name) - print(ec_type.name) - except: - pass - return available_backends - -_available_backends = collect_available_backends() - class Timer: def __init__(self): @@ -293,7 +278,7 @@ class TestPyECLib(unittest.TestCase): return format(throughput, '.10g') def test_xor_code(self): - if "flat_xor_hd" not in _available_backends: + if "flat_xor_hd" not in VALID_EC_TYPES: print("xor backend is not available in your enviromnet, skipping test") return @@ -324,7 +309,7 @@ class TestPyECLib(unittest.TestCase): (size_str, self.get_throughput(avg_time, size_str))) def test_shss(self): - if "shss" not in _available_backends: + if "shss" not in VALID_EC_TYPES: print("shss backend is not available in your enviromnet, skipping test") return @@ -395,7 +380,7 @@ class TestPyECLib(unittest.TestCase): def test_codes(self): for ec_type in self.rs_types: - if ec_type.name not in _available_backends: + if ec_type.name not in VALID_EC_TYPES: print("%s backend is not available in your enviromnet, skipping test" % ec_type.name) continue -- cgit v1.2.1