summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTushar Gohad <tushar.gohad@intel.com>2014-11-19 09:34:16 -0700
committerTushar Gohad <tushar.gohad@intel.com>2014-11-19 09:34:16 -0700
commit5026e16ab15e0247361e5d8cc2b14ed3fa4d4b63 (patch)
tree41c6e0f1218987c525325c26861e5665d684c85c
parent3254a95f0bcde0f0626569d390c0b45675d6cfd2 (diff)
downloadpyeclib-5026e16ab15e0247361e5d8cc2b14ed3fa4d4b63.tar.gz
Make library_import_str arg to ECDriver conditional
Signed-off-by: Tushar Gohad <tushar.gohad@intel.com>
-rw-r--r--README4
-rw-r--r--src/python/pyeclib/ec_iface.py15
-rw-r--r--test/test_pyeclib_api.py99
-rw-r--r--test/test_pyeclib_c.py4
-rw-r--r--tools/pyeclib_conf_tool.py6
-rw-r--r--tools/pyeclib_decode.py8
-rw-r--r--tools/pyeclib_encode.py9
-rw-r--r--tools/pyeclib_fragments_needed.py4
8 files changed, 48 insertions, 101 deletions
diff --git a/README b/README
index c739ef2..cec8cc0 100644
--- a/README
+++ b/README
@@ -32,14 +32,14 @@ Examples of using this library are provided in "tools" directory:
PyEClib initialization::
- ec_driver = ECDriver("pyeclib.core.ECPyECLibDriver",
- k=<num_encoded_data_fragments>,
+ ec_driver = ECDriver(k=<num_encoded_data_fragments>,
m=<num_encoded_parity_fragments>,
ec_type=<ec_scheme>))
Supported ``ec_type`` values:
* ``jerasure_rs_vand`` => Vandermonde Reed-Solomon encoding
+ * ``jerasure_rs_cauchy`` => Cauchy Reed-Solomon encoding (Jerasure variant)
* ``flat_xor_hd_3``, ``flat_xor_hd_4`` => Flat-XOR based HD combination codes
A configuration utility is provided to help compare available EC schemes in
diff --git a/src/python/pyeclib/ec_iface.py b/src/python/pyeclib/ec_iface.py
index 2eeeb85..ebc98b9 100644
--- a/src/python/pyeclib/ec_iface.py
+++ b/src/python/pyeclib/ec_iface.py
@@ -115,12 +115,11 @@ class ECDriverError(Exception):
# Main ECDriver class
class ECDriver(object):
- def __init__(self, library_import_str, *args, **kwargs):
+ def __init__(self, *args, **kwargs):
self.k = -1
self.m = -1
self.ec_type = None
self.chksum_type = None
- self.library_import_str = None
for (key, value) in kwargs.items():
if key == "k":
try:
@@ -149,17 +148,13 @@ class ECDriver(object):
raise ECDriverError(
"%s is not a valid checksum type for PyECLib!" % value)
- if library_import_str is not None:
- self.library_import_str = library_import_str
- else:
- raise ECDriverError(
- "Library import string (library_import_str) was not specified "
- "and is a required argument!")
+ self.library_import_str = kwargs.pop('library_import_str',
+ 'pyeclib.core.ECPyECLibDriver')
#
# Instantiate EC backend driver
#
self.ec_lib_reference = create_instance(
- library_import_str,
+ self.library_import_str,
k=self.k,
m=self.m,
ec_type=self.ec_type,
@@ -189,7 +184,7 @@ class ECDriver(object):
if len(not_implemented_str) > 0:
raise ECDriverError(
"The following required methods are not implemented "
- "in %s: %s" % (library_import_str, not_implemented_str))
+ "in %s: %s" % (self.library_import_str, not_implemented_str))
def encode(self, data_bytes):
"""
diff --git a/test/test_pyeclib_api.py b/test/test_pyeclib_api.py
index 75e4f38..71a5201 100644
--- a/test/test_pyeclib_api.py
+++ b/test/test_pyeclib_api.py
@@ -41,7 +41,8 @@ else:
class TestNullDriver(unittest.TestCase):
def setUp(self):
- self.null_driver = ECDriver("pyeclib.core.ECNullDriver", k=8, m=2)
+ self.null_driver = ECDriver(
+ library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
def tearDown(self):
pass
@@ -54,7 +55,8 @@ class TestNullDriver(unittest.TestCase):
class TestStripeDriver(unittest.TestCase):
def setUp(self):
- self.stripe_driver = ECDriver("pyeclib.core.ECStripingDriver", k=8, m=0)
+ self.stripe_driver = ECDriver(
+ library_import_str="pyeclib.core.ECStripingDriver", k=8, m=0)
def tearDown(self):
pass
@@ -102,12 +104,9 @@ class TestPyECLibDriver(unittest.TestCase):
def test_small_encode(self):
pyeclib_drivers = []
- pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="jerasure_rs_vand"))
- pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
- k=11, m=2, ec_type="jerasure_rs_vand"))
- pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=2, ec_type="jerasure_rs_vand"))
+ pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
+ pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
+ pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"]
@@ -121,17 +120,13 @@ class TestPyECLibDriver(unittest.TestCase):
def disabled_test_verify_fragment_algsig_chksum_fail(self):
pyeclib_drivers = []
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
+ ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
+ ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
+ ECDriver(k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
+ ECDriver(k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@@ -159,17 +154,13 @@ class TestPyECLibDriver(unittest.TestCase):
def disabled_test_verify_fragment_inline_succeed(self):
pyeclib_drivers = []
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
+ ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
+ ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
+ ECDriver(k=12, m=6, ec_type="flat_xor_hd", chksum_type="algsig"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
+ ECDriver(k=10, m=5, ec_type="flat_xor_hd", chksum_type="algsig"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@@ -190,17 +181,13 @@ class TestPyECLibDriver(unittest.TestCase):
def disabled_test_verify_fragment_inline_chksum_fail(self):
pyeclib_drivers = []
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
+ ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
+ ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
+ ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
+ ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@@ -233,17 +220,13 @@ class TestPyECLibDriver(unittest.TestCase):
def disabled_test_verify_fragment_inline_chksum_succeed(self):
pyeclib_drivers = []
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
+ ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
+ ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
+ ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
+ ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@@ -264,14 +247,11 @@ class TestPyECLibDriver(unittest.TestCase):
def test_get_segment_byterange_info(self):
pyeclib_drivers = []
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="jerasure_rs_vand"))
+ ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=11, m=2, ec_type="jerasure_rs_vand"))
+ ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=2, ec_type="jerasure_rs_vand"))
+ ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
file_size = 1024 * 1024
segment_size = 3 * 1024
@@ -300,14 +280,11 @@ class TestPyECLibDriver(unittest.TestCase):
def test_get_segment_info(self):
pyeclib_drivers = []
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="jerasure_rs_vand"))
+ ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=11, m=2, ec_type="jerasure_rs_vand"))
+ ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=2, ec_type="jerasure_rs_vand"))
+ ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
file_sizes = [
1024 * 1024,
@@ -375,23 +352,17 @@ class TestPyECLibDriver(unittest.TestCase):
def test_rs(self):
pyeclib_drivers = []
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="jerasure_rs_vand"))
+ ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="jerasure_rs_cauchy"))
+ ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=3, ec_type="jerasure_rs_vand"))
+ ECDriver(k=12, m=3, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=3, ec_type="jerasure_rs_cauchy"))
+ ECDriver(k=12, m=3, ec_type="jerasure_rs_cauchy"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=6, ec_type="flat_xor_hd"))
+ ECDriver(k=12, m=6, ec_type="flat_xor_hd"))
pyeclib_drivers.append(
- ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=5, ec_type="flat_xor_hd"))
+ ECDriver(k=10, m=5, ec_type="flat_xor_hd"))
for pyeclib_driver in pyeclib_drivers:
for file_size in self.file_sizes:
diff --git a/test/test_pyeclib_c.py b/test/test_pyeclib_c.py
index b273a84..eafb1f3 100644
--- a/test/test_pyeclib_c.py
+++ b/test/test_pyeclib_c.py
@@ -184,7 +184,7 @@ class TestPyECLib(unittest.TestCase):
fragments = pyeclib_c.encode(handle, whole_file_bytes)
orig_fragments = fragments[:]
-
+
for i in range(iterations):
missing_idxs = []
num_missing = hd - 1
@@ -367,7 +367,7 @@ class TestPyECLib(unittest.TestCase):
self.num_parities[i],
ec_type.value, self.num_parities[i] + 1,
size_str, self.iterations)
-
+
self.assertTrue(success)
print(("Range Decode (%s): %s" %
(size_str, self.get_throughput(avg_time, size_str))))
diff --git a/tools/pyeclib_conf_tool.py b/tools/pyeclib_conf_tool.py
index 84cd957..c21495a 100644
--- a/tools/pyeclib_conf_tool.py
+++ b/tools/pyeclib_conf_tool.py
@@ -58,12 +58,10 @@
# (limit 10)
#
-import pyeclib
from pyeclib.ec_iface import ECDriver
import random
import string
import sys
-import os
import argparse
import time
import math
@@ -238,9 +236,7 @@ for scheme in schemes:
string.ascii_uppercase + string.digits) for x in range(args.s))
try:
- ec_driver = ECDriver(
- "pyeclib.core.ECPyECLibDriver",
- k=scheme.k, m=scheme.m, ec_type=scheme.ec_type)
+ ec_driver = ECDriver(k=scheme.k, m=scheme.m, ec_type=scheme.ec_type)
except Exception as e:
print("Scheme %s is not defined (%s)." % (scheme, e))
continue
diff --git a/tools/pyeclib_decode.py b/tools/pyeclib_decode.py
index 4ec7f52..707a63a 100644
--- a/tools/pyeclib_decode.py
+++ b/tools/pyeclib_decode.py
@@ -21,12 +21,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import pyeclib
from pyeclib.ec_iface import ECDriver
-import random
-import string
-import sys
-import os
import argparse
parser = argparse.ArgumentParser(description='Decoder for PyECLib.')
@@ -44,8 +39,7 @@ print("ec_type = %s" % args.ec_type)
print("fragments = %s" % args.fragments)
print("filename = %s" % args.filename)
-ec_driver = ECDriver(
- "pyeclib.core.ECPyECLibDriver", k=args.k, m=args.m, ec_type=args.ec_type)
+ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
fragment_list = []
diff --git a/tools/pyeclib_encode.py b/tools/pyeclib_encode.py
index 28563c5..093d1cf 100644
--- a/tools/pyeclib_encode.py
+++ b/tools/pyeclib_encode.py
@@ -21,12 +21,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import pyeclib
from pyeclib.ec_iface import ECDriver
-import random
-import string
-import sys
-import os
import argparse
parser = argparse.ArgumentParser(description='Encoder for PyECLib.')
@@ -43,9 +38,7 @@ print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("filename = %s" % args.filename)
-ec_driver = ECDriver(
- "pyeclib.core.ECPyECLibDriver",
- k=args.k, m=args.m, ec_type=args.ec_type)
+ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
# read
with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp:
diff --git a/tools/pyeclib_fragments_needed.py b/tools/pyeclib_fragments_needed.py
index 92fd0b3..48d1bdc 100644
--- a/tools/pyeclib_fragments_needed.py
+++ b/tools/pyeclib_fragments_needed.py
@@ -40,9 +40,7 @@ parser.add_argument('missing_fragments', type=int, metavar='missing_fragment',
args = parser.parse_args()
-ec_driver = ECDriver(
- "pyeclib.core.ECPyECLibDriver",
- k=args.k, m=args.m, ec_type=args.ec_type)
+ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
fragments_needed = ec_driver.fragments_needed(args.missing_fragments)