summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTushar Gohad <tusharsg@gmail.com>2015-07-19 22:18:09 -0700
committerTushar Gohad <tusharsg@gmail.com>2015-07-19 22:18:09 -0700
commitcf04d34162c377c9ae08d49abacd6f23dd9e1430 (patch)
tree1fa597b178d942b086ef5d7cf8ce95b9619498d2
parent43868eaa34de93e03d849f521c21c84f3aa17b23 (diff)
parentf1524070ddf0b81f72166e425615a6cb0e7deef1 (diff)
downloadpyeclib-cf04d34162c377c9ae08d49abacd6f23dd9e1430.tar.gz
Merged extended_error_codes into master
-rw-r--r--pyeclib/core.py48
-rw-r--r--pyeclib/ec_iface.py90
-rw-r--r--src/c/pyeclib_c/pyeclib_c.c169
-rw-r--r--test/test_pyeclib_api.py233
4 files changed, 310 insertions, 230 deletions
diff --git a/pyeclib/core.py b/pyeclib/core.py
index d5acdac..d10e56e 100644
--- a/pyeclib/core.py
+++ b/pyeclib/core.py
@@ -21,14 +21,17 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-from ec_iface import PyECLib_FRAGHDRCHKSUM_Types, ECDriverError, ECDriverInsufficientFragmentError
+from ec_iface import ECDriverError
+from ec_iface import ECInsufficientFragments
+from ec_iface import PyECLib_FRAGHDRCHKSUM_Types
+
import math
import pyeclib_c
-from pyeclib_c import error as PyECLibError
import sys
pyver = float('%s.%s' % sys.version_info[:2])
+
class ECPyECLibDriver(object):
def __init__(self, k, m, ec_type,
@@ -80,21 +83,9 @@ class ECPyECLibDriver(object):
raise ECDriverError("Invalid fragment payload in ECPyECLibDriver.decode")
if len(fragment_payloads) < self.k:
- raise ECDriverInsufficientFragmentError("Not enough fragments given in ECPyECLibDriver.decode")
-
- try:
- ret = pyeclib_c.decode(self.handle, fragment_payloads, fragment_len, ranges, force_metadata_checks)
- except PyECLibError as e:
- if e.message.find("Insufficient number of fragments") < 0:
- raise ECDriverError(e)
- else:
- raise ECDriverInsufficientFragmentError(e)
+ raise ECInsufficientFragments("Not enough fragments given in ECPyECLibDriver.decode")
- # Was there an error decoding
- if ret is None:
- raise ECDriverError("Error decoding from fragments in ECPyECLibDriver.decode")
-
- return ret
+ return pyeclib_c.decode(self.handle, fragment_payloads, fragment_len, ranges, force_metadata_checks)
def reconstruct(self, fragment_payloads, indexes_to_reconstruct):
fragment_len = self._validate_and_return_fragment_size(fragment_payloads)
@@ -112,48 +103,33 @@ class ECPyECLibDriver(object):
while len(_indexes_to_reconstruct) > 0:
index = _indexes_to_reconstruct.pop(0)
- try:
- reconstructed = pyeclib_c.reconstruct(
- self.handle, _fragment_payloads, fragment_len, index)
- except PyECLibError as e:
- raise ECDriverError(e)
+ reconstructed = pyeclib_c.reconstruct(
+ self.handle, _fragment_payloads, fragment_len, index)
reconstructed_data.append(reconstructed)
_fragment_payloads.append(reconstructed)
return reconstructed_data
def fragments_needed(self, reconstruct_indexes, exclude_indexes):
- try:
- required_fragments = pyeclib_c.get_required_fragments(
+ required_fragments = pyeclib_c.get_required_fragments(
self.handle, reconstruct_indexes, exclude_indexes)
- return required_fragments
- except PyECLibError as e:
- raise ECDriverError(e)
+ return required_fragments
def min_parity_fragments_needed(self):
""" FIXME - fix this to return a function of HD """
return 1
def get_metadata(self, fragment, formatted = 0):
- try:
fragment_metadata = pyeclib_c.get_metadata(self.handle, fragment, formatted)
return fragment_metadata
- except PyECLibError as e:
- raise ECDriverError(e)
def verify_stripe_metadata(self, fragment_metadata_list):
- try:
success = pyeclib_c.check_metadata(self.handle, fragment_metadata_list)
return success
- except PyECLibError as e:
- raise ECDriverError(e)
def get_segment_info(self, data_len, segment_size):
- try:
segment_info = pyeclib_c.get_segment_info(self.handle, data_len, segment_size)
return segment_info
- except PyECLibError as e:
- raise ECDriverError(e)
class ECNullDriver(object):
@@ -245,7 +221,7 @@ class ECStripingDriver(object):
raise ECDriverError(
"Decode does not support metadata integrity checks in the striping driver.")
if len(fragment_payloads) != self.k:
- raise ECDriverError(
+ raise ECInsufficientFragments(
"Decode requires %d fragments, %d fragments were given" %
(len(fragment_payloads), self.k))
diff --git a/pyeclib/ec_iface.py b/pyeclib/ec_iface.py
index d802dd7..48f4fd0 100644
--- a/pyeclib/ec_iface.py
+++ b/pyeclib/ec_iface.py
@@ -111,25 +111,6 @@ class PyECLib_FRAGHDRCHKSUM_Types(PyECLibEnum):
inline_crc32 = 2
-# Generic ECDriverException
-class ECDriverError(Exception):
- def __init__(self, error):
- try:
- self.error_str = str(error)
- except:
- self.error_str = 'Error retrieving the error message from %s' \
- % error.__class__.__name__
- def __str__(self):
- return self.error_str
-
-# Exception specifying insufficient fragments specified for
-# decode or reconstruction
-class ECDriverInsufficientFragmentError(Exception):
- def __init__(self, *args):
- super(ECDriverInsufficientFragmentError, self).__init__(*args)
- def __str__(self):
- return self.error_str
-
# Main ECDriver class
class ECDriver(object):
@@ -158,7 +139,7 @@ class ECDriver(object):
self.ec_type = \
PyECLib_EC_Types.get_by_name(value)
else:
- raise ECDriverError(
+ raise ECBackendNotSupported(
"%s is not a valid EC type for PyECLib!" % value)
elif key == "chksum_type":
if PyECLib_FRAGHDRCHKSUM_Types.has_enum(value):
@@ -224,7 +205,7 @@ class ECDriver(object):
:param fragment_payloads: a list of buffers representing a subset of
the list generated by encode()
- :param ranges (optional): a list of byte ranges to return instead of
+ :param ranges (optional): a list of byte ranges to return instead of
the entire buffer
:param force_metadata_checks (optional): validate collective integrity
of the fragments before trying to decode
@@ -402,3 +383,70 @@ class ECDriver(object):
recipe[r] = segment_map
return recipe
+
+
+# PyECLib Exceptions
+
+# Generic ECDriverException
+class ECDriverError(Exception):
+ def __init__(self, error):
+ try:
+ self.error_str = str(error)
+ except Exception:
+ self.error_str = 'Error retrieving the error message from %s' \
+ % error.__class__.__name__
+
+ def __str__(self):
+ return self.error_str
+
+
+## More specific exceptions, mapped to liberasurecode error codes
+
+# Specified EC backend is not supported by PyECLib/liberasurecode
+class ECBackendNotSupported(ECDriverError):
+ pass
+
+
+# Unsupported EC method
+class ECMethodNotImplemented(ECDriverError):
+ pass
+
+
+# liberasurecode backend init error
+class ECBackendInitializationError(ECDriverError):
+ pass
+
+
+# Specified backend instance is invalid/unavailable
+class ECBackendInstanceNotAvailable(ECDriverError):
+ pass
+
+
+# Specified backend instance is busy
+class ECBackendInstanceInUse(ECDriverError):
+ pass
+
+
+# Invalid parameter passed to a method
+class ECInvalidParameter(ECDriverError):
+ pass
+
+
+# Invalid or incompatible fragment metadata
+class ECInvalidFragmentMetadata(ECDriverError):
+ pass
+
+
+# Fragment checksum verification failed
+class ECBadFragmentChecksum(ECDriverError):
+ pass
+
+
+# Insufficient fragments specified for decode or reconstruct operation
+class ECInsufficientFragments(ECDriverError):
+ pass
+
+
+# Out of memory
+class ECOutOfMemory(ECDriverError):
+ pass
diff --git a/src/c/pyeclib_c/pyeclib_c.c b/src/c/pyeclib_c/pyeclib_c.c
index 47dfa9f..554a8ed 100644
--- a/src/c/pyeclib_c/pyeclib_c.c
+++ b/src/c/pyeclib_c/pyeclib_c.c
@@ -63,8 +63,6 @@
#endif
-static PyObject *PyECLibError;
-
typedef struct pyeclib_byte_range {
uint64_t offset;
uint64_t length;
@@ -82,38 +80,63 @@ static PyObject * pyeclib_c_decode(PyObject *self, PyObject *args);
static PyObject * pyeclib_c_get_metadata(PyObject *self, PyObject *args);
static PyObject * pyeclib_c_check_metadata(PyObject *self, PyObject *args);
+static PyObject *import_class(const char *module, const char *cls)
+{
+ PyObject *s = PyImport_ImportModule(module);
+ return (PyObject *) PyObject_GetAttrString(s, cls);
+}
+
void
-liberasurecode_errstr(int ret, const char * prefix, char str[])
+pyeclib_c_seterr(int ret, const char * prefix)
{
- strcpy(str, prefix);
+ char err[255];
+ PyObject *eo = NULL;
+
+ strcpy(err, prefix);
switch (ret) {
case -EBACKENDNOTAVAIL:
- strcat(str, "Backend instance not found");
+ eo = import_class("pyeclib.ec_iface",
+ "ECBackendInstanceNotAvailable");
+ strcat(err, "Backend instance not found");
break;
case -EINSUFFFRAGS:
- strcat(str, "Insufficient number of fragments");
+ eo = import_class("pyeclib.ec_iface",
+ "ECInsufficientFragments");
+ strcat(err, "Insufficient number of fragments");
break;
case -EBACKENDNOTSUPP:
- strcat(str, "Backend not supported");
+ eo = import_class("pyeclib.ec_iface",
+ "ECBackendNotSupported");
+ strcat(err, "Backend not supported");
break;
case -EINVALIDPARAMS:
- strcat(str, "Invalid arguments");
+ eo = import_class("pyeclib.ec_iface",
+ "ECInvalidParameter");
+ strcat(err, "Invalid arguments");
break;
case -EBADCHKSUM:
- strcat(str, "Fragment integrity check failed");
+ eo = import_class("pyeclib.ec_iface",
+ "ECBadFragmentChecksum");
+ strcat(err, "Fragment integrity check failed");
break;
case -EBADHEADER:
- strcat(str, "Fragment integrity check failed");
+ eo = import_class("pyeclib.ec_iface",
+ "ECInvalidFragmentMetadata");
+ strcat(err, "Fragment integrity check failed");
break;
case -ENOMEM:
- strcat(str, "Out of memory");
+ eo = import_class("pyeclib.ec_iface",
+ "ECOutOfMemory");
+ strcat(err, "Out of memory");
break;
default:
- strcat(str, "Unknown error");
+ eo = import_class("pyeclib.ec_iface",
+ "ECDriverError");
+ strcat(err, "Unknown error");
break;
}
-
- strcat(str, ". Please inspect syslog for liberasurecode error report.");
+ strcat(err, ". Please inspect syslog for liberasurecode error report.");
+ PyErr_SetString(eo, err);
}
/**
@@ -136,13 +159,14 @@ pyeclib_c_init(PyObject *self, PyObject *args)
/* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "iii|iii", &k, &m, &backend_id, &hd, &use_inline_chksum, &use_algsig_chksum)) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.init");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_init ERROR: ");
return NULL;
}
/* Allocate and initialize the pyeclib object */
pyeclib_handle = (pyeclib_t *) alloc_zeroed_buffer(sizeof(pyeclib_t));
if (NULL == pyeclib_handle) {
+ pyeclib_c_seterr(-ENOMEM, "pyeclib_c_init ERROR: ");
goto error;
}
@@ -153,7 +177,7 @@ pyeclib_c_init(PyObject *self, PyObject *args)
pyeclib_handle->ec_desc = liberasurecode_instance_create(backend_id, &(pyeclib_handle->ec_args));
if (pyeclib_handle->ec_desc <= 0) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to liberasurecode_instance_create");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_init ERROR: ");
goto error;
}
@@ -169,7 +193,7 @@ pyeclib_c_init(PyObject *self, PyObject *args)
/* Clean up the allocated memory on error, or update the ref count */
if (pyeclib_obj_handle == NULL) {
- PyErr_SetString(PyECLibError, "Could not encapsulate pyeclib_handle into Python object in pyeclib.init");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_init ERROR: ");
goto error;
} else {
Py_INCREF(pyeclib_obj_handle);
@@ -195,13 +219,13 @@ pyeclib_c_destructor(PyObject *obj)
pyeclib_t *pyeclib_handle = NULL; /* pyeclib object to destroy */
if (!PyCapsule_CheckExact(obj)) {
- PyErr_SetString(PyECLibError, "Attempted to free a non-Capsule object in pyeclib");
+ pyeclib_c_seterr(-1, "pyeclib_c_destructor ERROR: ");
return;
}
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(obj, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Attempted to free an invalid reference to pyeclib_handle");
+ pyeclib_c_seterr(-1, "pyeclib_c_destructor ERROR: ");
} else {
check_and_free_buffer(pyeclib_handle);
}
@@ -252,12 +276,12 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args)
/* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "Oii", &pyeclib_obj_handle, &data_len, &segment_size)) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info ERROR: ");
return NULL;
}
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_segment_info ERROR: ");
return NULL;
}
@@ -324,7 +348,7 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args)
/* Create and return the python dictionary of segment info */
ret_dict = PyDict_New();
if (NULL == ret_dict) {
- PyErr_SetString(PyECLibError, "Error allocating python dict in get_segment_info");
+ pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_segment_info ERROR: ");
} else {
PyDict_SetItem(ret_dict, PyString_FromString("segment_size\0"), PyInt_FromLong(segment_size));
PyDict_SetItem(ret_dict, PyString_FromString("last_segment_size\0"), PyInt_FromLong(last_segment_size));
@@ -360,28 +384,25 @@ pyeclib_c_encode(PyObject *self, PyObject *args)
/* Assume binary data (force "byte array" input) */
if (!PyArg_ParseTuple(args, ENCODE_ARGS, &pyeclib_obj_handle, &data, &data_len)) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode ERROR: ");
return NULL;
}
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode ERROR: ");
return NULL;
}
ret = liberasurecode_encode(pyeclib_handle->ec_desc, data, data_len, &encoded_data, &encoded_parity, &fragment_len);
-
if (ret < 0) {
- char err[255];
- liberasurecode_errstr(ret, "Encode ERROR: ", err);
- PyErr_SetString(PyECLibError, err);
+ pyeclib_c_seterr(ret, "pyeclib_c_encode ERROR: ");
return NULL;
}
/* Create the python list of fragments to return */
list_of_strips = PyList_New(pyeclib_handle->ec_args.k + pyeclib_handle->ec_args.m);
if (NULL == list_of_strips) {
- PyErr_SetString(PyECLibError, "Error allocating python list in encode");
+ pyeclib_c_seterr(-ENOMEM, "pyeclib_c_encode ERROR: ");
return NULL;
}
@@ -431,12 +452,12 @@ pyeclib_c_get_required_fragments(PyObject *self, PyObject *args)
/* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "OOO", &pyeclib_obj_handle, &reconstruct_list, &exclude_list)) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.get_required_fragments");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_required_fragments ERROR: ");
return NULL;
}
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.get_required_fragments");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_required_fragments ERROR: ");
return NULL;
}
k = pyeclib_handle->ec_args.k;
@@ -446,6 +467,7 @@ pyeclib_c_get_required_fragments(PyObject *self, PyObject *args)
num_missing = (int) PyList_Size(reconstruct_list);
c_reconstruct_list = (int *) alloc_zeroed_buffer((num_missing + 1) * sizeof(int));
if (NULL == c_reconstruct_list) {
+ pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments ERROR: ");
return NULL;
}
c_reconstruct_list[num_missing] = -1;
@@ -458,6 +480,7 @@ pyeclib_c_get_required_fragments(PyObject *self, PyObject *args)
num_exclude = (int) PyList_Size(exclude_list);
c_exclude_list = (int *) alloc_zeroed_buffer((num_exclude + 1) * sizeof(int));
if (NULL == c_exclude_list) {
+ pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments ERROR: ");
goto exit;
}
c_exclude_list[num_exclude] = -1;
@@ -469,21 +492,21 @@ pyeclib_c_get_required_fragments(PyObject *self, PyObject *args)
fragments_needed = alloc_zeroed_buffer(sizeof(int) * (k + m));
if (NULL == fragments_needed) {
+ pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments ERROR: ");
goto exit;
}
ret = liberasurecode_fragments_needed(pyeclib_handle->ec_desc, c_reconstruct_list,
c_exclude_list, fragments_needed);
if (ret < 0) {
- char err[255];
- liberasurecode_errstr(ret, "Reconstruct_Fragments_Needed ERROR: ", err);
- PyErr_SetString(PyECLibError, err);
+ pyeclib_c_seterr(ret, "pyeclib_c_get_required_fragments ERROR: ");
goto exit;
}
/* Post-process into a Python list */
fragment_idx_list = PyList_New(0);
if (NULL == fragment_idx_list) {
+ pyeclib_c_seterr(-ENOMEM, "pyeclib_c_get_required_fragments ERROR: ");
goto exit;
}
@@ -532,18 +555,18 @@ pyeclib_c_reconstruct(PyObject *self, PyObject *args)
/* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "OOii", &pyeclib_obj_handle, &fragments,
&fragment_len, &destination_idx)) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.reconstruct");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_reconstruct ERROR: ");
return NULL;
}
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.reconstruct");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_reconstruct ERROR: ");
return NULL;
}
/* Pre-processing Python data structures */
if (!PyList_Check(fragments)) {
- PyErr_SetString(PyECLibError, "Invalid structure passed in for fragment list");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_reconstruct ERROR: ");
return NULL;
}
@@ -551,11 +574,13 @@ pyeclib_c_reconstruct(PyObject *self, PyObject *args)
c_fragments = (char **) alloc_zeroed_buffer(sizeof(char *) * num_fragments);
if (NULL == c_fragments) {
+ pyeclib_c_seterr(-ENOMEM, "pyeclib_c_reconstruct ERROR: ");
goto error;
}
c_reconstructed = (char*) alloc_zeroed_buffer(sizeof(char) * fragment_len);
if (NULL == c_fragments) {
+ pyeclib_c_seterr(-ENOMEM, "pyeclib_c_reconstruct ERROR: ");
goto error;
}
@@ -573,9 +598,7 @@ pyeclib_c_reconstruct(PyObject *self, PyObject *args)
destination_idx,
c_reconstructed);
if (ret < 0) {
- char err[255];
- liberasurecode_errstr(ret, "Reconstruct ERROR: ", err);
- PyErr_SetString(PyECLibError, err);
+ pyeclib_c_seterr(ret, "pyeclib_c_reconstruct ERROR: ");
reconstructed = NULL;
} else {
reconstructed = PY_BUILDVALUE_OBJ_LEN(c_reconstructed, fragment_len);
@@ -627,7 +650,7 @@ pyeclib_c_decode(PyObject *self, PyObject *args)
/* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "OOi|OO",&pyeclib_obj_handle, &fragments,
&fragment_len, &ranges, &metadata_checks_obj)) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.decode");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode ERROR: ");
return NULL;
}
@@ -643,11 +666,11 @@ pyeclib_c_decode(PyObject *self, PyObject *args)
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.decode");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode ERROR: ");
return NULL;
}
if (!PyList_Check(fragments)) {
- PyErr_SetString(PyECLibError, "Invalid structure passed in for available fragments in pyeclib.decode");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode ERROR: ");
return NULL;
}
@@ -658,14 +681,14 @@ pyeclib_c_decode(PyObject *self, PyObject *args)
}
if (pyeclib_handle->ec_args.k > num_fragments) {
- PyErr_SetString(PyECLibError, "The fragment list does not have enough entries in pyeclib.decode");
+ pyeclib_c_seterr(-EINSUFFFRAGS, "pyeclib_c_decode ERROR: ");
return NULL;
}
if (num_ranges > 0) {
c_ranges = (pyeclib_byte_range_t*)malloc(sizeof(pyeclib_byte_range_t) * num_ranges);
if (NULL == c_ranges) {
- PyErr_SetString(PyECLibError, "Could not allocate memory in pyeclib_c.decode");
+ pyeclib_c_seterr(-ENOMEM, "pyeclib_c_decode ERROR: ");
goto error;
}
for (i = 0; i < num_ranges; i++) {
@@ -675,7 +698,7 @@ pyeclib_c_decode(PyObject *self, PyObject *args)
PyObject *py_end = PyTuple_GetItem(tuple, 1);
if (!PyLong_Check(py_begin) || !PyLong_Check(py_end)) {
- PyErr_SetString(PyECLibError, "Invalid range passed to pyeclib.decode");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range ERROR: ");
goto error;
}
@@ -683,7 +706,7 @@ pyeclib_c_decode(PyObject *self, PyObject *args)
c_ranges[i].length = PyLong_AsLong(py_end) - c_ranges[i].offset + 1;
range_payload_size += c_ranges[i].length;
} else {
- PyErr_SetString(PyECLibError, "Invalid range passed to pyeclib.decode");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range ERROR: ");
goto error;
}
}
@@ -710,9 +733,7 @@ pyeclib_c_decode(PyObject *self, PyObject *args)
&orig_data_size);
if (ret < 0) {
- char err[255];
- liberasurecode_errstr(ret, "Decode ERROR: ", err);
- PyErr_SetString(PyECLibError, err);
+ pyeclib_c_seterr(ret, "pyeclib_c_decode ERROR: ");
goto error;
}
@@ -721,14 +742,14 @@ pyeclib_c_decode(PyObject *self, PyObject *args)
} else {
ret_payload = PyList_New(num_ranges);
if (NULL == ret_payload) {
- PyErr_SetString(PyECLibError, "Could not alloc list for range payloads in pyeclib.decode");
+ pyeclib_c_seterr(-ENOMEM, "pyeclib_c_decode ERROR: ");
goto error;
}
range_payload_size = 0;
for (i = 0; i < num_ranges; i++) {
/* Check that range is within the original buffer */
if (c_ranges[i].offset + c_ranges[i].length > orig_data_size) {
- PyErr_SetString(PyECLibError, "Invalid range passed to pyeclib.decode");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_decode invalid range ERROR: ");
goto error;
}
PyList_SET_ITEM(ret_payload, i,
@@ -846,32 +867,32 @@ fragment_metadata_to_dict(fragment_metadata_t *fragment_metadata)
metadata_dict = PyDict_New();
if (metadata_dict == NULL) {
- PyErr_SetString(PyECLibError, "Could not allocate dictionary for fragment metadata");
+ pyeclib_c_seterr(-ENOMEM, "fragment_metadata_to_dict ERROR: ");
return NULL;
}
if (PyDict_SetItemString(metadata_dict, "index",
PyLong_FromLong((unsigned long)fragment_metadata->idx)) < 0) {
- PyErr_SetString(PyECLibError, "Error parsing index from fragment metadata");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "fragment_metadata_to_dict index ERROR: ");
return NULL;
}
if (PyDict_SetItemString(metadata_dict, "size",
PyLong_FromLong(fragment_metadata->size)) < 0) {
- PyErr_SetString(PyECLibError, "Error parsing size from fragment metadata");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "fragment_metadata_to_dict size ERROR: ");
return NULL;
}
if (PyDict_SetItemString(metadata_dict, "orig_data_size",
PyLong_FromLong(fragment_metadata->orig_data_size)) < 0) {
- PyErr_SetString(PyECLibError, "Error parsing orig_data_size from fragment metadata");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "fragment_metadata_to_dict orig_data_size ERROR: ");
return NULL;
}
const char *chksum_type_str = chksum_type_to_str(fragment_metadata->chksum_type);
if (PyDict_SetItemString(metadata_dict, "chksum_type",
PyString_FromString(chksum_type_str)) < 0) {
- PyErr_SetString(PyECLibError, "Error parsing chksum_type from fragment metadata");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "fragment_metadata_to_dict chksum_type ERROR: ");
return NULL;
}
@@ -879,27 +900,26 @@ fragment_metadata_to_dict(fragment_metadata_t *fragment_metadata)
chksum_length(fragment_metadata->chksum_type));
if (PyDict_SetItemString(metadata_dict, "chksum",
PyString_FromString(encoded_chksum)) < 0) {
- PyErr_SetString(PyECLibError, "Error parsing chksum from fragment metadata");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "fragment_metadata_to_dict chksum ERROR: ");
return NULL;
}
if (PyDict_SetItemString(metadata_dict, "chksum_mismatch",
PyLong_FromLong((unsigned long)fragment_metadata->chksum_mismatch)) < 0) {
-
- PyErr_SetString(PyECLibError, "Error parsing chksum mismatch from fragment metadata");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "fragment_metadata_to_dict chksum_mismatch ERROR: ");
return NULL;
}
const char *backend_id_str = backend_id_to_str(fragment_metadata->backend_id);
if (PyDict_SetItemString(metadata_dict, "backend_id",
PyString_FromString(backend_id_str)) < 0) {
- PyErr_SetString(PyECLibError, "Error parsing backend id from fragment metadata");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "fragment_metadata_to_dict backend_id ERROR: ");
return NULL;
}
if (PyDict_SetItemString(metadata_dict, "backend_version",
PyLong_FromLong((unsigned long)fragment_metadata->backend_version)) < 0) {
- PyErr_SetString(PyECLibError, "Error parsing backend version from fragment metadata");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "fragment_metadata_to_dict backend_version ERROR: ");
return NULL;
}
@@ -928,28 +948,26 @@ pyeclib_c_get_metadata(PyObject *self, PyObject *args)
/* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, GET_METADATA_ARGS, &pyeclib_obj_handle, &fragment, &fragment_len, &formatted)) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.get_metadata");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_metadata ERROR: ");
return NULL;
}
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.get_required_fragments");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_get_metadata ERROR: ");
return NULL;
}
ret = liberasurecode_get_fragment_metadata(fragment, &c_fragment_metadata);
if (ret < 0) {
- char err[255];
- liberasurecode_errstr(ret, "Get_Fragment_Metadata ERROR: ", err);
- PyErr_SetString(PyECLibError, err);
+ pyeclib_c_seterr(ret, "pyeclib_c_get_metadata ERROR: ");
fragment_metadata = NULL;
} else {
if (formatted) {
fragment_metadata = fragment_metadata_to_dict(&c_fragment_metadata);
} else {
fragment_metadata = PY_BUILDVALUE_OBJ_LEN((char*)&c_fragment_metadata,
- sizeof(fragment_metadata_t));
+ sizeof(fragment_metadata_t));
}
}
@@ -986,19 +1004,19 @@ pyeclib_c_check_metadata(PyObject *self, PyObject *args)
/* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_metadata_list)) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode ERROR: ");
return NULL;
}
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode ERROR: ");
return NULL;
}
k = pyeclib_handle->ec_args.k;
m = pyeclib_handle->ec_args.m;
num_fragments = k + m;
if (num_fragments != PyList_Size(fragment_metadata_list)) {
- PyErr_SetString(PyECLibError, "Not enough fragment metadata to perform integrity check");
+ pyeclib_c_seterr(-EINVALIDPARAMS, "pyeclib_c_encode ERROR: ");
return NULL;
}
@@ -1006,6 +1024,7 @@ pyeclib_c_check_metadata(PyObject *self, PyObject *args)
size = sizeof(char * ) * num_fragments;
c_fragment_metadata_list = (char **) alloc_zeroed_buffer(size);
if (NULL == c_fragment_metadata_list) {
+ pyeclib_c_seterr(-ENOMEM, "pyeclib_c_encode ERROR: ");
goto error;
}
@@ -1072,13 +1091,5 @@ MOD_INIT(pyeclib_c)
if (m == NULL)
return MOD_ERROR_VAL;
- PyECLibError = PyErr_NewException("pyeclib.Error", NULL, NULL);
- if (PyECLibError == NULL) {
- fprintf(stderr, "Could not create default PyECLib exception object!\n");
- exit(2);
- }
- Py_INCREF(PyECLibError);
- PyModule_AddObject(m, "error", PyECLibError);
-
return MOD_SUCCESS_VAL(m);
}
diff --git a/test/test_pyeclib_api.py b/test/test_pyeclib_api.py
index 71552c1..21ebed6 100644
--- a/test/test_pyeclib_api.py
+++ b/test/test_pyeclib_api.py
@@ -26,12 +26,12 @@ from string import ascii_letters, ascii_uppercase, digits
import sys
import tempfile
import unittest
+from pyeclib.ec_iface import ECBackendNotSupported
from pyeclib.ec_iface import ECDriverError
+from pyeclib.ec_iface import ECInsufficientFragments
-from pyeclib.ec_iface import ECDriver, VALID_EC_TYPES, ECDriverError, \
- PyECLib_EC_Types
+from pyeclib.ec_iface import ECDriver, PyECLib_EC_Types
from test_pyeclib_c import _available_backends
-import pyeclib_c
if sys.version < '3':
def b2i(b):
@@ -113,18 +113,31 @@ class TestPyECLibDriver(unittest.TestCase):
print("Skipping test for %s backend" % _type)
continue
try:
- if _type is 'shss':
- _instance = ECDriver(k=10, m=4, ec_type=_type)
- else:
- _instance = ECDriver(k=10, m=5, ec_type=_type)
+ if _type is 'shss':
+ _instance = ECDriver(k=10, m=4, ec_type=_type)
+ else:
+ _instance = ECDriver(k=10, m=5, ec_type=_type)
except ECDriverError:
- self.fail("%s algorithm not supported" % _type)
+ self.fail("%p: %s algorithm not supported" % _instance, _type)
+
+ self.assertRaises(ECBackendNotSupported, ECDriver, k=10, m=5,
+ ec_type="invalid_algo")
def test_small_encode(self):
pyeclib_drivers = []
pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
+ pyeclib_drivers.append(ECDriver(k=8, m=4, ec_type="jerasure_rs_vand"))
+
+ pyeclib_drivers.append(ECDriver(k=12, m=2,
+ ec_type="liberasurecode_rs_vand"))
+ pyeclib_drivers.append(ECDriver(k=11, m=2,
+ ec_type="liberasurecode_rs_vand"))
+ pyeclib_drivers.append(ECDriver(k=10, m=2,
+ ec_type="liberasurecode_rs_vand"))
+ pyeclib_drivers.append(ECDriver(k=8, m=4,
+ ec_type="liberasurecode_rs_vand"))
encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"]
@@ -197,85 +210,102 @@ class TestPyECLibDriver(unittest.TestCase):
# pyeclib_driver.verify_stripe_metadata(fragment_metadata_list) == -1)
#
- def get_available_backend(self, k, m, ec_type, chksum_type = "inline_crc32"):
- if ec_type[:11] == "flat_xor_hd":
- return ECDriver(k=k, m=m, ec_type="flat_xor_hd", chksum_type=chksum_type)
- elif ec_type in self.available_backends:
- return ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type)
- else:
- return None
+ def check_metadata_formatted(self, k, m, ec_type, chksum_type):
- def test_get_metadata_formatted(self):
- pyeclib_driver = ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")
-
filesize = 1024 * 1024 * 3
- file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
+ file_str = ''.join(random.choice(ascii_letters)
+ for i in range(filesize))
file_bytes = file_str.encode('utf-8')
-
+
+ pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type,
+ chksum_type=chksum_type)
+
fragments = pyeclib_driver.encode(file_bytes)
- i = 0
+ f = 0
for fragment in fragments:
- metadata = pyeclib_driver.get_metadata(fragment, 1)
- if metadata.has_key('index'):
- self.assertEqual(metadata['index'], i)
- else:
- self.assertTrue(false)
-
- if metadata.has_key('chksum_mismatch'):
- self.assertEqual(metadata['chksum_mismatch'], 0)
- else:
- self.assertTrue(false)
-
- if metadata.has_key('backend_id'):
- self.assertEqual(metadata['backend_id'], 'jerasure_rs_vand')
- else:
- self.assertTrue(false)
-
- if metadata.has_key('orig_data_size'):
- self.assertEqual(metadata['orig_data_size'], 3145728)
- else:
- self.assertTrue(false)
-
- if metadata.has_key('chksum_type'):
- self.assertEqual(metadata['chksum_type'], 'crc32')
- else:
- self.assertTrue(false)
-
- if not metadata.has_key('backend_version'):
- self.assertTrue(false)
-
- if not metadata.has_key('chksum'):
- self.assertTrue(false)
-
- if not metadata.has_key('size'):
- self.assertTrue(false)
-
- i += 1
-
+ metadata = pyeclib_driver.get_metadata(fragment, 1)
+ if 'index' in metadata:
+ self.assertEqual(metadata['index'], f)
+ else:
+ self.assertTrue(False)
+
+ if 'chksum_mismatch' in metadata:
+ self.assertEqual(metadata['chksum_mismatch'], 0)
+ else:
+ self.assertTrue(False)
+
+ if 'backend_id' in metadata:
+ self.assertEqual(metadata['backend_id'], ec_type)
+ else:
+ self.assertTrue(False)
+ if 'orig_data_size' in metadata:
+ self.assertEqual(metadata['orig_data_size'], 3145728)
+ else:
+ self.assertTrue(False)
+
+ if 'chksum_type' in metadata:
+ self.assertEqual(metadata['chksum_type'], 'crc32')
+ else:
+ self.assertTrue(False)
+
+ if 'backend_version' not in metadata:
+ self.assertTrue(False)
+
+ if 'chksum' not in metadata:
+ self.assertTrue(False)
+
+ if 'size' not in metadata:
+ self.assertTrue(False)
+
+ f += 1
+
+ def test_get_metadata_formatted(self):
+ self.check_metadata_formatted(12, 2, "jerasure_rs_vand",
+ "inline_crc32")
+ self.check_metadata_formatted(12, 2, "liberasurecode_rs_vand",
+ "inline_crc32")
+ self.check_metadata_formatted(8, 4, "liberasurecode_rs_vand",
+ "inline_crc32")
def test_verify_fragment_inline_chksum_fail(self):
pyeclib_drivers = []
pyeclib_drivers.append(
- ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
+ ECDriver(k=12, m=2, ec_type="jerasure_rs_vand",
+ chksum_type="inline_crc32"))
pyeclib_drivers.append(
- ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
+ ECDriver(k=12, m=3, ec_type="jerasure_rs_vand",
+ chksum_type="inline_crc32"))
pyeclib_drivers.append(
- ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
+ ECDriver(k=12, m=4, ec_type="jerasure_rs_vand",
+ chksum_type="inline_crc32"))
pyeclib_drivers.append(
- ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
+ ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy",
+ chksum_type="inline_crc32"))
+
+ pyeclib_drivers.append(
+ ECDriver(k=12, m=2, ec_type="liberasurecode_rs_vand",
+ chksum_type="inline_crc32"))
+ pyeclib_drivers.append(
+ ECDriver(k=12, m=3, ec_type="liberasurecode_rs_vand",
+ chksum_type="inline_crc32"))
+ pyeclib_drivers.append(
+ ECDriver(k=12, m=4, ec_type="liberasurecode_rs_vand",
+ chksum_type="inline_crc32"))
+ pyeclib_drivers.append(
+ ECDriver(k=12, m=2, ec_type="liberasurecode_rs_vand",
+ chksum_type="inline_crc32"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
file_bytes = file_str.encode('utf-8')
-
for pyeclib_driver in pyeclib_drivers:
fragments = pyeclib_driver.encode(file_bytes)
fragment_metadata_list = []
-
+
first_fragment_to_corrupt = random.randint(0, len(fragments))
num_to_corrupt = 2
fragments_to_corrupt = [
@@ -340,6 +370,12 @@ class TestPyECLibDriver(unittest.TestCase):
ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
+ pyeclib_drivers.append(
+ ECDriver(k=12, m=2, ec_type="liberasurecode_rs_vand"))
+ pyeclib_drivers.append(
+ ECDriver(k=11, m=2, ec_type="liberasurecode_rs_vand"))
+ pyeclib_drivers.append(
+ ECDriver(k=10, m=2, ec_type="liberasurecode_rs_vand"))
file_size = 1024 * 1024
segment_size = 3 * 1024
@@ -373,6 +409,12 @@ class TestPyECLibDriver(unittest.TestCase):
ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
+ pyeclib_drivers.append(
+ ECDriver(k=12, m=2, ec_type="liberasurecode_rs_vand"))
+ pyeclib_drivers.append(
+ ECDriver(k=11, m=2, ec_type="liberasurecode_rs_vand"))
+ pyeclib_drivers.append(
+ ECDriver(k=10, m=2, ec_type="liberasurecode_rs_vand"))
file_sizes = [
1024 * 1024,
@@ -451,6 +493,12 @@ class TestPyECLibDriver(unittest.TestCase):
ECDriver(k=12, m=6, ec_type="flat_xor_hd"))
pyeclib_drivers.append(
ECDriver(k=10, m=5, ec_type="flat_xor_hd"))
+ pyeclib_drivers.append(
+ ECDriver(k=12, m=2, ec_type="liberasurecode_rs_vand"))
+ pyeclib_drivers.append(
+ ECDriver(k=8, m=6, ec_type="liberasurecode_rs_vand"))
+ pyeclib_drivers.append(
+ ECDriver(k=10, m=4, ec_type="liberasurecode_rs_vand"))
for pyeclib_driver in pyeclib_drivers:
for file_size in self.file_sizes:
@@ -458,7 +506,6 @@ class TestPyECLibDriver(unittest.TestCase):
tmp_file.seek(0)
whole_file_str = tmp_file.read()
whole_file_bytes = whole_file_str.encode('utf-8')
- got_exception = False
encode_input = whole_file_bytes
orig_fragments = pyeclib_driver.encode(encode_input)
@@ -512,35 +559,33 @@ class TestPyECLibDriver(unittest.TestCase):
fragments[i] = corrupted_fragment
i += 1
- try:
- decoded_string = pyeclib_driver.decode(fragments[:], force_metadata_checks=True)
- except:
- got_exception = True
- self.assertTrue(got_exception)
-
- def test_liberasurecode_error(self):
- pyeclib_driver = self.get_available_backend(k=10, m=5, ec_type="flat_xor_hd_3")
- file_size = self.file_sizes[0]
- tmp_file = self.files[file_size]
- tmp_file.seek(0)
- whole_file_str = tmp_file.read()
- whole_file_bytes = whole_file_str.encode('utf-8')
- hit_exception = False
-
- fragments = pyeclib_driver.encode(whole_file_bytes)
-
- #
- # Test reconstructor with insufficient fragments
- #
- try:
- pyeclib_driver.reconstruct([fragments[0]], [1,2,3,4,5,6])
- except ECDriverError as e:
- hit_exception = True
- print(e.error_str.__str__())
- self.assertTrue(e.error_str.__str__().find("Insufficient number of fragments") > -1)
-
- self.assertTrue(hit_exception)
-
+ self.assertRaises(ECInsufficientFragments,
+ pyeclib_driver.decode,
+ fragments[:], force_metadata_checks=True)
+
+ def get_available_backend(self, k, m, ec_type, chksum_type="inline_crc32"):
+ if ec_type[:11] == "flat_xor_hd":
+ return ECDriver(k=k, m=m, ec_type="flat_xor_hd",
+ chksum_type=chksum_type)
+ elif ec_type in _available_backends:
+ return ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type)
+ else:
+ return None
+
+ def test_liberasurecode_insufficient_frags_error(self):
+ file_size = self.file_sizes[0]
+ tmp_file = self.files[file_size]
+ tmp_file.seek(0)
+ whole_file_str = tmp_file.read()
+ whole_file_bytes = whole_file_str.encode('utf-8')
+ for ec_type in ['flat_xor_hd_3', 'liberasurecode_rs_vand']:
+ pyeclib_driver = self.get_available_backend(
+ k=10, m=5, ec_type=ec_type)
+ fragments = pyeclib_driver.encode(whole_file_bytes)
+ self.assertRaises(ECInsufficientFragments,
+ pyeclib_driver.reconstruct,
+ [fragments[0]], [1, 2, 3, 4, 5, 6])
+
def test_min_parity_fragments_needed(self):
pyeclib_drivers = []
pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))