summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKevin Greenan <kmgreen2@gmail.com>2014-06-03 10:24:15 -0700
committerKevin Greenan <kmgreen2@gmail.com>2014-06-03 10:24:15 -0700
commit596c2f04fe4d5c3ad3b6ed196441909221727aba (patch)
treeb6b07994640a5ce5bbbaf062cb463a7231e75dec
parent50f6b5769bc11b8016538f2cde94abde4e9e4150 (diff)
parent90b1d022ed70fa43edee58268813ea7ac7154ae5 (diff)
downloadpyeclib-596c2f04fe4d5c3ad3b6ed196441909221727aba.tar.gz
Merged in issue-43 (pull request #1)
Issue-43: Memory cleanup
-rw-r--r--src/c/pyeclib_c/pyeclib_c.c1582
1 files changed, 900 insertions, 682 deletions
diff --git a/src/c/pyeclib_c/pyeclib_c.c b/src/c/pyeclib_c/pyeclib_c.c
index ecb74db..787d055 100644
--- a/src/c/pyeclib_c/pyeclib_c.c
+++ b/src/c/pyeclib_c/pyeclib_c.c
@@ -38,7 +38,6 @@
#include <pyeclib_c.h>
#include <bytesobject.h>
-
/* Python 3 compatibility macros */
#if PY_MAJOR_VERSION >= 3
#define MOD_ERROR_VAL NULL
@@ -89,6 +88,22 @@
static PyObject *PyECLibError;
+
+/**
+ * Prototypes for Python/C API methods
+ */
+static PyObject * pyeclib_c_init(PyObject *self, PyObject *args);
+static void pyeclib_c_destructor(PyObject *obj);
+static PyObject * pyeclib_c_get_segment_info(PyObject *self, PyObject *args);
+static PyObject * pyeclib_c_encode(PyObject *self, PyObject *args);
+static PyObject * pyeclib_c_fragments_to_string(PyObject *self, PyObject *args);
+static PyObject * pyeclib_c_get_fragment_partition(PyObject *self, PyObject *args);
+static PyObject * pyeclib_c_reconstruct(PyObject *self, PyObject *args);
+static PyObject * pyeclib_c_decode(PyObject *self, PyObject *args);
+static PyObject * pyeclib_c_get_metadata(PyObject *self, PyObject *args);
+static PyObject * pyeclib_c_check_metadata(PyObject *self, PyObject *args);
+
+
/*
* Determine if an address is aligned to a particular boundary
*/
@@ -172,10 +187,75 @@ void init_fragment_header(char *buf)
header->magic = PYECC_HEADER_MAGIC;
}
+
+/**
+ * Memory Management Methods
+ *
+ * The following methods provide wrappers for allocating and deallocating
+ * memory.
+ *
+ * Future discussions may want to consider moving closer to the recommended
+ * guidelines in the Python\C API reference manual. One potential issue,
+ * however, may be how we enforce memory alignment in the Python heap.
+ *
+ * 2.7: https://docs.python.org/2.7/c-api/memory.html
+ * 3.4: https://docs.python.org/3.4/c-api/memory.html
+ */
+
+/**
+ * Allocate a zero-ed buffer of a specific size. This method allocates from
+ * the Python stack in order to comply with the recommendations of the
+ * Python\C API. On error, return NULL and call PyErr_NoMemory.
+ *
+ * @param size integer size in bytes of buffer to allocate
+ * @return pointer to start of allocated buffer or NULL on error
+ */
static
-void* get_aligned_buffer16(int size)
+void * alloc_zeroed_buffer(int size)
{
- void *buf;
+ void * buf = NULL; /* buffer to allocate and return */
+
+ /* Allocate and zero the buffer, or set the appropriate error */
+ buf = PyMem_Malloc((size_t) size);
+ if (buf) {
+ buf = memset(buf, 0, (size_t) size);
+ } else {
+ buf = (void *) PyErr_NoMemory();
+ }
+
+ return buf;
+}
+
+
+/**
+ * Deallocate memory buffer. This methods returns NULL so that you can free
+ * and reset a buffer using a single line as follows:
+ *
+ * my_ptr = free_buffer(my_ptr);
+ *
+ * @return NULL
+ */
+static
+void * free_buffer(void * buf)
+{
+ if (buf) PyMem_Free(buf);
+
+ return NULL;
+}
+
+
+/**
+ * Allocate and zero out a buffer aligned to a 16-byte boundary in order
+ * to support 128-bit operations. On error, call PyErr_NoMemory to set
+ * the appropriate error string and return NULL.
+ *
+ * @param size integer size in bytes of buffer to allocate
+ * @return pointer to start of allocated buffer, or NULL on error
+ */
+static
+void* alloc_aligned_buffer16(int size)
+{
+ void *buf = NULL;
/*
* Ensure all memory is aligned to
@@ -183,43 +263,107 @@ void* get_aligned_buffer16(int size)
* 128-bit operations
*/
if (posix_memalign(&buf, 16, size) < 0) {
- return NULL;
+ return PyErr_NoMemory();
+ } else {
+ memset(buf, 0, size);
}
- bzero(buf, size);
return buf;
}
+
+/**
+ * Allocate an initialized fragment buffer. On error, return NULL and
+ * preserve call to PyErr_NoMemory. Note, all allocated memory is aligned
+ * to 16-bytes boundaries in order to support 128-bit operations.
+ *
+ * @param size integer size in bytes of buffer to allocate
+ * @return pointer to start of allocated fragment or NULL on error
+ */
static
char *alloc_fragment_buffer(int size)
{
- char *buf;
- fragment_header_t* header = NULL;
+ char *buf = NULL;
+ fragment_header_t *header = NULL;
+ /* Calculate size needed for fragment header + data */
size += sizeof(fragment_header_t);
- /*
- * Ensure all memory is aligned to
- * 16-byte boundaries to support
- * 128-bit operations
- */
- if (posix_memalign((void**)&buf, 16, size) < 0) {
- return NULL;
+ /* Allocate and init the aligned buffer, or set the appropriate error */
+ buf = alloc_aligned_buffer16(size);
+ if (buf) {
+ init_fragment_header(buf);
+ }
+
+ return buf;
+}
+
+
+/**
+ * Deallocate a fragment buffer. This method confirms via magic number that
+ * the passed in buffer is a proper fragment buffer.
+ *
+ * @param buf pointer to fragment buffer
+ * @return 0 on successful free, -1 on error
+ */
+static
+int free_fragment_buffer(char *buf)
+{
+ fragment_header_t *header;
+ if (buf == NULL) {
+ return -1;
}
- bzero(buf, size);
+
+ buf -= sizeof(fragment_header_t);
header = (fragment_header_t*)buf;
- header->magic = PYECC_HEADER_MAGIC;
+ if (header->magic != PYECC_HEADER_MAGIC) {
+ PyErr_SetString(PyECLibError, "Invalid fragment header (free fragment)!");
+ return -1;
+ }
+ free(buf);
- return buf;
+ return 0;
}
+
+/**
+ * Allocate a zero-ed Python string of a specific size. On error, call
+ * PyErr_NoMemory and return NULL.
+ *
+ * @param size integer size in bytes of zero string to create
+ * @return pointer to start of allocated zero string or NULL on error
+ */
+static
+PyObject * alloc_zero_string(int size)
+{
+ char *tmp_data = NULL; /* tmp buffer used in PyObject creation */
+ PyObject *zero_string = NULL; /* zero string to return */
+
+ /* Create the zero-out c-string buffer */
+ tmp_data = (char *) alloc_zeroed_buffer(size);
+ if (NULL == tmp_data) {
+ return PyErr_NoMemory();
+ }
+
+ /* Create the python value to return */
+ zero_string = PY_BUILDVALUE_OBJ_LEN(tmp_data, size);
+ free_buffer(tmp_data);
+
+ return zero_string;
+}
+
+
static
char* get_data_ptr_from_fragment(char *buf)
{
- buf += sizeof(fragment_header_t);
+ char * data_ptr = NULL;
- return buf;
+ if (NULL != buf) {
+ data_ptr = buf + sizeof(fragment_header_t);
+ }
+
+ return data_ptr;
}
static
@@ -385,74 +529,6 @@ int validate_fragment(char *buf)
return 0;
}
-static
-int free_fragment_buffer(char *buf)
-{
- fragment_header_t *header;
- if (buf == NULL) {
- return -1;
- }
-
- buf -= sizeof(fragment_header_t);
-
- header = (fragment_header_t*)buf;
- if (header->magic != PYECC_HEADER_MAGIC) {
- PyErr_SetString(PyECLibError, "Invalid fragment header (free fragment)!");
- return -1;
- }
- free(buf);
-
- return 0;
-}
-
-static void pyeclib_c_destructor(PyObject *obj)
-{
- pyeclib_t *pyeclib_handle = NULL;
-
- if (!PyCapsule_CheckExact(obj)) {
- PyErr_SetString(PyECLibError, "Attempted to free a non-Capsule object in pyeclib");
- return;
- }
-
- pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(obj, PYECC_HANDLE_NAME);
-
- if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Attempted to free an invalid reference to pyeclib_handle");
- return;
- }
-}
-
-static
-int get_fragment_metadata(pyeclib_t *pyeclib_handle, char *fragment_buf, fragment_metadata_t *fragment_metadata)
-{
- char *fragment_data = get_data_ptr_from_fragment(fragment_buf);
- int fragment_size = get_fragment_size(fragment_buf);
- int fragment_idx = get_fragment_idx(fragment_buf);
-
- memset(fragment_metadata, 0, sizeof(fragment_metadata_t));
-
- fragment_metadata->size = fragment_size;
- fragment_metadata->idx = fragment_idx;
-
- /*
- * If w \in [8, 16] and using RS_VAND or XOR_CODE, then use Alg_sig
- * Else use CRC32
- */
- if (supports_alg_sig(pyeclib_handle)) {
- // Compute algebraic signature
- compute_alg_sig(pyeclib_handle->alg_sig_desc, fragment_data, fragment_size, fragment_metadata->signature);
- } else if (use_inline_chksum(pyeclib_handle)) {
- int stored_chksum = get_chksum(fragment_buf);
- int computed_chksum = crc32(0, fragment_data, fragment_size);
-
- if (stored_chksum != computed_chksum) {
- fragment_metadata->chksum_mismatch = 1;
- }
- }
-
- return 0;
-}
-
/*
* Buffers for data, parity and missing_idxs
@@ -574,41 +650,116 @@ static int get_decoding_info(pyeclib_t *pyeclib_handle,
return 0;
}
+
+/**
+ * Compute a size aligned to the number of data and the underlying wordsize
+ * of the EC algorithm.
+ *
+ * @param pyeclib_handle eclib object with EC configurations
+ * @param data_len integer length of data in bytes
+ * @return integer data length aligned with wordsize of EC algorithm
+ */
+static int
+get_aligned_data_size(pyeclib_t* pyeclib_handle, int data_len)
+{
+ int word_size = pyeclib_handle->w / 8;
+ int alignment_multiple;
+ int aligned_size = 0;
+
+ /*
+ * For Cauchy reed-solomon align to k*word_size*packet_size
+ * For Vandermonde reed-solomon and flat-XOR, align to k*word_size
+ */
+ if (pyeclib_handle->type == PYECC_RS_CAUCHY_ORIG) {
+ alignment_multiple = pyeclib_handle->k * pyeclib_handle->w * PYECC_CAUCHY_PACKETSIZE;
+ } else {
+ alignment_multiple = pyeclib_handle->k * word_size;
+ }
+
+ aligned_size = (int)ceill((double)data_len / alignment_multiple) * alignment_multiple;
+
+ return aligned_size;
+}
+
+
+static
+int get_minimum_encode_size(pyeclib_t *pyeclib_handle)
+{
+ return get_aligned_data_size(pyeclib_handle, 1);
+}
+
+
+static
+int get_fragment_metadata(pyeclib_t *pyeclib_handle, char *fragment_buf, fragment_metadata_t *fragment_metadata)
+{
+ char *fragment_data = get_data_ptr_from_fragment(fragment_buf);
+ int fragment_size = get_fragment_size(fragment_buf);
+ int fragment_idx = get_fragment_idx(fragment_buf);
+
+ memset(fragment_metadata, 0, sizeof(fragment_metadata_t));
+
+ fragment_metadata->size = fragment_size;
+ fragment_metadata->idx = fragment_idx;
+
+ /*
+ * If w \in [8, 16] and using RS_VAND or XOR_CODE, then use Alg_sig
+ * Else use CRC32
+ */
+ if (supports_alg_sig(pyeclib_handle)) {
+ // Compute algebraic signature
+ compute_alg_sig(pyeclib_handle->alg_sig_desc, fragment_data, fragment_size, fragment_metadata->signature);
+ } else if (use_inline_chksum(pyeclib_handle)) {
+ int stored_chksum = get_chksum(fragment_buf);
+ int computed_chksum = crc32(0, fragment_data, fragment_size);
+
+ if (stored_chksum != computed_chksum) {
+ fragment_metadata->chksum_mismatch = 1;
+ }
+ }
+
+ return 0;
+}
+
+
+/**
+ * Constructor method for creating a new pyeclib object using the given parameters.
+ *
+ * @param k integer number of data elements
+ * @param m integer number of checksum elements
+ * @param w integer word size in bytes
+ * @param type_str string name of erasure coding algorithm
+ * @return pointer to PyObject or NULL on error
+ */
static PyObject *
pyeclib_c_init(PyObject *self, PyObject *args)
{
- pyeclib_t *pyeclib_handle;
- PyObject* pyeclib_obj_handle;
+ pyeclib_t *pyeclib_handle = NULL;
+ PyObject *pyeclib_obj_handle = NULL;
int k, m, w;
int use_inline_chksum = 0, use_algsig_chksum = 0;
const char *type_str;
pyeclib_type_t type;
+ /* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "iiis|ii", &k, &m, &w, &type_str, &use_inline_chksum, &use_algsig_chksum)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.init");
return NULL;
}
-
- // Validate the arguments
type = get_ecc_type(type_str);
if (type == PYECC_NOT_FOUND) {
PyErr_SetString(PyECLibError, "Invalid type passed to pyeclib.init");
return NULL;
}
-
if (!validate_args(k, m, w, type)) {
PyErr_SetString(PyECLibError, "Invalid args passed to pyeclib.init");
return NULL;
}
-
- pyeclib_handle = (pyeclib_t*)malloc(sizeof(pyeclib_t));
- if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "malloc() returned NULL in pyeclib.init");
+
+ /* Allocate and initialize the pyeclib object */
+ pyeclib_handle = (pyeclib_t *) alloc_zeroed_buffer(sizeof(pyeclib_t));
+ if (!pyeclib_handle) {
return NULL;
}
-
- memset(pyeclib_handle, 0, sizeof(pyeclib_t));
-
pyeclib_handle->k = k;
pyeclib_handle->m = m;
pyeclib_handle->w = w;
@@ -645,59 +796,54 @@ pyeclib_c_init(PyObject *self, PyObject *args)
break;
}
+ /* Prepare the python object to return */
#ifdef Py_CAPSULE_H
pyeclib_obj_handle = PyCapsule_New(pyeclib_handle, PYECC_HANDLE_NAME,
pyeclib_c_destructor);
#else
pyeclib_obj_handle = PyCObject_FromVoidPtrAndDesc(pyeclib_handle,
- (void *) PYECC_HANDLE_NAME,
- pyeclib_c_destructor);
+ (void *) PYECC_HANDLE_NAME,
+ pyeclib_c_destructor);
#endif /* Py_CAPSULE_H */
+ /* Clean up the allocated memory on error, or update the ref count */
if (pyeclib_obj_handle == NULL) {
PyErr_SetString(PyECLibError, "Could not encapsulate pyeclib_handle into Python object in pyeclib.init");
- return NULL;
+ free_buffer(pyeclib_handle);
+ } else {
+ Py_INCREF(pyeclib_obj_handle);
}
-
- Py_INCREF(pyeclib_obj_handle);
-
+
return pyeclib_obj_handle;
}
-static int
-get_aligned_data_size(pyeclib_t* pyeclib_handle, int data_len)
+
+/**
+ * Destructor method for cleaning up pyeclib object.
+ */
+static void
+pyeclib_c_destructor(PyObject *obj)
{
- int word_size = pyeclib_handle->w / 8;
- int alignment_multiple;
- int aligned_size = 0;
+ pyeclib_t *pyeclib_handle = NULL; /* pyeclib object to destroy */
- /*
- * For Cauchy reed-solomon align to k*word_size*packet_size
- *
- * For Vandermonde reed-solomon and flat-XOR, align to k*word_size
- */
- if (pyeclib_handle->type == PYECC_RS_CAUCHY_ORIG) {
- alignment_multiple = pyeclib_handle->k * pyeclib_handle->w * PYECC_CAUCHY_PACKETSIZE;
- } else {
- alignment_multiple = pyeclib_handle->k * word_size;
+ if (!PyCapsule_CheckExact(obj)) {
+ PyErr_SetString(PyECLibError, "Attempted to free a non-Capsule object in pyeclib");
+ return;
}
- aligned_size = (int)ceill((double)data_len / alignment_multiple) * alignment_multiple;
-
- return aligned_size;
-}
-
-static
-int get_minimum_encode_size(pyeclib_t *pyeclib_handle)
-{
- return get_aligned_data_size(pyeclib_handle, 1);
+ pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(obj, PYECC_HANDLE_NAME);
+ if (pyeclib_handle == NULL) {
+ PyErr_SetString(PyECLibError, "Attempted to free an invalid reference to pyeclib_handle");
+ } else {
+ free_buffer(pyeclib_handle);
+ }
+
+ return;
}
/**
- *
- * This function takes data length and
- * a segment size and returns an object
+ * This function takes data length and a segment size and returns an object
* containing:
*
* segment_size: size of the payload to give to encode()
@@ -706,28 +852,30 @@ int get_minimum_encode_size(pyeclib_t *pyeclib_handle)
* last_fragment_size: the fragment size returned by encode()
* num_segments: number of segments
*
- * This allows the caller to prepare requests
- * when segmenting a data stream to be EC'd.
+ * This allows the caller to prepare requests when segmenting a data stream
+ * to be EC'd.
*
- * Since the data length will rarely be aligned
- * to the segment size, the last segment will be
- * a different size than the others.
+ * Since the data length will rarely be aligned to the segment size, the last
+ * segment will be a different size than the others.
*
- * There are restrictions on the length given to encode(),
- * so calling this before encode is highly recommended when
- * segmenting a data stream.
+ * There are restrictions on the length given to encode(), so calling this
+ * before encode is highly recommended when segmenting a data stream.
*
- * Minimum segment size depends on the underlying EC type
- * (if it is less than this, then the last segment will be
- * slightly larger than the others, otherwise it will be smaller).
+ * Minimum segment size depends on the underlying EC type (if it is less
+ * than this, then the last segment will be slightly larger than the others,
+ * otherwise it will be smaller).
+ *
+ * @param pyeclib_obj_handle
+ * @param data_len integer length of data in bytes
+ * @param segment_size integer length of segment in bytes
*
*/
static PyObject *
pyeclib_c_get_segment_info(PyObject *self, PyObject *args)
{
- PyObject *pyeclib_obj_handle;
- PyObject *ret_dict;
- pyeclib_t *pyeclib_handle;
+ PyObject *pyeclib_obj_handle = NULL;
+ PyObject *ret_dict = NULL;
+ pyeclib_t *pyeclib_handle = NULL;
int data_len;
int segment_size, last_segment_size;
int num_segments;
@@ -736,11 +884,11 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args)
int aligned_segment_size;
int aligned_data_len;
+ /* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "Oii", &pyeclib_obj_handle, &data_len, &segment_size)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode");
return NULL;
}
-
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode");
@@ -829,212 +977,225 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args)
last_fragment_size += sizeof(fragment_header_t);
fragment_size += sizeof(fragment_header_t);
+ /* Create and return the python dictionary of segment info */
ret_dict = PyDict_New();
-
- PyDict_SetItem(ret_dict, PyString_FromString("segment_size\0"), PyInt_FromLong(segment_size));
- PyDict_SetItem(ret_dict, PyString_FromString("last_segment_size\0"), PyInt_FromLong(last_segment_size));
- PyDict_SetItem(ret_dict, PyString_FromString("fragment_size\0"), PyInt_FromLong(fragment_size));
- PyDict_SetItem(ret_dict, PyString_FromString("last_fragment_size\0"), PyInt_FromLong(last_fragment_size));
- PyDict_SetItem(ret_dict, PyString_FromString("num_segments\0"), PyInt_FromLong(num_segments));
-
+ if (NULL == ret_dict) {
+ PyErr_SetString(PyECLibError, "Error allocating python dict in get_segment_info");
+ } else {
+ PyDict_SetItem(ret_dict, PyString_FromString("segment_size\0"), PyInt_FromLong(segment_size));
+ PyDict_SetItem(ret_dict, PyString_FromString("last_segment_size\0"), PyInt_FromLong(last_segment_size));
+ PyDict_SetItem(ret_dict, PyString_FromString("fragment_size\0"), PyInt_FromLong(fragment_size));
+ PyDict_SetItem(ret_dict, PyString_FromString("last_fragment_size\0"), PyInt_FromLong(last_fragment_size));
+ PyDict_SetItem(ret_dict, PyString_FromString("num_segments\0"), PyInt_FromLong(num_segments));
+ }
+
return ret_dict;
}
+
+/**
+ * Erasure encode a data buffer.
+ *
+ * @param pyeclib_obj_handle
+ * @param data to encode
+ * @return python list of encoded data and parity elements
+ */
static PyObject *
pyeclib_c_encode(PyObject *self, PyObject *args)
{
- PyObject *pyeclib_obj_handle;
- pyeclib_t *pyeclib_handle;
- char *data;
- int data_len;
- int aligned_data_len;
- int orig_data_size;
- int blocksize;
- char **data_to_encode;
- char **encoded_parity;
- int i;
- PyObject *list_of_strips;
+ PyObject *pyeclib_obj_handle = NULL;
+ pyeclib_t *pyeclib_handle= NULL;
+ char **data_to_encode = NULL; /* array of k data buffers */
+ char **encoded_parity = NULL; /* array of m parity buffers */
+ PyObject *list_of_strips = NULL; /* list of encoded strips to return */
+ char *data; /* param, data buffer to encode */
+ int data_len; /* param, length of data buffer */
+ int aligned_data_len; /* EC algorithm compatible data length */
+ int orig_data_size; /* data length to write to headers */
+ int blocksize; /* length of each of k data elements */
+ int i; /* a counter */
/* Assume binary data (force "byte array" input) */
if (!PyArg_ParseTuple(args, ENCODE_ARGS, &pyeclib_obj_handle, &data, &data_len)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode");
return NULL;
}
-
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode");
return NULL;
}
-
- /*
- * Grab the original size to put in the headers
- */
- orig_data_size = data_len;
- /*
- * This will copmpute a size algined to the number of data
- * and the underlying wordsize of the EC algorithm.
- */
+ /* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/
+ orig_data_size = data_len;
aligned_data_len = get_aligned_data_size(pyeclib_handle, data_len);
-
- /*
- * aligned_data_len is guaranteed to be divisible by k
- */
blocksize = aligned_data_len / pyeclib_handle->k;
- data_to_encode = (char**)malloc(sizeof(char*)*pyeclib_handle->k);
- if (data_to_encode == NULL) {
- PyErr_SetString(PyECLibError, "Could not allocate memory in pyeclib.encode");
- return NULL;
+ /* Allocate and initialize an array of zero'd out data buffers */
+ data_to_encode = (char**) alloc_zeroed_buffer(sizeof(char*) * pyeclib_handle->k);
+ if (NULL == data_to_encode) {
+ goto error;
}
-
- for (i=0; i < pyeclib_handle->k; i++) {
- char *fragment = alloc_fragment_buffer(blocksize);
+ for (i = 0; i < pyeclib_handle->k; i++) {
int payload_size = data_len > blocksize ? blocksize : data_len;
- data_to_encode[i] = get_data_ptr_from_fragment(fragment);
- if (data_to_encode[i] == NULL) {
- PyErr_SetString(PyECLibError, "Could not allocate memory in pyeclib.encode");
- return NULL;
+ char *fragment = alloc_fragment_buffer(blocksize);
+ if (NULL == fragment) {
+ goto error;
}
-
- /*
- * Only do the memcpy if there is data to copy
- *
- * alloc_fragment_buffer() does a bzero(), so the padding
- * will be all 0s.
- */
+
+ /* Copy existing data into clean, zero'd out buffer */
+ data_to_encode[i] = get_data_ptr_from_fragment(fragment);
if (data_len > 0) {
memcpy(data_to_encode[i], data, payload_size);
}
- /*
- * Fragment size will always be the same (may be able to get rid of this)
- */
+ /* Fragment size will always be the same (may be able to get rid of this) */
set_fragment_size(fragment, blocksize);
-
data += payload_size;
data_len -= payload_size;
}
- encoded_parity = (char**)malloc(sizeof(char*)*pyeclib_handle->m);
- if (encoded_parity == NULL) {
- PyErr_SetString(PyECLibError, "Could not allocate memory in pyeclib.encode");
- return NULL;
+ /* Allocate and initialize an array of zero'd out parity buffers */
+ encoded_parity = (char**) alloc_zeroed_buffer(sizeof(char*) * pyeclib_handle->m);
+ if (NULL == encoded_parity) {
+ goto error;
}
-
- for (i=0; i < pyeclib_handle->m; i++) {
+ for (i = 0; i < pyeclib_handle->m; i++) {
char *fragment = alloc_fragment_buffer(blocksize);
- encoded_parity[i] = get_data_ptr_from_fragment(fragment);
- if (encoded_parity[i] == NULL) {
- PyErr_SetString(PyECLibError, "Could not allocate memory in pyeclib.encode");
- return NULL;
+ if (NULL == fragment) {
+ goto error;
}
+
+ encoded_parity[i] = get_data_ptr_from_fragment(fragment);
set_fragment_size(fragment, blocksize);
}
+ /* Run the erasure coding algorithm to generate the parity fragments */
switch (pyeclib_handle->type) {
case PYECC_RS_CAUCHY_ORIG:
- jerasure_bitmatrix_encode(pyeclib_handle->k, pyeclib_handle->m, pyeclib_handle->w, pyeclib_handle->bitmatrix, data_to_encode, encoded_parity, blocksize, PYECC_CAUCHY_PACKETSIZE);
+ jerasure_bitmatrix_encode(pyeclib_handle->k, pyeclib_handle->m,
+ pyeclib_handle->w, pyeclib_handle->bitmatrix,
+ data_to_encode, encoded_parity, blocksize,
+ PYECC_CAUCHY_PACKETSIZE);
break;
case PYECC_XOR_HD_3:
case PYECC_XOR_HD_4:
- pyeclib_handle->xor_code_desc->encode(pyeclib_handle->xor_code_desc, data_to_encode, encoded_parity, blocksize);
+ pyeclib_handle->xor_code_desc->encode(pyeclib_handle->xor_code_desc,
+ data_to_encode, encoded_parity,
+ blocksize);
break;
case PYECC_RS_VAND:
default:
- jerasure_matrix_encode(pyeclib_handle->k, pyeclib_handle->m, pyeclib_handle->w, pyeclib_handle->matrix, data_to_encode, encoded_parity, blocksize);
+ jerasure_matrix_encode(pyeclib_handle->k, pyeclib_handle->m,
+ pyeclib_handle->w, pyeclib_handle->matrix,
+ data_to_encode, encoded_parity, blocksize);
break;
}
+ /* Create the python list of fragments to return */
list_of_strips = PyList_New(pyeclib_handle->k + pyeclib_handle->m);
+ if (NULL == list_of_strips) {
+ PyErr_SetString(PyECLibError, "Error allocating python list in encode");
+ goto error;
+ }
- for (i=0; i < pyeclib_handle->k; i++) {
+ /* Finalize data fragments and add them to the python list to return */
+ for (i = 0; i < pyeclib_handle->k; i++) {
char *fragment_ptr = get_fragment_ptr_from_data(data_to_encode[i]);
- int fragment_size = blocksize+sizeof(fragment_header_t);
+ int fragment_size = blocksize + sizeof(fragment_header_t);
set_fragment_idx(fragment_ptr, i);
set_orig_data_size(fragment_ptr, orig_data_size);
if (use_inline_chksum(pyeclib_handle)) {
int chksum = crc32(0, data_to_encode[i], blocksize);
set_chksum(fragment_ptr, chksum);
}
- PyList_SET_ITEM(list_of_strips, i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size));
- free_fragment_buffer(data_to_encode[i]);
+ PyList_SET_ITEM(list_of_strips, i,
+ PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size));
}
- free(data_to_encode);
- for (i=0; i < pyeclib_handle->m; i++) {
+ /* Finalize parity fragments and add them to the python list to return */
+ for (i = 0; i < pyeclib_handle->m; i++) {
char *fragment_ptr = get_fragment_ptr_from_data(encoded_parity[i]);
- int fragment_size = blocksize+sizeof(fragment_header_t);
+ int fragment_size = blocksize + sizeof(fragment_header_t);
set_fragment_idx(fragment_ptr, pyeclib_handle->k+i);
set_orig_data_size(fragment_ptr, orig_data_size);
if (use_inline_chksum(pyeclib_handle)) {
int chksum = crc32(0, encoded_parity[i], blocksize);
set_chksum(fragment_ptr, chksum);
}
- PyList_SET_ITEM(list_of_strips, pyeclib_handle->k + i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size));
- free_fragment_buffer(encoded_parity[i]);
+ PyList_SET_ITEM(list_of_strips, pyeclib_handle->k + i,
+ PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size));
}
- free(encoded_parity);
+
+ goto exit;
+error:
+ list_of_strips = NULL;
+
+exit:
+ if (data_to_encode) {
+ for (i = 0; i < pyeclib_handle->k; i++) {
+ if (data_to_encode[i]) free_fragment_buffer(data_to_encode[i]);
+ }
+ free_buffer(data_to_encode);
+ }
+ if (encoded_parity) {
+ for (i = 0; i < pyeclib_handle->m; i++) {
+ if (encoded_parity[i]) free_fragment_buffer(encoded_parity[i]);
+ }
+ free_buffer(encoded_parity);
+ }
+
return list_of_strips;
}
+
/*
- * Convert a set of fragments into a string. If,
- * all k data elements are not present, return NULL;
- * otherwise return a string.
+ * Convert a set of fragments into a string. If, less than k data fragments
+ * are present, return None. Return NULL on error.
+ *
+ * @param pyeclib_obj_handle
+ * @param list of fragments
+ * @param python string or None, NULL on error
*/
static PyObject *
pyeclib_c_fragments_to_string(PyObject *self, PyObject *args)
{
- PyObject *pyeclib_obj_handle;
- PyObject *fragment_list;
- PyObject *ret_string;
- pyeclib_t* pyeclib_handle;
- char *ret_cstring;
- char **data;
- int string_len = 0;
- int string_off = 0;
- int i = 0;
- int num_fragments = 0;
- int num_data = 0;
- int orig_data_size = -1;
- int ret_data_size;
-
+ PyObject *pyeclib_obj_handle = NULL;
+ pyeclib_t *pyeclib_handle = NULL;
+ PyObject *fragment_list = NULL; /* param, python list of fragments */
+ PyObject *ret_string = NULL; /* python string to return */
+ char *ret_cstring = NULL; /* c string to build return string from */
+ int ret_data_size; /* size of return string in bytes */
+ char **data = NULL; /* array of data buffers */
+ int string_off = 0; /* offset into cstring */
+ int num_fragments = 0; /* num of fragments provided by caller */
+ int num_data = 0; /* num of fragments that are data */
+ int orig_data_size = -1; /* data size from fragment header */
+ int i = 0; /* a counter */
+
+ /* Collect and validate the method arguments */
if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_list)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.fragments_to_string");
return NULL;
}
-
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.fragments_to_string");
return NULL;
}
-
if (!PyList_Check(fragment_list)) {
PyErr_SetString(PyECLibError, "Invalid structure passed in for fragment list in pyeclib.fragments_to_string");
return NULL;
}
- // Set the number of fragments we have
- num_fragments = (int)PyList_Size(fragment_list);
-
- /*
- * If we have less than k fragments, then we definitely need to call decode, so return NULL
- */
+ /* Return None if there's insufficient fragments */
+ num_fragments = (int) PyList_Size(fragment_list);
if (pyeclib_handle->k > num_fragments) {
return Py_BuildValue("");
}
-
- data = (char**)malloc(sizeof(char*) * pyeclib_handle->k);
- if (data == NULL) {
- PyErr_SetString(PyECLibError, "Could not allocate memory for data buffers");
- return NULL;
- }
-
+
/*
* NOTE: Update to only copy original size out of the buffers
*/
@@ -1043,356 +1204,348 @@ pyeclib_c_fragments_to_string(PyObject *self, PyObject *args)
* Iterate over the fragments. If we have all k data fragments, then we can
* concatenate them into a string and return it; otherwise, we return NULL
*/
- for (i=0; i < num_fragments && num_data < pyeclib_handle->k; i++) {
+ data = (char **) alloc_zeroed_buffer(sizeof(char *) * pyeclib_handle->k);
+ if (NULL == data) {
+ return NULL;
+ }
+ for (i = 0; i < num_fragments && num_data < pyeclib_handle->k; i++) {
PyObject *tmp_data = PyList_GetItem(fragment_list, i);
char *tmp_buf;
int index;
int data_size;
Py_ssize_t len;
+ /* Get and validate the fragment index and size */
PyBytes_AsStringAndSize(tmp_data, &tmp_buf, &len);
-
- /*
- * Get fragment index and size
- */
index = get_fragment_idx(tmp_buf);
- if (index < 0) {
- ret_string = NULL;
- goto out;
- }
data_size = get_fragment_size(tmp_buf);
- if (data_size < 0) {
+ if ((index < 0) || (data_size < 0)) {
ret_string = NULL;
- goto out;
+ goto exit;
}
- /*
- * If we got this far, then we shouold
- * find a valid original data size.
- */
+ /* Validate the original data size */
if (orig_data_size < 0) {
orig_data_size = get_orig_data_size(tmp_buf);
} else {
- int my_orig_data_size = get_orig_data_size(tmp_buf);
- if (my_orig_data_size != orig_data_size) {
+ if (get_orig_data_size(tmp_buf) != orig_data_size) {
PyErr_SetString(PyECLibError, "Inconsistent orig data sizes found in headers");
ret_string = NULL;
- goto out;
+ goto exit;
}
}
- /*
- * Skip over parity fragments
- */
+ /* Skip parity fragments, put data fragments in index order */
if (index >= pyeclib_handle->k) {
continue;
+ } else {
+ data[index] = tmp_buf;
+ num_data++;
}
-
- /*
- * Put fragment reference in proper index of data
- */
- data[index] = tmp_buf;
-
- // Increment the number of data fragments we have
- num_data++;
- string_len += data_size;
}
+ /* Return None if there still isn't insufficient fragments */
if (num_data != pyeclib_handle->k) {
ret_string = Py_BuildValue("");
- goto out;
+ goto exit;
+ }
+
+ /* Create the c-string to return */
+ ret_cstring = (char *) alloc_zeroed_buffer(orig_data_size);
+ if (NULL == ret_cstring) {
+ ret_string = NULL;
+ goto exit;
}
-
- ret_cstring = (char*)malloc(orig_data_size);
-
ret_data_size = orig_data_size;
- /*
- * Copy data payloads into a cstring. The
- * fragments should be ordered by index in data.
- */
- for (i=0; i < num_data && orig_data_size > 0; i++) {
+ /* Copy fragment data into cstring (fragments should be in index order) */
+ for (i = 0; i < num_data && orig_data_size > 0; i++) {
char* fragment_data = get_data_ptr_from_fragment(data[i]);
int fragment_size = get_fragment_size(data[i]);
int payload_size = orig_data_size > fragment_size ? fragment_size : orig_data_size;
memcpy(ret_cstring + string_off, fragment_data, payload_size);
-
orig_data_size -= payload_size;
string_off += payload_size;
}
-
ret_string = PY_BUILDVALUE_OBJ_LEN(ret_cstring, ret_data_size);
- free(ret_cstring);
-out:
- free(data);
+exit:
+ free_buffer(data);
+ free_buffer(ret_cstring);
+
return ret_string;
}
+
/*
- * Return a tuple containing a refernece to a data fragment
- * list, a parity fragment list and a missing fragment index list,
- * including all '0' fragment for missing fragments. Copy
- * references to the existing fragments and create new string
- * for missing fragments.
+ * Return a tuple containing a reference to a data fragment list, a parity
+ * fragment list and a missing fragment index list, including all '0'
+ * fragment for missing fragments. Copy references to the existing
+ * fragments and create new string for missing fragments.
+ *
+ * @param pyeclib_obj_handle
+ * @param list of fragments
+ * @return (data fragments, parity fragments, missing indexes) tuple
*/
static PyObject *
pyeclib_c_get_fragment_partition(PyObject *self, PyObject *args)
{
- PyObject *pyeclib_obj_handle;
- PyObject *fragment_list;
- PyObject *data_list;
- PyObject *parity_list;
- PyObject *missing_list;
- PyObject *return_lists;
- PyObject **data;
- PyObject **parity;
- int *missing;
- pyeclib_t* pyeclib_handle;
- int num_fragments;
- int num_missing = 0;
- int i;
- int fragment_size = 0;
-
+ PyObject *pyeclib_obj_handle = NULL;
+ pyeclib_t* pyeclib_handle = NULL;
+ PyObject *fragment_list = NULL; /* param, list of fragments */
+ PyObject *data_list = NULL; /* data fragments for return tuple */
+ PyObject *parity_list = NULL; /* parity fragments for return tuple */
+ PyObject *fragment_string = NULL; /* fragment string for lists */
+ PyObject *missing_list = NULL; /* missing indexes for return tuple */
+ PyObject *return_lists = NULL; /* Python tuple to return */
+ PyObject **data = NULL; /* array of k data fragment buffers*/
+ PyObject **parity = NULL; /* array of m parity fragment buffers */
+ int *missing = NULL; /* indicies of missing fragments */
+ int num_missing = 0; /* num of missing fragments */
+ int num_fragments; /* num of frags provided by caller */
+ int fragment_size = 0; /* size in bytes of fragments */
+ int i = 0; /* a counter */
+
+ /* Collect and validate the method arguments */
if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_list)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.get_fragment_partition");
return NULL;
}
-
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.get_fragment_partition");
return NULL;
}
-
if (!PyList_Check(fragment_list)) {
PyErr_SetString(PyECLibError, "Invalid structure passed in for fragment list in pyeclib.get_fragment_partition");
return NULL;
}
- // Set the number of fragments we have
- num_fragments = (int)PyList_Size(fragment_list);
-
- data = (PyObject**)malloc(pyeclib_handle->k * sizeof(PyObject*));
- parity = (PyObject**)malloc(pyeclib_handle->m * sizeof(PyObject*));
-
- for (i=0; i < pyeclib_handle->k; i++) {
- data[i] = NULL;
+ /* Create the arrays need to hold the data and parity fragments */
+ data = (PyObject **) alloc_zeroed_buffer(pyeclib_handle->k * sizeof(PyObject*));
+ if (NULL == data) {
+ goto error;
}
-
- for (i=0; i < pyeclib_handle->m; i++) {
- parity[i] = NULL;
+ parity = (PyObject **) alloc_zeroed_buffer(pyeclib_handle->m * sizeof(PyObject*));
+ if (NULL == parity) {
+ goto error;
}
- /*
- * ASSUMPTION: We will never try to do anything when we have more
- * than k missing fragments
- */
- missing = (int*)malloc(pyeclib_handle->k * sizeof(int));
-
- /*
- * Fill in the data, parity and missing indexes
- */
- for (i=0; i < num_fragments; i++) {
+
+ /* Fill in the data and parity pointers to reveal missing fragments */
+ num_fragments = (int) PyList_Size(fragment_list);
+ for (i = 0; i < num_fragments; i++) {
PyObject *tmp_data = PyList_GetItem(fragment_list, i);
char *c_buf;
int index;
Py_ssize_t len;
-
+
+ /* ASSUMPTION: The fragment_size is the max of the fragments we read */
PyBytes_AsStringAndSize(tmp_data, &c_buf, &len);
-
- /*
- * Assume the fragment_size is the max of
- * the fragments we read
- */
if (len > fragment_size) {
- fragment_size = (int)len;
+ fragment_size = (int) len;
}
- /*
- * Get fragment index and size
- */
+ /* Get fragment index and set the index in data or parity */
index = get_fragment_idx(c_buf);
-
if (index < pyeclib_handle->k) {
- // Data element
data[index] = tmp_data;
-
} else {
- // Parity element
parity[index - pyeclib_handle->k] = tmp_data;
}
}
-
- /*
- * If there are missing bufs, figure out which indexes are missing
+
+ /* If there are missing bufs, figure out which indexes are missing
+ *
+ * ASSUMPTION: We will never try to do anything when we have more
+ * than k missing fragments
*/
- if (missing != NULL) {
+ missing = (int *) alloc_zeroed_buffer(pyeclib_handle->k * sizeof(int));
+ if (NULL == missing) {
+ goto error;
+ } else {
num_missing = 0;
- for (i=0; i < pyeclib_handle->k; i++) {
+ for (i = 0; i < pyeclib_handle->k; i++) {
if (data[i] == NULL) {
missing[num_missing] = i;
num_missing++;
}
}
- for (i=0; i < pyeclib_handle->m; i++) {
+ for (i = 0; i < pyeclib_handle->m; i++) {
if (parity[i] == NULL) {
missing[num_missing] = i + pyeclib_handle->k;
num_missing++;
}
}
}
-
+ /* Create the python objects to return */
data_list = PyList_New(pyeclib_handle->k);
parity_list = PyList_New(pyeclib_handle->m);
missing_list = PyList_New(num_missing);
+ return_lists = PyTuple_New(3);
+ if (!data_list || !parity_list || !missing_list || !return_lists) {
+ return_lists = PyErr_NoMemory();
+ goto exit;
+ }
- /*
- * Fill in the data fragments
- */
- for (i=0; i < pyeclib_handle->k; i++) {
+ /* Fill in the data fragments, create zero fragment if missing */
+ for (i = 0; i < pyeclib_handle->k; i++) {
if (data[i] != NULL) {
- // This is a borrowed reference, so increment the ref count
- // before inserting into list
+ /* BORROWED REF: increment ref count before inserting into list */
Py_INCREF(data[i]);
- PyList_SET_ITEM(data_list, i, data[i]);
+ fragment_string = data[i];
} else {
- char *tmp_data = (char*)malloc(fragment_size);
- PyObject *zero_string;
- memset(tmp_data, 0, fragment_size);
- zero_string = PY_BUILDVALUE_OBJ_LEN(tmp_data, fragment_size);
- //Py_INCREF(zero_string);
- PyList_SET_ITEM(data_list, i, zero_string);
- free(tmp_data);
+ fragment_string = alloc_zero_string(fragment_size);
+ if (NULL == fragment_string) {
+ goto error;
+ }
}
+ PyList_SET_ITEM(data_list, i, fragment_string);
}
- free(data);
-
- /*
- * Fill in the parity fragments
- */
- for (i=0; i < pyeclib_handle->m; i++) {
+ /* Fill in the parity fragments, create zero fragment if missing */
+ for (i = 0; i < pyeclib_handle->m; i++) {
if (parity[i] != NULL) {
- // This is a borrowed reference, so increment the ref count
- // before inserting into list
+ /* BORROWED REF: increment ref count before inserting into list */
Py_INCREF(parity[i]);
- PyList_SET_ITEM(parity_list, i, parity[i]);
+ fragment_string = parity[i];
} else {
- char *tmp_parity = (char*)malloc(fragment_size);
- PyObject *zero_string;
- memset(tmp_parity, 0, fragment_size);
- zero_string = PY_BUILDVALUE_OBJ_LEN(tmp_parity, fragment_size);
- //Py_INCREF(zero_string);
- PyList_SET_ITEM(parity_list, i, zero_string);
- free(tmp_parity);
+ fragment_string = alloc_zero_string(fragment_size);
+ if (NULL == fragment_string) {
+ goto error;
+ }
}
+ PyList_SET_ITEM(parity_list, i, fragment_string);
}
- free(parity);
-
- for( i=0;i < num_missing; i++) {
+ /* Fill in the list of missing indexes */
+ for (i = 0;i < num_missing; i++) {
PyList_SET_ITEM(missing_list, i, Py_BuildValue("i", missing[i]));
}
-
- free(missing);
-
- return_lists = PyTuple_New(3);
Py_INCREF(data_list);
Py_INCREF(parity_list);
Py_INCREF(missing_list);
-
+
PyTuple_SetItem(return_lists, 0, data_list);
PyTuple_SetItem(return_lists, 1, parity_list);
PyTuple_SetItem(return_lists, 2, missing_list);
-
//Py_INCREF(return_lists);
+ goto exit;
+
+error:
+ return_lists = NULL;
+
+exit:
+ free_buffer(data);
+ free_buffer(parity);
+ free_buffer(missing);
+
return return_lists;
}
+
+/**
+ * Return a list of lists with valid rebuild indexes given an EC algorithm
+ * and a list of missing indexes.
+ *
+ * @param pyeclib_obj_handle
+ * @param missing_list indexes of missing fragments
+ * @return a list of lists of indexes to rebuild data from
+ */
static PyObject *
pyeclib_c_get_required_fragments(PyObject *self, PyObject *args)
{
- PyObject *pyeclib_obj_handle;
- PyObject *missing_list;
- PyObject *fragment_idx_list = NULL;
- pyeclib_t* pyeclib_handle;
- int num_missing;
- int *c_missing_list;
- int i, j;
- unsigned long missing_bm = 0;
-
+ PyObject *pyeclib_obj_handle = NULL;
+ pyeclib_t *pyeclib_handle = NULL;
+ PyObject *missing_list = NULL; /* param, */
+ PyObject *fragment_idx_list = NULL; /* list of req'd indexes to return */
+ int *c_missing_list = NULL; /* c-array of missing indexes */
+ int num_missing; /* size of passed in missing list */
+ int i = 0, j = 0; /* counters */
+ int k, m; /* EC algorithm parameters */
+ unsigned long missing_bm = 0; /* bitmap of missing indexes */
+ int *fragments_needed = NULL; /* indexes of xor code fragments */
+ int ret; /* return value for xor code */
+
+ /* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &missing_list)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.get_required_fragments");
return NULL;
}
-
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.get_required_fragments");
return NULL;
}
+ k = pyeclib_handle->k;
+ m = pyeclib_handle->m;
- num_missing = (int)PyList_Size(missing_list);
-
- c_missing_list = (int*)malloc((num_missing+1)*sizeof(int));
-
- for (i=0; i < num_missing; i++) {
+ /* Generate -1 terminated c-array and bitmap of missing indexes */
+ num_missing = (int) PyList_Size(missing_list);
+ c_missing_list = (int *) alloc_zeroed_buffer((num_missing + 1) * sizeof(int));
+ if (NULL == c_missing_list) {
+ return NULL;
+ }
+ c_missing_list[num_missing] = -1;
+ for (i = 0; i < num_missing; i++) {
PyObject *obj_idx = PyList_GetItem(missing_list, i);
long idx = PyLong_AsLong(obj_idx);
- c_missing_list[i] = (int)idx;
+ c_missing_list[i] = (int) idx;
}
-
- c_missing_list[num_missing] = -1;
-
missing_bm = convert_list_to_bitmap(c_missing_list);
-
+
+ /* Generate the python list of lists of rebuild indexes to return */
fragment_idx_list = PyList_New(0);
-
+ if (NULL == fragment_idx_list) {
+ goto exit;
+ }
+ j = 0;
switch(pyeclib_handle->type) {
case PYECC_RS_CAUCHY_ORIG:
case PYECC_RS_VAND:
- j=0;
- for (i=0; i < pyeclib_handle->k + pyeclib_handle->m; i++) {
+ for (i = 0; i < (k + m); i++) {
if (!(missing_bm & (1 << i))) {
PyList_Append(fragment_idx_list, Py_BuildValue("i", i));
j++;
}
- if (j == pyeclib_handle->k) {
+ if (j == k) {
break;
}
}
- if (j != pyeclib_handle->k) {
- // Let the garbage collector clean this up
+ if (j != k) {
+ /* Let the garbage collector clean this up */
Py_DECREF(fragment_idx_list);
- PyErr_Format(PyECLibError, "Not enough fragments for pyeclib.get_required_fragments (need at least %d, %d are given)", pyeclib_handle->k, j);
+ PyErr_Format(PyECLibError, "Not enough fragments for pyeclib.get_required_fragments (need at least %d, %d are given)", k, j);
fragment_idx_list = NULL;
}
break;
case PYECC_XOR_HD_3:
case PYECC_XOR_HD_4:
{
- int *fragments_needed = (int*)malloc(sizeof(int)*(pyeclib_handle->k+pyeclib_handle->m));
- int ret = pyeclib_handle->xor_code_desc->fragments_needed(pyeclib_handle->xor_code_desc, c_missing_list, fragments_needed);
+ fragments_needed = alloc_zeroed_buffer(sizeof(int) * (k + m));
+ if (NULL == fragments_needed) {
+ fragment_idx_list = NULL;
+ goto exit;
+ }
+ ret = pyeclib_handle->xor_code_desc->fragments_needed(pyeclib_handle->xor_code_desc,
+ c_missing_list,
+ fragments_needed);
if (ret < 0) {
Py_DECREF(fragment_idx_list);
PyErr_Format(PyECLibError, "Not enough fragments for pyeclib.get_required_fragments!");
fragment_idx_list = NULL;
- free(fragments_needed);
break;
}
- j=0;
while (fragments_needed[j] > -1) {
PyList_Append(fragment_idx_list, Py_BuildValue("i", fragments_needed[j]));
j++;
- }
- free(fragments_needed);
+ }
break;
}
default:
@@ -1400,158 +1553,159 @@ pyeclib_c_get_required_fragments(PyObject *self, PyObject *args)
break;
}
- // Free the temporary list
- free(c_missing_list);
+exit:
+ free_buffer(c_missing_list);
+ free_buffer(fragments_needed);
return fragment_idx_list;
}
-/*
- * TODO: If we are reconstructing a parity element, ensure that all of the data elements are available!
+
+/**
+ * Reconstruct a missing fragment from the the remaining fragments.
+ *
+ * TODO: If we are reconstructing a parity element, ensure that all of the
+ * data elements are available!
+ *
+ * @param pyeclib_obj_handle
+ * @param data_list k length list of data elements
+ * @param parity_list m length list of parity elements
+ * @param missing_idx_list list of the indexes of missing elements
+ * @param destination_idx index of fragment to reconstruct
+ * @param fragment_size size in bytes of the fragments
+ * @return reconstructed destination fragment or NULL on error
*/
static PyObject *
pyeclib_c_reconstruct(PyObject *self, PyObject *args)
{
- PyObject *pyeclib_obj_handle;
- PyObject *data_list;
- PyObject *parity_list;
- PyObject *missing_idx_list;
- PyObject *reconstructed;
- int *erased;
- pyeclib_t *pyeclib_handle;
- int fragment_size;
- int blocksize;
- char **data;
- char **parity;
- int *missing_idxs;
- int destination_idx;
- unsigned long long realloc_bm = 0; // Identifies symbols that had to be allocated for alignment
- int orig_data_size = -1;
- int missing_size;
- int *decoding_matrix;
- int *decoding_row;
- int *dm_ids;
- int ret;
- int i;
-
- if (!PyArg_ParseTuple(args, "OOOOii", &pyeclib_obj_handle, &data_list, &parity_list, &missing_idx_list, &destination_idx, &fragment_size)) {
+ PyObject *pyeclib_obj_handle = NULL;
+ pyeclib_t *pyeclib_handle = NULL;
+ PyObject *data_list = NULL; /* param, list of data fragments */
+ PyObject *parity_list = NULL; /* param, list of parity fragments */
+ PyObject *missing_idx_list = NULL; /* param, list of missing indexes */
+ PyObject *reconstructed = NULL; /* reconstructed object to return */
+ int *erased = NULL; /* jerasure notation of erased devs */
+ int fragment_size; /* param, size in bytes of fragment */
+ int blocksize; /* size in bytes, fragment - header */
+ char **data = NULL; /* k length array of data buffers */
+ char **parity = NULL; /* m length array of parity buffers */
+ int *missing_idxs = NULL; /* array of missing indexes */
+ int destination_idx; /* param, index to reconstruct */
+ unsigned long long realloc_bm = 0; /* bitmap, which fragments were realloc'ed */
+ int orig_data_size = -1; /* data size (B),from fragment hdr */
+ int missing_size; /* number of missing indexes */
+ int *decoding_matrix = NULL; /* reconstruct specific decode matrix */
+ int *decoding_row = NULL; /* required row from decode matrix */
+ int *dm_ids = NULL; /* k length array of surviving indexes */
+ int k, m, w; /* EC algorithm parameters */
+ int ret; /* decode matrix creation return val */
+ int i = 0; /* a counter */
+
+ /* Obtain and validate the method parameters */
+ if (!PyArg_ParseTuple(args, "OOOOii", &pyeclib_obj_handle, &data_list,
+ &parity_list, &missing_idx_list, &destination_idx,
+ &fragment_size)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode");
return NULL;
}
-
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode");
return NULL;
}
-
+ k = pyeclib_handle->k;
+ m = pyeclib_handle->m;
+ w = pyeclib_handle->w;
if (!PyList_Check(data_list) || !PyList_Check(parity_list) || !PyList_Check(missing_idx_list)) {
PyErr_SetString(PyECLibError, "Invalid structure passed in for data, parity and/or missing_idx list");
return NULL;
}
-
- if (pyeclib_handle->k != PyList_Size(data_list)) {
+ if (k != PyList_Size(data_list)) {
PyErr_SetString(PyECLibError, "The data list does not have the correct number of entries");
return NULL;
}
-
- if (pyeclib_handle->m != PyList_Size(parity_list)) {
+ if (m != PyList_Size(parity_list)) {
PyErr_SetString(PyECLibError, "The parity list does not have the correct number of entries");
return NULL;
}
+ /* Allocate data structures needed for reconstruction */
blocksize = FRAGSIZE_2_BLOCKSIZE(fragment_size);
-
- missing_size = (int)PyList_Size(missing_idx_list);
- missing_idxs = (int*)malloc(sizeof(int)*(missing_size+1));
- if (missing_idxs == NULL) {
- PyErr_SetString(PyECLibError, "Could not allocate memory for missing indexes");
- return NULL;
- }
-
- data = (char**)malloc(sizeof(char*) * pyeclib_handle->k);
- if (data == NULL) {
- PyErr_SetString(PyECLibError, "Could not allocate memory for data buffers");
- free(missing_idxs);
- return NULL;
+ missing_size = (int) PyList_Size(missing_idx_list);
+ missing_idxs = (int *) alloc_zeroed_buffer(sizeof(int) * (missing_size + 1));
+ data = (char **) alloc_zeroed_buffer(sizeof(char *) * k);
+ parity = (char **) alloc_zeroed_buffer(sizeof(char *) * m);
+ if (NULL == missing_idxs || NULL == data || NULL == parity) {
+ goto error;
}
- parity = (char**)malloc(sizeof(char*) * pyeclib_handle->m);
- if (parity == NULL) {
- PyErr_SetString(PyECLibError, "Could not allocate memory for parity buffers");
- free(missing_idxs);
- free(data);
- return NULL;
- }
-
- if (get_decoding_info(pyeclib_handle, data_list, parity_list, missing_idx_list, data, parity, missing_idxs, &orig_data_size, fragment_size, &realloc_bm)) {
+ /* Prepare for decoding, no need to go further on error */
+ if (get_decoding_info(pyeclib_handle, data_list, parity_list,
+ missing_idx_list, data, parity, missing_idxs,
+ &orig_data_size, fragment_size, &realloc_bm)) {
PyErr_SetString(PyECLibError, "Could not extract adequate decoding info from data, parity and missing lists");
- reconstructed = NULL;
- goto out;
+ goto error;
}
- erased = jerasure_erasures_to_erased(pyeclib_handle->k, pyeclib_handle->m, missing_idxs);
-
+ /* Create the decoding matrix, and attempt reconstruction */
+ erased = jerasure_erasures_to_erased(k, m, missing_idxs);
switch (pyeclib_handle->type) {
case PYECC_RS_CAUCHY_ORIG:
-
- if (destination_idx < pyeclib_handle->k) {
- decoding_matrix = (int*)malloc(sizeof(int*)*pyeclib_handle->k*pyeclib_handle->k*pyeclib_handle->w*pyeclib_handle->w);
- dm_ids = (int*)malloc(sizeof(int)*pyeclib_handle->k);
- ret = jerasure_make_decoding_bitmatrix(pyeclib_handle->k, pyeclib_handle->m, pyeclib_handle->w, pyeclib_handle->bitmatrix, erased, decoding_matrix, dm_ids);
- decoding_row = decoding_matrix + (destination_idx*pyeclib_handle->k*pyeclib_handle->w*pyeclib_handle->w);
+ if (destination_idx < k) {
+ decoding_matrix = (int *) alloc_zeroed_buffer(sizeof(int*) * k * k * w * w);
+ dm_ids = (int *) alloc_zeroed_buffer(sizeof(int) * k);
+ if (NULL == decoding_matrix || NULL == dm_ids) {
+ goto error;
+ }
+ ret = jerasure_make_decoding_bitmatrix(k, m, w, pyeclib_handle->bitmatrix,
+ erased, decoding_matrix, dm_ids);
+ decoding_row = decoding_matrix + (destination_idx * k * w * w);
} else {
- dm_ids = NULL;
- decoding_row = pyeclib_handle->bitmatrix + ((destination_idx - pyeclib_handle->k)*pyeclib_handle->k*pyeclib_handle->w*pyeclib_handle->w);
ret = 0;
+ decoding_row = pyeclib_handle->bitmatrix + ((destination_idx - k) * k * w * w);
}
+
if (ret == 0) {
- jerasure_bitmatrix_dotprod(pyeclib_handle->k, pyeclib_handle->w, decoding_row, dm_ids, destination_idx, data, parity, blocksize, PYECC_CAUCHY_PACKETSIZE);
+ jerasure_bitmatrix_dotprod(k, w, decoding_row, dm_ids, destination_idx,
+ data, parity, blocksize,
+ PYECC_CAUCHY_PACKETSIZE);
}
-
- if (destination_idx < pyeclib_handle->k) {
- free(decoding_matrix);
- }
- free(dm_ids);
-
break;
case PYECC_RS_VAND:
-
- if (destination_idx < pyeclib_handle->k) {
- decoding_matrix = (int*)malloc(sizeof(int*)*pyeclib_handle->k*pyeclib_handle->k);
- dm_ids = (int*)malloc(sizeof(int)*pyeclib_handle->k);
- ret = jerasure_make_decoding_matrix(pyeclib_handle->k, pyeclib_handle->m, pyeclib_handle->w, pyeclib_handle->matrix, erased, decoding_matrix, dm_ids);
- decoding_row = decoding_matrix + (destination_idx * pyeclib_handle->k);
+ if (destination_idx < k) {
+ decoding_matrix = (int *) alloc_zeroed_buffer(sizeof(int*) * k * k);
+ dm_ids = (int *) alloc_zeroed_buffer(sizeof(int) * k);
+ if (NULL == decoding_matrix || NULL == dm_ids) {
+ goto error;
+ }
+ ret = jerasure_make_decoding_matrix(k, m, w, pyeclib_handle->matrix,
+ erased, decoding_matrix, dm_ids);
+ decoding_row = decoding_matrix + (destination_idx * k);
} else {
- dm_ids = NULL;
- decoding_row = pyeclib_handle->matrix + ((destination_idx - pyeclib_handle->k) * pyeclib_handle->k);
ret = 0;
+ decoding_row = pyeclib_handle->matrix + ((destination_idx - k) * k);
}
+
if (ret == 0) {
- jerasure_matrix_dotprod(pyeclib_handle->k, pyeclib_handle->w, decoding_row, dm_ids, destination_idx, data, parity, blocksize);
- }
-
- if (destination_idx < pyeclib_handle->k) {
- free(decoding_matrix);
- free(dm_ids);
+ jerasure_matrix_dotprod(k, w, decoding_row, dm_ids, destination_idx,
+ data, parity, blocksize);
}
-
break;
-
case PYECC_XOR_HD_3:
case PYECC_XOR_HD_4:
- xor_reconstruct_one(pyeclib_handle->xor_code_desc, data, parity, missing_idxs, destination_idx, blocksize);
ret = 0;
+ xor_reconstruct_one(pyeclib_handle->xor_code_desc, data, parity,
+ missing_idxs, destination_idx, blocksize);
break;
-
default:
ret = -1;
break;
}
+ /* Set the metadata on the reconstructed fragment */
if (ret == 0) {
- char *fragment_ptr;
- if (destination_idx < pyeclib_handle->k) {
+ char *fragment_ptr = NULL;
+ if (destination_idx < k) {
fragment_ptr = get_fragment_ptr_from_data_novalidate(data[destination_idx]);
init_fragment_header(fragment_ptr);
set_fragment_idx(fragment_ptr, destination_idx);
@@ -1562,139 +1716,155 @@ pyeclib_c_reconstruct(PyObject *self, PyObject *args)
set_chksum(fragment_ptr, chksum);
}
} else {
- fragment_ptr = get_fragment_ptr_from_data_novalidate(parity[destination_idx - pyeclib_handle->k]);
+ fragment_ptr = get_fragment_ptr_from_data_novalidate(parity[destination_idx - k]);
init_fragment_header(fragment_ptr);
set_fragment_idx(fragment_ptr, destination_idx);
set_orig_data_size(fragment_ptr, orig_data_size);
set_fragment_size(fragment_ptr, blocksize);
if (use_inline_chksum(pyeclib_handle)) {
- int chksum = crc32(0, parity[destination_idx - pyeclib_handle->k], blocksize);
+ int chksum = crc32(0, parity[destination_idx - k], blocksize);
set_chksum(fragment_ptr, chksum);
}
}
reconstructed = PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size);
-
} else {
reconstructed = NULL;
}
-
- for (i=0; i < pyeclib_handle->k; i++) {
+
+ goto out;
+
+error:
+ reconstructed = NULL;
+
+out:
+ /* Free fragment buffers that needed to be reallocated for alignment */
+ for (i = 0; i < k; i++) {
if (realloc_bm & (1 << i)) {
- char *ptr = get_fragment_ptr_from_data_novalidate(data[i]);
- free(ptr);
+ free(get_fragment_ptr_from_data_novalidate(data[i]));
}
}
- for (i=0; i < pyeclib_handle->m; i++) {
- if (realloc_bm & (1 << (i + pyeclib_handle->k))) {
- char *ptr = get_fragment_ptr_from_data_novalidate(parity[i]);
- free(ptr);
+ for (i = 0; i < m; i++) {
+ if (realloc_bm & (1 << (i + k))) {
+ free(get_fragment_ptr_from_data_novalidate(parity[i]));
}
}
-
-out:
- free(missing_idxs);
- free(data);
- free(parity);
+ free_buffer(missing_idxs);
+ free_buffer(data);
+ free_buffer(parity);
+ free_buffer(decoding_matrix);
+ free_buffer(dm_ids);
return reconstructed;
}
+
+/**
+ * Reconstruct all of the missing fragments from a set of fragments.
+ *
+ * TODO: There's a lot of duplicated code between this and the
+ * reconstruct method. Consider refactoring these methods.
+ *
+ * @param pyeclib_obj_handle
+ * @param data_list k length list of data elements
+ * @param parity_list m length list of parity elements
+ * @param missing_idx_list list of the indexes of missing elements
+ * @param fragment_size size in bytes of the fragments
+ * @return list of fragments
+ */
static PyObject *
pyeclib_c_decode(PyObject *self, PyObject *args)
{
- PyObject *list_of_strips;
- PyObject *pyeclib_obj_handle;
- PyObject *data_list;
- PyObject *parity_list;
- PyObject *missing_idx_list;
- pyeclib_t *pyeclib_handle;
- int fragment_size;
- int blocksize;
- unsigned long long realloc_bm = 0; // Identifies symbols that had to be allocated for alignment
- char **data;
- char **parity;
- int *missing_idxs;
- int missing_size;
- int orig_data_size = -1;
- int i, j;
-
+ PyObject *pyeclib_obj_handle = NULL;
+ pyeclib_t *pyeclib_handle = NULL;
+ PyObject *list_of_strips = NULL; /* list of strips to return */
+ PyObject *data_list = NULL; /* param, list of data fragments */
+ PyObject *parity_list = NULL; /* param, list of data fragments */
+ PyObject *missing_idx_list = NULL; /* param, list of missing indexes */
+ int fragment_size; /* param, size in bytes of fragment */
+ int blocksize; /* size in bytes, fragment - header */
+ unsigned long long realloc_bm = 0; /* bitmap, which fragments were realloc'ed */
+ char **data = NULL; /* k length array of data buffers */
+ char **parity = NULL; /* m length array of parity buffers */
+ int *missing_idxs = NULL; /* array of missing indexes */
+ int missing_size; /* number of missing indexes */
+ int orig_data_size = -1; /* data size in bytes ,from fragment hdr */
+ int k, m, w; /* EC algorithm parameters */
+ int i = 0, j = 0; /* counters */
+
+ /* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "OOOOi", &pyeclib_obj_handle, &data_list, &parity_list, &missing_idx_list, &fragment_size)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode");
return NULL;
}
-
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode");
return NULL;
}
-
+ k = pyeclib_handle->k;
+ m = pyeclib_handle->m;
+ w = pyeclib_handle->w;
if (!PyList_Check(data_list) || !PyList_Check(parity_list) || !PyList_Check(missing_idx_list)) {
PyErr_SetString(PyECLibError, "Invalid structure passed in for data, parity and/or missing_idx list");
return NULL;
}
-
- if (pyeclib_handle->k != PyList_Size(data_list)) {
+ if (k != PyList_Size(data_list)) {
PyErr_SetString(PyECLibError, "The data list does not have the correct number of entries");
return NULL;
}
-
- if (pyeclib_handle->m != PyList_Size(parity_list)) {
+ if (m != PyList_Size(parity_list)) {
PyErr_SetString(PyECLibError, "The parity list does not have the correct number of entries");
return NULL;
}
+ /* Allocate data structures needed for reconstruction */
blocksize = FRAGSIZE_2_BLOCKSIZE(fragment_size);
-
- missing_size = (int)PyList_Size(missing_idx_list);
- missing_idxs = (int*)malloc(sizeof(int)*(missing_size+1));
- if (missing_idxs == NULL) {
- PyErr_SetString(PyECLibError, "Could not allocate memory for missing indexes");
- return NULL;
- }
-
- data = (char**)malloc(sizeof(char*) * pyeclib_handle->k);
- if (data == NULL) {
- PyErr_SetString(PyECLibError, "Could not allocate memory for data buffers");
- return NULL;
- }
-
- parity = (char**)malloc(sizeof(char*) * pyeclib_handle->m);
- if (parity == NULL) {
- PyErr_SetString(PyECLibError, "Could not allocate memory for parity buffers");
- return NULL;
- }
-
- if (get_decoding_info(pyeclib_handle, data_list, parity_list, missing_idx_list, data, parity, missing_idxs, &orig_data_size, fragment_size, &realloc_bm)) {
+ missing_size = (int) PyList_Size(missing_idx_list);
+ missing_idxs = (int *) alloc_zeroed_buffer(sizeof(int) * (missing_size + 1));
+ data = (char **) alloc_zeroed_buffer(sizeof(char *) * k);
+ parity = (char **) alloc_zeroed_buffer(sizeof(char *) * m);
+ if (NULL == missing_idxs || NULL == data || NULL == parity) {
+ goto error;
+ }
+
+ /* Prepare for decoding, no need to go further on error */
+ if (get_decoding_info(pyeclib_handle, data_list, parity_list, missing_idx_list,
+ data, parity, missing_idxs, &orig_data_size,
+ fragment_size, &realloc_bm)) {
PyErr_SetString(PyECLibError, "Could not extract adequate decoding info from data, parity and missing lists");
return NULL;
}
+ /* Reconstruct the missing fragments */
switch (pyeclib_handle->type) {
case PYECC_RS_CAUCHY_ORIG:
- jerasure_bitmatrix_decode(pyeclib_handle->k, pyeclib_handle->m, pyeclib_handle->w, pyeclib_handle->bitmatrix, 0, missing_idxs, data, parity, blocksize, PYECC_CAUCHY_PACKETSIZE);
+ jerasure_bitmatrix_decode(k, m, w, pyeclib_handle->bitmatrix, 0, missing_idxs,
+ data, parity, blocksize, PYECC_CAUCHY_PACKETSIZE);
break;
case PYECC_XOR_HD_3:
case PYECC_XOR_HD_4:
- pyeclib_handle->xor_code_desc->decode(pyeclib_handle->xor_code_desc, data, parity, missing_idxs, blocksize, 1);
+ pyeclib_handle->xor_code_desc->decode(pyeclib_handle->xor_code_desc, data,
+ parity, missing_idxs, blocksize, 1);
break;
case PYECC_RS_VAND:
default:
- jerasure_matrix_decode(pyeclib_handle->k, pyeclib_handle->m, pyeclib_handle->w, pyeclib_handle->matrix, 1, missing_idxs, data, parity, blocksize);
+ jerasure_matrix_decode(k, m, w, pyeclib_handle->matrix, 1, missing_idxs,
+ data, parity, blocksize);
break;
}
- list_of_strips = PyList_New(pyeclib_handle->k + pyeclib_handle->m);
+ /* Create the python list to return */
+ list_of_strips = PyList_New(k + m);
+ if (NULL == list_of_strips) {
+ goto error;
+ }
- /*
- * Create headers for the newly decoded elements
- */
- j=0;
+ /* Create headers for the newly decoded elements */
while (missing_idxs[j] >= 0) {
int missing_idx = missing_idxs[j];
- if (missing_idx < pyeclib_handle->k) {
+ if (missing_idx < k) {
char *fragment_ptr = get_fragment_ptr_from_data_novalidate(data[missing_idx]);
init_fragment_header(fragment_ptr);
set_fragment_idx(fragment_ptr, missing_idx);
@@ -1704,8 +1874,8 @@ pyeclib_c_decode(PyObject *self, PyObject *args)
int chksum = crc32(0, data[missing_idx], blocksize);
set_chksum(fragment_ptr, chksum);
}
- } else if (missing_idx >= pyeclib_handle->k) {
- int parity_idx = missing_idx - pyeclib_handle->k;
+ } else if (missing_idx >= k) {
+ int parity_idx = missing_idx - k;
char *fragment_ptr = get_fragment_ptr_from_data_novalidate(parity[parity_idx]);
init_fragment_header(fragment_ptr);
set_fragment_idx(fragment_ptr, missing_idx);
@@ -1719,106 +1889,144 @@ pyeclib_c_decode(PyObject *self, PyObject *args)
j++;
}
- /*
- * Fill in the data fragments
- */
- for (i=0; i < pyeclib_handle->k; i++) {
+ /* Fill in the data fragments */
+ for (i = 0; i < k; i++) {
char *fragment_ptr = get_fragment_ptr_from_data(data[i]);
PyList_SET_ITEM(list_of_strips, i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size));
+ }
+ /* Fill in the parity fragments */
+ for (i = 0; i < m; i++) {
+ char *fragment_ptr = get_fragment_ptr_from_data(parity[i]);
+ PyList_SET_ITEM(list_of_strips, k + i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size));
+ }
+
+ goto exit;
+
+error:
+ list_of_strips = NULL;
+
+exit:
+ /* Free fragment buffers that needed to be reallocated for alignment */
+ for (i = 0; i < k; i++) {
if (realloc_bm & (1 << i)) {
- free(fragment_ptr);
+ free(get_fragment_ptr_from_data_novalidate(data[i]));
}
}
- /*
- * Fill in the parity fragments
- */
- for (i=0; i < pyeclib_handle->m; i++) {
- char *fragment_ptr = get_fragment_ptr_from_data(parity[i]);
- PyList_SET_ITEM(list_of_strips, pyeclib_handle->k + i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size));
- if (realloc_bm & (1 << (i + pyeclib_handle->k))) {
- free(fragment_ptr);
+ for (i = 0; i < m; i++) {
+ if (realloc_bm & (1 << (i + k))) {
+ free(get_fragment_ptr_from_data_novalidate(parity[i]));
}
}
-
- free(missing_idxs);
- free(data);
- free(parity);
+
+ free_buffer(missing_idxs);
+ free_buffer(data);
+ free_buffer(parity);
return list_of_strips;
}
+
+/**
+ * Obtain the metadata from a fragment.
+ *
+ * @param pyeclib_obj_handle
+ * @param data fragment from user to extract metadata from
+ * @param data_len size in bytes of the data fragment
+ * @return fragment metadata or NULL on error
+ */
static PyObject *
pyeclib_c_get_metadata(PyObject *self, PyObject *args)
{
- PyObject *pyeclib_obj_handle;
- pyeclib_t* pyeclib_handle;
- char *data;
- int data_len;
- fragment_metadata_t *fragment_metadata;
- PyObject *ret_fragment_metadata;
-
+ PyObject *pyeclib_obj_handle = NULL;
+ pyeclib_t* pyeclib_handle = NULL;
+ char *data = NULL; /* param, fragment from caller */
+ int data_len; /* param, data len (B) */
+ int metadata_len; /* metadata header size (B) */
+ fragment_metadata_t *fragment_metadata = NULL; /* buffer to hold metadata */
+ PyObject *ret_fragment_metadata = NULL; /* metadata object to return */
+
+ /* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, GET_METADATA_ARGS, &pyeclib_obj_handle, &data, &data_len)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.get_metadata");
return NULL;
}
-
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.get_required_fragments");
return NULL;
}
- fragment_metadata = (fragment_metadata_t*)malloc(sizeof(fragment_metadata_t));
-
- get_fragment_metadata(pyeclib_handle, data, fragment_metadata);
-
- ret_fragment_metadata = PY_BUILDVALUE_OBJ_LEN((char*)fragment_metadata, sizeof(fragment_metadata_t));
-
- free(fragment_metadata);
+ /* Obtain the metadata from the data and build a object to return */
+ metadata_len = sizeof(fragment_metadata_t);
+ fragment_metadata = (fragment_metadata_t *) alloc_zeroed_buffer(metadata_len);
+ if (NULL == fragment_metadata) {
+ ret_fragment_metadata = NULL;
+ } else {
+ get_fragment_metadata(pyeclib_handle, data, fragment_metadata);
+ ret_fragment_metadata = PY_BUILDVALUE_OBJ_LEN((char*)fragment_metadata,
+ metadata_len);
+ free_buffer(fragment_metadata);
+ }
return ret_fragment_metadata;
}
+
+/**
+ * Confirm the health of the fragment metadata.
+ *
+ * TODO: Return a list containing tuples (index, problem). An empty list means
+ * everything is OK.
+ *
+ * @param pyeclib_obj_handle
+ * @param fragment_metadata_list list of fragment metadata headers
+ * @return -1 if no errors, or the index of the first problem checksum
+ */
static PyObject*
pyeclib_c_check_metadata(PyObject *self, PyObject *args)
{
- PyObject *pyeclib_obj_handle;
- PyObject *fragment_metadata_list;
- pyeclib_t *pyeclib_handle;
- int i;
- int ret = -1;
- int num_fragments;
- fragment_metadata_t** c_fragment_metadata_list;
- char **c_fragment_signatures;
-
+ PyObject *pyeclib_obj_handle = NULL;
+ pyeclib_t *pyeclib_handle = NULL;
+ PyObject *fragment_metadata_list = NULL; /* param, fragment metadata */
+ fragment_metadata_t **c_fragment_metadata_list = NULL; /* c version of metadata */
+ int num_fragments; /* k + m from EC algorithm */
+ char **c_fragment_signatures = NULL; /* array of alg. signatures */
+ int k, m, w; /* EC algorithm params */
+ int size; /* size for buf allocation */
+ int ret = -1; /* c return value */
+ PyObject *ret_obj = NULL; /* python long to return */
+ int i = 0; /* a counter */
+
+ /* Obtain and validate the method parameters */
if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_metadata_list)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode");
return NULL;
}
-
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode");
return NULL;
}
-
- num_fragments = pyeclib_handle->k + pyeclib_handle->m;
-
+ k = pyeclib_handle->k;
+ m = pyeclib_handle->m;
+ w = pyeclib_handle->w;
+ num_fragments = k + m;
if (num_fragments != PyList_Size(fragment_metadata_list)) {
PyErr_SetString(PyECLibError, "Not enough fragment metadata to perform integrity check");
return NULL;
}
-
- c_fragment_metadata_list = (fragment_metadata_t**)malloc(sizeof(fragment_metadata_t*)*num_fragments);
- memset(c_fragment_metadata_list, 0, sizeof(fragment_metadata_t*)*num_fragments);
- c_fragment_signatures = (char**)malloc(sizeof(char*)*num_fragments);
- memset(c_fragment_signatures, 0, sizeof(char*)*num_fragments);
+ /* Allocate space for fragment signatures */
+ size = sizeof(fragment_metadata_t * ) * num_fragments;
+ c_fragment_metadata_list = (fragment_metadata_t**) alloc_zeroed_buffer(size);
+ size = sizeof(char *) * num_fragments;
+ c_fragment_signatures = (char **) alloc_zeroed_buffer(size);
+ if (NULL == c_fragment_metadata_list || NULL == c_fragment_signatures) {
+ goto error;
+ }
- /*
- * Populate and order the metadata
- */
- for (i=0; i < num_fragments; i++) {
+ /* Populate and order the metadata */
+ for (i = 0; i < num_fragments; i++) {
PyObject *tmp_data = PyList_GetItem(fragment_metadata_list, i);
Py_ssize_t len = 0;
char *c_buf = NULL;
@@ -1829,69 +2037,80 @@ pyeclib_c_check_metadata(PyObject *self, PyObject *args)
c_fragment_metadata_list[fragment_metadata->idx] = fragment_metadata;
if (supports_alg_sig(pyeclib_handle)) {
- c_fragment_signatures[fragment_metadata->idx] = (char*)get_aligned_buffer16(PYCC_MAX_SIG_LEN);
+ c_fragment_signatures[fragment_metadata->idx] = (char*)alloc_aligned_buffer16(PYCC_MAX_SIG_LEN);
memcpy(c_fragment_signatures[fragment_metadata->idx], fragment_metadata->signature, PYCC_MAX_SIG_LEN);
} else {
c_fragment_signatures[fragment_metadata->idx] = fragment_metadata->signature;
}
}
- /*
- * Ensure all fragments are here and check integrity using alg signatures
- */
+ /* Ensure all fragments are here and check integrity using alg signatures */
if (supports_alg_sig(pyeclib_handle)) {
- char **parity_sigs = (char**)malloc(sizeof(char**)*pyeclib_handle->m);
-
- for (i=0; i < pyeclib_handle->m; i++) {
- parity_sigs[i] = (char*)get_aligned_buffer16(PYCC_MAX_SIG_LEN);
- memset(parity_sigs[i], 0, PYCC_MAX_SIG_LEN);
+ char **parity_sigs = (char **) alloc_zeroed_buffer(sizeof(char **) * m);
+ if (NULL == parity_sigs) {
+ goto error;
+ }
+ for (i = 0; i < m; i++) {
+ parity_sigs[i] = (char *) alloc_aligned_buffer16(PYCC_MAX_SIG_LEN);
+ if (NULL == parity_sigs[i]) {
+ for (int j = 0; j < i; j++) free(parity_sigs[j]);
+ goto error;
+ } else {
+ memset(parity_sigs[i], 0, PYCC_MAX_SIG_LEN);
+ }
}
+ /* Calculate the parity of the signatures */
if (pyeclib_handle->type == PYECC_RS_VAND) {
- jerasure_matrix_encode(pyeclib_handle->k,
- pyeclib_handle->m,
- pyeclib_handle->w,
- pyeclib_handle->matrix,
- c_fragment_signatures,
- parity_sigs,
- PYCC_MAX_SIG_LEN);
+ jerasure_matrix_encode(k, m, w, pyeclib_handle->matrix,
+ c_fragment_signatures, parity_sigs, PYCC_MAX_SIG_LEN);
} else {
pyeclib_handle->xor_code_desc->encode(pyeclib_handle->xor_code_desc,
- c_fragment_signatures,
+ c_fragment_signatures,
parity_sigs,
PYCC_MAX_SIG_LEN);
}
-
- for (i=0; i < pyeclib_handle->m; i++) {
- if (memcmp(parity_sigs[i], c_fragment_signatures[pyeclib_handle->k + i], PYCC_MAX_SIG_LEN) != 0) {
+
+ /* Compare the parity of the signatures, and the signature of the parity */
+ for (i = 0; i < m; i++) {
+ if (memcmp(parity_sigs[i], c_fragment_signatures[k + i], PYCC_MAX_SIG_LEN) != 0) {
ret = i;
break;
}
}
- for (i=0; i < pyeclib_handle->m; i++) {
+ /* Clean up memory used in algebraic signature checking */
+ for (i = 0; i < m; i++) {
free(parity_sigs[i]);
}
- for (i=0; i < pyeclib_handle->k; i++) {
+ free_buffer(parity_sigs);
+ for (i = 0; i < k; i++) {
free(c_fragment_signatures[i]);
}
- free(parity_sigs);
} else if (use_inline_chksum(pyeclib_handle)) {
- for (i=0; i < num_fragments; i++) {
+ for (i = 0; i < num_fragments; i++) {
if (c_fragment_metadata_list[i]->chksum_mismatch == 1) {
ret = i;
break;
}
}
}
-
+
+ /* Return index of first checksum signature error */
+ ret_obj = PyLong_FromLong((long)ret);
+ goto exit;
+
+error:
+ ret_obj = NULL;
+
+exit:
free(c_fragment_signatures);
free(c_fragment_metadata_list);
-
- // TODO: Return a list containing tuples (index, problem). An empty list means everything is OK.
- return PyLong_FromLong((long)ret);
+
+ return ret_obj;
}
+
static PyMethodDef PyECLibMethods[] = {
{"init", pyeclib_c_init, METH_VARARGS, "Initialize a new erasure encoder/decoder"},
{"encode", pyeclib_c_encode, METH_VARARGS, "Create parity using source data"},
@@ -1925,4 +2144,3 @@ MOD_INIT(pyeclib_c)
return MOD_SUCCESS_VAL(m);
}
-