diff options
author | Kevin Greenan <kmgreen2@gmail.com> | 2014-06-03 10:24:15 -0700 |
---|---|---|
committer | Kevin Greenan <kmgreen2@gmail.com> | 2014-06-03 10:24:15 -0700 |
commit | 596c2f04fe4d5c3ad3b6ed196441909221727aba (patch) | |
tree | b6b07994640a5ce5bbbaf062cb463a7231e75dec | |
parent | 50f6b5769bc11b8016538f2cde94abde4e9e4150 (diff) | |
parent | 90b1d022ed70fa43edee58268813ea7ac7154ae5 (diff) | |
download | pyeclib-596c2f04fe4d5c3ad3b6ed196441909221727aba.tar.gz |
Merged in issue-43 (pull request #1)
Issue-43: Memory cleanup
-rw-r--r-- | src/c/pyeclib_c/pyeclib_c.c | 1582 |
1 files changed, 900 insertions, 682 deletions
diff --git a/src/c/pyeclib_c/pyeclib_c.c b/src/c/pyeclib_c/pyeclib_c.c index ecb74db..787d055 100644 --- a/src/c/pyeclib_c/pyeclib_c.c +++ b/src/c/pyeclib_c/pyeclib_c.c @@ -38,7 +38,6 @@ #include <pyeclib_c.h> #include <bytesobject.h> - /* Python 3 compatibility macros */ #if PY_MAJOR_VERSION >= 3 #define MOD_ERROR_VAL NULL @@ -89,6 +88,22 @@ static PyObject *PyECLibError; + +/** + * Prototypes for Python/C API methods + */ +static PyObject * pyeclib_c_init(PyObject *self, PyObject *args); +static void pyeclib_c_destructor(PyObject *obj); +static PyObject * pyeclib_c_get_segment_info(PyObject *self, PyObject *args); +static PyObject * pyeclib_c_encode(PyObject *self, PyObject *args); +static PyObject * pyeclib_c_fragments_to_string(PyObject *self, PyObject *args); +static PyObject * pyeclib_c_get_fragment_partition(PyObject *self, PyObject *args); +static PyObject * pyeclib_c_reconstruct(PyObject *self, PyObject *args); +static PyObject * pyeclib_c_decode(PyObject *self, PyObject *args); +static PyObject * pyeclib_c_get_metadata(PyObject *self, PyObject *args); +static PyObject * pyeclib_c_check_metadata(PyObject *self, PyObject *args); + + /* * Determine if an address is aligned to a particular boundary */ @@ -172,10 +187,75 @@ void init_fragment_header(char *buf) header->magic = PYECC_HEADER_MAGIC; } + +/** + * Memory Management Methods + * + * The following methods provide wrappers for allocating and deallocating + * memory. + * + * Future discussions may want to consider moving closer to the recommended + * guidelines in the Python\C API reference manual. One potential issue, + * however, may be how we enforce memory alignment in the Python heap. + * + * 2.7: https://docs.python.org/2.7/c-api/memory.html + * 3.4: https://docs.python.org/3.4/c-api/memory.html + */ + +/** + * Allocate a zero-ed buffer of a specific size. This method allocates from + * the Python stack in order to comply with the recommendations of the + * Python\C API. On error, return NULL and call PyErr_NoMemory. + * + * @param size integer size in bytes of buffer to allocate + * @return pointer to start of allocated buffer or NULL on error + */ static -void* get_aligned_buffer16(int size) +void * alloc_zeroed_buffer(int size) { - void *buf; + void * buf = NULL; /* buffer to allocate and return */ + + /* Allocate and zero the buffer, or set the appropriate error */ + buf = PyMem_Malloc((size_t) size); + if (buf) { + buf = memset(buf, 0, (size_t) size); + } else { + buf = (void *) PyErr_NoMemory(); + } + + return buf; +} + + +/** + * Deallocate memory buffer. This methods returns NULL so that you can free + * and reset a buffer using a single line as follows: + * + * my_ptr = free_buffer(my_ptr); + * + * @return NULL + */ +static +void * free_buffer(void * buf) +{ + if (buf) PyMem_Free(buf); + + return NULL; +} + + +/** + * Allocate and zero out a buffer aligned to a 16-byte boundary in order + * to support 128-bit operations. On error, call PyErr_NoMemory to set + * the appropriate error string and return NULL. + * + * @param size integer size in bytes of buffer to allocate + * @return pointer to start of allocated buffer, or NULL on error + */ +static +void* alloc_aligned_buffer16(int size) +{ + void *buf = NULL; /* * Ensure all memory is aligned to @@ -183,43 +263,107 @@ void* get_aligned_buffer16(int size) * 128-bit operations */ if (posix_memalign(&buf, 16, size) < 0) { - return NULL; + return PyErr_NoMemory(); + } else { + memset(buf, 0, size); } - bzero(buf, size); return buf; } + +/** + * Allocate an initialized fragment buffer. On error, return NULL and + * preserve call to PyErr_NoMemory. Note, all allocated memory is aligned + * to 16-bytes boundaries in order to support 128-bit operations. + * + * @param size integer size in bytes of buffer to allocate + * @return pointer to start of allocated fragment or NULL on error + */ static char *alloc_fragment_buffer(int size) { - char *buf; - fragment_header_t* header = NULL; + char *buf = NULL; + fragment_header_t *header = NULL; + /* Calculate size needed for fragment header + data */ size += sizeof(fragment_header_t); - /* - * Ensure all memory is aligned to - * 16-byte boundaries to support - * 128-bit operations - */ - if (posix_memalign((void**)&buf, 16, size) < 0) { - return NULL; + /* Allocate and init the aligned buffer, or set the appropriate error */ + buf = alloc_aligned_buffer16(size); + if (buf) { + init_fragment_header(buf); + } + + return buf; +} + + +/** + * Deallocate a fragment buffer. This method confirms via magic number that + * the passed in buffer is a proper fragment buffer. + * + * @param buf pointer to fragment buffer + * @return 0 on successful free, -1 on error + */ +static +int free_fragment_buffer(char *buf) +{ + fragment_header_t *header; + if (buf == NULL) { + return -1; } - bzero(buf, size); + + buf -= sizeof(fragment_header_t); header = (fragment_header_t*)buf; - header->magic = PYECC_HEADER_MAGIC; + if (header->magic != PYECC_HEADER_MAGIC) { + PyErr_SetString(PyECLibError, "Invalid fragment header (free fragment)!"); + return -1; + } + free(buf); - return buf; + return 0; } + +/** + * Allocate a zero-ed Python string of a specific size. On error, call + * PyErr_NoMemory and return NULL. + * + * @param size integer size in bytes of zero string to create + * @return pointer to start of allocated zero string or NULL on error + */ +static +PyObject * alloc_zero_string(int size) +{ + char *tmp_data = NULL; /* tmp buffer used in PyObject creation */ + PyObject *zero_string = NULL; /* zero string to return */ + + /* Create the zero-out c-string buffer */ + tmp_data = (char *) alloc_zeroed_buffer(size); + if (NULL == tmp_data) { + return PyErr_NoMemory(); + } + + /* Create the python value to return */ + zero_string = PY_BUILDVALUE_OBJ_LEN(tmp_data, size); + free_buffer(tmp_data); + + return zero_string; +} + + static char* get_data_ptr_from_fragment(char *buf) { - buf += sizeof(fragment_header_t); + char * data_ptr = NULL; - return buf; + if (NULL != buf) { + data_ptr = buf + sizeof(fragment_header_t); + } + + return data_ptr; } static @@ -385,74 +529,6 @@ int validate_fragment(char *buf) return 0; } -static -int free_fragment_buffer(char *buf) -{ - fragment_header_t *header; - if (buf == NULL) { - return -1; - } - - buf -= sizeof(fragment_header_t); - - header = (fragment_header_t*)buf; - if (header->magic != PYECC_HEADER_MAGIC) { - PyErr_SetString(PyECLibError, "Invalid fragment header (free fragment)!"); - return -1; - } - free(buf); - - return 0; -} - -static void pyeclib_c_destructor(PyObject *obj) -{ - pyeclib_t *pyeclib_handle = NULL; - - if (!PyCapsule_CheckExact(obj)) { - PyErr_SetString(PyECLibError, "Attempted to free a non-Capsule object in pyeclib"); - return; - } - - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(obj, PYECC_HANDLE_NAME); - - if (pyeclib_handle == NULL) { - PyErr_SetString(PyECLibError, "Attempted to free an invalid reference to pyeclib_handle"); - return; - } -} - -static -int get_fragment_metadata(pyeclib_t *pyeclib_handle, char *fragment_buf, fragment_metadata_t *fragment_metadata) -{ - char *fragment_data = get_data_ptr_from_fragment(fragment_buf); - int fragment_size = get_fragment_size(fragment_buf); - int fragment_idx = get_fragment_idx(fragment_buf); - - memset(fragment_metadata, 0, sizeof(fragment_metadata_t)); - - fragment_metadata->size = fragment_size; - fragment_metadata->idx = fragment_idx; - - /* - * If w \in [8, 16] and using RS_VAND or XOR_CODE, then use Alg_sig - * Else use CRC32 - */ - if (supports_alg_sig(pyeclib_handle)) { - // Compute algebraic signature - compute_alg_sig(pyeclib_handle->alg_sig_desc, fragment_data, fragment_size, fragment_metadata->signature); - } else if (use_inline_chksum(pyeclib_handle)) { - int stored_chksum = get_chksum(fragment_buf); - int computed_chksum = crc32(0, fragment_data, fragment_size); - - if (stored_chksum != computed_chksum) { - fragment_metadata->chksum_mismatch = 1; - } - } - - return 0; -} - /* * Buffers for data, parity and missing_idxs @@ -574,41 +650,116 @@ static int get_decoding_info(pyeclib_t *pyeclib_handle, return 0; } + +/** + * Compute a size aligned to the number of data and the underlying wordsize + * of the EC algorithm. + * + * @param pyeclib_handle eclib object with EC configurations + * @param data_len integer length of data in bytes + * @return integer data length aligned with wordsize of EC algorithm + */ +static int +get_aligned_data_size(pyeclib_t* pyeclib_handle, int data_len) +{ + int word_size = pyeclib_handle->w / 8; + int alignment_multiple; + int aligned_size = 0; + + /* + * For Cauchy reed-solomon align to k*word_size*packet_size + * For Vandermonde reed-solomon and flat-XOR, align to k*word_size + */ + if (pyeclib_handle->type == PYECC_RS_CAUCHY_ORIG) { + alignment_multiple = pyeclib_handle->k * pyeclib_handle->w * PYECC_CAUCHY_PACKETSIZE; + } else { + alignment_multiple = pyeclib_handle->k * word_size; + } + + aligned_size = (int)ceill((double)data_len / alignment_multiple) * alignment_multiple; + + return aligned_size; +} + + +static +int get_minimum_encode_size(pyeclib_t *pyeclib_handle) +{ + return get_aligned_data_size(pyeclib_handle, 1); +} + + +static +int get_fragment_metadata(pyeclib_t *pyeclib_handle, char *fragment_buf, fragment_metadata_t *fragment_metadata) +{ + char *fragment_data = get_data_ptr_from_fragment(fragment_buf); + int fragment_size = get_fragment_size(fragment_buf); + int fragment_idx = get_fragment_idx(fragment_buf); + + memset(fragment_metadata, 0, sizeof(fragment_metadata_t)); + + fragment_metadata->size = fragment_size; + fragment_metadata->idx = fragment_idx; + + /* + * If w \in [8, 16] and using RS_VAND or XOR_CODE, then use Alg_sig + * Else use CRC32 + */ + if (supports_alg_sig(pyeclib_handle)) { + // Compute algebraic signature + compute_alg_sig(pyeclib_handle->alg_sig_desc, fragment_data, fragment_size, fragment_metadata->signature); + } else if (use_inline_chksum(pyeclib_handle)) { + int stored_chksum = get_chksum(fragment_buf); + int computed_chksum = crc32(0, fragment_data, fragment_size); + + if (stored_chksum != computed_chksum) { + fragment_metadata->chksum_mismatch = 1; + } + } + + return 0; +} + + +/** + * Constructor method for creating a new pyeclib object using the given parameters. + * + * @param k integer number of data elements + * @param m integer number of checksum elements + * @param w integer word size in bytes + * @param type_str string name of erasure coding algorithm + * @return pointer to PyObject or NULL on error + */ static PyObject * pyeclib_c_init(PyObject *self, PyObject *args) { - pyeclib_t *pyeclib_handle; - PyObject* pyeclib_obj_handle; + pyeclib_t *pyeclib_handle = NULL; + PyObject *pyeclib_obj_handle = NULL; int k, m, w; int use_inline_chksum = 0, use_algsig_chksum = 0; const char *type_str; pyeclib_type_t type; + /* Obtain and validate the method parameters */ if (!PyArg_ParseTuple(args, "iiis|ii", &k, &m, &w, &type_str, &use_inline_chksum, &use_algsig_chksum)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.init"); return NULL; } - - // Validate the arguments type = get_ecc_type(type_str); if (type == PYECC_NOT_FOUND) { PyErr_SetString(PyECLibError, "Invalid type passed to pyeclib.init"); return NULL; } - if (!validate_args(k, m, w, type)) { PyErr_SetString(PyECLibError, "Invalid args passed to pyeclib.init"); return NULL; } - - pyeclib_handle = (pyeclib_t*)malloc(sizeof(pyeclib_t)); - if (pyeclib_handle == NULL) { - PyErr_SetString(PyECLibError, "malloc() returned NULL in pyeclib.init"); + + /* Allocate and initialize the pyeclib object */ + pyeclib_handle = (pyeclib_t *) alloc_zeroed_buffer(sizeof(pyeclib_t)); + if (!pyeclib_handle) { return NULL; } - - memset(pyeclib_handle, 0, sizeof(pyeclib_t)); - pyeclib_handle->k = k; pyeclib_handle->m = m; pyeclib_handle->w = w; @@ -645,59 +796,54 @@ pyeclib_c_init(PyObject *self, PyObject *args) break; } + /* Prepare the python object to return */ #ifdef Py_CAPSULE_H pyeclib_obj_handle = PyCapsule_New(pyeclib_handle, PYECC_HANDLE_NAME, pyeclib_c_destructor); #else pyeclib_obj_handle = PyCObject_FromVoidPtrAndDesc(pyeclib_handle, - (void *) PYECC_HANDLE_NAME, - pyeclib_c_destructor); + (void *) PYECC_HANDLE_NAME, + pyeclib_c_destructor); #endif /* Py_CAPSULE_H */ + /* Clean up the allocated memory on error, or update the ref count */ if (pyeclib_obj_handle == NULL) { PyErr_SetString(PyECLibError, "Could not encapsulate pyeclib_handle into Python object in pyeclib.init"); - return NULL; + free_buffer(pyeclib_handle); + } else { + Py_INCREF(pyeclib_obj_handle); } - - Py_INCREF(pyeclib_obj_handle); - + return pyeclib_obj_handle; } -static int -get_aligned_data_size(pyeclib_t* pyeclib_handle, int data_len) + +/** + * Destructor method for cleaning up pyeclib object. + */ +static void +pyeclib_c_destructor(PyObject *obj) { - int word_size = pyeclib_handle->w / 8; - int alignment_multiple; - int aligned_size = 0; + pyeclib_t *pyeclib_handle = NULL; /* pyeclib object to destroy */ - /* - * For Cauchy reed-solomon align to k*word_size*packet_size - * - * For Vandermonde reed-solomon and flat-XOR, align to k*word_size - */ - if (pyeclib_handle->type == PYECC_RS_CAUCHY_ORIG) { - alignment_multiple = pyeclib_handle->k * pyeclib_handle->w * PYECC_CAUCHY_PACKETSIZE; - } else { - alignment_multiple = pyeclib_handle->k * word_size; + if (!PyCapsule_CheckExact(obj)) { + PyErr_SetString(PyECLibError, "Attempted to free a non-Capsule object in pyeclib"); + return; } - aligned_size = (int)ceill((double)data_len / alignment_multiple) * alignment_multiple; - - return aligned_size; -} - -static -int get_minimum_encode_size(pyeclib_t *pyeclib_handle) -{ - return get_aligned_data_size(pyeclib_handle, 1); + pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(obj, PYECC_HANDLE_NAME); + if (pyeclib_handle == NULL) { + PyErr_SetString(PyECLibError, "Attempted to free an invalid reference to pyeclib_handle"); + } else { + free_buffer(pyeclib_handle); + } + + return; } /** - * - * This function takes data length and - * a segment size and returns an object + * This function takes data length and a segment size and returns an object * containing: * * segment_size: size of the payload to give to encode() @@ -706,28 +852,30 @@ int get_minimum_encode_size(pyeclib_t *pyeclib_handle) * last_fragment_size: the fragment size returned by encode() * num_segments: number of segments * - * This allows the caller to prepare requests - * when segmenting a data stream to be EC'd. + * This allows the caller to prepare requests when segmenting a data stream + * to be EC'd. * - * Since the data length will rarely be aligned - * to the segment size, the last segment will be - * a different size than the others. + * Since the data length will rarely be aligned to the segment size, the last + * segment will be a different size than the others. * - * There are restrictions on the length given to encode(), - * so calling this before encode is highly recommended when - * segmenting a data stream. + * There are restrictions on the length given to encode(), so calling this + * before encode is highly recommended when segmenting a data stream. * - * Minimum segment size depends on the underlying EC type - * (if it is less than this, then the last segment will be - * slightly larger than the others, otherwise it will be smaller). + * Minimum segment size depends on the underlying EC type (if it is less + * than this, then the last segment will be slightly larger than the others, + * otherwise it will be smaller). + * + * @param pyeclib_obj_handle + * @param data_len integer length of data in bytes + * @param segment_size integer length of segment in bytes * */ static PyObject * pyeclib_c_get_segment_info(PyObject *self, PyObject *args) { - PyObject *pyeclib_obj_handle; - PyObject *ret_dict; - pyeclib_t *pyeclib_handle; + PyObject *pyeclib_obj_handle = NULL; + PyObject *ret_dict = NULL; + pyeclib_t *pyeclib_handle = NULL; int data_len; int segment_size, last_segment_size; int num_segments; @@ -736,11 +884,11 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args) int aligned_segment_size; int aligned_data_len; + /* Obtain and validate the method parameters */ if (!PyArg_ParseTuple(args, "Oii", &pyeclib_obj_handle, &data_len, &segment_size)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode"); return NULL; } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); if (pyeclib_handle == NULL) { PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode"); @@ -829,212 +977,225 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args) last_fragment_size += sizeof(fragment_header_t); fragment_size += sizeof(fragment_header_t); + /* Create and return the python dictionary of segment info */ ret_dict = PyDict_New(); - - PyDict_SetItem(ret_dict, PyString_FromString("segment_size\0"), PyInt_FromLong(segment_size)); - PyDict_SetItem(ret_dict, PyString_FromString("last_segment_size\0"), PyInt_FromLong(last_segment_size)); - PyDict_SetItem(ret_dict, PyString_FromString("fragment_size\0"), PyInt_FromLong(fragment_size)); - PyDict_SetItem(ret_dict, PyString_FromString("last_fragment_size\0"), PyInt_FromLong(last_fragment_size)); - PyDict_SetItem(ret_dict, PyString_FromString("num_segments\0"), PyInt_FromLong(num_segments)); - + if (NULL == ret_dict) { + PyErr_SetString(PyECLibError, "Error allocating python dict in get_segment_info"); + } else { + PyDict_SetItem(ret_dict, PyString_FromString("segment_size\0"), PyInt_FromLong(segment_size)); + PyDict_SetItem(ret_dict, PyString_FromString("last_segment_size\0"), PyInt_FromLong(last_segment_size)); + PyDict_SetItem(ret_dict, PyString_FromString("fragment_size\0"), PyInt_FromLong(fragment_size)); + PyDict_SetItem(ret_dict, PyString_FromString("last_fragment_size\0"), PyInt_FromLong(last_fragment_size)); + PyDict_SetItem(ret_dict, PyString_FromString("num_segments\0"), PyInt_FromLong(num_segments)); + } + return ret_dict; } + +/** + * Erasure encode a data buffer. + * + * @param pyeclib_obj_handle + * @param data to encode + * @return python list of encoded data and parity elements + */ static PyObject * pyeclib_c_encode(PyObject *self, PyObject *args) { - PyObject *pyeclib_obj_handle; - pyeclib_t *pyeclib_handle; - char *data; - int data_len; - int aligned_data_len; - int orig_data_size; - int blocksize; - char **data_to_encode; - char **encoded_parity; - int i; - PyObject *list_of_strips; + PyObject *pyeclib_obj_handle = NULL; + pyeclib_t *pyeclib_handle= NULL; + char **data_to_encode = NULL; /* array of k data buffers */ + char **encoded_parity = NULL; /* array of m parity buffers */ + PyObject *list_of_strips = NULL; /* list of encoded strips to return */ + char *data; /* param, data buffer to encode */ + int data_len; /* param, length of data buffer */ + int aligned_data_len; /* EC algorithm compatible data length */ + int orig_data_size; /* data length to write to headers */ + int blocksize; /* length of each of k data elements */ + int i; /* a counter */ /* Assume binary data (force "byte array" input) */ if (!PyArg_ParseTuple(args, ENCODE_ARGS, &pyeclib_obj_handle, &data, &data_len)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode"); return NULL; } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); if (pyeclib_handle == NULL) { PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode"); return NULL; } - - /* - * Grab the original size to put in the headers - */ - orig_data_size = data_len; - /* - * This will copmpute a size algined to the number of data - * and the underlying wordsize of the EC algorithm. - */ + /* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/ + orig_data_size = data_len; aligned_data_len = get_aligned_data_size(pyeclib_handle, data_len); - - /* - * aligned_data_len is guaranteed to be divisible by k - */ blocksize = aligned_data_len / pyeclib_handle->k; - data_to_encode = (char**)malloc(sizeof(char*)*pyeclib_handle->k); - if (data_to_encode == NULL) { - PyErr_SetString(PyECLibError, "Could not allocate memory in pyeclib.encode"); - return NULL; + /* Allocate and initialize an array of zero'd out data buffers */ + data_to_encode = (char**) alloc_zeroed_buffer(sizeof(char*) * pyeclib_handle->k); + if (NULL == data_to_encode) { + goto error; } - - for (i=0; i < pyeclib_handle->k; i++) { - char *fragment = alloc_fragment_buffer(blocksize); + for (i = 0; i < pyeclib_handle->k; i++) { int payload_size = data_len > blocksize ? blocksize : data_len; - data_to_encode[i] = get_data_ptr_from_fragment(fragment); - if (data_to_encode[i] == NULL) { - PyErr_SetString(PyECLibError, "Could not allocate memory in pyeclib.encode"); - return NULL; + char *fragment = alloc_fragment_buffer(blocksize); + if (NULL == fragment) { + goto error; } - - /* - * Only do the memcpy if there is data to copy - * - * alloc_fragment_buffer() does a bzero(), so the padding - * will be all 0s. - */ + + /* Copy existing data into clean, zero'd out buffer */ + data_to_encode[i] = get_data_ptr_from_fragment(fragment); if (data_len > 0) { memcpy(data_to_encode[i], data, payload_size); } - /* - * Fragment size will always be the same (may be able to get rid of this) - */ + /* Fragment size will always be the same (may be able to get rid of this) */ set_fragment_size(fragment, blocksize); - data += payload_size; data_len -= payload_size; } - encoded_parity = (char**)malloc(sizeof(char*)*pyeclib_handle->m); - if (encoded_parity == NULL) { - PyErr_SetString(PyECLibError, "Could not allocate memory in pyeclib.encode"); - return NULL; + /* Allocate and initialize an array of zero'd out parity buffers */ + encoded_parity = (char**) alloc_zeroed_buffer(sizeof(char*) * pyeclib_handle->m); + if (NULL == encoded_parity) { + goto error; } - - for (i=0; i < pyeclib_handle->m; i++) { + for (i = 0; i < pyeclib_handle->m; i++) { char *fragment = alloc_fragment_buffer(blocksize); - encoded_parity[i] = get_data_ptr_from_fragment(fragment); - if (encoded_parity[i] == NULL) { - PyErr_SetString(PyECLibError, "Could not allocate memory in pyeclib.encode"); - return NULL; + if (NULL == fragment) { + goto error; } + + encoded_parity[i] = get_data_ptr_from_fragment(fragment); set_fragment_size(fragment, blocksize); } + /* Run the erasure coding algorithm to generate the parity fragments */ switch (pyeclib_handle->type) { case PYECC_RS_CAUCHY_ORIG: - jerasure_bitmatrix_encode(pyeclib_handle->k, pyeclib_handle->m, pyeclib_handle->w, pyeclib_handle->bitmatrix, data_to_encode, encoded_parity, blocksize, PYECC_CAUCHY_PACKETSIZE); + jerasure_bitmatrix_encode(pyeclib_handle->k, pyeclib_handle->m, + pyeclib_handle->w, pyeclib_handle->bitmatrix, + data_to_encode, encoded_parity, blocksize, + PYECC_CAUCHY_PACKETSIZE); break; case PYECC_XOR_HD_3: case PYECC_XOR_HD_4: - pyeclib_handle->xor_code_desc->encode(pyeclib_handle->xor_code_desc, data_to_encode, encoded_parity, blocksize); + pyeclib_handle->xor_code_desc->encode(pyeclib_handle->xor_code_desc, + data_to_encode, encoded_parity, + blocksize); break; case PYECC_RS_VAND: default: - jerasure_matrix_encode(pyeclib_handle->k, pyeclib_handle->m, pyeclib_handle->w, pyeclib_handle->matrix, data_to_encode, encoded_parity, blocksize); + jerasure_matrix_encode(pyeclib_handle->k, pyeclib_handle->m, + pyeclib_handle->w, pyeclib_handle->matrix, + data_to_encode, encoded_parity, blocksize); break; } + /* Create the python list of fragments to return */ list_of_strips = PyList_New(pyeclib_handle->k + pyeclib_handle->m); + if (NULL == list_of_strips) { + PyErr_SetString(PyECLibError, "Error allocating python list in encode"); + goto error; + } - for (i=0; i < pyeclib_handle->k; i++) { + /* Finalize data fragments and add them to the python list to return */ + for (i = 0; i < pyeclib_handle->k; i++) { char *fragment_ptr = get_fragment_ptr_from_data(data_to_encode[i]); - int fragment_size = blocksize+sizeof(fragment_header_t); + int fragment_size = blocksize + sizeof(fragment_header_t); set_fragment_idx(fragment_ptr, i); set_orig_data_size(fragment_ptr, orig_data_size); if (use_inline_chksum(pyeclib_handle)) { int chksum = crc32(0, data_to_encode[i], blocksize); set_chksum(fragment_ptr, chksum); } - PyList_SET_ITEM(list_of_strips, i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size)); - free_fragment_buffer(data_to_encode[i]); + PyList_SET_ITEM(list_of_strips, i, + PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size)); } - free(data_to_encode); - for (i=0; i < pyeclib_handle->m; i++) { + /* Finalize parity fragments and add them to the python list to return */ + for (i = 0; i < pyeclib_handle->m; i++) { char *fragment_ptr = get_fragment_ptr_from_data(encoded_parity[i]); - int fragment_size = blocksize+sizeof(fragment_header_t); + int fragment_size = blocksize + sizeof(fragment_header_t); set_fragment_idx(fragment_ptr, pyeclib_handle->k+i); set_orig_data_size(fragment_ptr, orig_data_size); if (use_inline_chksum(pyeclib_handle)) { int chksum = crc32(0, encoded_parity[i], blocksize); set_chksum(fragment_ptr, chksum); } - PyList_SET_ITEM(list_of_strips, pyeclib_handle->k + i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size)); - free_fragment_buffer(encoded_parity[i]); + PyList_SET_ITEM(list_of_strips, pyeclib_handle->k + i, + PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size)); } - free(encoded_parity); + + goto exit; +error: + list_of_strips = NULL; + +exit: + if (data_to_encode) { + for (i = 0; i < pyeclib_handle->k; i++) { + if (data_to_encode[i]) free_fragment_buffer(data_to_encode[i]); + } + free_buffer(data_to_encode); + } + if (encoded_parity) { + for (i = 0; i < pyeclib_handle->m; i++) { + if (encoded_parity[i]) free_fragment_buffer(encoded_parity[i]); + } + free_buffer(encoded_parity); + } + return list_of_strips; } + /* - * Convert a set of fragments into a string. If, - * all k data elements are not present, return NULL; - * otherwise return a string. + * Convert a set of fragments into a string. If, less than k data fragments + * are present, return None. Return NULL on error. + * + * @param pyeclib_obj_handle + * @param list of fragments + * @param python string or None, NULL on error */ static PyObject * pyeclib_c_fragments_to_string(PyObject *self, PyObject *args) { - PyObject *pyeclib_obj_handle; - PyObject *fragment_list; - PyObject *ret_string; - pyeclib_t* pyeclib_handle; - char *ret_cstring; - char **data; - int string_len = 0; - int string_off = 0; - int i = 0; - int num_fragments = 0; - int num_data = 0; - int orig_data_size = -1; - int ret_data_size; - + PyObject *pyeclib_obj_handle = NULL; + pyeclib_t *pyeclib_handle = NULL; + PyObject *fragment_list = NULL; /* param, python list of fragments */ + PyObject *ret_string = NULL; /* python string to return */ + char *ret_cstring = NULL; /* c string to build return string from */ + int ret_data_size; /* size of return string in bytes */ + char **data = NULL; /* array of data buffers */ + int string_off = 0; /* offset into cstring */ + int num_fragments = 0; /* num of fragments provided by caller */ + int num_data = 0; /* num of fragments that are data */ + int orig_data_size = -1; /* data size from fragment header */ + int i = 0; /* a counter */ + + /* Collect and validate the method arguments */ if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_list)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.fragments_to_string"); return NULL; } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); if (pyeclib_handle == NULL) { PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.fragments_to_string"); return NULL; } - if (!PyList_Check(fragment_list)) { PyErr_SetString(PyECLibError, "Invalid structure passed in for fragment list in pyeclib.fragments_to_string"); return NULL; } - // Set the number of fragments we have - num_fragments = (int)PyList_Size(fragment_list); - - /* - * If we have less than k fragments, then we definitely need to call decode, so return NULL - */ + /* Return None if there's insufficient fragments */ + num_fragments = (int) PyList_Size(fragment_list); if (pyeclib_handle->k > num_fragments) { return Py_BuildValue(""); } - - data = (char**)malloc(sizeof(char*) * pyeclib_handle->k); - if (data == NULL) { - PyErr_SetString(PyECLibError, "Could not allocate memory for data buffers"); - return NULL; - } - + /* * NOTE: Update to only copy original size out of the buffers */ @@ -1043,356 +1204,348 @@ pyeclib_c_fragments_to_string(PyObject *self, PyObject *args) * Iterate over the fragments. If we have all k data fragments, then we can * concatenate them into a string and return it; otherwise, we return NULL */ - for (i=0; i < num_fragments && num_data < pyeclib_handle->k; i++) { + data = (char **) alloc_zeroed_buffer(sizeof(char *) * pyeclib_handle->k); + if (NULL == data) { + return NULL; + } + for (i = 0; i < num_fragments && num_data < pyeclib_handle->k; i++) { PyObject *tmp_data = PyList_GetItem(fragment_list, i); char *tmp_buf; int index; int data_size; Py_ssize_t len; + /* Get and validate the fragment index and size */ PyBytes_AsStringAndSize(tmp_data, &tmp_buf, &len); - - /* - * Get fragment index and size - */ index = get_fragment_idx(tmp_buf); - if (index < 0) { - ret_string = NULL; - goto out; - } data_size = get_fragment_size(tmp_buf); - if (data_size < 0) { + if ((index < 0) || (data_size < 0)) { ret_string = NULL; - goto out; + goto exit; } - /* - * If we got this far, then we shouold - * find a valid original data size. - */ + /* Validate the original data size */ if (orig_data_size < 0) { orig_data_size = get_orig_data_size(tmp_buf); } else { - int my_orig_data_size = get_orig_data_size(tmp_buf); - if (my_orig_data_size != orig_data_size) { + if (get_orig_data_size(tmp_buf) != orig_data_size) { PyErr_SetString(PyECLibError, "Inconsistent orig data sizes found in headers"); ret_string = NULL; - goto out; + goto exit; } } - /* - * Skip over parity fragments - */ + /* Skip parity fragments, put data fragments in index order */ if (index >= pyeclib_handle->k) { continue; + } else { + data[index] = tmp_buf; + num_data++; } - - /* - * Put fragment reference in proper index of data - */ - data[index] = tmp_buf; - - // Increment the number of data fragments we have - num_data++; - string_len += data_size; } + /* Return None if there still isn't insufficient fragments */ if (num_data != pyeclib_handle->k) { ret_string = Py_BuildValue(""); - goto out; + goto exit; + } + + /* Create the c-string to return */ + ret_cstring = (char *) alloc_zeroed_buffer(orig_data_size); + if (NULL == ret_cstring) { + ret_string = NULL; + goto exit; } - - ret_cstring = (char*)malloc(orig_data_size); - ret_data_size = orig_data_size; - /* - * Copy data payloads into a cstring. The - * fragments should be ordered by index in data. - */ - for (i=0; i < num_data && orig_data_size > 0; i++) { + /* Copy fragment data into cstring (fragments should be in index order) */ + for (i = 0; i < num_data && orig_data_size > 0; i++) { char* fragment_data = get_data_ptr_from_fragment(data[i]); int fragment_size = get_fragment_size(data[i]); int payload_size = orig_data_size > fragment_size ? fragment_size : orig_data_size; memcpy(ret_cstring + string_off, fragment_data, payload_size); - orig_data_size -= payload_size; string_off += payload_size; } - ret_string = PY_BUILDVALUE_OBJ_LEN(ret_cstring, ret_data_size); - free(ret_cstring); -out: - free(data); +exit: + free_buffer(data); + free_buffer(ret_cstring); + return ret_string; } + /* - * Return a tuple containing a refernece to a data fragment - * list, a parity fragment list and a missing fragment index list, - * including all '0' fragment for missing fragments. Copy - * references to the existing fragments and create new string - * for missing fragments. + * Return a tuple containing a reference to a data fragment list, a parity + * fragment list and a missing fragment index list, including all '0' + * fragment for missing fragments. Copy references to the existing + * fragments and create new string for missing fragments. + * + * @param pyeclib_obj_handle + * @param list of fragments + * @return (data fragments, parity fragments, missing indexes) tuple */ static PyObject * pyeclib_c_get_fragment_partition(PyObject *self, PyObject *args) { - PyObject *pyeclib_obj_handle; - PyObject *fragment_list; - PyObject *data_list; - PyObject *parity_list; - PyObject *missing_list; - PyObject *return_lists; - PyObject **data; - PyObject **parity; - int *missing; - pyeclib_t* pyeclib_handle; - int num_fragments; - int num_missing = 0; - int i; - int fragment_size = 0; - + PyObject *pyeclib_obj_handle = NULL; + pyeclib_t* pyeclib_handle = NULL; + PyObject *fragment_list = NULL; /* param, list of fragments */ + PyObject *data_list = NULL; /* data fragments for return tuple */ + PyObject *parity_list = NULL; /* parity fragments for return tuple */ + PyObject *fragment_string = NULL; /* fragment string for lists */ + PyObject *missing_list = NULL; /* missing indexes for return tuple */ + PyObject *return_lists = NULL; /* Python tuple to return */ + PyObject **data = NULL; /* array of k data fragment buffers*/ + PyObject **parity = NULL; /* array of m parity fragment buffers */ + int *missing = NULL; /* indicies of missing fragments */ + int num_missing = 0; /* num of missing fragments */ + int num_fragments; /* num of frags provided by caller */ + int fragment_size = 0; /* size in bytes of fragments */ + int i = 0; /* a counter */ + + /* Collect and validate the method arguments */ if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_list)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.get_fragment_partition"); return NULL; } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); if (pyeclib_handle == NULL) { PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.get_fragment_partition"); return NULL; } - if (!PyList_Check(fragment_list)) { PyErr_SetString(PyECLibError, "Invalid structure passed in for fragment list in pyeclib.get_fragment_partition"); return NULL; } - // Set the number of fragments we have - num_fragments = (int)PyList_Size(fragment_list); - - data = (PyObject**)malloc(pyeclib_handle->k * sizeof(PyObject*)); - parity = (PyObject**)malloc(pyeclib_handle->m * sizeof(PyObject*)); - - for (i=0; i < pyeclib_handle->k; i++) { - data[i] = NULL; + /* Create the arrays need to hold the data and parity fragments */ + data = (PyObject **) alloc_zeroed_buffer(pyeclib_handle->k * sizeof(PyObject*)); + if (NULL == data) { + goto error; } - - for (i=0; i < pyeclib_handle->m; i++) { - parity[i] = NULL; + parity = (PyObject **) alloc_zeroed_buffer(pyeclib_handle->m * sizeof(PyObject*)); + if (NULL == parity) { + goto error; } - /* - * ASSUMPTION: We will never try to do anything when we have more - * than k missing fragments - */ - missing = (int*)malloc(pyeclib_handle->k * sizeof(int)); - - /* - * Fill in the data, parity and missing indexes - */ - for (i=0; i < num_fragments; i++) { + + /* Fill in the data and parity pointers to reveal missing fragments */ + num_fragments = (int) PyList_Size(fragment_list); + for (i = 0; i < num_fragments; i++) { PyObject *tmp_data = PyList_GetItem(fragment_list, i); char *c_buf; int index; Py_ssize_t len; - + + /* ASSUMPTION: The fragment_size is the max of the fragments we read */ PyBytes_AsStringAndSize(tmp_data, &c_buf, &len); - - /* - * Assume the fragment_size is the max of - * the fragments we read - */ if (len > fragment_size) { - fragment_size = (int)len; + fragment_size = (int) len; } - /* - * Get fragment index and size - */ + /* Get fragment index and set the index in data or parity */ index = get_fragment_idx(c_buf); - if (index < pyeclib_handle->k) { - // Data element data[index] = tmp_data; - } else { - // Parity element parity[index - pyeclib_handle->k] = tmp_data; } } - - /* - * If there are missing bufs, figure out which indexes are missing + + /* If there are missing bufs, figure out which indexes are missing + * + * ASSUMPTION: We will never try to do anything when we have more + * than k missing fragments */ - if (missing != NULL) { + missing = (int *) alloc_zeroed_buffer(pyeclib_handle->k * sizeof(int)); + if (NULL == missing) { + goto error; + } else { num_missing = 0; - for (i=0; i < pyeclib_handle->k; i++) { + for (i = 0; i < pyeclib_handle->k; i++) { if (data[i] == NULL) { missing[num_missing] = i; num_missing++; } } - for (i=0; i < pyeclib_handle->m; i++) { + for (i = 0; i < pyeclib_handle->m; i++) { if (parity[i] == NULL) { missing[num_missing] = i + pyeclib_handle->k; num_missing++; } } } - + /* Create the python objects to return */ data_list = PyList_New(pyeclib_handle->k); parity_list = PyList_New(pyeclib_handle->m); missing_list = PyList_New(num_missing); + return_lists = PyTuple_New(3); + if (!data_list || !parity_list || !missing_list || !return_lists) { + return_lists = PyErr_NoMemory(); + goto exit; + } - /* - * Fill in the data fragments - */ - for (i=0; i < pyeclib_handle->k; i++) { + /* Fill in the data fragments, create zero fragment if missing */ + for (i = 0; i < pyeclib_handle->k; i++) { if (data[i] != NULL) { - // This is a borrowed reference, so increment the ref count - // before inserting into list + /* BORROWED REF: increment ref count before inserting into list */ Py_INCREF(data[i]); - PyList_SET_ITEM(data_list, i, data[i]); + fragment_string = data[i]; } else { - char *tmp_data = (char*)malloc(fragment_size); - PyObject *zero_string; - memset(tmp_data, 0, fragment_size); - zero_string = PY_BUILDVALUE_OBJ_LEN(tmp_data, fragment_size); - //Py_INCREF(zero_string); - PyList_SET_ITEM(data_list, i, zero_string); - free(tmp_data); + fragment_string = alloc_zero_string(fragment_size); + if (NULL == fragment_string) { + goto error; + } } + PyList_SET_ITEM(data_list, i, fragment_string); } - free(data); - - /* - * Fill in the parity fragments - */ - for (i=0; i < pyeclib_handle->m; i++) { + /* Fill in the parity fragments, create zero fragment if missing */ + for (i = 0; i < pyeclib_handle->m; i++) { if (parity[i] != NULL) { - // This is a borrowed reference, so increment the ref count - // before inserting into list + /* BORROWED REF: increment ref count before inserting into list */ Py_INCREF(parity[i]); - PyList_SET_ITEM(parity_list, i, parity[i]); + fragment_string = parity[i]; } else { - char *tmp_parity = (char*)malloc(fragment_size); - PyObject *zero_string; - memset(tmp_parity, 0, fragment_size); - zero_string = PY_BUILDVALUE_OBJ_LEN(tmp_parity, fragment_size); - //Py_INCREF(zero_string); - PyList_SET_ITEM(parity_list, i, zero_string); - free(tmp_parity); + fragment_string = alloc_zero_string(fragment_size); + if (NULL == fragment_string) { + goto error; + } } + PyList_SET_ITEM(parity_list, i, fragment_string); } - free(parity); - - for( i=0;i < num_missing; i++) { + /* Fill in the list of missing indexes */ + for (i = 0;i < num_missing; i++) { PyList_SET_ITEM(missing_list, i, Py_BuildValue("i", missing[i])); } - - free(missing); - - return_lists = PyTuple_New(3); Py_INCREF(data_list); Py_INCREF(parity_list); Py_INCREF(missing_list); - + PyTuple_SetItem(return_lists, 0, data_list); PyTuple_SetItem(return_lists, 1, parity_list); PyTuple_SetItem(return_lists, 2, missing_list); - //Py_INCREF(return_lists); + goto exit; + +error: + return_lists = NULL; + +exit: + free_buffer(data); + free_buffer(parity); + free_buffer(missing); + return return_lists; } + +/** + * Return a list of lists with valid rebuild indexes given an EC algorithm + * and a list of missing indexes. + * + * @param pyeclib_obj_handle + * @param missing_list indexes of missing fragments + * @return a list of lists of indexes to rebuild data from + */ static PyObject * pyeclib_c_get_required_fragments(PyObject *self, PyObject *args) { - PyObject *pyeclib_obj_handle; - PyObject *missing_list; - PyObject *fragment_idx_list = NULL; - pyeclib_t* pyeclib_handle; - int num_missing; - int *c_missing_list; - int i, j; - unsigned long missing_bm = 0; - + PyObject *pyeclib_obj_handle = NULL; + pyeclib_t *pyeclib_handle = NULL; + PyObject *missing_list = NULL; /* param, */ + PyObject *fragment_idx_list = NULL; /* list of req'd indexes to return */ + int *c_missing_list = NULL; /* c-array of missing indexes */ + int num_missing; /* size of passed in missing list */ + int i = 0, j = 0; /* counters */ + int k, m; /* EC algorithm parameters */ + unsigned long missing_bm = 0; /* bitmap of missing indexes */ + int *fragments_needed = NULL; /* indexes of xor code fragments */ + int ret; /* return value for xor code */ + + /* Obtain and validate the method parameters */ if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &missing_list)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.get_required_fragments"); return NULL; } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); if (pyeclib_handle == NULL) { PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.get_required_fragments"); return NULL; } + k = pyeclib_handle->k; + m = pyeclib_handle->m; - num_missing = (int)PyList_Size(missing_list); - - c_missing_list = (int*)malloc((num_missing+1)*sizeof(int)); - - for (i=0; i < num_missing; i++) { + /* Generate -1 terminated c-array and bitmap of missing indexes */ + num_missing = (int) PyList_Size(missing_list); + c_missing_list = (int *) alloc_zeroed_buffer((num_missing + 1) * sizeof(int)); + if (NULL == c_missing_list) { + return NULL; + } + c_missing_list[num_missing] = -1; + for (i = 0; i < num_missing; i++) { PyObject *obj_idx = PyList_GetItem(missing_list, i); long idx = PyLong_AsLong(obj_idx); - c_missing_list[i] = (int)idx; + c_missing_list[i] = (int) idx; } - - c_missing_list[num_missing] = -1; - missing_bm = convert_list_to_bitmap(c_missing_list); - + + /* Generate the python list of lists of rebuild indexes to return */ fragment_idx_list = PyList_New(0); - + if (NULL == fragment_idx_list) { + goto exit; + } + j = 0; switch(pyeclib_handle->type) { case PYECC_RS_CAUCHY_ORIG: case PYECC_RS_VAND: - j=0; - for (i=0; i < pyeclib_handle->k + pyeclib_handle->m; i++) { + for (i = 0; i < (k + m); i++) { if (!(missing_bm & (1 << i))) { PyList_Append(fragment_idx_list, Py_BuildValue("i", i)); j++; } - if (j == pyeclib_handle->k) { + if (j == k) { break; } } - if (j != pyeclib_handle->k) { - // Let the garbage collector clean this up + if (j != k) { + /* Let the garbage collector clean this up */ Py_DECREF(fragment_idx_list); - PyErr_Format(PyECLibError, "Not enough fragments for pyeclib.get_required_fragments (need at least %d, %d are given)", pyeclib_handle->k, j); + PyErr_Format(PyECLibError, "Not enough fragments for pyeclib.get_required_fragments (need at least %d, %d are given)", k, j); fragment_idx_list = NULL; } break; case PYECC_XOR_HD_3: case PYECC_XOR_HD_4: { - int *fragments_needed = (int*)malloc(sizeof(int)*(pyeclib_handle->k+pyeclib_handle->m)); - int ret = pyeclib_handle->xor_code_desc->fragments_needed(pyeclib_handle->xor_code_desc, c_missing_list, fragments_needed); + fragments_needed = alloc_zeroed_buffer(sizeof(int) * (k + m)); + if (NULL == fragments_needed) { + fragment_idx_list = NULL; + goto exit; + } + ret = pyeclib_handle->xor_code_desc->fragments_needed(pyeclib_handle->xor_code_desc, + c_missing_list, + fragments_needed); if (ret < 0) { Py_DECREF(fragment_idx_list); PyErr_Format(PyECLibError, "Not enough fragments for pyeclib.get_required_fragments!"); fragment_idx_list = NULL; - free(fragments_needed); break; } - j=0; while (fragments_needed[j] > -1) { PyList_Append(fragment_idx_list, Py_BuildValue("i", fragments_needed[j])); j++; - } - free(fragments_needed); + } break; } default: @@ -1400,158 +1553,159 @@ pyeclib_c_get_required_fragments(PyObject *self, PyObject *args) break; } - // Free the temporary list - free(c_missing_list); +exit: + free_buffer(c_missing_list); + free_buffer(fragments_needed); return fragment_idx_list; } -/* - * TODO: If we are reconstructing a parity element, ensure that all of the data elements are available! + +/** + * Reconstruct a missing fragment from the the remaining fragments. + * + * TODO: If we are reconstructing a parity element, ensure that all of the + * data elements are available! + * + * @param pyeclib_obj_handle + * @param data_list k length list of data elements + * @param parity_list m length list of parity elements + * @param missing_idx_list list of the indexes of missing elements + * @param destination_idx index of fragment to reconstruct + * @param fragment_size size in bytes of the fragments + * @return reconstructed destination fragment or NULL on error */ static PyObject * pyeclib_c_reconstruct(PyObject *self, PyObject *args) { - PyObject *pyeclib_obj_handle; - PyObject *data_list; - PyObject *parity_list; - PyObject *missing_idx_list; - PyObject *reconstructed; - int *erased; - pyeclib_t *pyeclib_handle; - int fragment_size; - int blocksize; - char **data; - char **parity; - int *missing_idxs; - int destination_idx; - unsigned long long realloc_bm = 0; // Identifies symbols that had to be allocated for alignment - int orig_data_size = -1; - int missing_size; - int *decoding_matrix; - int *decoding_row; - int *dm_ids; - int ret; - int i; - - if (!PyArg_ParseTuple(args, "OOOOii", &pyeclib_obj_handle, &data_list, &parity_list, &missing_idx_list, &destination_idx, &fragment_size)) { + PyObject *pyeclib_obj_handle = NULL; + pyeclib_t *pyeclib_handle = NULL; + PyObject *data_list = NULL; /* param, list of data fragments */ + PyObject *parity_list = NULL; /* param, list of parity fragments */ + PyObject *missing_idx_list = NULL; /* param, list of missing indexes */ + PyObject *reconstructed = NULL; /* reconstructed object to return */ + int *erased = NULL; /* jerasure notation of erased devs */ + int fragment_size; /* param, size in bytes of fragment */ + int blocksize; /* size in bytes, fragment - header */ + char **data = NULL; /* k length array of data buffers */ + char **parity = NULL; /* m length array of parity buffers */ + int *missing_idxs = NULL; /* array of missing indexes */ + int destination_idx; /* param, index to reconstruct */ + unsigned long long realloc_bm = 0; /* bitmap, which fragments were realloc'ed */ + int orig_data_size = -1; /* data size (B),from fragment hdr */ + int missing_size; /* number of missing indexes */ + int *decoding_matrix = NULL; /* reconstruct specific decode matrix */ + int *decoding_row = NULL; /* required row from decode matrix */ + int *dm_ids = NULL; /* k length array of surviving indexes */ + int k, m, w; /* EC algorithm parameters */ + int ret; /* decode matrix creation return val */ + int i = 0; /* a counter */ + + /* Obtain and validate the method parameters */ + if (!PyArg_ParseTuple(args, "OOOOii", &pyeclib_obj_handle, &data_list, + &parity_list, &missing_idx_list, &destination_idx, + &fragment_size)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode"); return NULL; } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); if (pyeclib_handle == NULL) { PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode"); return NULL; } - + k = pyeclib_handle->k; + m = pyeclib_handle->m; + w = pyeclib_handle->w; if (!PyList_Check(data_list) || !PyList_Check(parity_list) || !PyList_Check(missing_idx_list)) { PyErr_SetString(PyECLibError, "Invalid structure passed in for data, parity and/or missing_idx list"); return NULL; } - - if (pyeclib_handle->k != PyList_Size(data_list)) { + if (k != PyList_Size(data_list)) { PyErr_SetString(PyECLibError, "The data list does not have the correct number of entries"); return NULL; } - - if (pyeclib_handle->m != PyList_Size(parity_list)) { + if (m != PyList_Size(parity_list)) { PyErr_SetString(PyECLibError, "The parity list does not have the correct number of entries"); return NULL; } + /* Allocate data structures needed for reconstruction */ blocksize = FRAGSIZE_2_BLOCKSIZE(fragment_size); - - missing_size = (int)PyList_Size(missing_idx_list); - missing_idxs = (int*)malloc(sizeof(int)*(missing_size+1)); - if (missing_idxs == NULL) { - PyErr_SetString(PyECLibError, "Could not allocate memory for missing indexes"); - return NULL; - } - - data = (char**)malloc(sizeof(char*) * pyeclib_handle->k); - if (data == NULL) { - PyErr_SetString(PyECLibError, "Could not allocate memory for data buffers"); - free(missing_idxs); - return NULL; + missing_size = (int) PyList_Size(missing_idx_list); + missing_idxs = (int *) alloc_zeroed_buffer(sizeof(int) * (missing_size + 1)); + data = (char **) alloc_zeroed_buffer(sizeof(char *) * k); + parity = (char **) alloc_zeroed_buffer(sizeof(char *) * m); + if (NULL == missing_idxs || NULL == data || NULL == parity) { + goto error; } - parity = (char**)malloc(sizeof(char*) * pyeclib_handle->m); - if (parity == NULL) { - PyErr_SetString(PyECLibError, "Could not allocate memory for parity buffers"); - free(missing_idxs); - free(data); - return NULL; - } - - if (get_decoding_info(pyeclib_handle, data_list, parity_list, missing_idx_list, data, parity, missing_idxs, &orig_data_size, fragment_size, &realloc_bm)) { + /* Prepare for decoding, no need to go further on error */ + if (get_decoding_info(pyeclib_handle, data_list, parity_list, + missing_idx_list, data, parity, missing_idxs, + &orig_data_size, fragment_size, &realloc_bm)) { PyErr_SetString(PyECLibError, "Could not extract adequate decoding info from data, parity and missing lists"); - reconstructed = NULL; - goto out; + goto error; } - erased = jerasure_erasures_to_erased(pyeclib_handle->k, pyeclib_handle->m, missing_idxs); - + /* Create the decoding matrix, and attempt reconstruction */ + erased = jerasure_erasures_to_erased(k, m, missing_idxs); switch (pyeclib_handle->type) { case PYECC_RS_CAUCHY_ORIG: - - if (destination_idx < pyeclib_handle->k) { - decoding_matrix = (int*)malloc(sizeof(int*)*pyeclib_handle->k*pyeclib_handle->k*pyeclib_handle->w*pyeclib_handle->w); - dm_ids = (int*)malloc(sizeof(int)*pyeclib_handle->k); - ret = jerasure_make_decoding_bitmatrix(pyeclib_handle->k, pyeclib_handle->m, pyeclib_handle->w, pyeclib_handle->bitmatrix, erased, decoding_matrix, dm_ids); - decoding_row = decoding_matrix + (destination_idx*pyeclib_handle->k*pyeclib_handle->w*pyeclib_handle->w); + if (destination_idx < k) { + decoding_matrix = (int *) alloc_zeroed_buffer(sizeof(int*) * k * k * w * w); + dm_ids = (int *) alloc_zeroed_buffer(sizeof(int) * k); + if (NULL == decoding_matrix || NULL == dm_ids) { + goto error; + } + ret = jerasure_make_decoding_bitmatrix(k, m, w, pyeclib_handle->bitmatrix, + erased, decoding_matrix, dm_ids); + decoding_row = decoding_matrix + (destination_idx * k * w * w); } else { - dm_ids = NULL; - decoding_row = pyeclib_handle->bitmatrix + ((destination_idx - pyeclib_handle->k)*pyeclib_handle->k*pyeclib_handle->w*pyeclib_handle->w); ret = 0; + decoding_row = pyeclib_handle->bitmatrix + ((destination_idx - k) * k * w * w); } + if (ret == 0) { - jerasure_bitmatrix_dotprod(pyeclib_handle->k, pyeclib_handle->w, decoding_row, dm_ids, destination_idx, data, parity, blocksize, PYECC_CAUCHY_PACKETSIZE); + jerasure_bitmatrix_dotprod(k, w, decoding_row, dm_ids, destination_idx, + data, parity, blocksize, + PYECC_CAUCHY_PACKETSIZE); } - - if (destination_idx < pyeclib_handle->k) { - free(decoding_matrix); - } - free(dm_ids); - break; case PYECC_RS_VAND: - - if (destination_idx < pyeclib_handle->k) { - decoding_matrix = (int*)malloc(sizeof(int*)*pyeclib_handle->k*pyeclib_handle->k); - dm_ids = (int*)malloc(sizeof(int)*pyeclib_handle->k); - ret = jerasure_make_decoding_matrix(pyeclib_handle->k, pyeclib_handle->m, pyeclib_handle->w, pyeclib_handle->matrix, erased, decoding_matrix, dm_ids); - decoding_row = decoding_matrix + (destination_idx * pyeclib_handle->k); + if (destination_idx < k) { + decoding_matrix = (int *) alloc_zeroed_buffer(sizeof(int*) * k * k); + dm_ids = (int *) alloc_zeroed_buffer(sizeof(int) * k); + if (NULL == decoding_matrix || NULL == dm_ids) { + goto error; + } + ret = jerasure_make_decoding_matrix(k, m, w, pyeclib_handle->matrix, + erased, decoding_matrix, dm_ids); + decoding_row = decoding_matrix + (destination_idx * k); } else { - dm_ids = NULL; - decoding_row = pyeclib_handle->matrix + ((destination_idx - pyeclib_handle->k) * pyeclib_handle->k); ret = 0; + decoding_row = pyeclib_handle->matrix + ((destination_idx - k) * k); } + if (ret == 0) { - jerasure_matrix_dotprod(pyeclib_handle->k, pyeclib_handle->w, decoding_row, dm_ids, destination_idx, data, parity, blocksize); - } - - if (destination_idx < pyeclib_handle->k) { - free(decoding_matrix); - free(dm_ids); + jerasure_matrix_dotprod(k, w, decoding_row, dm_ids, destination_idx, + data, parity, blocksize); } - break; - case PYECC_XOR_HD_3: case PYECC_XOR_HD_4: - xor_reconstruct_one(pyeclib_handle->xor_code_desc, data, parity, missing_idxs, destination_idx, blocksize); ret = 0; + xor_reconstruct_one(pyeclib_handle->xor_code_desc, data, parity, + missing_idxs, destination_idx, blocksize); break; - default: ret = -1; break; } + /* Set the metadata on the reconstructed fragment */ if (ret == 0) { - char *fragment_ptr; - if (destination_idx < pyeclib_handle->k) { + char *fragment_ptr = NULL; + if (destination_idx < k) { fragment_ptr = get_fragment_ptr_from_data_novalidate(data[destination_idx]); init_fragment_header(fragment_ptr); set_fragment_idx(fragment_ptr, destination_idx); @@ -1562,139 +1716,155 @@ pyeclib_c_reconstruct(PyObject *self, PyObject *args) set_chksum(fragment_ptr, chksum); } } else { - fragment_ptr = get_fragment_ptr_from_data_novalidate(parity[destination_idx - pyeclib_handle->k]); + fragment_ptr = get_fragment_ptr_from_data_novalidate(parity[destination_idx - k]); init_fragment_header(fragment_ptr); set_fragment_idx(fragment_ptr, destination_idx); set_orig_data_size(fragment_ptr, orig_data_size); set_fragment_size(fragment_ptr, blocksize); if (use_inline_chksum(pyeclib_handle)) { - int chksum = crc32(0, parity[destination_idx - pyeclib_handle->k], blocksize); + int chksum = crc32(0, parity[destination_idx - k], blocksize); set_chksum(fragment_ptr, chksum); } } reconstructed = PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size); - } else { reconstructed = NULL; } - - for (i=0; i < pyeclib_handle->k; i++) { + + goto out; + +error: + reconstructed = NULL; + +out: + /* Free fragment buffers that needed to be reallocated for alignment */ + for (i = 0; i < k; i++) { if (realloc_bm & (1 << i)) { - char *ptr = get_fragment_ptr_from_data_novalidate(data[i]); - free(ptr); + free(get_fragment_ptr_from_data_novalidate(data[i])); } } - for (i=0; i < pyeclib_handle->m; i++) { - if (realloc_bm & (1 << (i + pyeclib_handle->k))) { - char *ptr = get_fragment_ptr_from_data_novalidate(parity[i]); - free(ptr); + for (i = 0; i < m; i++) { + if (realloc_bm & (1 << (i + k))) { + free(get_fragment_ptr_from_data_novalidate(parity[i])); } } - -out: - free(missing_idxs); - free(data); - free(parity); + free_buffer(missing_idxs); + free_buffer(data); + free_buffer(parity); + free_buffer(decoding_matrix); + free_buffer(dm_ids); return reconstructed; } + +/** + * Reconstruct all of the missing fragments from a set of fragments. + * + * TODO: There's a lot of duplicated code between this and the + * reconstruct method. Consider refactoring these methods. + * + * @param pyeclib_obj_handle + * @param data_list k length list of data elements + * @param parity_list m length list of parity elements + * @param missing_idx_list list of the indexes of missing elements + * @param fragment_size size in bytes of the fragments + * @return list of fragments + */ static PyObject * pyeclib_c_decode(PyObject *self, PyObject *args) { - PyObject *list_of_strips; - PyObject *pyeclib_obj_handle; - PyObject *data_list; - PyObject *parity_list; - PyObject *missing_idx_list; - pyeclib_t *pyeclib_handle; - int fragment_size; - int blocksize; - unsigned long long realloc_bm = 0; // Identifies symbols that had to be allocated for alignment - char **data; - char **parity; - int *missing_idxs; - int missing_size; - int orig_data_size = -1; - int i, j; - + PyObject *pyeclib_obj_handle = NULL; + pyeclib_t *pyeclib_handle = NULL; + PyObject *list_of_strips = NULL; /* list of strips to return */ + PyObject *data_list = NULL; /* param, list of data fragments */ + PyObject *parity_list = NULL; /* param, list of data fragments */ + PyObject *missing_idx_list = NULL; /* param, list of missing indexes */ + int fragment_size; /* param, size in bytes of fragment */ + int blocksize; /* size in bytes, fragment - header */ + unsigned long long realloc_bm = 0; /* bitmap, which fragments were realloc'ed */ + char **data = NULL; /* k length array of data buffers */ + char **parity = NULL; /* m length array of parity buffers */ + int *missing_idxs = NULL; /* array of missing indexes */ + int missing_size; /* number of missing indexes */ + int orig_data_size = -1; /* data size in bytes ,from fragment hdr */ + int k, m, w; /* EC algorithm parameters */ + int i = 0, j = 0; /* counters */ + + /* Obtain and validate the method parameters */ if (!PyArg_ParseTuple(args, "OOOOi", &pyeclib_obj_handle, &data_list, &parity_list, &missing_idx_list, &fragment_size)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode"); return NULL; } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); if (pyeclib_handle == NULL) { PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode"); return NULL; } - + k = pyeclib_handle->k; + m = pyeclib_handle->m; + w = pyeclib_handle->w; if (!PyList_Check(data_list) || !PyList_Check(parity_list) || !PyList_Check(missing_idx_list)) { PyErr_SetString(PyECLibError, "Invalid structure passed in for data, parity and/or missing_idx list"); return NULL; } - - if (pyeclib_handle->k != PyList_Size(data_list)) { + if (k != PyList_Size(data_list)) { PyErr_SetString(PyECLibError, "The data list does not have the correct number of entries"); return NULL; } - - if (pyeclib_handle->m != PyList_Size(parity_list)) { + if (m != PyList_Size(parity_list)) { PyErr_SetString(PyECLibError, "The parity list does not have the correct number of entries"); return NULL; } + /* Allocate data structures needed for reconstruction */ blocksize = FRAGSIZE_2_BLOCKSIZE(fragment_size); - - missing_size = (int)PyList_Size(missing_idx_list); - missing_idxs = (int*)malloc(sizeof(int)*(missing_size+1)); - if (missing_idxs == NULL) { - PyErr_SetString(PyECLibError, "Could not allocate memory for missing indexes"); - return NULL; - } - - data = (char**)malloc(sizeof(char*) * pyeclib_handle->k); - if (data == NULL) { - PyErr_SetString(PyECLibError, "Could not allocate memory for data buffers"); - return NULL; - } - - parity = (char**)malloc(sizeof(char*) * pyeclib_handle->m); - if (parity == NULL) { - PyErr_SetString(PyECLibError, "Could not allocate memory for parity buffers"); - return NULL; - } - - if (get_decoding_info(pyeclib_handle, data_list, parity_list, missing_idx_list, data, parity, missing_idxs, &orig_data_size, fragment_size, &realloc_bm)) { + missing_size = (int) PyList_Size(missing_idx_list); + missing_idxs = (int *) alloc_zeroed_buffer(sizeof(int) * (missing_size + 1)); + data = (char **) alloc_zeroed_buffer(sizeof(char *) * k); + parity = (char **) alloc_zeroed_buffer(sizeof(char *) * m); + if (NULL == missing_idxs || NULL == data || NULL == parity) { + goto error; + } + + /* Prepare for decoding, no need to go further on error */ + if (get_decoding_info(pyeclib_handle, data_list, parity_list, missing_idx_list, + data, parity, missing_idxs, &orig_data_size, + fragment_size, &realloc_bm)) { PyErr_SetString(PyECLibError, "Could not extract adequate decoding info from data, parity and missing lists"); return NULL; } + /* Reconstruct the missing fragments */ switch (pyeclib_handle->type) { case PYECC_RS_CAUCHY_ORIG: - jerasure_bitmatrix_decode(pyeclib_handle->k, pyeclib_handle->m, pyeclib_handle->w, pyeclib_handle->bitmatrix, 0, missing_idxs, data, parity, blocksize, PYECC_CAUCHY_PACKETSIZE); + jerasure_bitmatrix_decode(k, m, w, pyeclib_handle->bitmatrix, 0, missing_idxs, + data, parity, blocksize, PYECC_CAUCHY_PACKETSIZE); break; case PYECC_XOR_HD_3: case PYECC_XOR_HD_4: - pyeclib_handle->xor_code_desc->decode(pyeclib_handle->xor_code_desc, data, parity, missing_idxs, blocksize, 1); + pyeclib_handle->xor_code_desc->decode(pyeclib_handle->xor_code_desc, data, + parity, missing_idxs, blocksize, 1); break; case PYECC_RS_VAND: default: - jerasure_matrix_decode(pyeclib_handle->k, pyeclib_handle->m, pyeclib_handle->w, pyeclib_handle->matrix, 1, missing_idxs, data, parity, blocksize); + jerasure_matrix_decode(k, m, w, pyeclib_handle->matrix, 1, missing_idxs, + data, parity, blocksize); break; } - list_of_strips = PyList_New(pyeclib_handle->k + pyeclib_handle->m); + /* Create the python list to return */ + list_of_strips = PyList_New(k + m); + if (NULL == list_of_strips) { + goto error; + } - /* - * Create headers for the newly decoded elements - */ - j=0; + /* Create headers for the newly decoded elements */ while (missing_idxs[j] >= 0) { int missing_idx = missing_idxs[j]; - if (missing_idx < pyeclib_handle->k) { + if (missing_idx < k) { char *fragment_ptr = get_fragment_ptr_from_data_novalidate(data[missing_idx]); init_fragment_header(fragment_ptr); set_fragment_idx(fragment_ptr, missing_idx); @@ -1704,8 +1874,8 @@ pyeclib_c_decode(PyObject *self, PyObject *args) int chksum = crc32(0, data[missing_idx], blocksize); set_chksum(fragment_ptr, chksum); } - } else if (missing_idx >= pyeclib_handle->k) { - int parity_idx = missing_idx - pyeclib_handle->k; + } else if (missing_idx >= k) { + int parity_idx = missing_idx - k; char *fragment_ptr = get_fragment_ptr_from_data_novalidate(parity[parity_idx]); init_fragment_header(fragment_ptr); set_fragment_idx(fragment_ptr, missing_idx); @@ -1719,106 +1889,144 @@ pyeclib_c_decode(PyObject *self, PyObject *args) j++; } - /* - * Fill in the data fragments - */ - for (i=0; i < pyeclib_handle->k; i++) { + /* Fill in the data fragments */ + for (i = 0; i < k; i++) { char *fragment_ptr = get_fragment_ptr_from_data(data[i]); PyList_SET_ITEM(list_of_strips, i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size)); + } + /* Fill in the parity fragments */ + for (i = 0; i < m; i++) { + char *fragment_ptr = get_fragment_ptr_from_data(parity[i]); + PyList_SET_ITEM(list_of_strips, k + i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size)); + } + + goto exit; + +error: + list_of_strips = NULL; + +exit: + /* Free fragment buffers that needed to be reallocated for alignment */ + for (i = 0; i < k; i++) { if (realloc_bm & (1 << i)) { - free(fragment_ptr); + free(get_fragment_ptr_from_data_novalidate(data[i])); } } - /* - * Fill in the parity fragments - */ - for (i=0; i < pyeclib_handle->m; i++) { - char *fragment_ptr = get_fragment_ptr_from_data(parity[i]); - PyList_SET_ITEM(list_of_strips, pyeclib_handle->k + i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size)); - if (realloc_bm & (1 << (i + pyeclib_handle->k))) { - free(fragment_ptr); + for (i = 0; i < m; i++) { + if (realloc_bm & (1 << (i + k))) { + free(get_fragment_ptr_from_data_novalidate(parity[i])); } } - - free(missing_idxs); - free(data); - free(parity); + + free_buffer(missing_idxs); + free_buffer(data); + free_buffer(parity); return list_of_strips; } + +/** + * Obtain the metadata from a fragment. + * + * @param pyeclib_obj_handle + * @param data fragment from user to extract metadata from + * @param data_len size in bytes of the data fragment + * @return fragment metadata or NULL on error + */ static PyObject * pyeclib_c_get_metadata(PyObject *self, PyObject *args) { - PyObject *pyeclib_obj_handle; - pyeclib_t* pyeclib_handle; - char *data; - int data_len; - fragment_metadata_t *fragment_metadata; - PyObject *ret_fragment_metadata; - + PyObject *pyeclib_obj_handle = NULL; + pyeclib_t* pyeclib_handle = NULL; + char *data = NULL; /* param, fragment from caller */ + int data_len; /* param, data len (B) */ + int metadata_len; /* metadata header size (B) */ + fragment_metadata_t *fragment_metadata = NULL; /* buffer to hold metadata */ + PyObject *ret_fragment_metadata = NULL; /* metadata object to return */ + + /* Obtain and validate the method parameters */ if (!PyArg_ParseTuple(args, GET_METADATA_ARGS, &pyeclib_obj_handle, &data, &data_len)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.get_metadata"); return NULL; } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); if (pyeclib_handle == NULL) { PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.get_required_fragments"); return NULL; } - fragment_metadata = (fragment_metadata_t*)malloc(sizeof(fragment_metadata_t)); - - get_fragment_metadata(pyeclib_handle, data, fragment_metadata); - - ret_fragment_metadata = PY_BUILDVALUE_OBJ_LEN((char*)fragment_metadata, sizeof(fragment_metadata_t)); - - free(fragment_metadata); + /* Obtain the metadata from the data and build a object to return */ + metadata_len = sizeof(fragment_metadata_t); + fragment_metadata = (fragment_metadata_t *) alloc_zeroed_buffer(metadata_len); + if (NULL == fragment_metadata) { + ret_fragment_metadata = NULL; + } else { + get_fragment_metadata(pyeclib_handle, data, fragment_metadata); + ret_fragment_metadata = PY_BUILDVALUE_OBJ_LEN((char*)fragment_metadata, + metadata_len); + free_buffer(fragment_metadata); + } return ret_fragment_metadata; } + +/** + * Confirm the health of the fragment metadata. + * + * TODO: Return a list containing tuples (index, problem). An empty list means + * everything is OK. + * + * @param pyeclib_obj_handle + * @param fragment_metadata_list list of fragment metadata headers + * @return -1 if no errors, or the index of the first problem checksum + */ static PyObject* pyeclib_c_check_metadata(PyObject *self, PyObject *args) { - PyObject *pyeclib_obj_handle; - PyObject *fragment_metadata_list; - pyeclib_t *pyeclib_handle; - int i; - int ret = -1; - int num_fragments; - fragment_metadata_t** c_fragment_metadata_list; - char **c_fragment_signatures; - + PyObject *pyeclib_obj_handle = NULL; + pyeclib_t *pyeclib_handle = NULL; + PyObject *fragment_metadata_list = NULL; /* param, fragment metadata */ + fragment_metadata_t **c_fragment_metadata_list = NULL; /* c version of metadata */ + int num_fragments; /* k + m from EC algorithm */ + char **c_fragment_signatures = NULL; /* array of alg. signatures */ + int k, m, w; /* EC algorithm params */ + int size; /* size for buf allocation */ + int ret = -1; /* c return value */ + PyObject *ret_obj = NULL; /* python long to return */ + int i = 0; /* a counter */ + + /* Obtain and validate the method parameters */ if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_metadata_list)) { PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode"); return NULL; } - pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME); if (pyeclib_handle == NULL) { PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode"); return NULL; } - - num_fragments = pyeclib_handle->k + pyeclib_handle->m; - + k = pyeclib_handle->k; + m = pyeclib_handle->m; + w = pyeclib_handle->w; + num_fragments = k + m; if (num_fragments != PyList_Size(fragment_metadata_list)) { PyErr_SetString(PyECLibError, "Not enough fragment metadata to perform integrity check"); return NULL; } - - c_fragment_metadata_list = (fragment_metadata_t**)malloc(sizeof(fragment_metadata_t*)*num_fragments); - memset(c_fragment_metadata_list, 0, sizeof(fragment_metadata_t*)*num_fragments); - c_fragment_signatures = (char**)malloc(sizeof(char*)*num_fragments); - memset(c_fragment_signatures, 0, sizeof(char*)*num_fragments); + /* Allocate space for fragment signatures */ + size = sizeof(fragment_metadata_t * ) * num_fragments; + c_fragment_metadata_list = (fragment_metadata_t**) alloc_zeroed_buffer(size); + size = sizeof(char *) * num_fragments; + c_fragment_signatures = (char **) alloc_zeroed_buffer(size); + if (NULL == c_fragment_metadata_list || NULL == c_fragment_signatures) { + goto error; + } - /* - * Populate and order the metadata - */ - for (i=0; i < num_fragments; i++) { + /* Populate and order the metadata */ + for (i = 0; i < num_fragments; i++) { PyObject *tmp_data = PyList_GetItem(fragment_metadata_list, i); Py_ssize_t len = 0; char *c_buf = NULL; @@ -1829,69 +2037,80 @@ pyeclib_c_check_metadata(PyObject *self, PyObject *args) c_fragment_metadata_list[fragment_metadata->idx] = fragment_metadata; if (supports_alg_sig(pyeclib_handle)) { - c_fragment_signatures[fragment_metadata->idx] = (char*)get_aligned_buffer16(PYCC_MAX_SIG_LEN); + c_fragment_signatures[fragment_metadata->idx] = (char*)alloc_aligned_buffer16(PYCC_MAX_SIG_LEN); memcpy(c_fragment_signatures[fragment_metadata->idx], fragment_metadata->signature, PYCC_MAX_SIG_LEN); } else { c_fragment_signatures[fragment_metadata->idx] = fragment_metadata->signature; } } - /* - * Ensure all fragments are here and check integrity using alg signatures - */ + /* Ensure all fragments are here and check integrity using alg signatures */ if (supports_alg_sig(pyeclib_handle)) { - char **parity_sigs = (char**)malloc(sizeof(char**)*pyeclib_handle->m); - - for (i=0; i < pyeclib_handle->m; i++) { - parity_sigs[i] = (char*)get_aligned_buffer16(PYCC_MAX_SIG_LEN); - memset(parity_sigs[i], 0, PYCC_MAX_SIG_LEN); + char **parity_sigs = (char **) alloc_zeroed_buffer(sizeof(char **) * m); + if (NULL == parity_sigs) { + goto error; + } + for (i = 0; i < m; i++) { + parity_sigs[i] = (char *) alloc_aligned_buffer16(PYCC_MAX_SIG_LEN); + if (NULL == parity_sigs[i]) { + for (int j = 0; j < i; j++) free(parity_sigs[j]); + goto error; + } else { + memset(parity_sigs[i], 0, PYCC_MAX_SIG_LEN); + } } + /* Calculate the parity of the signatures */ if (pyeclib_handle->type == PYECC_RS_VAND) { - jerasure_matrix_encode(pyeclib_handle->k, - pyeclib_handle->m, - pyeclib_handle->w, - pyeclib_handle->matrix, - c_fragment_signatures, - parity_sigs, - PYCC_MAX_SIG_LEN); + jerasure_matrix_encode(k, m, w, pyeclib_handle->matrix, + c_fragment_signatures, parity_sigs, PYCC_MAX_SIG_LEN); } else { pyeclib_handle->xor_code_desc->encode(pyeclib_handle->xor_code_desc, - c_fragment_signatures, + c_fragment_signatures, parity_sigs, PYCC_MAX_SIG_LEN); } - - for (i=0; i < pyeclib_handle->m; i++) { - if (memcmp(parity_sigs[i], c_fragment_signatures[pyeclib_handle->k + i], PYCC_MAX_SIG_LEN) != 0) { + + /* Compare the parity of the signatures, and the signature of the parity */ + for (i = 0; i < m; i++) { + if (memcmp(parity_sigs[i], c_fragment_signatures[k + i], PYCC_MAX_SIG_LEN) != 0) { ret = i; break; } } - for (i=0; i < pyeclib_handle->m; i++) { + /* Clean up memory used in algebraic signature checking */ + for (i = 0; i < m; i++) { free(parity_sigs[i]); } - for (i=0; i < pyeclib_handle->k; i++) { + free_buffer(parity_sigs); + for (i = 0; i < k; i++) { free(c_fragment_signatures[i]); } - free(parity_sigs); } else if (use_inline_chksum(pyeclib_handle)) { - for (i=0; i < num_fragments; i++) { + for (i = 0; i < num_fragments; i++) { if (c_fragment_metadata_list[i]->chksum_mismatch == 1) { ret = i; break; } } } - + + /* Return index of first checksum signature error */ + ret_obj = PyLong_FromLong((long)ret); + goto exit; + +error: + ret_obj = NULL; + +exit: free(c_fragment_signatures); free(c_fragment_metadata_list); - - // TODO: Return a list containing tuples (index, problem). An empty list means everything is OK. - return PyLong_FromLong((long)ret); + + return ret_obj; } + static PyMethodDef PyECLibMethods[] = { {"init", pyeclib_c_init, METH_VARARGS, "Initialize a new erasure encoder/decoder"}, {"encode", pyeclib_c_encode, METH_VARARGS, "Create parity using source data"}, @@ -1925,4 +2144,3 @@ MOD_INIT(pyeclib_c) return MOD_SUCCESS_VAL(m); } - |