summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKevin Greenan <kmgreen2@gmail.com>2014-07-30 21:54:07 -0700
committerKevin Greenan <kmgreen2@gmail.com>2014-08-02 10:11:51 -0700
commit9d67b8726dfee9615d676847145fee6ed9f2f40d (patch)
treec77d6f9edd9f22c5d3f8a740487bd001f221390b
parentf81a59b59248f96ba2e1b5ab09fdce4cebcf67d9 (diff)
downloadpyeclib-9d67b8726dfee9615d676847145fee6ed9f2f40d.tar.gz
Plugging liberasurecode functions into pyeclib
Python changes needed to plug-in liberasurecode. Changes needed to get PyECLib working with liberasurecode and some of the unit tests passing.
-rw-r--r--README4
-rw-r--r--src/c/pyeclib_c/pyeclib_c.c1752
-rw-r--r--src/c/pyeclib_c/pyeclib_c.h112
-rw-r--r--src/python/pyeclib/core.py68
-rw-r--r--src/python/pyeclib/ec_iface.py8
-rwxr-xr-xtest/ec_pyeclib_file_test.sh8
-rw-r--r--test/test_pyeclib_api.py64
-rw-r--r--test/test_pyeclib_c.py10
-rw-r--r--tools/pyeclib_conf_tool.py24
9 files changed, 264 insertions, 1786 deletions
diff --git a/README b/README
index 4d12de0..031bc65 100644
--- a/README
+++ b/README
@@ -39,8 +39,8 @@ PyEClib initialization::
Supported ``ec_type`` values:
- * ``rs_vand`` => Vandermonde Reed-Solomon encoding
- * ``flat_xor_3``, ``flat_xor_4`` => Flat-XOR based HD combination codes
+ * ``jerasure_rs_vand`` => Vandermonde Reed-Solomon encoding
+ * ``flat_xor_hd_3``, ``flat_xor_hd_4`` => Flat-XOR based HD combination codes
A configuration utility is provided to help compare available EC schemes in
terms of performance and redundancy:: tools/pyeclib_conf_tool.py
diff --git a/src/c/pyeclib_c/pyeclib_c.c b/src/c/pyeclib_c/pyeclib_c.c
index 6827c09..4a29fd5 100644
--- a/src/c/pyeclib_c/pyeclib_c.c
+++ b/src/c/pyeclib_c/pyeclib_c.c
@@ -29,13 +29,8 @@
/* Compat layer for python <= 2.6 */
#include "capsulethunk.h"
-#include <xor_code.h>
-#include <reed_sol.h>
-#include <alg_sig.h>
-#include <cauchy.h>
-#include <jerasure.h>
-#include <liberation.h>
-#include <galois.h>
+#include <erasurecode.h>
+#include <erasurecode_helpers.h>
#include <math.h>
#include <pyeclib_c.h>
#include <bytesobject.h>
@@ -68,26 +63,6 @@
#endif
-/*
- * TODO (kmg): Cauchy restriction (k*w*PACKETSIZE) < data_len / k, otherwise you could
- * end up with full-blocks of padding
- *
- */
-
-/*
- * TODO (kmg): Do a big cleanup: standardize naming, types (typedef stuff), comments, standard error
- * handling and refactor where needed.
- */
-
-/*
- * TODO (kmg): Need to clean-up all of the reference counting related stuff. That is, INCREF before
- * calling a function that will "steal" a reference and create functions to DECREF stuff.
- */
-
-/*
- * Make the buffer alignment code in 'encode' generic and simple... It sucks right now.
- */
-
static PyObject *PyECLibError;
@@ -98,667 +73,11 @@ static PyObject * pyeclib_c_init(PyObject *self, PyObject *args);
static void pyeclib_c_destructor(PyObject *obj);
static PyObject * pyeclib_c_get_segment_info(PyObject *self, PyObject *args);
static PyObject * pyeclib_c_encode(PyObject *self, PyObject *args);
-static PyObject * pyeclib_c_fragments_to_string(PyObject *self, PyObject *args);
-static PyObject * pyeclib_c_get_fragment_partition(PyObject *self, PyObject *args);
static PyObject * pyeclib_c_reconstruct(PyObject *self, PyObject *args);
static PyObject * pyeclib_c_decode(PyObject *self, PyObject *args);
static PyObject * pyeclib_c_get_metadata(PyObject *self, PyObject *args);
static PyObject * pyeclib_c_check_metadata(PyObject *self, PyObject *args);
-
-/*
- * Determine if an address is aligned to a particular boundary
- */
-static int is_addr_aligned(unsigned long addr, int align)
-{
- return (addr & (align-1)) == 0;
-}
-
-/*
- * Validate the basic erasure code parameters
- */
-static int validate_args(int k, int m, int w, pyeclib_type_t type)
-{
-
- switch (type) {
- case PYECC_RS_CAUCHY_ORIG:
- {
- if (w < 31 && (k+m) > (1 << w)) {
- return 0;
- }
- }
- case PYECC_XOR_HD_4:
- case PYECC_XOR_HD_3:
- return 1;
- case PYECC_RS_VAND:
- default:
- {
- long long max_symbols;
-
- if (w != 8 && w != 16 && w != 32) {
- return 0;
- }
- max_symbols = 1LL << w;
- if ((k+m) > max_symbols) {
- return 0;
- }
- }
- }
- return 1;
-}
-
-/*
- * Convert an int list into a bitmap
- * Assume the list is '-1' terminated.
- */
-static unsigned long long convert_list_to_bitmap(int *list)
-{
- int i = 0;
- unsigned long long bm = 0;
-
- while (list[i] > -1) {
- /*
- * TODO: Assert list[i] < 64
- */
- bm |= (1 << list[i]);
- i++;
- }
-
- return bm;
-}
-
-static
-void init_fragment_header(char *buf)
-{
- fragment_header_t* header = (fragment_header_t*)buf;
-
- header->magic = PYECC_HEADER_MAGIC;
-}
-
-
-/**
- * Memory Management Methods
- *
- * The following methods provide wrappers for allocating and deallocating
- * memory.
- *
- * Future discussions may want to consider moving closer to the recommended
- * guidelines in the Python\C API reference manual. One potential issue,
- * however, may be how we enforce memory alignment in the Python heap.
- *
- * 2.7: https://docs.python.org/2.7/c-api/memory.html
- * 3.4: https://docs.python.org/3.4/c-api/memory.html
- */
-
-/**
- * Allocate a zero-ed buffer of a specific size. This method allocates from
- * the Python stack in order to comply with the recommendations of the
- * Python\C API. On error, return NULL and call PyErr_NoMemory.
- *
- * @param size integer size in bytes of buffer to allocate
- * @return pointer to start of allocated buffer or NULL on error
- */
-static
-void * alloc_zeroed_buffer(int size)
-{
- void * buf = NULL; /* buffer to allocate and return */
-
- /* Allocate and zero the buffer, or set the appropriate error */
- buf = PyMem_Malloc((size_t) size);
- if (buf) {
- buf = memset(buf, 0, (size_t) size);
- } else {
- buf = (void *) PyErr_NoMemory();
- }
-
- return buf;
-}
-
-
-/**
- * Deallocate memory buffer if it's not NULL. This methods returns NULL so
- * that you can free and reset a buffer using a single line as follows:
- *
- * my_ptr = check_and_free_buffer(my_ptr);
- *
- * @return NULL
- */
-static
-void * check_and_free_buffer(void * buf)
-{
- if (buf) PyMem_Free(buf);
-
- return NULL;
-}
-
-
-/**
- * Allocates the resources required for the algebraic signatures. On error,
- * PyErr_NoMemory is called to set the appropriate error response and NULL
- * is returned.
- *
- * NOTE: It is the caller's responsibility to free the memory allocated here!
- *
- * @param sig_len integer signature length
- * @param gf_w integer galois field size
- * @return ptr to allocated and initialized algebraic signature tables
- */
-static
-alg_sig_t *alloc_alg_sig(int sig_len, int gf_w)
-{
- alg_sig_t *buf = NULL; /* algebraic signature handle to allocate */
-
- buf = init_alg_sig(sig_len, gf_w);
- if (NULL == buf) {
- PyErr_NoMemory();
- }
-
- return buf;
-}
-
-
-/**
- * Deallocate the resources used for algebraic signatures.
- *
- * @return NULL
- */
-static
-void * check_and_free_alg_sig(alg_sig_t *buf)
-{
- if (buf) destroy_alg_sig(buf);
-
- return NULL;
-}
-
-
-/**
- * Allocate and zero out a buffer aligned to a 16-byte boundary in order
- * to support 128-bit operations. On error, call PyErr_NoMemory to set
- * the appropriate error string and return NULL.
- *
- * @param size integer size in bytes of buffer to allocate
- * @return pointer to start of allocated buffer, or NULL on error
- */
-static
-void* alloc_aligned_buffer16(int size)
-{
- void *buf = NULL;
-
- /*
- * Ensure all memory is aligned to
- * 16-byte boundaries to support
- * 128-bit operations
- */
- if (posix_memalign(&buf, 16, size) != 0) {
- return PyErr_NoMemory();
- } else {
- memset(buf, 0, size);
- }
-
- return buf;
-}
-
-
-/**
- * Allocate an initialized fragment buffer. On error, return NULL and
- * preserve call to PyErr_NoMemory. Note, all allocated memory is aligned
- * to 16-bytes boundaries in order to support 128-bit operations.
- *
- * @param size integer size in bytes of buffer to allocate
- * @return pointer to start of allocated fragment or NULL on error
- */
-static
-char *alloc_fragment_buffer(int size)
-{
- char *buf = NULL;
- fragment_header_t *header = NULL;
-
- /* Calculate size needed for fragment header + data */
- size += sizeof(fragment_header_t);
-
- /* Allocate and init the aligned buffer, or set the appropriate error */
- buf = alloc_aligned_buffer16(size);
- if (buf) {
- init_fragment_header(buf);
- }
-
- return buf;
-}
-
-
-/**
- * Deallocate a fragment buffer. This method confirms via magic number that
- * the passed in buffer is a proper fragment buffer.
- *
- * @param buf pointer to fragment buffer
- * @return 0 on successful free, -1 on error
- */
-static
-int free_fragment_buffer(char *buf)
-{
- fragment_header_t *header;
- if (buf == NULL) {
- return -1;
- }
-
- buf -= sizeof(fragment_header_t);
-
- header = (fragment_header_t*)buf;
- if (header->magic != PYECC_HEADER_MAGIC) {
- PyErr_SetString(PyECLibError, "Invalid fragment header (free fragment)!");
- return -1;
- }
- free(buf);
-
- return 0;
-}
-
-
-/**
- * Allocate a zero-ed Python string of a specific size. On error, call
- * PyErr_NoMemory and return NULL.
- *
- * @param size integer size in bytes of zero string to create
- * @return pointer to start of allocated zero string or NULL on error
- */
-static
-PyObject * alloc_zero_string(int size)
-{
- char *tmp_data = NULL; /* tmp buffer used in PyObject creation */
- PyObject *zero_string = NULL; /* zero string to return */
-
- /* Create the zero-out c-string buffer */
- tmp_data = (char *) alloc_zeroed_buffer(size);
- if (NULL == tmp_data) {
- return PyErr_NoMemory();
- }
-
- /* Create the python value to return */
- zero_string = PY_BUILDVALUE_OBJ_LEN(tmp_data, size);
- check_and_free_buffer(tmp_data);
-
- return zero_string;
-}
-
-
-static
-char* get_data_ptr_from_fragment(char *buf)
-{
- char * data_ptr = NULL;
-
- if (NULL != buf) {
- data_ptr = buf + sizeof(fragment_header_t);
- }
-
- return data_ptr;
-}
-
-static
-char* get_fragment_ptr_from_data_novalidate(char *buf)
-{
- buf -= sizeof(fragment_header_t);
-
- return buf;
-}
-
-static
-int use_inline_chksum(pyeclib_t *pyeclib_handle)
-{
- return pyeclib_handle->inline_chksum;
-}
-
-static
-int supports_alg_sig(pyeclib_t *pyeclib_handle)
-{
- if (!pyeclib_handle->algsig_chksum) return 0;
- if (pyeclib_handle->alg_sig_desc != NULL) return 1;
- return 0;
-}
-
-static
-char* get_fragment_ptr_from_data(char *buf)
-{
- fragment_header_t *header;
-
- buf -= sizeof(fragment_header_t);
-
- header = (fragment_header_t*)buf;
-
- if (header->magic != PYECC_HEADER_MAGIC) {
- fprintf(stderr, "Invalid fragment header (get header ptr)!\n");
- return NULL;
- }
-
- return buf;
-}
-
-static
-int set_chksum(char *buf, int chksum)
-{
- fragment_header_t* header = (fragment_header_t*)buf;
-
- if (header->magic != PYECC_HEADER_MAGIC) {
- fprintf(stderr, "Invalid fragment header (set chksum)!\n");
- return -1;
- }
-
- header->chksum = chksum;
-
- return 0;
-}
-
-static
-int get_chksum(char *buf)
-{
- fragment_header_t* header = (fragment_header_t*)buf;
-
- if (header->magic != PYECC_HEADER_MAGIC) {
- PyErr_SetString(PyECLibError, "Invalid fragment header (get chksum)!");
- return -1;
- }
-
- return header->chksum;
-}
-
-static
-int set_fragment_idx(char *buf, int idx)
-{
- fragment_header_t* header = (fragment_header_t*)buf;
-
- if (header->magic != PYECC_HEADER_MAGIC) {
- fprintf(stderr, "Invalid fragment header (idx check)!\n");
- return -1;
- }
-
- header->idx = idx;
-
- return 0;
-}
-
-static
-int get_fragment_idx(char *buf)
-{
- fragment_header_t* header = (fragment_header_t*)buf;
-
- if (header->magic != PYECC_HEADER_MAGIC) {
- PyErr_SetString(PyECLibError, "Invalid fragment header (get idx)!");
- return -1;
- }
-
- return header->idx;
-}
-
-static
-int set_fragment_size(char *buf, int size)
-{
- fragment_header_t* header = (fragment_header_t*)buf;
-
- if (header->magic != PYECC_HEADER_MAGIC) {
- PyErr_SetString(PyECLibError, "Invalid fragment header (size check)!");
- return -1;
- }
-
- header->size = size;
-
- return 0;
-}
-
-static
-int get_fragment_size(char *buf)
-{
- fragment_header_t* header = (fragment_header_t*)buf;
-
- if (header->magic != PYECC_HEADER_MAGIC) {
- PyErr_SetString(PyECLibError, "Invalid fragment header (get size)!");
- return -1;
- }
-
- return header->size;
-}
-
-static
-int set_orig_data_size(char *buf, int orig_data_size)
-{
- fragment_header_t* header = (fragment_header_t*)buf;
-
- if (header->magic != PYECC_HEADER_MAGIC) {
- PyErr_SetString(PyECLibError, "Invalid fragment header (set orig data check)!");
- return -1;
- }
-
- header->orig_data_size = orig_data_size;
-
- return 0;
-}
-
-static
-int get_orig_data_size(char *buf)
-{
- fragment_header_t* header = (fragment_header_t*)buf;
-
- if (header->magic != PYECC_HEADER_MAGIC) {
- PyErr_SetString(PyECLibError, "Invalid fragment header (get orig data check)!");
- return -1;
- }
-
- return header->orig_data_size;
-}
-
-static
-int validate_fragment(char *buf)
-{
- fragment_header_t* header = (fragment_header_t*)buf;
-
- if (header->magic != PYECC_HEADER_MAGIC) {
- return -1;
- }
-
- return 0;
-}
-
-
-/*
- * Prepares the fragments and helper data structures for decoding. Note,
- * buffers for data, parity and missing_idxs must be allocated by the caller.
- *
- * @param *pyeclib_handle
- * @param *data_list python list of data fragments
- * @param *parity_list python list of parity fragments
- * @param *missing_idx_list python list of indexes of missing fragments
- * @param **data allocated k-length array of data buffers
- * @param **parity allocated m-length array of parity buffers
- * @param *missing_idxs array of missing indexes with extra space for -1 terminator
- * @param *orig_size original size of object from the fragment header
- * @param fragment_size integer size in bytes of fragment
- * @return 0 on success, -1 on error
- */
-static int get_decoding_info(pyeclib_t *pyeclib_handle,
- PyObject *data_list,
- PyObject *parity_list,
- PyObject *missing_idx_list,
- char **data,
- char **parity,
- int *missing_idxs,
- int *orig_size,
- int fragment_size,
- unsigned long long *realloc_bm)
-{
- int i; /* a counter */
- int data_size = 0; /* number of data fragments provided */
- int parity_size = 0; /* number of parity fragments provided */
- int missing_size = 0; /* number of missing index entries */
- int orig_data_size = -1; /* data size (B) from fragment header */
- unsigned long long missing_bm; /* bitmap form of missing indexes list */
- int needs_addr_alignment = PYECLIB_NEEDS_ADDR_ALIGNMENT(pyeclib_handle->type);
-
- /* Validate the list sizes */
- data_size = (int)PyList_Size(data_list);
- parity_size = (int)PyList_Size(parity_list);
- missing_size = (int)PyList_Size(missing_idx_list);
- if (data_size != pyeclib_handle->k) {
- return -1;
- }
- if (parity_size != pyeclib_handle->m) {
- return -1;
- }
-
- /* Record missing indexes in a -1 terminated array and bitmap */
- for (i = 0; i < missing_size; i++) {
- PyObject *obj_idx = PyList_GetItem(missing_idx_list, i);
-
- missing_idxs[i] = (int) PyLong_AsLong(obj_idx);;
- }
- missing_idxs[i] = -1;
- missing_bm = convert_list_to_bitmap(missing_idxs);
-
- /*
- * Determine if each data fragment is:
- * 1.) Alloc'd: if not, alloc new buffer (for missing fragments)
- * 2.) Aligned to 16-byte boundaries: if not, alloc a new buffer
- * memcpy the contents and free the old buffer
- */
- for (i = 0; i < pyeclib_handle->k; i++) {
- PyObject *tmp_data = PyList_GetItem(data_list, i);
- Py_ssize_t len = 0;
- PyBytes_AsStringAndSize(tmp_data, &(data[i]), &len);
-
- /*
- * Allocate or replace with aligned buffer if the buffer was not aligned.
- * DO NOT FREE: the python GC should free the original when cleaning up 'data_list'
- */
- if (len == 0 || !data[i]) {
- data[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
- if (NULL == data[i]) {
- return -1;
- }
- *realloc_bm = *realloc_bm | (1 << i);
- } else if ((needs_addr_alignment && !is_addr_aligned((unsigned long)data[i], 16))) {
- char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
- memcpy(tmp_buf, data[i], fragment_size);
- data[i] = tmp_buf;
- *realloc_bm = *realloc_bm | (1 << i);
- }
-
- /* Need to determine the size of the original data */
- if (((missing_bm & (1 << i)) == 0) && orig_data_size < 0) {
- orig_data_size = get_orig_data_size(data[i]);
- if (orig_data_size < 0) {
- return -1;
- }
- }
-
- /* Set the data element to the fragment payload */
- data[i] = get_data_ptr_from_fragment(data[i]);
- if (data[i] == NULL) {
- return -1;
- }
- }
-
- /* Perform the same allocation, alignment checks on the parity fragments */
- for (i=0; i < pyeclib_handle->m; i++) {
- PyObject *tmp_parity = PyList_GetItem(parity_list, i);
- Py_ssize_t len = 0;
- PyBytes_AsStringAndSize(tmp_parity, &(parity[i]), &len);
-
- /*
- * Allocate or replace with aligned buffer, if the buffer was not aligned.
- * DO NOT FREE: the python GC should free the original when cleaning up 'data_list'
- */
- if (len == 0 || !parity[i]) {
- parity[i] = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
- if (NULL == parity[i]) {
- return -1;
- }
- *realloc_bm = *realloc_bm | (1 << (pyeclib_handle->k + i));
- } else if (needs_addr_alignment && !is_addr_aligned((unsigned long)parity[i], 16)) {
- char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
- memcpy(tmp_buf, parity[i], fragment_size);
- parity[i] = tmp_buf;
- *realloc_bm = *realloc_bm | (1 << (pyeclib_handle->k + i));
- }
-
- /* Set the parity element to the fragment payload */
- parity[i] = get_data_ptr_from_fragment(parity[i]);
- if (parity[i] == NULL) {
- return -1;
- }
- }
-
- *orig_size = orig_data_size;
- return 0;
-}
-
-
-/**
- * Compute a size aligned to the number of data and the underlying wordsize
- * of the EC algorithm.
- *
- * @param pyeclib_handle eclib object with EC configurations
- * @param data_len integer length of data in bytes
- * @return integer data length aligned with wordsize of EC algorithm
- */
-static int
-get_aligned_data_size(pyeclib_t* pyeclib_handle, int data_len)
-{
- int word_size = pyeclib_handle->w / 8;
- int alignment_multiple;
- int aligned_size = 0;
-
- /*
- * For Cauchy reed-solomon align to k*word_size*packet_size
- * For Vandermonde reed-solomon and flat-XOR, align to k*word_size
- */
- if (pyeclib_handle->type == PYECC_RS_CAUCHY_ORIG) {
- alignment_multiple = pyeclib_handle->k * pyeclib_handle->w * PYECC_CAUCHY_PACKETSIZE;
- } else {
- alignment_multiple = pyeclib_handle->k * word_size;
- }
-
- aligned_size = (int)ceill((double)data_len / alignment_multiple) * alignment_multiple;
-
- return aligned_size;
-}
-
-
-static
-int get_minimum_encode_size(pyeclib_t *pyeclib_handle)
-{
- return get_aligned_data_size(pyeclib_handle, 1);
-}
-
-
-static
-int get_fragment_metadata(pyeclib_t *pyeclib_handle, char *fragment_buf, fragment_metadata_t *fragment_metadata)
-{
- char *fragment_data = get_data_ptr_from_fragment(fragment_buf);
- int fragment_size = get_fragment_size(fragment_buf);
- int fragment_idx = get_fragment_idx(fragment_buf);
-
- memset(fragment_metadata, 0, sizeof(fragment_metadata_t));
-
- fragment_metadata->size = fragment_size;
- fragment_metadata->idx = fragment_idx;
-
- /*
- * If w \in [8, 16] and using RS_VAND or XOR_CODE, then use Alg_sig
- * Else use CRC32
- */
- if (supports_alg_sig(pyeclib_handle)) {
- // Compute algebraic signature
- compute_alg_sig(pyeclib_handle->alg_sig_desc, fragment_data, fragment_size, fragment_metadata->signature);
- } else if (use_inline_chksum(pyeclib_handle)) {
- int stored_chksum = get_chksum(fragment_buf);
- int computed_chksum = crc32(0, fragment_data, fragment_size);
-
- if (stored_chksum != computed_chksum) {
- fragment_metadata->chksum_mismatch = 1;
- }
- }
-
- return 0;
-}
-
-
/**
* Constructor method for creating a new pyeclib object using the given parameters.
*
@@ -773,78 +92,32 @@ pyeclib_c_init(PyObject *self, PyObject *args)
{
pyeclib_t *pyeclib_handle = NULL;
PyObject *pyeclib_obj_handle = NULL;
- int k, m, w;
+ int k, m, hd=0;
int use_inline_chksum = 0, use_algsig_chksum = 0;
const char *type_str;
- pyeclib_type_t type;
/* Obtain and validate the method parameters */
- if (!PyArg_ParseTuple(args, "iis|ii", &k, &m, &type_str, &use_inline_chksum, &use_algsig_chksum)) {
+ if (!PyArg_ParseTuple(args, "iis|iii", &k, &m, &type_str, &hd, &use_inline_chksum, &use_algsig_chksum)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.init");
return NULL;
}
- type = get_ecc_type(type_str);
- if (type == PYECC_NOT_FOUND) {
- PyErr_SetString(PyECLibError, "Invalid type passed to pyeclib.init");
- return NULL;
- }
-
- w = get_best_w_for_ecc_type(type);
- if (w < 0) {
- /* this should not happen */
- PyErr_SetString(PyECLibError, "Invalid w value. pyeclib internal error");
- return NULL;
- }
-
- if (!validate_args(k, m, w, type)) {
- PyErr_SetString(PyECLibError, "Invalid args passed to pyeclib.init");
- return NULL;
- }
/* Allocate and initialize the pyeclib object */
pyeclib_handle = (pyeclib_t *) alloc_zeroed_buffer(sizeof(pyeclib_t));
if (NULL == pyeclib_handle) {
goto error;
}
- pyeclib_handle->k = k;
- pyeclib_handle->m = m;
- pyeclib_handle->w = w;
- pyeclib_handle->type = type;
- pyeclib_handle->inline_chksum = use_inline_chksum;
- pyeclib_handle->algsig_chksum = use_algsig_chksum;
- pyeclib_handle->alg_sig_desc = NULL;
-
- /* Allocate and initialize additional resources */
- switch (type) {
- case PYECC_RS_CAUCHY_ORIG:
- pyeclib_handle->matrix = cauchy_original_coding_matrix(k, m, w);
- pyeclib_handle->bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, pyeclib_handle->matrix);
- pyeclib_handle->schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, pyeclib_handle->bitmatrix);
- break;
- case PYECC_XOR_HD_3:
- pyeclib_handle->xor_code_desc = init_xor_hd_code(k, m, 3);
- if (pyeclib_handle->algsig_chksum) {
- pyeclib_handle->alg_sig_desc = alloc_alg_sig(32, 16);
- if (NULL == pyeclib_handle->alg_sig_desc) goto error;
- }
- break;
- case PYECC_XOR_HD_4:
- pyeclib_handle->xor_code_desc = init_xor_hd_code(k, m, 4);
- if (pyeclib_handle->algsig_chksum) {
- pyeclib_handle->alg_sig_desc = alloc_alg_sig(32, 16);
- if (NULL == pyeclib_handle->alg_sig_desc) goto error;
- }
- break;
- case PYECC_RS_VAND:
- default:
- pyeclib_handle->matrix = reed_sol_vandermonde_coding_matrix(k, m, w);
- if (pyeclib_handle->algsig_chksum && w == 8) {
- pyeclib_handle->alg_sig_desc = alloc_alg_sig(32, 8);
- if (NULL == pyeclib_handle->alg_sig_desc) goto error;
- } else if (pyeclib_handle->algsig_chksum && w == 16) {
- pyeclib_handle->alg_sig_desc = alloc_alg_sig(32, 16);
- if (NULL == pyeclib_handle->alg_sig_desc) goto error;
- }
+
+ pyeclib_handle->ec_args.k = k;
+ pyeclib_handle->ec_args.m = m;
+ pyeclib_handle->ec_args.hd = hd;
+ pyeclib_handle->ec_args.inline_chksum = use_inline_chksum;
+ pyeclib_handle->ec_args.algsig_chksum = use_algsig_chksum;
+
+ pyeclib_handle->ec_desc = liberasurecode_instance_create(type_str, &(pyeclib_handle->ec_args));
+ if (pyeclib_handle->ec_desc <= 0) {
+ PyErr_SetString(PyECLibError, "Invalid arguments passed to liberasurecode_instance_create");
+ goto error;
}
/* Prepare the python object to return */
@@ -868,9 +141,6 @@ pyeclib_c_init(PyObject *self, PyObject *args)
goto exit;
error:
- if (NULL != pyeclib_handle) {
- check_and_free_alg_sig(pyeclib_handle->alg_sig_desc);
- }
check_and_free_buffer(pyeclib_handle);
pyeclib_obj_handle = NULL;
@@ -896,10 +166,8 @@ pyeclib_c_destructor(PyObject *obj)
if (pyeclib_handle == NULL) {
PyErr_SetString(PyECLibError, "Attempted to free an invalid reference to pyeclib_handle");
} else {
- check_and_free_alg_sig(pyeclib_handle->alg_sig_desc);
check_and_free_buffer(pyeclib_handle);
}
-
return;
}
@@ -959,7 +227,7 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args)
}
/* The minimum segment size depends on the EC algorithm */
- min_segment_size = get_minimum_encode_size(pyeclib_handle);
+ min_segment_size = liberasurecode_get_minimum_encode_size(pyeclib_handle->ec_desc);
/* Get the number of segments */
num_segments = (int)ceill((double)data_len / segment_size);
@@ -983,10 +251,10 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args)
* This will copmpute a size algined to the number of data
* and the underlying wordsize of the EC algorithm.
*/
- aligned_data_len = get_aligned_data_size(pyeclib_handle, data_len);
+ aligned_data_len = liberasurecode_get_aligned_data_size(pyeclib_handle->ec_desc, data_len);
/* aligned_data_len is guaranteed to be divisible by k */
- fragment_size = aligned_data_len / pyeclib_handle->k;
+ fragment_size = aligned_data_len / pyeclib_handle->ec_args.k;
/* Segment size is the user-provided segment size */
segment_size = data_len;
@@ -998,10 +266,10 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args)
* the minimum segment size.
*/
- aligned_segment_size = get_aligned_data_size(pyeclib_handle, segment_size);
+ aligned_segment_size = liberasurecode_get_aligned_data_size(pyeclib_handle->ec_desc, segment_size);
/* aligned_data_len is guaranteed to be divisible by k */
- fragment_size = aligned_segment_size / pyeclib_handle->k;
+ fragment_size = aligned_segment_size / pyeclib_handle->ec_args.k;
last_segment_size = data_len - (segment_size * (num_segments - 1));
@@ -1017,10 +285,10 @@ pyeclib_c_get_segment_info(PyObject *self, PyObject *args)
last_segment_size = last_segment_size + segment_size;
}
- aligned_segment_size = get_aligned_data_size(pyeclib_handle, last_segment_size);
+ aligned_segment_size = liberasurecode_get_aligned_data_size(pyeclib_handle->ec_desc, last_segment_size);
/* Compute the last fragment size from the last segment size */
- last_fragment_size = aligned_segment_size / pyeclib_handle->k;
+ last_fragment_size = aligned_segment_size / pyeclib_handle->ec_args.k;
}
/* Add header to fragment sizes */
@@ -1055,14 +323,12 @@ pyeclib_c_encode(PyObject *self, PyObject *args)
{
PyObject *pyeclib_obj_handle = NULL;
pyeclib_t *pyeclib_handle= NULL;
- char **data_to_encode = NULL; /* array of k data buffers */
+ char **encoded_data = NULL; /* array of k data buffers */
char **encoded_parity = NULL; /* array of m parity buffers */
PyObject *list_of_strips = NULL; /* list of encoded strips to return */
char *data; /* param, data buffer to encode */
int data_len; /* param, length of data buffer */
- int aligned_data_len; /* EC algorithm compatible data length */
- int orig_data_size; /* data length to write to headers */
- int blocksize; /* length of each of k data elements */
+ uint64_t fragment_len; /* length, in bytes of the fragments */
int i; /* a counter */
/* Assume binary data (force "byte array" input) */
@@ -1075,428 +341,37 @@ pyeclib_c_encode(PyObject *self, PyObject *args)
PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode");
return NULL;
}
-
- /* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/
- orig_data_size = data_len;
- aligned_data_len = get_aligned_data_size(pyeclib_handle, data_len);
- blocksize = aligned_data_len / pyeclib_handle->k;
-
- /* Allocate and initialize an array of zero'd out data buffers */
- data_to_encode = (char**) alloc_zeroed_buffer(sizeof(char*) * pyeclib_handle->k);
- if (NULL == data_to_encode) {
- goto error;
- }
- for (i = 0; i < pyeclib_handle->k; i++) {
- int payload_size = data_len > blocksize ? blocksize : data_len;
- char *fragment = alloc_fragment_buffer(blocksize);
- if (NULL == fragment) {
- goto error;
- }
-
- /* Copy existing data into clean, zero'd out buffer */
- data_to_encode[i] = get_data_ptr_from_fragment(fragment);
- if (data_len > 0) {
- memcpy(data_to_encode[i], data, payload_size);
- }
- /* Fragment size will always be the same (may be able to get rid of this) */
- set_fragment_size(fragment, blocksize);
+ if (liberasurecode_encode(pyeclib_handle->ec_desc,
+ data, data_len, &encoded_data, &encoded_parity, &fragment_len) < 0) {
- data += payload_size;
- data_len -= payload_size;
- }
-
- /* Allocate and initialize an array of zero'd out parity buffers */
- encoded_parity = (char**) alloc_zeroed_buffer(sizeof(char*) * pyeclib_handle->m);
- if (NULL == encoded_parity) {
- goto error;
- }
- for (i = 0; i < pyeclib_handle->m; i++) {
- char *fragment = alloc_fragment_buffer(blocksize);
- if (NULL == fragment) {
- goto error;
- }
-
- encoded_parity[i] = get_data_ptr_from_fragment(fragment);
- set_fragment_size(fragment, blocksize);
- }
-
- /* Run the erasure coding algorithm to generate the parity fragments */
- switch (pyeclib_handle->type) {
- case PYECC_RS_CAUCHY_ORIG:
- jerasure_bitmatrix_encode(pyeclib_handle->k, pyeclib_handle->m,
- pyeclib_handle->w, pyeclib_handle->bitmatrix,
- data_to_encode, encoded_parity, blocksize,
- PYECC_CAUCHY_PACKETSIZE);
- break;
- case PYECC_XOR_HD_3:
- case PYECC_XOR_HD_4:
- pyeclib_handle->xor_code_desc->encode(pyeclib_handle->xor_code_desc,
- data_to_encode, encoded_parity,
- blocksize);
- break;
- case PYECC_RS_VAND:
- default:
- jerasure_matrix_encode(pyeclib_handle->k, pyeclib_handle->m,
- pyeclib_handle->w, pyeclib_handle->matrix,
- data_to_encode, encoded_parity, blocksize);
- break;
+ PyErr_SetString(PyECLibError, "Encountered error in liberasurecode_encode");
+ return NULL;
}
-
+
/* Create the python list of fragments to return */
- list_of_strips = PyList_New(pyeclib_handle->k + pyeclib_handle->m);
+ list_of_strips = PyList_New(pyeclib_handle->ec_args.k + pyeclib_handle->ec_args.m);
if (NULL == list_of_strips) {
PyErr_SetString(PyECLibError, "Error allocating python list in encode");
- goto error;
+ return NULL;
}
- /* Finalize data fragments and add them to the python list to return */
- for (i = 0; i < pyeclib_handle->k; i++) {
- char *fragment_ptr = get_fragment_ptr_from_data(data_to_encode[i]);
- int fragment_size = blocksize + sizeof(fragment_header_t);
- set_fragment_idx(fragment_ptr, i);
- set_orig_data_size(fragment_ptr, orig_data_size);
- if (use_inline_chksum(pyeclib_handle)) {
- int chksum = crc32(0, data_to_encode[i], blocksize);
- set_chksum(fragment_ptr, chksum);
- }
+ /* Add data fragments to the python list to return */
+ for (i = 0; i < pyeclib_handle->ec_args.k; i++) {
PyList_SET_ITEM(list_of_strips, i,
- PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size));
+ PY_BUILDVALUE_OBJ_LEN(encoded_data[i], fragment_len));
}
- /* Finalize parity fragments and add them to the python list to return */
- for (i = 0; i < pyeclib_handle->m; i++) {
- char *fragment_ptr = get_fragment_ptr_from_data(encoded_parity[i]);
- int fragment_size = blocksize + sizeof(fragment_header_t);
- set_fragment_idx(fragment_ptr, pyeclib_handle->k+i);
- set_orig_data_size(fragment_ptr, orig_data_size);
- if (use_inline_chksum(pyeclib_handle)) {
- int chksum = crc32(0, encoded_parity[i], blocksize);
- set_chksum(fragment_ptr, chksum);
- }
- PyList_SET_ITEM(list_of_strips, pyeclib_handle->k + i,
- PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size));
- }
-
- goto exit;
-
-error:
- list_of_strips = NULL;
-
-exit:
- if (data_to_encode) {
- for (i = 0; i < pyeclib_handle->k; i++) {
- if (data_to_encode[i]) free_fragment_buffer(data_to_encode[i]);
- }
- check_and_free_buffer(data_to_encode);
- }
- if (encoded_parity) {
- for (i = 0; i < pyeclib_handle->m; i++) {
- if (encoded_parity[i]) free_fragment_buffer(encoded_parity[i]);
- }
- check_and_free_buffer(encoded_parity);
+ /* Add parity fragments to the python list to return */
+ for (i = 0; i < pyeclib_handle->ec_args.m; i++) {
+ PyList_SET_ITEM(list_of_strips, pyeclib_handle->ec_args.k + i,
+ PY_BUILDVALUE_OBJ_LEN(encoded_parity[i], fragment_len));
}
return list_of_strips;
}
-/*
- * Convert a set of fragments into a string. If, less than k data fragments
- * are present, return None. Return NULL on error.
- *
- * @param pyeclib_obj_handle
- * @param list of fragments
- * @param python string or None, NULL on error
- */
-static PyObject *
-pyeclib_c_fragments_to_string(PyObject *self, PyObject *args)
-{
- PyObject *pyeclib_obj_handle = NULL;
- pyeclib_t *pyeclib_handle = NULL;
- PyObject *fragment_list = NULL; /* param, python list of fragments */
- PyObject *ret_string = NULL; /* python string to return */
- char *ret_cstring = NULL; /* c string to build return string from */
- int ret_data_size; /* size of return string in bytes */
- char **data = NULL; /* array of data buffers */
- int string_off = 0; /* offset into cstring */
- int num_fragments = 0; /* num of fragments provided by caller */
- int num_data = 0; /* num of fragments that are data */
- int orig_data_size = -1; /* data size from fragment header */
- int i = 0; /* a counter */
-
- /* Collect and validate the method arguments */
- if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_list)) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.fragments_to_string");
- return NULL;
- }
- pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
- if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.fragments_to_string");
- return NULL;
- }
- if (!PyList_Check(fragment_list)) {
- PyErr_SetString(PyECLibError, "Invalid structure passed in for fragment list in pyeclib.fragments_to_string");
- return NULL;
- }
-
- /* Return None if there's insufficient fragments */
- num_fragments = (int) PyList_Size(fragment_list);
- if (pyeclib_handle->k > num_fragments) {
- return Py_BuildValue("");
- }
-
- /*
- * NOTE: Update to only copy original size out of the buffers
- */
-
- /*
- * Iterate over the fragments. If we have all k data fragments, then we can
- * concatenate them into a string and return it; otherwise, we return NULL
- */
- data = (char **) alloc_zeroed_buffer(sizeof(char *) * pyeclib_handle->k);
- if (NULL == data) {
- return NULL;
- }
- for (i = 0; i < num_fragments && num_data < pyeclib_handle->k; i++) {
- PyObject *tmp_data = PyList_GetItem(fragment_list, i);
- char *tmp_buf;
- int index;
- int data_size;
- Py_ssize_t len;
-
- /* Get and validate the fragment index and size */
- PyBytes_AsStringAndSize(tmp_data, &tmp_buf, &len);
- index = get_fragment_idx(tmp_buf);
- data_size = get_fragment_size(tmp_buf);
- if ((index < 0) || (data_size < 0)) {
- ret_string = NULL;
- goto exit;
- }
-
- /* Validate the original data size */
- if (orig_data_size < 0) {
- orig_data_size = get_orig_data_size(tmp_buf);
- } else {
- if (get_orig_data_size(tmp_buf) != orig_data_size) {
- PyErr_SetString(PyECLibError, "Inconsistent orig data sizes found in headers");
- ret_string = NULL;
- goto exit;
- }
- }
-
- /* Skip parity fragments, put data fragments in index order */
- if (index >= pyeclib_handle->k) {
- continue;
- } else {
- data[index] = tmp_buf;
- num_data++;
- }
- }
-
- /* Return None if there still isn't insufficient fragments */
- if (num_data != pyeclib_handle->k) {
- ret_string = Py_BuildValue("");
- goto exit;
- }
-
- /* Create the c-string to return */
- ret_cstring = (char *) alloc_zeroed_buffer(orig_data_size);
- if (NULL == ret_cstring) {
- ret_string = NULL;
- goto exit;
- }
- ret_data_size = orig_data_size;
-
- /* Copy fragment data into cstring (fragments should be in index order) */
- for (i = 0; i < num_data && orig_data_size > 0; i++) {
- char* fragment_data = get_data_ptr_from_fragment(data[i]);
- int fragment_size = get_fragment_size(data[i]);
- int payload_size = orig_data_size > fragment_size ? fragment_size : orig_data_size;
-
- memcpy(ret_cstring + string_off, fragment_data, payload_size);
- orig_data_size -= payload_size;
- string_off += payload_size;
- }
- ret_string = PY_BUILDVALUE_OBJ_LEN(ret_cstring, ret_data_size);
-
-exit:
- check_and_free_buffer(data);
- check_and_free_buffer(ret_cstring);
-
- return ret_string;
-}
-
-
-/*
- * Return a tuple containing a reference to a data fragment list, a parity
- * fragment list and a missing fragment index list, including all '0'
- * fragment for missing fragments. Copy references to the existing
- * fragments and create new string for missing fragments.
- *
- * @param pyeclib_obj_handle
- * @param list of fragments
- * @return (data fragments, parity fragments, missing indexes) tuple
- */
-static PyObject *
-pyeclib_c_get_fragment_partition(PyObject *self, PyObject *args)
-{
- PyObject *pyeclib_obj_handle = NULL;
- pyeclib_t* pyeclib_handle = NULL;
- PyObject *fragment_list = NULL; /* param, list of fragments */
- PyObject *data_list = NULL; /* data fragments for return tuple */
- PyObject *parity_list = NULL; /* parity fragments for return tuple */
- PyObject *fragment_string = NULL; /* fragment string for lists */
- PyObject *missing_list = NULL; /* missing indexes for return tuple */
- PyObject *return_lists = NULL; /* Python tuple to return */
- PyObject **data = NULL; /* array of k data fragment buffers*/
- PyObject **parity = NULL; /* array of m parity fragment buffers */
- int *missing = NULL; /* indicies of missing fragments */
- int num_missing = 0; /* num of missing fragments */
- int num_fragments; /* num of frags provided by caller */
- int fragment_size = 0; /* size in bytes of fragments */
- int i = 0; /* a counter */
-
- /* Collect and validate the method arguments */
- if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_list)) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.get_fragment_partition");
- return NULL;
- }
- pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
- if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.get_fragment_partition");
- return NULL;
- }
- if (!PyList_Check(fragment_list)) {
- PyErr_SetString(PyECLibError, "Invalid structure passed in for fragment list in pyeclib.get_fragment_partition");
- return NULL;
- }
-
- /* Create the arrays need to hold the data and parity fragments */
- data = (PyObject **) alloc_zeroed_buffer(pyeclib_handle->k * sizeof(PyObject*));
- if (NULL == data) {
- goto error;
- }
- parity = (PyObject **) alloc_zeroed_buffer(pyeclib_handle->m * sizeof(PyObject*));
- if (NULL == parity) {
- goto error;
- }
-
- /* Fill in the data and parity pointers to reveal missing fragments */
- num_fragments = (int) PyList_Size(fragment_list);
- for (i = 0; i < num_fragments; i++) {
- PyObject *tmp_data = PyList_GetItem(fragment_list, i);
- char *c_buf;
- int index;
- Py_ssize_t len;
-
- /* ASSUMPTION: The fragment_size is the max of the fragments we read */
- PyBytes_AsStringAndSize(tmp_data, &c_buf, &len);
- if (len > fragment_size) {
- fragment_size = (int) len;
- }
-
- /* Get fragment index and set the index in data or parity */
- index = get_fragment_idx(c_buf);
- if (index < pyeclib_handle->k) {
- data[index] = tmp_data;
- } else {
- parity[index - pyeclib_handle->k] = tmp_data;
- }
- }
-
- /* If there are missing bufs, figure out which indexes are missing
- *
- * ASSUMPTION: We will never try to do anything when we have more
- * than k missing fragments
- */
- missing = (int *) alloc_zeroed_buffer(pyeclib_handle->k * sizeof(int));
- if (NULL == missing) {
- goto error;
- } else {
- num_missing = 0;
- for (i = 0; i < pyeclib_handle->k; i++) {
- if (data[i] == NULL) {
- missing[num_missing] = i;
- num_missing++;
- }
- }
- for (i = 0; i < pyeclib_handle->m; i++) {
- if (parity[i] == NULL) {
- missing[num_missing] = i + pyeclib_handle->k;
- num_missing++;
- }
- }
- }
-
- /* Create the python objects to return */
- data_list = PyList_New(pyeclib_handle->k);
- parity_list = PyList_New(pyeclib_handle->m);
- missing_list = PyList_New(num_missing);
- return_lists = PyTuple_New(3);
- if (!data_list || !parity_list || !missing_list || !return_lists) {
- return_lists = PyErr_NoMemory();
- goto exit;
- }
-
- /* Fill in the data fragments, create zero fragment if missing */
- for (i = 0; i < pyeclib_handle->k; i++) {
- if (data[i] != NULL) {
- /* BORROWED REF: increment ref count before inserting into list */
- Py_INCREF(data[i]);
- fragment_string = data[i];
- } else {
- fragment_string = alloc_zero_string(fragment_size);
- if (NULL == fragment_string) {
- goto error;
- }
- }
- PyList_SET_ITEM(data_list, i, fragment_string);
- }
-
- /* Fill in the parity fragments, create zero fragment if missing */
- for (i = 0; i < pyeclib_handle->m; i++) {
- if (parity[i] != NULL) {
- /* BORROWED REF: increment ref count before inserting into list */
- Py_INCREF(parity[i]);
- fragment_string = parity[i];
- } else {
- fragment_string = alloc_zero_string(fragment_size);
- if (NULL == fragment_string) {
- goto error;
- }
- }
- PyList_SET_ITEM(parity_list, i, fragment_string);
- }
-
- /* Fill in the list of missing indexes */
- for (i = 0;i < num_missing; i++) {
- PyList_SET_ITEM(missing_list, i, Py_BuildValue("i", missing[i]));
- }
-
- Py_INCREF(data_list);
- Py_INCREF(parity_list);
- Py_INCREF(missing_list);
-
- PyTuple_SetItem(return_lists, 0, data_list);
- PyTuple_SetItem(return_lists, 1, parity_list);
- PyTuple_SetItem(return_lists, 2, missing_list);
- //Py_INCREF(return_lists);
-
- goto exit;
-
-error:
- return_lists = NULL;
-
-exit:
- check_and_free_buffer(data);
- check_and_free_buffer(parity);
- check_and_free_buffer(missing);
-
- return return_lists;
-}
-
-
/**
* Return a list of lists with valid rebuild indexes given an EC algorithm
* and a list of missing indexes.
@@ -1514,9 +389,8 @@ pyeclib_c_get_required_fragments(PyObject *self, PyObject *args)
PyObject *fragment_idx_list = NULL; /* list of req'd indexes to return */
int *c_missing_list = NULL; /* c-array of missing indexes */
int num_missing; /* size of passed in missing list */
- int i = 0, j = 0; /* counters */
+ int i = 0; /* counters */
int k, m; /* EC algorithm parameters */
- unsigned long missing_bm = 0; /* bitmap of missing indexes */
int *fragments_needed = NULL; /* indexes of xor code fragments */
int ret; /* return value for xor code */
@@ -1530,8 +404,8 @@ pyeclib_c_get_required_fragments(PyObject *self, PyObject *args)
PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.get_required_fragments");
return NULL;
}
- k = pyeclib_handle->k;
- m = pyeclib_handle->m;
+ k = pyeclib_handle->ec_args.k;
+ m = pyeclib_handle->ec_args.m;
/* Generate -1 terminated c-array and bitmap of missing indexes */
num_missing = (int) PyList_Size(missing_list);
@@ -1545,62 +419,27 @@ pyeclib_c_get_required_fragments(PyObject *self, PyObject *args)
long idx = PyLong_AsLong(obj_idx);
c_missing_list[i] = (int) idx;
}
- missing_bm = convert_list_to_bitmap(c_missing_list);
+
+ fragments_needed = alloc_zeroed_buffer(sizeof(int) * (k + m));
+ if (NULL == fragments_needed) {
+ goto exit;
+ }
+
+ ret = liberasurecode_fragments_needed(pyeclib_handle->ec_desc, c_missing_list, fragments_needed);
+ if (ret < 0) {
+ PyErr_Format(PyECLibError, "Not enough fragments for pyeclib.get_required_fragments!");
+ goto exit;
+ }
- /* Generate the python list of lists of rebuild indexes to return */
+ /* Post-process into a Python list */
fragment_idx_list = PyList_New(0);
if (NULL == fragment_idx_list) {
goto exit;
}
- j = 0;
- switch(pyeclib_handle->type) {
- case PYECC_RS_CAUCHY_ORIG:
- case PYECC_RS_VAND:
- for (i = 0; i < (k + m); i++) {
- if (!(missing_bm & (1 << i))) {
- PyList_Append(fragment_idx_list, Py_BuildValue("i", i));
- j++;
- }
- if (j == k) {
- break;
- }
- }
-
- if (j != k) {
- /* Let the garbage collector clean this up */
- Py_DECREF(fragment_idx_list);
- PyErr_Format(PyECLibError, "Not enough fragments for pyeclib.get_required_fragments (need at least %d, %d are given)", k, j);
- fragment_idx_list = NULL;
- }
- break;
- case PYECC_XOR_HD_3:
- case PYECC_XOR_HD_4:
- {
- fragments_needed = alloc_zeroed_buffer(sizeof(int) * (k + m));
- if (NULL == fragments_needed) {
- fragment_idx_list = NULL;
- goto exit;
- }
- ret = pyeclib_handle->xor_code_desc->fragments_needed(pyeclib_handle->xor_code_desc,
- c_missing_list,
- fragments_needed);
-
- if (ret < 0) {
- Py_DECREF(fragment_idx_list);
- PyErr_Format(PyECLibError, "Not enough fragments for pyeclib.get_required_fragments!");
- fragment_idx_list = NULL;
- break;
- }
-
- while (fragments_needed[j] > -1) {
- PyList_Append(fragment_idx_list, Py_BuildValue("i", fragments_needed[j]));
- j++;
- }
- break;
- }
- default:
- PyErr_SetString(PyECLibError, "Invalid EC type used in pyeclib.get_required_fragments");
- break;
+
+ while (fragments_needed[i] > -1) {
+ PyList_Append(fragment_idx_list, Py_BuildValue("i", fragments_needed[i]));
+ i++;
}
exit:
@@ -1630,156 +469,63 @@ pyeclib_c_reconstruct(PyObject *self, PyObject *args)
{
PyObject *pyeclib_obj_handle = NULL;
pyeclib_t *pyeclib_handle = NULL;
- PyObject *data_list = NULL; /* param, list of data fragments */
- PyObject *parity_list = NULL; /* param, list of parity fragments */
- PyObject *missing_idx_list = NULL; /* param, list of missing indexes */
+ PyObject *fragments = NULL; /* param, list of fragments */
PyObject *reconstructed = NULL; /* reconstructed object to return */
- int *erased = NULL; /* jerasure notation of erased devs */
- int fragment_size; /* param, size in bytes of fragment */
- int blocksize; /* size in bytes, fragment - header */
- char **data = NULL; /* k length array of data buffers */
- char **parity = NULL; /* m length array of parity buffers */
- int *missing_idxs = NULL; /* array of missing indexes */
+ char * c_reconstructed = NULL; /* C string of reconstructed fragment */
+ int fragment_len; /* param, size in bytes of fragment */
+ int num_fragments; /* number of fragments passed in */
+ char **c_fragments = NULL; /* C array containing the fragment payloads */
int destination_idx; /* param, index to reconstruct */
- unsigned long long realloc_bm = 0; /* bitmap, which fragments were realloc'ed */
- int orig_data_size = -1; /* data size (B),from fragment hdr */
- int missing_size; /* number of missing indexes */
- int *decoding_matrix = NULL; /* reconstruct specific decode matrix */
- int *decoding_row = NULL; /* required row from decode matrix */
- int *dm_ids = NULL; /* k length array of surviving indexes */
- int k, m, w; /* EC algorithm parameters */
int ret; /* decode matrix creation return val */
int i = 0; /* a counter */
/* Obtain and validate the method parameters */
- if (!PyArg_ParseTuple(args, "OOOOii", &pyeclib_obj_handle, &data_list,
- &parity_list, &missing_idx_list, &destination_idx,
- &fragment_size)) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode");
+ if (!PyArg_ParseTuple(args, "OOii", &pyeclib_obj_handle, &fragments,
+ &fragment_len, &destination_idx)) {
+ PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.reconstruct");
return NULL;
}
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode");
- return NULL;
- }
- k = pyeclib_handle->k;
- m = pyeclib_handle->m;
- w = pyeclib_handle->w;
- if (!PyList_Check(data_list) || !PyList_Check(parity_list) || !PyList_Check(missing_idx_list)) {
- PyErr_SetString(PyECLibError, "Invalid structure passed in for data, parity and/or missing_idx list");
+ PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.reconstruct");
return NULL;
}
- if (k != PyList_Size(data_list)) {
- PyErr_SetString(PyECLibError, "The data list does not have the correct number of entries");
- return NULL;
- }
- if (m != PyList_Size(parity_list)) {
- PyErr_SetString(PyECLibError, "The parity list does not have the correct number of entries");
+
+ /* Pre-processing Python data structures */
+ if (!PyList_Check(fragments)) {
+ PyErr_SetString(PyECLibError, "Invalid structure passed in for fragment list");
return NULL;
}
- /* Allocate data structures needed for reconstruction */
- blocksize = FRAGSIZE_2_BLOCKSIZE(fragment_size);
- missing_size = (int) PyList_Size(missing_idx_list);
- missing_idxs = (int *) alloc_zeroed_buffer(sizeof(int) * (missing_size + 1));
- data = (char **) alloc_zeroed_buffer(sizeof(char *) * k);
- parity = (char **) alloc_zeroed_buffer(sizeof(char *) * m);
- if (NULL == missing_idxs || NULL == data || NULL == parity) {
+ num_fragments = PyList_Size(fragments);
+
+ c_fragments = (char **) alloc_zeroed_buffer(sizeof(char *) * num_fragments);
+ if (NULL == c_fragments) {
goto error;
}
-
- /* Prepare for decoding, no need to go further on error */
- if (get_decoding_info(pyeclib_handle, data_list, parity_list,
- missing_idx_list, data, parity, missing_idxs,
- &orig_data_size, fragment_size, &realloc_bm)) {
- PyErr_SetString(PyECLibError, "Could not extract adequate decoding info from data, parity and missing lists");
+
+ c_reconstructed = (char*) alloc_zeroed_buffer(sizeof(char) * fragment_len);
+ if (NULL == c_fragments) {
goto error;
}
- /* Create the decoding matrix, and attempt reconstruction */
- erased = jerasure_erasures_to_erased(k, m, missing_idxs);
- switch (pyeclib_handle->type) {
- case PYECC_RS_CAUCHY_ORIG:
- if (destination_idx < k) {
- decoding_matrix = (int *) alloc_zeroed_buffer(sizeof(int*) * k * k * w * w);
- dm_ids = (int *) alloc_zeroed_buffer(sizeof(int) * k);
- if (NULL == decoding_matrix || NULL == dm_ids) {
- goto error;
- }
- ret = jerasure_make_decoding_bitmatrix(k, m, w, pyeclib_handle->bitmatrix,
- erased, decoding_matrix, dm_ids);
- decoding_row = decoding_matrix + (destination_idx * k * w * w);
- } else {
- ret = 0;
- decoding_row = pyeclib_handle->bitmatrix + ((destination_idx - k) * k * w * w);
- }
-
- if (ret == 0) {
- jerasure_bitmatrix_dotprod(k, w, decoding_row, dm_ids, destination_idx,
- data, parity, blocksize,
- PYECC_CAUCHY_PACKETSIZE);
- }
- break;
- case PYECC_RS_VAND:
- if (destination_idx < k) {
- decoding_matrix = (int *) alloc_zeroed_buffer(sizeof(int*) * k * k);
- dm_ids = (int *) alloc_zeroed_buffer(sizeof(int) * k);
- if (NULL == decoding_matrix || NULL == dm_ids) {
- goto error;
- }
- ret = jerasure_make_decoding_matrix(k, m, w, pyeclib_handle->matrix,
- erased, decoding_matrix, dm_ids);
- decoding_row = decoding_matrix + (destination_idx * k);
- } else {
- ret = 0;
- decoding_row = pyeclib_handle->matrix + ((destination_idx - k) * k);
- }
-
- if (ret == 0) {
- jerasure_matrix_dotprod(k, w, decoding_row, dm_ids, destination_idx,
- data, parity, blocksize);
- }
- break;
- case PYECC_XOR_HD_3:
- case PYECC_XOR_HD_4:
- ret = 0;
- xor_reconstruct_one(pyeclib_handle->xor_code_desc, data, parity,
- missing_idxs, destination_idx, blocksize);
- break;
- default:
- ret = -1;
- break;
+ /* Put the fragments into an array of C strings */
+ for (i = 0; i < num_fragments; i++) {
+ PyObject *tmp_data = PyList_GetItem(fragments, i);
+ Py_ssize_t len = 0;
+ PyBytes_AsStringAndSize(tmp_data, &(c_fragments[i]), &len);
}
-
- /* Set the metadata on the reconstructed fragment */
- if (ret == 0) {
- char *fragment_ptr = NULL;
- if (destination_idx < k) {
- fragment_ptr = get_fragment_ptr_from_data_novalidate(data[destination_idx]);
- init_fragment_header(fragment_ptr);
- set_fragment_idx(fragment_ptr, destination_idx);
- set_orig_data_size(fragment_ptr, orig_data_size);
- set_fragment_size(fragment_ptr, blocksize);
- if (use_inline_chksum(pyeclib_handle)) {
- int chksum = crc32(0, data[destination_idx], blocksize);
- set_chksum(fragment_ptr, chksum);
- }
- } else {
- fragment_ptr = get_fragment_ptr_from_data_novalidate(parity[destination_idx - k]);
- init_fragment_header(fragment_ptr);
- set_fragment_idx(fragment_ptr, destination_idx);
- set_orig_data_size(fragment_ptr, orig_data_size);
- set_fragment_size(fragment_ptr, blocksize);
- if (use_inline_chksum(pyeclib_handle)) {
- int chksum = crc32(0, parity[destination_idx - k], blocksize);
- set_chksum(fragment_ptr, chksum);
- }
- }
-
- reconstructed = PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size);
- } else {
+
+ ret = liberasurecode_reconstruct_fragment(pyeclib_handle->ec_desc,
+ c_fragments,
+ num_fragments,
+ fragment_len,
+ destination_idx,
+ c_reconstructed);
+ if (ret < 0) {
reconstructed = NULL;
+ } else {
+ reconstructed = PY_BUILDVALUE_OBJ_LEN(c_reconstructed, fragment_len);
}
goto out;
@@ -1788,34 +534,14 @@ error:
reconstructed = NULL;
out:
- free(erased);
- /* Free fragment buffers that needed to be reallocated for alignment */
- for (i = 0; i < k; i++) {
- if (realloc_bm & (1 << i)) {
- free(get_fragment_ptr_from_data_novalidate(data[i]));
- }
- }
- for (i = 0; i < m; i++) {
- if (realloc_bm & (1 << (i + k))) {
- free(get_fragment_ptr_from_data_novalidate(parity[i]));
- }
- }
-
- check_and_free_buffer(missing_idxs);
- check_and_free_buffer(data);
- check_and_free_buffer(parity);
- check_and_free_buffer(decoding_matrix);
- check_and_free_buffer(dm_ids);
+ check_and_free_buffer(c_fragments);
+ check_and_free_buffer(c_reconstructed);
return reconstructed;
}
-
/**
- * Reconstruct all of the missing fragments from a set of fragments.
- *
- * TODO: There's a lot of duplicated code between this and the
- * reconstruct method. Consider refactoring these methods.
+ * Decode a set of fragments into the original string
*
* @param pyeclib_obj_handle
* @param data_list k length list of data elements
@@ -1829,151 +555,68 @@ pyeclib_c_decode(PyObject *self, PyObject *args)
{
PyObject *pyeclib_obj_handle = NULL;
pyeclib_t *pyeclib_handle = NULL;
- PyObject *list_of_strips = NULL; /* list of strips to return */
- PyObject *data_list = NULL; /* param, list of data fragments */
- PyObject *parity_list = NULL; /* param, list of data fragments */
- PyObject *missing_idx_list = NULL; /* param, list of missing indexes */
- int fragment_size; /* param, size in bytes of fragment */
- int blocksize; /* size in bytes, fragment - header */
- unsigned long long realloc_bm = 0; /* bitmap, which fragments were realloc'ed */
- char **data = NULL; /* k length array of data buffers */
- char **parity = NULL; /* m length array of parity buffers */
- int *missing_idxs = NULL; /* array of missing indexes */
- int missing_size; /* number of missing indexes */
- int orig_data_size = -1; /* data size in bytes ,from fragment hdr */
- int k, m, w; /* EC algorithm parameters */
- int i = 0, j = 0; /* counters */
+ PyObject *fragments = NULL; /* param, list of missing indexes */
+ PyObject *orig_payload = NULL; /* buffer to store original payload in */
+ int fragment_len; /* param, size in bytes of fragment */
+ char **c_fragments = NULL; /* k length array of data buffers */
+ int num_fragments; /* param, number of fragments */
+ char *c_orig_payload = NULL; /* buffer to store original payload in */
+ uint64_t orig_data_size = 0; /* data size in bytes ,from fragment hdr */
+ int i = 0; /* counters */
+ int ret = 0;
/* Obtain and validate the method parameters */
- if (!PyArg_ParseTuple(args, "OOOOi", &pyeclib_obj_handle, &data_list, &parity_list, &missing_idx_list, &fragment_size)) {
- PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode");
+ if (!PyArg_ParseTuple(args, "OOi", &pyeclib_obj_handle, &fragments, &fragment_len)) {
+ PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.decode");
return NULL;
}
pyeclib_handle = (pyeclib_t*)PyCapsule_GetPointer(pyeclib_obj_handle, PYECC_HANDLE_NAME);
if (pyeclib_handle == NULL) {
- PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode");
- return NULL;
- }
- k = pyeclib_handle->k;
- m = pyeclib_handle->m;
- w = pyeclib_handle->w;
- if (!PyList_Check(data_list) || !PyList_Check(parity_list) || !PyList_Check(missing_idx_list)) {
- PyErr_SetString(PyECLibError, "Invalid structure passed in for data, parity and/or missing_idx list");
- return NULL;
- }
- if (k != PyList_Size(data_list)) {
- PyErr_SetString(PyECLibError, "The data list does not have the correct number of entries");
+ PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.decode");
return NULL;
}
- if (m != PyList_Size(parity_list)) {
- PyErr_SetString(PyECLibError, "The parity list does not have the correct number of entries");
+ if (!PyList_Check(fragments)) {
+ PyErr_SetString(PyECLibError, "Invalid structure passed in for available fragments in pyeclib.decode");
return NULL;
}
+
+ num_fragments = PyList_Size(fragments);
- /* Allocate data structures needed for reconstruction */
- blocksize = FRAGSIZE_2_BLOCKSIZE(fragment_size);
- missing_size = (int) PyList_Size(missing_idx_list);
- missing_idxs = (int *) alloc_zeroed_buffer(sizeof(int) * (missing_size + 1));
- data = (char **) alloc_zeroed_buffer(sizeof(char *) * k);
- parity = (char **) alloc_zeroed_buffer(sizeof(char *) * m);
- if (NULL == missing_idxs || NULL == data || NULL == parity) {
- goto error;
- }
-
- /* Prepare for decoding, no need to go further on error */
- if (get_decoding_info(pyeclib_handle, data_list, parity_list, missing_idx_list,
- data, parity, missing_idxs, &orig_data_size,
- fragment_size, &realloc_bm)) {
- PyErr_SetString(PyECLibError, "Could not extract adequate decoding info from data, parity and missing lists");
+ if (pyeclib_handle->ec_args.k > num_fragments) {
+ PyErr_SetString(PyECLibError, "The fragment list does not have enough entries in pyeclib.decode");
return NULL;
}
-
- /* Reconstruct the missing fragments */
- switch (pyeclib_handle->type) {
- case PYECC_RS_CAUCHY_ORIG:
- jerasure_bitmatrix_decode(k, m, w, pyeclib_handle->bitmatrix, 0, missing_idxs,
- data, parity, blocksize, PYECC_CAUCHY_PACKETSIZE);
- break;
- case PYECC_XOR_HD_3:
- case PYECC_XOR_HD_4:
- pyeclib_handle->xor_code_desc->decode(pyeclib_handle->xor_code_desc, data,
- parity, missing_idxs, blocksize, 1);
- break;
- case PYECC_RS_VAND:
- default:
- jerasure_matrix_decode(k, m, w, pyeclib_handle->matrix, 1, missing_idxs,
- data, parity, blocksize);
- break;
- }
-
- /* Create the python list to return */
- list_of_strips = PyList_New(k + m);
- if (NULL == list_of_strips) {
+
+ c_fragments = (char **) alloc_zeroed_buffer(sizeof(char *) * num_fragments);
+ if (NULL == c_fragments) {
goto error;
}
- /* Create headers for the newly decoded elements */
- while (missing_idxs[j] >= 0) {
- int missing_idx = missing_idxs[j];
- if (missing_idx < k) {
- char *fragment_ptr = get_fragment_ptr_from_data_novalidate(data[missing_idx]);
- init_fragment_header(fragment_ptr);
- set_fragment_idx(fragment_ptr, missing_idx);
- set_orig_data_size(fragment_ptr, orig_data_size);
- set_fragment_size(fragment_ptr, blocksize);
- if (use_inline_chksum(pyeclib_handle)) {
- int chksum = crc32(0, data[missing_idx], blocksize);
- set_chksum(fragment_ptr, chksum);
- }
- } else if (missing_idx >= k) {
- int parity_idx = missing_idx - k;
- char *fragment_ptr = get_fragment_ptr_from_data_novalidate(parity[parity_idx]);
- init_fragment_header(fragment_ptr);
- set_fragment_idx(fragment_ptr, missing_idx);
- set_orig_data_size(fragment_ptr, orig_data_size);
- set_fragment_size(fragment_ptr, blocksize);
- if (use_inline_chksum(pyeclib_handle)) {
- int chksum = crc32(0, parity[parity_idx], blocksize);
- set_chksum(fragment_ptr, chksum);
- }
- }
- j++;
- }
-
- /* Fill in the data fragments */
- for (i = 0; i < k; i++) {
- char *fragment_ptr = get_fragment_ptr_from_data(data[i]);
- PyList_SET_ITEM(list_of_strips, i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size));
- }
- /* Fill in the parity fragments */
- for (i = 0; i < m; i++) {
- char *fragment_ptr = get_fragment_ptr_from_data(parity[i]);
- PyList_SET_ITEM(list_of_strips, k + i, PY_BUILDVALUE_OBJ_LEN(fragment_ptr, fragment_size));
+ /* Put the fragments into an array of C strings */
+ for (i = 0; i < num_fragments; i++) {
+ PyObject *tmp_data = PyList_GetItem(fragments, i);
+ Py_ssize_t len = 0;
+ PyBytes_AsStringAndSize(tmp_data, &(c_fragments[i]), &len);
}
+ ret = liberasurecode_decode(pyeclib_handle->ec_desc,
+ c_fragments,
+ num_fragments,
+ fragment_len,
+ &c_orig_payload,
+ &orig_data_size);
+
+ orig_payload = PY_BUILDVALUE_OBJ_LEN(c_orig_payload, orig_data_size);
goto exit;
error:
- list_of_strips = NULL;
+ orig_payload = NULL;
exit:
- /* Free fragment buffers that needed to be reallocated for alignment */
- for (i = 0; i < k; i++) {
- if (realloc_bm & (1 << i)) {
- free(get_fragment_ptr_from_data_novalidate(data[i]));
- }
- }
- for (i = 0; i < m; i++) {
- if (realloc_bm & (1 << (i + k))) {
- free(get_fragment_ptr_from_data_novalidate(parity[i]));
- }
- }
-
- check_and_free_buffer(missing_idxs);
- check_and_free_buffer(data);
- check_and_free_buffer(parity);
+ check_and_free_buffer(c_fragments);
+ check_and_free_buffer(c_orig_payload);
- return list_of_strips;
+ return orig_payload;
}
@@ -1990,14 +633,14 @@ pyeclib_c_get_metadata(PyObject *self, PyObject *args)
{
PyObject *pyeclib_obj_handle = NULL;
pyeclib_t* pyeclib_handle = NULL;
- char *data = NULL; /* param, fragment from caller */
- int data_len; /* param, data len (B) */
- int metadata_len; /* metadata header size (B) */
- fragment_metadata_t *fragment_metadata = NULL; /* buffer to hold metadata */
- PyObject *ret_fragment_metadata = NULL; /* metadata object to return */
+ char *fragment = NULL; /* param, fragment from caller */
+ int fragment_metadata_len; /* metadata header size (B) */
+ char *c_fragment_metadata = NULL; /* buffer to hold metadata */
+ PyObject *fragment_metadata = NULL; /* metadata object to return */
+ int ret;
/* Obtain and validate the method parameters */
- if (!PyArg_ParseTuple(args, GET_METADATA_ARGS, &pyeclib_obj_handle, &data, &data_len)) {
+ if (!PyArg_ParseTuple(args, GET_METADATA_ARGS, &pyeclib_obj_handle, &fragment)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.get_metadata");
return NULL;
}
@@ -2007,19 +650,17 @@ pyeclib_c_get_metadata(PyObject *self, PyObject *args)
return NULL;
}
- /* Obtain the metadata from the data and build a object to return */
- metadata_len = sizeof(fragment_metadata_t);
- fragment_metadata = (fragment_metadata_t *) alloc_zeroed_buffer(metadata_len);
- if (NULL == fragment_metadata) {
- ret_fragment_metadata = NULL;
+ ret = liberasurecode_get_fragment_metadata(fragment, &c_fragment_metadata, &fragment_metadata_len);
+
+ if (ret < 0) {
+ fragment_metadata = NULL;
+ PyErr_SetString(PyECLibError, "Failed to get metadata in pyeclib.get_metadata");
} else {
- get_fragment_metadata(pyeclib_handle, data, fragment_metadata);
- ret_fragment_metadata = PY_BUILDVALUE_OBJ_LEN((char*)fragment_metadata,
- metadata_len);
- check_and_free_buffer(fragment_metadata);
+ fragment_metadata = PY_BUILDVALUE_OBJ_LEN((char*)c_fragment_metadata,
+ fragment_metadata_len);
}
- return ret_fragment_metadata;
+ return fragment_metadata;
}
@@ -2039,17 +680,17 @@ pyeclib_c_check_metadata(PyObject *self, PyObject *args)
PyObject *pyeclib_obj_handle = NULL;
pyeclib_t *pyeclib_handle = NULL;
PyObject *fragment_metadata_list = NULL; /* param, fragment metadata */
- fragment_metadata_t **c_fragment_metadata_list = NULL; /* c version of metadata */
+ char **c_fragment_metadata_list = NULL; /* c version of metadata */
+ int fragment_metadata_size; /* size of the metadata payload */
int num_fragments; /* k + m from EC algorithm */
- char **c_fragment_signatures = NULL; /* array of alg. signatures */
- int k, m, w; /* EC algorithm params */
+ int k, m; /* EC algorithm params */
int size; /* size for buf allocation */
int ret = -1; /* c return value */
PyObject *ret_obj = NULL; /* python long to return */
int i = 0; /* a counter */
/* Obtain and validate the method parameters */
- if (!PyArg_ParseTuple(args, "OO", &pyeclib_obj_handle, &fragment_metadata_list)) {
+ if (!PyArg_ParseTuple(args, "OOi", &pyeclib_obj_handle, &fragment_metadata_list, &fragment_metadata_size)) {
PyErr_SetString(PyECLibError, "Invalid arguments passed to pyeclib.encode");
return NULL;
}
@@ -2058,9 +699,8 @@ pyeclib_c_check_metadata(PyObject *self, PyObject *args)
PyErr_SetString(PyECLibError, "Invalid handle passed to pyeclib.encode");
return NULL;
}
- k = pyeclib_handle->k;
- m = pyeclib_handle->m;
- w = pyeclib_handle->w;
+ k = pyeclib_handle->ec_args.k;
+ m = pyeclib_handle->ec_args.m;
num_fragments = k + m;
if (num_fragments != PyList_Size(fragment_metadata_list)) {
PyErr_SetString(PyECLibError, "Not enough fragment metadata to perform integrity check");
@@ -2068,95 +708,29 @@ pyeclib_c_check_metadata(PyObject *self, PyObject *args)
}
/* Allocate space for fragment signatures */
- size = sizeof(fragment_metadata_t * ) * num_fragments;
- c_fragment_metadata_list = (fragment_metadata_t**) alloc_zeroed_buffer(size);
- size = sizeof(char *) * num_fragments;
- c_fragment_signatures = (char **) alloc_zeroed_buffer(size);
- if (NULL == c_fragment_metadata_list || NULL == c_fragment_signatures) {
+ size = sizeof(char * ) * num_fragments;
+ c_fragment_metadata_list = (char **) alloc_zeroed_buffer(size);
+ if (NULL == c_fragment_metadata_list) {
goto error;
}
- /* Populate and order the metadata */
+ /* Populate the metadata into a C array */
for (i = 0; i < num_fragments; i++) {
PyObject *tmp_data = PyList_GetItem(fragment_metadata_list, i);
Py_ssize_t len = 0;
- char *c_buf = NULL;
- fragment_metadata_t *fragment_metadata;
- PyBytes_AsStringAndSize(tmp_data, &(c_buf), &len);
-
- fragment_metadata = (fragment_metadata_t*)c_buf;
- c_fragment_metadata_list[fragment_metadata->idx] = fragment_metadata;
-
- if (supports_alg_sig(pyeclib_handle)) {
- c_fragment_signatures[fragment_metadata->idx] = (char*)alloc_aligned_buffer16(PYCC_MAX_SIG_LEN);
- memcpy(c_fragment_signatures[fragment_metadata->idx], fragment_metadata->signature, PYCC_MAX_SIG_LEN);
- } else {
- c_fragment_signatures[fragment_metadata->idx] = fragment_metadata->signature;
- }
+ PyBytes_AsStringAndSize(tmp_data, &(c_fragment_metadata_list[i]), &len);
}
-
- /* Ensure all fragments are here and check integrity using alg signatures */
- if (supports_alg_sig(pyeclib_handle)) {
- char **parity_sigs = (char **) alloc_zeroed_buffer(sizeof(char **) * m);
- if (NULL == parity_sigs) {
- goto error;
- }
- for (i = 0; i < m; i++) {
- parity_sigs[i] = (char *) alloc_aligned_buffer16(PYCC_MAX_SIG_LEN);
- if (NULL == parity_sigs[i]) {
- int j;
- for (j = 0; j < i; j++) free(parity_sigs[j]);
- goto error;
- } else {
- memset(parity_sigs[i], 0, PYCC_MAX_SIG_LEN);
- }
- }
-
- /* Calculate the parity of the signatures */
- if (pyeclib_handle->type == PYECC_RS_VAND) {
- jerasure_matrix_encode(k, m, w, pyeclib_handle->matrix,
- c_fragment_signatures, parity_sigs, PYCC_MAX_SIG_LEN);
- } else {
- pyeclib_handle->xor_code_desc->encode(pyeclib_handle->xor_code_desc,
- c_fragment_signatures,
- parity_sigs,
- PYCC_MAX_SIG_LEN);
- }
-
- /* Compare the parity of the signatures, and the signature of the parity */
- for (i = 0; i < m; i++) {
- if (memcmp(parity_sigs[i], c_fragment_signatures[k + i], PYCC_MAX_SIG_LEN) != 0) {
- ret = i;
- break;
- }
- }
-
- /* Clean up memory used in algebraic signature checking */
- for (i = 0; i < m; i++) {
- free(parity_sigs[i]);
- }
- check_and_free_buffer(parity_sigs);
- for (i = 0; i < k; i++) {
- free(c_fragment_signatures[i]);
- }
- } else if (use_inline_chksum(pyeclib_handle)) {
- for (i = 0; i < num_fragments; i++) {
- if (c_fragment_metadata_list[i]->chksum_mismatch == 1) {
- ret = i;
- break;
- }
- }
- }
-
- /* Return index of first checksum signature error */
+
+ ret = liberasurecode_verify_stripe_metadata(c_fragment_metadata_list, num_fragments, fragment_metadata_size);
+
ret_obj = PyLong_FromLong((long)ret);
+
goto exit;
error:
ret_obj = NULL;
exit:
- free(c_fragment_signatures);
free(c_fragment_metadata_list);
return ret_obj;
@@ -2168,8 +742,6 @@ static PyMethodDef PyECLibMethods[] = {
{"encode", pyeclib_c_encode, METH_VARARGS, "Create parity using source data"},
{"decode", pyeclib_c_decode, METH_VARARGS, "Recover all lost data/parity"},
{"reconstruct", pyeclib_c_reconstruct, METH_VARARGS, "Recover selective data/parity"},
- {"fragments_to_string", pyeclib_c_fragments_to_string, METH_VARARGS, "Try to transform a set of fragments into original string without calling the decoder"},
- {"get_fragment_partition", pyeclib_c_get_fragment_partition, METH_VARARGS, "Parition fragments into data and parity, also returns a list of missing indexes"},
{"get_required_fragments", pyeclib_c_get_required_fragments, METH_VARARGS, "Return the fragments required to reconstruct a set of missing fragments"},
{"get_segment_info", pyeclib_c_get_segment_info, METH_VARARGS, "Return segment and fragment size information needed when encoding a segmented stream"},
{"get_metadata", pyeclib_c_get_metadata, METH_VARARGS, "Get the integrity checking metadata for a fragment"},
diff --git a/src/c/pyeclib_c/pyeclib_c.h b/src/c/pyeclib_c/pyeclib_c.h
index f7d8439..4b7b3db 100644
--- a/src/c/pyeclib_c/pyeclib_c.h
+++ b/src/c/pyeclib_c/pyeclib_c.h
@@ -30,122 +30,14 @@
/* For exact-width integer types */
#include <stdint.h>
-/*
- * Make sure these enum values match those exposed from the Python EC interface
- * src/python/pyeclib/ec_iface.py
- */
-#define PYECLIB_MAX_DATA 32
-#define PYECLIB_MAX_PARITY 32
-
-typedef enum {
- PYECC_NOT_FOUND = 0,
- PYECC_RS_VAND = 1,
- PYECC_RS_CAUCHY_ORIG = 2,
- PYECC_XOR_HD_3 = 3,
- PYECC_XOR_HD_4 = 4,
- PYECC_NUM_TYPES = 4,
-} pyeclib_type_t;
-
-const char *pyeclib_type_str[] = {
- "not_found",
- "jerasure_rs_vand",
- "jerasure_rs_cauchy_orig",
- "flat_xor_3",
- "flat_xor_4",
-};
-
-const int pyeclib_type_word_size_bytes[] = {
- 0,
- sizeof(long), /* rs_vand */
- sizeof(long), /* rs_cauchy_orig */
- sizeof(long), /* flat_xor_3 */
- sizeof(long) /* flat_xor_4 */
-};
-
-const int pyeclib_type_best_w_size_bytes[] = {
- 32, /* default */
- 16, /* rs_vand */
- 4, /* rs_cauchy_orig */
- 32, /* flat_xor_3 */
- 32 /* flat_xor_4 */
-};
-
-// Unconditionally enforce alignment for now.
-// This is needed for the SIMD extentions.
-// TODO (kmg): Parse cpuinfo and determine if it is necessary...
-const int pyeclib_type_needs_addr_align[] = {
- -1,
- 1,
- 1,
- 1,
- 1
-};
-
typedef struct pyeclib_s
{
- int k;
- int m;
- int w;
- int *matrix;
- int *bitmatrix;
- int **schedule;
- xor_code_t *xor_code_desc;
- alg_sig_t *alg_sig_desc;
- pyeclib_type_t type;
- int inline_chksum;
- int algsig_chksum;
+ int ec_desc;
+ struct ec_args ec_args;
} pyeclib_t;
-/*
- * Convert the string ECC type to the enum value
- * Start lookup at index 1
- */
-static inline
-pyeclib_type_t get_ecc_type(const char *str_type)
-{
- int i;
- for (i = 1; i <= PYECC_NUM_TYPES; i++) {
- if (strcmp(str_type, pyeclib_type_str[i]) == 0) {
- return i;
- }
- }
- return PYECC_NOT_FOUND;
-}
-/*
- * Convert the string ECC type to the w value
- * Start lookup at index 1
- */
-static inline
-unsigned int get_best_w_for_ecc_type(pyeclib_type_t type)
-{
- if (type > PYECC_NUM_TYPES)
- return -1;
-
- return pyeclib_type_best_w_size_bytes[type];
-}
-
-
-#define PYECC_FLAGS_MASK 0x1
-#define PYECC_FLAGS_READ_VERIFY 0x1
#define PYECC_HANDLE_NAME "pyeclib_handle"
-#define PYECC_CAUCHY_PACKETSIZE (sizeof(long) * 128)
-
-#define PYCC_MAX_SIG_LEN 8
-
-typedef struct fragment_metadata_s
-{
- int size;
- int idx;
- char signature[PYCC_MAX_SIG_LEN];
- int chksum_mismatch;
-} fragment_metadata_t;
-
-#define FRAGSIZE_2_BLOCKSIZE(fragment_size) (fragment_size - sizeof(fragment_header_t))
-
-#define PYECLIB_WORD_SIZE(type) pyeclib_type_word_size_bytes[type]
-
-#define PYECLIB_NEEDS_ADDR_ALIGNMENT(type) pyeclib_type_needs_addr_align[type]
#endif
diff --git a/src/python/pyeclib/core.py b/src/python/pyeclib/core.py
index dd1c4db..4385d0a 100644
--- a/src/python/pyeclib/core.py
+++ b/src/python/pyeclib/core.py
@@ -47,6 +47,7 @@ class ECPyECLibDriver(object):
self.m = m
self.ec_type = ec_type
self.chksum_type = chksum_type
+ hd = m
self.inline_chksum = 0
self.algsig_chksum = 0
@@ -56,39 +57,53 @@ class ECPyECLibDriver(object):
elif self.chksum_type is PyECLib_FRAGHDRCHKSUM_Types.algsig:
self.algsig_chksum = 1
+ name = self.ec_type.name
+
+ if name == "flat_xor_hd_3":
+ hd = 3
+ name = "flat_xor_hd"
+ elif name == "flat_xor_hd_4":
+ hd = 4
+ name = "flat_xor_hd"
+
self.handle = pyeclib_c.init(
self.k,
self.m,
- self.ec_type.name,
+ name,
+ hd,
self.inline_chksum,
self.algsig_chksum)
def encode(self, data_bytes):
return pyeclib_c.encode(self.handle, data_bytes)
+ def _validate_and_return_fragment_size(self, fragments):
+ if len(fragments) > 0 and len(fragments[0]) == 0:
+ return -1
+ fragment_len = len(fragments[0])
+
+ for fragment in fragments[1:]:
+ if len(fragment) != fragment_len:
+ return -1
+
+ return fragment_len
+
def decode(self, fragment_payloads):
- try:
- ret_string = pyeclib_c.fragments_to_string(
- self.handle,
- fragment_payloads)
- except Exception as e:
- raise ECPyECLibException("Error in ECPyECLibDriver.decode")
-
- if ret_string is None:
- (data_frags,
- parity_frags,
- missing_idxs) = pyeclib_c.get_fragment_partition(
- self.handle, fragment_payloads)
- decoded_fragments = pyeclib_c.decode(
- self.handle, data_frags, parity_frags, missing_idxs,
- len(data_frags[0]))
- ret_string = pyeclib_c.fragments_to_string(
- self.handle,
- decoded_fragments)
+ fragment_len = self._validate_and_return_fragment_size(fragment_payloads)
+ if fragment_len < 0:
+ raise ECPyECLibException("Invalid fragment payload in ECPyECLibDriver.decode")
- return ret_string
+ if len(fragment_payloads) < self.k:
+ raise ECPyECLibException("Not enough fragments given in ECPyECLibDriver.decode")
+
+ return pyeclib_c.decode(self.handle, fragment_payloads, fragment_len)
+
def reconstruct(self, fragment_payloads, indexes_to_reconstruct):
+ fragment_len = self._validate_and_return_fragment_size(fragment_payloads)
+ if fragment_len < 0:
+ raise ECPyECLibException("Invalid fragment payload in ECPyECLibDriver.reconstruct")
+
reconstructed_data = []
# Reconstruct the data, then the parity
@@ -99,13 +114,8 @@ class ECPyECLibDriver(object):
while len(_indexes_to_reconstruct) > 0:
index = _indexes_to_reconstruct.pop(0)
- (data_frags,
- parity_frags,
- missing_idxs) = pyeclib_c.get_fragment_partition(
- self.handle, fragment_payloads)
reconstructed = pyeclib_c.reconstruct(
- self.handle, data_frags, parity_frags, missing_idxs,
- index, len(data_frags[0]))
+ self.handle, fragment_payloads, index, fragment_len)
reconstructed_data.append(reconstructed)
return reconstructed_data
@@ -118,7 +128,11 @@ class ECPyECLibDriver(object):
return pyeclib_c.get_metadata(self.handle, fragment)
def verify_stripe_metadata(self, fragment_metadata_list):
- return pyeclib_c.check_metadata(self.handle, fragment_metadata_list)
+ metadata_len = self._validate_and_return_fragment_size(fragment_payloads)
+ if metadata_len < 0:
+ raise ECPyECLibException("Invalid fragment payload in ECPyECLibDriver.verify_metadata")
+
+ return pyeclib_c.check_metadata(self.handle, fragment_metadata_list, metadata_len)
def get_segment_info(self, data_len, segment_size):
return pyeclib_c.get_segment_info(self.handle, data_len, segment_size)
diff --git a/src/python/pyeclib/ec_iface.py b/src/python/pyeclib/ec_iface.py
index a80d895..5dde2b4 100644
--- a/src/python/pyeclib/ec_iface.py
+++ b/src/python/pyeclib/ec_iface.py
@@ -82,10 +82,10 @@ class PyECLibEnum(Enum):
class PyECLib_EC_Types(PyECLibEnum):
# Note: the Enum start value defaults to 1 as the starting value and not 0
# 0 is False in the boolean sense but enum members evaluate to True
- rs_vand = 1
- rs_cauchy_orig = 2
- flat_xor_3 = 3
- flat_xor_4 = 4
+ jerasure_rs_vand = 1
+ jerasure_rs_cauchy = 2
+ flat_xor_hd_3 = 3
+ flat_xor_hd_4 = 4
# Output of Erasure (en)Coding process are data "fragments". Fragment data
diff --git a/test/ec_pyeclib_file_test.sh b/test/ec_pyeclib_file_test.sh
index 9914466..afd09b9 100755
--- a/test/ec_pyeclib_file_test.sh
+++ b/test/ec_pyeclib_file_test.sh
@@ -37,7 +37,7 @@ if [ ! -d ${FRAGMENT_DIR} ]; then
mkdir ${FRAGMENT_DIR}
fi
-TYPES="flat_xor_4 flat_xor_3 rs_vand rs_cauchy_orig"
+TYPES="flat_xor_hd_4 flat_xor_hd_3 jerasure_rs_vand jerasure_rs_cauchy"
NUM_DATAS="10 11 12"
RS_NUM_PARITIES="2 3 4"
XOR_NUM_PARITIES="6"
@@ -51,16 +51,16 @@ for TYPE in ${TYPES}; do
rm ${DECODED_DIR}/*
rm ${FRAGMENT_DIR}/*
NUM_PARITIES=${RS_NUM_PARITIES}
- if [[ `echo flat_xor_4 flat_xor_3 | grep ${TYPE}` ]]; then
+ if [[ `echo flat_xor_hd_4 flat_xor_hd_3 | grep ${TYPE}` ]]; then
NUM_PARITIES=${XOR_NUM_PARITIES}
fi
for NUM_PARITY in ${NUM_PARITIES}; do
let NUM_TOTAL=$(( NUM_DATA + NUM_PARITY))
FAULT_TOL=${NUM_PARITY}
- if [[ ${TYPE} == "flat_xor_4" ]]; then
+ if [[ ${TYPE} == "flat_xor_hd_4" ]]; then
FAULT_TOL="3"
fi
- if [[ ${TYPE} == "flat_xor_3" ]]; then
+ if [[ ${TYPE} == "flat_xor_hd_3" ]]; then
FAULT_TOL="2"
fi
for file in `cd ${FILES}; echo *; cd ..`; do
diff --git a/test/test_pyeclib_api.py b/test/test_pyeclib_api.py
index 82356a4..1e618a2 100644
--- a/test/test_pyeclib_api.py
+++ b/test/test_pyeclib_api.py
@@ -103,11 +103,11 @@ class TestPyECLibDriver(unittest.TestCase):
def test_small_encode(self):
pyeclib_drivers = []
pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="rs_vand"))
+ k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
- k=11, m=2, ec_type="rs_vand"))
+ k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=2, ec_type="rs_vand"))
+ k=10, m=2, ec_type="jerasure_rs_vand"))
encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"]
@@ -122,19 +122,19 @@ class TestPyECLibDriver(unittest.TestCase):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="rs_vand", chksum_type="algsig"))
+ k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=3, ec_type="rs_vand", chksum_type="algsig"))
+ k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=6, ec_type="flat_xor_4", chksum_type="algsig"))
+ k=12, m=6, ec_type="flat_xor_hd_4", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=5, ec_type="flat_xor_4", chksum_type="algsig"))
+ k=10, m=5, ec_type="flat_xor_hd_4", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=5, ec_type="flat_xor_3", chksum_type="algsig"))
+ k=10, m=5, ec_type="flat_xor_hd_3", chksum_type="algsig"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@@ -163,19 +163,19 @@ class TestPyECLibDriver(unittest.TestCase):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="rs_vand", chksum_type="algsig"))
+ k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=3, ec_type="rs_vand", chksum_type="algsig"))
+ k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=6, ec_type="flat_xor_4", chksum_type="algsig"))
+ k=12, m=6, ec_type="flat_xor_hd_4", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=5, ec_type="flat_xor_4", chksum_type="algsig"))
+ k=10, m=5, ec_type="flat_xor_hd_4", chksum_type="algsig"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=5, ec_type="flat_xor_3", chksum_type="algsig"))
+ k=10, m=5, ec_type="flat_xor_hd_3", chksum_type="algsig"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@@ -197,16 +197,16 @@ class TestPyECLibDriver(unittest.TestCase):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="rs_vand", chksum_type="inline_crc32"))
+ k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=3, ec_type="rs_vand", chksum_type="inline_crc32"))
+ k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=4, ec_type="rs_vand", chksum_type="inline_crc32"))
+ k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="rs_cauchy_orig", chksum_type="inline_crc32"))
+ k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@@ -240,16 +240,16 @@ class TestPyECLibDriver(unittest.TestCase):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="rs_vand", chksum_type="inline_crc32"))
+ k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=3, ec_type="rs_vand", chksum_type="inline_crc32"))
+ k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=4, ec_type="rs_vand", chksum_type="inline_crc32"))
+ k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="rs_cauchy_orig", chksum_type="inline_crc32"))
+ k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
@@ -271,13 +271,13 @@ class TestPyECLibDriver(unittest.TestCase):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="rs_vand"))
+ k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=11, m=2, ec_type="rs_vand"))
+ k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=2, ec_type="rs_vand"))
+ k=10, m=2, ec_type="jerasure_rs_vand"))
file_sizes = [
1024 * 1024,
@@ -346,28 +346,28 @@ class TestPyECLibDriver(unittest.TestCase):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="rs_vand"))
+ k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=2, ec_type="rs_cauchy_orig"))
+ k=12, m=2, ec_type="jerasure_rs_cauchy"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=3, ec_type="rs_vand"))
+ k=12, m=3, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=3, ec_type="rs_cauchy_orig"))
+ k=12, m=3, ec_type="jerasure_rs_cauchy"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=12, m=6, ec_type="flat_xor_4"))
+ k=12, m=6, ec_type="flat_xor_hd_4"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=5, ec_type="flat_xor_4"))
+ k=10, m=5, ec_type="flat_xor_hd_4"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=10, m=5, ec_type="flat_xor_3"))
+ k=10, m=5, ec_type="flat_xor_hd_3"))
pyeclib_drivers.append(
ECDriver("pyeclib.core.ECPyECLibDriver",
- k=9, m=5, ec_type="flat_xor_3"))
+ k=9, m=5, ec_type="flat_xor_hd_3"))
for pyeclib_driver in pyeclib_drivers:
for file_size in self.file_sizes:
diff --git a/test/test_pyeclib_c.py b/test/test_pyeclib_c.py
index 81f7177..707be89 100644
--- a/test/test_pyeclib_c.py
+++ b/test/test_pyeclib_c.py
@@ -58,10 +58,10 @@ class TestPyECLib(unittest.TestCase):
self.iterations = 100
# EC algorithm and config parameters
- self.rs_types = [("rs_vand"), ("rs_cauchy_orig")]
- self.xor_types = [("flat_xor_4", 12, 6, 4),
- ("flat_xor_4", 10, 5, 4),
- ("flat_xor_3", 10, 5, 3)]
+ self.rs_types = [("jerasure_rs_vand"), ("jerasure_rs_cauchy")]
+ self.xor_types = [("flat_xor_hd_4", 12, 6, 4),
+ ("flat_xor_hd_4", 10, 5, 4),
+ ("flat_xor_hd_3", 10, 5, 3)]
# Input temp files for testing
self.sizes = ["101-K", "202-K", "303-K"]
@@ -332,7 +332,7 @@ class TestPyECLib(unittest.TestCase):
#
# MDS codes need any k fragments
#
- if ec_type in ["rs_vand", "rs_cauchy_orig"]:
+ if ec_type in ["jerasure_rs_vand", "jerasure_rs_cauchy"]:
expected_fragments = [i for i in range(num_data + num_parity)]
missing_fragments = []
diff --git a/tools/pyeclib_conf_tool.py b/tools/pyeclib_conf_tool.py
index 2d8890b..84cd957 100644
--- a/tools/pyeclib_conf_tool.py
+++ b/tools/pyeclib_conf_tool.py
@@ -44,8 +44,8 @@
# ======== swift.conf ============
# [storage-policy:10]
# type = erasure_coding
-# name = ec_rs_cauchy_orig_12_2
-# ec_type = rs_cauchy_orig
+# name = ec_jerasure_rs_cauchy_12_2
+# ec_type = jerasure_rs_cauchy
# ec_k = 12
# ec_m = 2
# ============================
@@ -109,11 +109,11 @@ class ECScheme:
def __str__(self):
return "k=%d m=%d w=%d ec_type=%s" % (self.k, self.m, self.w, self.ec_type)
-valid_flat_xor_3 = [(6, 6), (7, 6), (8, 6), (9, 6),
+valid_flat_xor_hd_3 = [(6, 6), (7, 6), (8, 6), (9, 6),
(10, 6), (11, 6), (12, 6), (13, 6),
(14, 6), (15, 6)]
-valid_flat_xor_4 = [(6, 6), (7, 6), (8, 6), (9, 6),
+valid_flat_xor_hd_4 = [(6, 6), (7, 6), (8, 6), (9, 6),
(10, 6), (11, 6), (12, 6), (13, 6),
(14, 6), (15, 6), (16, 6), (17, 6),
(18, 6), (19, 6), (20, 6)]
@@ -150,11 +150,11 @@ def get_viable_schemes(
#
for w in [8, 16, 32]:
list_of_schemes.append(
- ECScheme(k, max_num_frags - k, w, "rs_vand_%d" % w))
+ ECScheme(k, max_num_frags - k, w, "jerasure_rs_vand_%d" % w))
for w in [4, 8]:
list_of_schemes.append(
- ECScheme(k, max_num_frags - k, w, "rs_cauchy_orig_%d" % w))
+ ECScheme(k, max_num_frags - k, w, "jerasure_rs_cauchy_%d" % w))
#
# The XOR codes are a little tricker
@@ -163,24 +163,24 @@ def get_viable_schemes(
# Constraint for 2: k <= (m choose 2)
# Constraint for 3: k <= (m choose 3)
#
- # The '3' flat_xor_3 (and '4' in flat_xor_4) refers to the Hamming
+ # The '3' flat_xor_hd_3 (and '4' in flat_xor_hd_4) refers to the Hamming
# distance, which means the code guarantees the reconstruction of any
- # 2 lost fragments (or 3 in the case of flat_xor_4).
+ # 2 lost fragments (or 3 in the case of flat_xor_hd_4).
#
# So, only consider the XOR code if the fault_tolerance matches and
# the additional constraint is met
#
if fault_tolerance == 2:
max_k = nCr(max_num_frags - k, 2)
- if k <= max_k and (k, max_num_frags - k) in valid_flat_xor_3:
+ if k <= max_k and (k, max_num_frags - k) in valid_flat_xor_hd_3:
list_of_schemes.append(
- ECScheme(k, max_num_frags - k, 0, "flat_xor_3"))
+ ECScheme(k, max_num_frags - k, 0, "flat_xor_hd_3"))
if fault_tolerance == 3:
max_k = nCr(max_num_frags - k, 3)
- if k <= max_k and (k, max_num_frags - k) in valid_flat_xor_4:
+ if k <= max_k and (k, max_num_frags - k) in valid_flat_xor_hd_4:
list_of_schemes.append(
- ECScheme(k, max_num_frags - k, 0, "flat_xor_4"))
+ ECScheme(k, max_num_frags - k, 0, "flat_xor_hd_4"))
return list_of_schemes