# Copyright (c) 2013, 2014, Kevin Greenan (kmgreen2@gmail.com) # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. # # Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. THIS SOFTWARE IS # PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN # NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from .ec_iface import ECDriverError from .ec_iface import ECInsufficientFragments from .ec_iface import PyECLib_FRAGHDRCHKSUM_Types import math import pyeclib_c import sys pyver = float('%s.%s' % sys.version_info[:2]) class ECPyECLibDriver(object): def __init__(self, k, m, hd, ec_type, chksum_type=PyECLib_FRAGHDRCHKSUM_Types.none, validate=False): self.k = k self.m = m self.hd = hd self.ec_type = ec_type self.chksum_type = chksum_type self.inline_chksum = 0 self.algsig_chksum = 0 # crc32 is the only inline checksum type currently supported if self.chksum_type is PyECLib_FRAGHDRCHKSUM_Types.inline_crc32: self.inline_chksum = 1 self.handle = pyeclib_c.init( self.k, self.m, ec_type.value, self.hd, self.inline_chksum, self.algsig_chksum, validate) def __repr__(self): return '%s(k=%r, m=%r, hd=%r, ec_type=%r, chksum_type=%r)' % ( self.__class__.__name__, self.k, self.m, self.hd, self.ec_type, self.chksum_type) def encode(self, data_bytes): return pyeclib_c.encode(self.handle, data_bytes) def _validate_and_return_fragment_size(self, fragments): if len(fragments) == 0 or len(fragments[0]) == 0: return -1 fragment_len = len(fragments[0]) for fragment in fragments[1:]: if len(fragment) != fragment_len: return -1 return fragment_len def decode(self, fragment_payloads, ranges=None, force_metadata_checks=False): _fragment_payloads = list(fragment_payloads) fragment_len = self._validate_and_return_fragment_size( _fragment_payloads) if fragment_len < 0: raise ECDriverError( "Invalid fragment payload in ECPyECLibDriver.decode") if len(_fragment_payloads) < self.k: raise ECInsufficientFragments( "Not enough fragments given in ECPyECLibDriver.decode") return pyeclib_c.decode(self.handle, _fragment_payloads, fragment_len, ranges, force_metadata_checks) def reconstruct(self, fragment_payloads, indexes_to_reconstruct): _fragment_payloads = list(fragment_payloads) fragment_len = self._validate_and_return_fragment_size( _fragment_payloads) if fragment_len < 0: raise ECDriverError( "Invalid fragment payload in ECPyECLibDriver.reconstruct") reconstructed_data = [] # Reconstruct the data, then the parity # The parity cannot be reconstructed until # after all data is reconstructed indexes_to_reconstruct.sort() _indexes_to_reconstruct = indexes_to_reconstruct[:] while len(_indexes_to_reconstruct) > 0: index = _indexes_to_reconstruct.pop(0) reconstructed = pyeclib_c.reconstruct( self.handle, _fragment_payloads, fragment_len, index) reconstructed_data.append(reconstructed) _fragment_payloads.append(reconstructed) return reconstructed_data def fragments_needed(self, reconstruct_indexes, exclude_indexes): required_fragments = pyeclib_c.get_required_fragments( self.handle, reconstruct_indexes, exclude_indexes) return required_fragments def min_parity_fragments_needed(self): """ FIXME - fix this to return a function of HD """ return 1 def get_metadata(self, fragment, formatted=0): fragment_metadata = pyeclib_c.get_metadata(self.handle, fragment, formatted) return fragment_metadata def verify_stripe_metadata(self, fragment_metadata_list): success = pyeclib_c.check_metadata(self.handle, fragment_metadata_list) return success def get_segment_info(self, data_len, segment_size): segment_info = pyeclib_c.get_segment_info(self.handle, data_len, segment_size) return segment_info class ECNullDriver(object): def __init__(self, k, m, hd, ec_type=None, chksum_type=None, validate=False): self.k = k self.m = m self.hd = hd def encode(self, data_bytes): pass def decode(self, fragment_payloads, ranges, force_metadata_checks): pass def reconstruct(self, available_fragment_payloads, missing_fragment_indexes): pass def fragments_needed(self, missing_fragment_indexes): pass def get_metadata(self, fragment, formatted=0): pass def min_parity_fragments_needed(self): pass def verify_stripe_metadata(self, fragment_metadata_list): pass def get_segment_info(self, data_len, segment_size): pass # # A striping-only driver for EC. This is # pretty much RAID 0. # class ECStripingDriver(object): def __init__(self, k, m, hd, ec_type=None, chksum_type=None, validate=False): """Stripe an arbitrary-sized string into k fragments :param k: the number of data fragments to stripe :param m: the number of parity fragments to stripe :raises: ECDriverError if there is an error during encoding """ self.k = k if m != 0: raise ECDriverError("This driver only supports m=0") self.m = m self.hd = hd def encode(self, data_bytes): """Stripe an arbitrary-sized string into k fragments :param data_bytes: the buffer to encode :returns: a list of k buffers (data only) :raises: ECDriverError if there is an error during encoding """ # Main fragment size fragment_size = math.ceil(len(data_bytes) / float(self.k)) # Size of last fragment last_fragment_size = len(data_bytes) - (fragment_size * self.k - 1) fragments = [] offset = 0 for i in range(self.k - 1): fragments.append(data_bytes[offset:fragment_size]) offset += fragment_size fragments.append(data_bytes[offset:last_fragment_size]) return fragments def decode(self, fragment_payloads, ranges=None, force_metadata_checks=False): """Convert a k-fragment data stripe into a string :param fragment_payloads: fragments (in order) to convert into a string :param ranges (unsupported): range decode :param force_metadata_checks (unsupported): verify fragment metadata :returns: a string containing original data :raises: ECDriverError if there is an error during decoding """ if ranges is not None: raise ECDriverError("Decode does not support range requests in the" " striping driver.") if force_metadata_checks is not False: raise ECDriverError( "Decode does not support metadata integrity checks in the " " striping driver.") if len(fragment_payloads) != self.k: raise ECInsufficientFragments( "Decode requires %d fragments, %d fragments were given" % (len(fragment_payloads), self.k)) ret_string = '' for fragment in fragment_payloads: ret_string += fragment return ret_string def reconstruct(self, available_fragment_payloads, missing_fragment_indexes): """We cannot reconstruct a fragment using other fragments. This means that reconstruction means all fragments must be specified, otherwise we cannot reconstruct and must raise an error. :param available_fragment_payloads: available fragments (in order) :param missing_fragment_indexes: indexes of missing fragments :returns: a string containing the original data :raises: ECDriverError if there is an error during reconstruction """ if len(available_fragment_payloads) != self.k: raise ECDriverError( "Reconstruction requires %d fragments, %d fragments given" % (len(available_fragment_payloads), self.k)) return available_fragment_payloads def fragments_needed(self, missing_fragment_indexes): """By definition, all missing fragment indexes are needed to reconstruct, so just return the list handed to this function. :param missing_fragment_indexes: indexes of missing fragments :returns: missing_fragment_indexes """ return missing_fragment_indexes def min_parity_fragments_needed(self): pass def get_metadata(self, fragment, formatted=0): """This driver does not include fragment metadata, so return empty string :param fragment: a fragment :returns: empty string """ return '' def verify_stripe_metadata(self, fragment_metadata_list): """This driver does not include fragment metadata, so return true :param fragment_metadata_list: a list of fragments :returns: True """ return True def get_segment_info(self, data_len, segment_size): pass