diff options
author | Sebastian Thiel <byronimo@gmail.com> | 2015-01-04 11:19:57 +0100 |
---|---|---|
committer | Sebastian Thiel <byronimo@gmail.com> | 2015-01-04 11:19:57 +0100 |
commit | 84929ed811142e366d6c5916125302c1419acad6 (patch) | |
tree | 3bf33015838b0fc3c10df5c8eb0e959ad8f2aebe /smmap | |
parent | eb40b44ce4a6e646aabf7b7091d876738336c42f (diff) | |
download | smmap-84929ed811142e366d6c5916125302c1419acad6.tar.gz |
Applied autopep8
autopep8 -v -j 8 --max-line-length 120 --in-place --recursive
Diffstat (limited to 'smmap')
-rw-r--r-- | smmap/buf.py | 23 | ||||
-rw-r--r-- | smmap/exc.py | 2 | ||||
-rw-r--r-- | smmap/mman.py | 109 | ||||
-rw-r--r-- | smmap/test/lib.py | 8 | ||||
-rw-r--r-- | smmap/test/test_buf.py | 20 | ||||
-rw-r--r-- | smmap/test/test_mman.py | 59 | ||||
-rw-r--r-- | smmap/test/test_tutorial.py | 8 | ||||
-rw-r--r-- | smmap/test/test_util.py | 20 | ||||
-rw-r--r-- | smmap/util.py | 61 |
9 files changed, 161 insertions, 149 deletions
diff --git a/smmap/buf.py b/smmap/buf.py index 66029cb..17d2d36 100644 --- a/smmap/buf.py +++ b/smmap/buf.py @@ -10,6 +10,7 @@ except NameError: class SlidingWindowMapBuffer(object): + """A buffer like object which allows direct byte-wise object and slicing into memory of a mapped file. The mapping is controlled by the provided cursor. @@ -20,9 +21,9 @@ class SlidingWindowMapBuffer(object): underneath, it can unfortunately not be used in any non-pure python method which needs a buffer or string""" __slots__ = ( - '_c', # our cursor - '_size', # our supposed size - ) + '_c', # our cursor + '_size', # our supposed size + ) def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0): """Initalize the instance to operate on the given cursor. @@ -57,7 +58,7 @@ class SlidingWindowMapBuffer(object): if not c.includes_ofs(i): c.use_region(i, 1) # END handle region usage - return c.buffer()[i-c.ofs_begin()] + return c.buffer()[i - c.ofs_begin()] def __getslice__(self, i, j): c = self._c @@ -72,9 +73,9 @@ class SlidingWindowMapBuffer(object): j = self._size + j if (c.ofs_begin() <= i) and (j < c.ofs_end()): b = c.ofs_begin() - return c.buffer()[i-b:j-b] + return c.buffer()[i - b:j - b] else: - l = j-i # total length + l = j - i # total length ofs = i # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower # in the previous iteration of this code @@ -86,7 +87,7 @@ class SlidingWindowMapBuffer(object): ofs += len(d) l -= len(d) md.append(d) - #END while there are bytes to read + # END while there are bytes to read return bytes().join(md) # END fast or slow path #{ Interface @@ -100,7 +101,7 @@ class SlidingWindowMapBuffer(object): :return: True if the buffer can be used""" if cursor: self._c = cursor - #END update our cursor + # END update our cursor # reuse existing cursors if possible if self._c is not None and self._c.is_associated(): @@ -112,9 +113,9 @@ class SlidingWindowMapBuffer(object): # If not, the user is in trouble. if size > self._c.file_size(): size = self._c.file_size() - offset - #END handle size + # END handle size self._size = size - #END set size + # END set size return res # END use our cursor return False @@ -128,7 +129,7 @@ class SlidingWindowMapBuffer(object): self._size = 0 if self._c is not None: self._c.unuse_region() - #END unuse region + # END unuse region def cursor(self): """:return: the currently set cursor which provides access to the data""" diff --git a/smmap/exc.py b/smmap/exc.py index 5e90cf7..1176645 100644 --- a/smmap/exc.py +++ b/smmap/exc.py @@ -2,8 +2,10 @@ class MemoryManagerError(Exception): + """Base class for all exceptions thrown by the memory manager""" class RegionCollectionError(MemoryManagerError): + """Thrown if a memory region could not be collected, or if no region for collection was found""" diff --git a/smmap/mman.py b/smmap/mman.py index 6663687..c7a4595 100644 --- a/smmap/mman.py +++ b/smmap/mman.py @@ -1,12 +1,12 @@ """Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" from .util import ( - MapWindow, - MapRegion, - MapRegionList, - is_64_bit, - string_types, - buffer, - ) + MapWindow, + MapRegion, + MapRegionList, + is_64_bit, + string_types, + buffer, +) from weakref import ref import sys @@ -19,6 +19,7 @@ __all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"] class WindowCursor(object): + """ Pointer into the mapped region of the memory manager, keeping the map alive until it is destroyed and no other client uses it. @@ -29,12 +30,12 @@ class WindowCursor(object): that it must be suited for the somewhat quite different sliding manager. It could be improved, but I see no real need to do so.""" __slots__ = ( - '_manager', # the manger keeping all file regions - '_rlist', # a regions list with regions for our file + '_manager', # the manger keeping all file regions + '_rlist', # a regions list with regions for our file '_region', # our current region or None '_ofs', # relative offset from the actually mapped area to our start area '_size' # maximum size we should provide - ) + ) def __init__(self, manager=None, regions=None): self._manager = manager @@ -65,8 +66,8 @@ class WindowCursor(object): # this python problem (for now). # The next step is to get rid of the error prone getrefcount alltogether. pass - #END exception handling - #END handle regions + # END exception handling + # END handle regions def _copy_from(self, rhs): """Copy all data from rhs into this instance, handles usage count""" @@ -121,11 +122,11 @@ class WindowCursor(object): # offset too large ? if offset >= fsize: return self - #END handle offset + # END handle offset if need_region: self._region = man._obtain_region(self._rlist, offset, size, flags, False) - #END need region handling + # END need region handling self._region.increment_usage_count() self._ofs = offset - self._region._b @@ -221,13 +222,14 @@ class WindowCursor(object): :raise ValueError: if the mapping was not created by a file descriptor""" if isinstance(self._rlist.path_or_fd(), string_types()): raise ValueError("File descriptor queried although mapping was generated from path") - #END handle type + # END handle type return self._rlist.path_or_fd() #} END interface class StaticWindowMapManager(object): + """Provides a manager which will produce single size cursors that are allowed to always map the whole file. @@ -240,13 +242,13 @@ class StaticWindowMapManager(object): accommodate this fact""" __slots__ = [ - '_fdict', # mapping of path -> StorageHelper (of some kind - '_window_size', # maximum size of a window - '_max_memory_size', # maximum amount of memory we may allocate - '_max_handle_count', # maximum amount of handles to keep open - '_memory_size', # currently allocated memory size - '_handle_count', # amount of currently allocated file handles - ] + '_fdict', # mapping of path -> StorageHelper (of some kind + '_window_size', # maximum size of a window + '_max_memory_size', # maximum amount of memory we may allocate + '_max_handle_count', # maximum amount of handles to keep open + '_memory_size', # currently allocated memory size + '_handle_count', # amount of currently allocated file handles + ] #{ Configuration MapRegionListCls = MapRegionList @@ -279,7 +281,7 @@ class StaticWindowMapManager(object): coeff = 64 if is_64_bit(): coeff = 1024 - #END handle arch + # END handle arch self._window_size = coeff * self._MB_in_bytes # END handle max window size @@ -287,9 +289,9 @@ class StaticWindowMapManager(object): coeff = 1024 if is_64_bit(): coeff = 8192 - #END handle arch + # END handle arch self._max_memory_size = coeff * self._MB_in_bytes - #END handle max memory size + # END handle max memory size #{ Internal Methods @@ -310,23 +312,23 @@ class StaticWindowMapManager(object): for regions in self._fdict.values(): for region in regions: # check client count - consider that we keep one reference ourselves ! - if (region.client_count()-2 == 0 and - (lru_region is None or region._uc < lru_region._uc)): + if (region.client_count() - 2 == 0 and + (lru_region is None or region._uc < lru_region._uc)): lru_region = region lru_list = regions # END update lru_region - #END for each region - #END for each regions list + # END for each region + # END for each regions list if lru_region is None: break - #END handle region not found + # END handle region not found num_found += 1 del(lru_list[lru_list.index(lru_region)]) self._memory_size -= lru_region.size() self._handle_count -= 1 - #END while there is more memory to free + # END while there is more memory to free return num_found def _obtain_region(self, a, offset, size, flags, is_recursive): @@ -336,7 +338,7 @@ class StaticWindowMapManager(object): :return: The newly created region""" if self._memory_size + size > self._max_memory_size: self._collect_lru_region(size) - #END handle collection + # END handle collection r = None if a: @@ -354,10 +356,10 @@ class StaticWindowMapManager(object): # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it raise - #END handle existing recursion + # END handle existing recursion self._collect_lru_region(0) return self._obtain_region(a, offset, size, flags, True) - #END handle exceptions + # END handle exceptions self._handle_count += 1 self._memory_size += r.size() @@ -404,7 +406,7 @@ class StaticWindowMapManager(object): def num_open_files(self): """Amount of opened files in the system""" - return reduce(lambda x, y: x+y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) + return reduce(lambda x, y: x + y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0) def window_size(self): """:return: size of each window when allocating new regions""" @@ -441,7 +443,7 @@ class StaticWindowMapManager(object): **Note:** does nothing on non-windows platforms""" if sys.platform != 'win32': return - #END early bailout + # END early bailout num_closed = 0 for path, rlist in self._fdict.items(): @@ -449,13 +451,14 @@ class StaticWindowMapManager(object): for region in rlist: region._mf.close() num_closed += 1 - #END path matches - #END for each path + # END path matches + # END for each path return num_closed #} END special purpose interface class SlidingWindowMapManager(StaticWindowMapManager): + """Maintains a list of ranges of mapped memory regions in one or more files and allows to easily obtain additional regions assuring there is no overlap. Once a certain memory limit is reached globally, or if there cannot be more open file handles @@ -482,18 +485,18 @@ class SlidingWindowMapManager(StaticWindowMapManager): lo = 0 hi = len(a) while lo < hi: - mid = (lo+hi)//2 + mid = (lo + hi) // 2 ofs = a[mid]._b if ofs <= offset: if a[mid].includes_ofs(offset): r = a[mid] break - #END have region - lo = mid+1 + # END have region + lo = mid + 1 else: hi = mid - #END handle position - #END while bisecting + # END handle position + # END while bisecting if r is None: window_size = self._window_size @@ -506,7 +509,7 @@ class SlidingWindowMapManager(StaticWindowMapManager): # Save calls ! if self._memory_size + window_size > self._max_memory_size: self._collect_lru_region(window_size) - #END handle collection + # END handle collection # we assume the list remains sorted by offset insert_pos = 0 @@ -514,7 +517,7 @@ class SlidingWindowMapManager(StaticWindowMapManager): if len_regions == 1: if a[0]._b <= offset: insert_pos = 1 - #END maintain sort + # END maintain sort else: # find insert position insert_pos = len_regions @@ -522,8 +525,8 @@ class SlidingWindowMapManager(StaticWindowMapManager): if region._b > offset: insert_pos = i break - #END if insert position is correct - #END for each region + # END if insert position is correct + # END for each region # END obtain insert pos # adjust the actual offset and size values to create the largest @@ -531,13 +534,13 @@ class SlidingWindowMapManager(StaticWindowMapManager): if insert_pos == 0: if len_regions: right = self.MapWindowCls.from_region(a[insert_pos]) - #END adjust right side + # END adjust right side else: if insert_pos != len_regions: right = self.MapWindowCls.from_region(a[insert_pos]) # END adjust right window left = self.MapWindowCls.from_region(a[insert_pos - 1]) - #END adjust surrounding windows + # END adjust surrounding windows mid.extend_left_to(left, window_size) mid.extend_right_to(right, window_size) @@ -546,13 +549,13 @@ class SlidingWindowMapManager(StaticWindowMapManager): # it can happen that we align beyond the end of the file if mid.ofs_end() > right.ofs: mid.size = right.ofs - mid.ofs - #END readjust size + # END readjust size # insert new region at the right offset to keep the order try: if self._handle_count >= self._max_handle_count: raise Exception - #END assert own imposed max file handles + # END assert own imposed max file handles r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags) except Exception: # apparently we are out of system resources or hit a limit @@ -563,10 +566,10 @@ class SlidingWindowMapManager(StaticWindowMapManager): # we already tried this, and still have no success in obtaining # a mapping. This is an exception, so we propagate it raise - #END handle existing recursion + # END handle existing recursion self._collect_lru_region(0) return self._obtain_region(a, offset, size, flags, True) - #END handle exceptions + # END handle exceptions self._handle_count += 1 self._memory_size += r.size() diff --git a/smmap/test/lib.py b/smmap/test/lib.py index 67aec63..93cb09a 100644 --- a/smmap/test/lib.py +++ b/smmap/test/lib.py @@ -9,6 +9,7 @@ __all__ = ['TestBase', 'FileCreator'] #{ Utilities class FileCreator(object): + """A instance which creates a temporary file with a prefix and a given size and provides this info to the user. Once it gets deleted, it will remove the temporary file as well.""" @@ -21,7 +22,7 @@ class FileCreator(object): self._size = size fp = open(self._path, "wb") - fp.seek(size-1) + fp.seek(size - 1) fp.write(b'1') fp.close() @@ -32,7 +33,7 @@ class FileCreator(object): os.remove(self.path) except OSError: pass - #END exception handling + # END exception handling @property def path(self): @@ -46,6 +47,7 @@ class FileCreator(object): class TestBase(TestCase): + """Foundation used by all tests""" #{ Configuration @@ -58,7 +60,7 @@ class TestBase(TestCase): # nothing for now pass - #END overrides + # END overrides #{ Interface diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py index d07b7f4..0337715 100644 --- a/smmap/test/test_buf.py +++ b/smmap/test/test_buf.py @@ -3,9 +3,9 @@ from __future__ import print_function from .lib import TestBase, FileCreator from smmap.mman import ( - SlidingWindowMapManager, - StaticWindowMapManager - ) + SlidingWindowMapManager, + StaticWindowMapManager +) from smmap.buf import SlidingWindowMapBuffer from random import randint @@ -57,11 +57,11 @@ class TestBuf(TestBase): with open(fc.path, 'rb') as fp: data = fp.read() assert data[offset] == buf[0] - assert data[offset:offset*2] == buf[0:offset] + assert data[offset:offset * 2] == buf[0:offset] # negative indices, partial slices - assert buf[-1] == buf[len(buf)-1] - assert buf[-10:] == buf[len(buf)-10:len(buf)] + assert buf[-1] == buf[len(buf) - 1] + assert buf[-10:] == buf[len(buf) - 10:len(buf)] # end access makes its cursor invalid buf.end_access() @@ -97,7 +97,7 @@ class TestBuf(TestBase): buf.begin_access() while num_accesses_left: num_accesses_left -= 1 - if access_mode: # multi + if access_mode: # multi ofs_start = randint(0, fsize) ofs_end = randint(ofs_start, fsize) d = buf[ofs_start:ofs_end] @@ -108,7 +108,7 @@ class TestBuf(TestBase): pos = randint(0, fsize) assert buf[pos] == data[pos] num_bytes += 1 - #END handle mode + # END handle mode # END handle num accesses buf.end_access() @@ -116,10 +116,10 @@ class TestBuf(TestBase): assert manager.collect() assert manager.num_file_handles() == 0 elapsed = max(time() - st, 0.001) # prevent zero division errors on windows - mb = float(1000*1000) + mb = float(1000 * 1000) mode_str = (access_mode and "slice") or "single byte" print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" - % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed), + % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), file=sys.stderr) # END handle access mode # END for each manager diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py index d903af6..b718b06 100644 --- a/smmap/test/test_mman.py +++ b/smmap/test/test_mman.py @@ -3,10 +3,10 @@ from __future__ import print_function from .lib import TestBase, FileCreator from smmap.mman import ( - WindowCursor, - SlidingWindowMapManager, - StaticWindowMapManager - ) + WindowCursor, + SlidingWindowMapManager, + StaticWindowMapManager +) from smmap.util import align_to_mmap from random import randint @@ -29,7 +29,7 @@ class TestMMan(TestBase): cv = man.make_cursor(fc.path) assert not cv.is_valid() # no region mapped yet - assert cv.is_associated()# but it know where to map it from + assert cv.is_associated() # but it know where to map it from assert cv.file_size() == fc.size assert cv.path() == fc.path @@ -60,7 +60,7 @@ class TestMMan(TestBase): winsize_cmp_val = 0 if isinstance(man, StaticWindowMapManager): winsize_cmp_val = -1 - #END handle window size + # END handle window size assert man.window_size() > winsize_cmp_val assert man.mapped_memory_size() == 0 assert man.max_mapped_memory_size() > 0 @@ -89,8 +89,8 @@ class TestMMan(TestBase): self.assertRaises(ValueError, c.path) else: self.assertRaises(ValueError, c.fd) - #END handle value error - #END for each input + # END handle value error + # END for each input os.close(fd) # END for each manager type @@ -101,7 +101,7 @@ class TestMMan(TestBase): data = fp.read() fd = os.open(fc.path, os.O_RDONLY) max_num_handles = 15 - #small_size = + # small_size = for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)), (SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),): for item in (fc.path, fd): @@ -120,22 +120,23 @@ class TestMMan(TestBase): size = man.window_size() // 2 assert c.use_region(base_offset, size).is_valid() rr = c.region_ref() - assert rr().client_count() == 2 # the manager and the cursor and us + assert rr().client_count() == 2 # the manager and the cursor and us assert man.num_open_files() == 1 assert man.num_file_handles() == 1 assert man.mapped_memory_size() == rr().size() - #assert c.size() == size # the cursor may overallocate in its static version + # assert c.size() == size # the cursor may overallocate in its static version assert c.ofs_begin() == base_offset assert rr().ofs_begin() == 0 # it was aligned and expanded if man.window_size(): - assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned) + # but isn't larger than the max window (aligned) + assert rr().size() == align_to_mmap(man.window_size(), True) else: assert rr().size() == fc.size - #END ignore static managers which dont use windows and are aligned to file boundaries + # END ignore static managers which dont use windows and are aligned to file boundaries - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] # obtain second window, which spans the first part of the file - it is a still the same window nsize = (size or fc.size) - 10 @@ -153,16 +154,16 @@ class TestMMan(TestBase): if man.window_size(): assert man.num_file_handles() == 2 assert c.size() < size - assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore - assert rr().client_count() == 1 # only held by manager + assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore + assert rr().client_count() == 1 # only held by manager else: assert c.size() < fc.size - #END ignore static managers which only have one handle per file + # END ignore static managers which only have one handle per file rr = c.region_ref() - assert rr().client_count() == 2 # manager + cursor - assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left - assert rr().ofs_end() <= fc.size # it cannot be larger than the file - assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())] + assert rr().client_count() == 2 # manager + cursor + assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left + assert rr().ofs_end() <= fc.size # it cannot be larger than the file + assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())] # unising a region makes the cursor invalid c.unuse_region() @@ -171,7 +172,7 @@ class TestMMan(TestBase): # but doesn't change anything regarding the handle count - we cache it and only # remove mapped regions if we have to assert man.num_file_handles() == 2 - #END ignore this for static managers + # END ignore this for static managers # iterate through the windows, verify data contents # this will trigger map collection after a while @@ -193,21 +194,21 @@ class TestMMan(TestBase): # precondition if man.window_size(): assert max_mapped_memory_size >= mapped_memory_size() - #END statics will overshoot, which is fine + # END statics will overshoot, which is fine assert max_file_handles >= num_file_handles() assert c.use_region(base_offset, (size or c.size())).is_valid() csize = c.size() - assert c.buffer()[:] == data[base_offset:base_offset+csize] + assert c.buffer()[:] == data[base_offset:base_offset + csize] memory_read += csize assert includes_ofs(base_offset) - assert includes_ofs(base_offset+csize-1) - assert not includes_ofs(base_offset+csize) + assert includes_ofs(base_offset + csize - 1) + assert not includes_ofs(base_offset + csize) # END while we should do an access - elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows + elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows mb = float(1000 * 1000) print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n" - % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed), + % (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed), file=sys.stderr) # an offset as large as the size doesn't work ! @@ -217,6 +218,6 @@ class TestMMan(TestBase): assert man.num_file_handles() assert man.collect() assert man.num_file_handles() == 0 - #END for each item + # END for each item # END for each manager type os.close(fd) diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py index 5c931de..f7a2128 100644 --- a/smmap/test/test_tutorial.py +++ b/smmap/test/test_tutorial.py @@ -20,7 +20,7 @@ class TestTutorial(TestBase): # Cursors ########## import smmap.test.lib - fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file") + fc = smmap.test.lib.FileCreator(1024 * 1024 * 8, "test_file") # obtain a cursor to access some file. c = mman.make_cursor(fc.path) @@ -40,7 +40,7 @@ class TestTutorial(TestBase): assert c.size() c.buffer()[0] # first byte c.buffer()[1:10] # first 9 bytes - c.buffer()[c.size()-1] # last byte + c.buffer()[c.size() - 1] # last byte # its recommended not to create big slices when feeding the buffer # into consumers (e.g. struct or zlib). @@ -72,8 +72,8 @@ class TestTutorial(TestBase): assert buf.cursor().is_valid() buf[0] # access the first byte - buf[-1] # access the last ten bytes on the file - buf[-10:]# access the last ten bytes + buf[-1] # access the last ten bytes on the file + buf[-10:] # access the last ten bytes # If you want to keep the instance between different accesses, use the # dedicated methods diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py index 745fedf..0bbf91b 100644 --- a/smmap/test/test_util.py +++ b/smmap/test/test_util.py @@ -1,13 +1,13 @@ from .lib import TestBase, FileCreator from smmap.util import ( - MapWindow, - MapRegion, - MapRegionList, - ALLOCATIONGRANULARITY, - is_64_bit, - align_to_mmap - ) + MapWindow, + MapRegion, + MapRegionList, + ALLOCATIONGRANULARITY, + is_64_bit, + align_to_mmap +) import os import sys @@ -74,7 +74,7 @@ class TestMMan(TestBase): assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size - assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size) + assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size) assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize) # with the values we have, this test only works on windows where an alignment # size of 4096 is assumed. @@ -83,7 +83,7 @@ class TestMMan(TestBase): # argument of mmap. if sys.platform != 'win32': assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0) - #END handle platforms + # END handle platforms # auto-refcount assert rfull.client_count() == 1 @@ -111,7 +111,7 @@ class TestMMan(TestBase): assert len(ml) == 0 assert ml.path_or_fd() == item assert ml.file_size() == fc.size - #END handle input + # END handle input os.close(fd) def test_util(self): diff --git a/smmap/util.py b/smmap/util.py index 394e6b1..e079a52 100644 --- a/smmap/util.py +++ b/smmap/util.py @@ -10,10 +10,10 @@ except ImportError: # in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly # useful for aligning the offset. The offset argument doesn't exist there though from mmap import PAGESIZE as ALLOCATIONGRANULARITY -#END handle pythons missing quality assurance +# END handle pythons missing quality assurance __all__ = ["align_to_mmap", "is_64_bit", "buffer", - "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] + "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities @@ -25,7 +25,7 @@ except NameError: def buffer(obj, offset, size): # return memoryview(obj)[offset:offset+size] # doing it directly is much faster ! - return obj[offset:offset+size] + return obj[offset:offset + size] def string_types(): @@ -45,7 +45,7 @@ def align_to_mmap(num, round_up): res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY if round_up and (res != num): res += ALLOCATIONGRANULARITY - #END handle size + # END handle size return res @@ -59,11 +59,12 @@ def is_64_bit(): #{ Utility Classes class MapWindow(object): + """Utility type which is used to snap windows towards each other, and to adjust their size""" __slots__ = ( - 'ofs', # offset into the file in bytes - 'size' # size of the window in bytes - ) + 'ofs', # offset into the file in bytes + 'size' # size of the window in bytes + ) def __init__(self, offset, size): self.ofs = offset @@ -104,21 +105,22 @@ class MapWindow(object): class MapRegion(object): + """Defines a mapped region of memory, aligned to pagesizes **Note:** deallocates used region automatically on destruction""" __slots__ = [ - '_b', # beginning of mapping - '_mf', # mapped memory chunk (as returned by mmap) - '_uc', # total amount of usages - '_size', # cached size of our memory map - '__weakref__' - ] + '_b', # beginning of mapping + '_mf', # mapped memory chunk (as returned by mmap) + '_uc', # total amount of usages + '_size', # cached size of our memory map + '__weakref__' + ] _need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6 if _need_compat_layer: __slots__.append('_mfb') # mapped memory buffer to provide offset - #END handle additional slot + # END handle additional slot #{ Configuration # Used for testing only. If True, all data will be loaded into memory at once. @@ -142,7 +144,7 @@ class MapRegion(object): fd = path_or_fd else: fd = os.open(path_or_fd, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) - #END handle fd + # END handle fd try: kwargs = dict(access=ACCESS_READ, offset=ofs) @@ -162,18 +164,18 @@ class MapRegion(object): self._mf = self._read_into_memory(fd, ofs, actual_size) else: self._mf = mmap(fd, actual_size, **kwargs) - #END handle memory mode + # END handle memory mode self._size = len(self._mf) if self._need_compat_layer: self._mfb = buffer(self._mf, ofs, self._size) - #END handle buffer wrapping + # END handle buffer wrapping finally: if isinstance(path_or_fd, string_types()): os.close(fd) - #END only close it if we opened it - #END close file handle + # END only close it if we opened it + # END close file handle def _read_into_memory(self, fd, offset, size): """:return: string data as read from the given file descriptor, offset and size """ @@ -181,11 +183,11 @@ class MapRegion(object): mf = '' bytes_todo = size while bytes_todo: - chunk = 1024*1024 + chunk = 1024 * 1024 d = os.read(fd, chunk) bytes_todo -= len(d) mf += d - #END loop copy items + # END loop copy items return mf def __repr__(self): @@ -221,7 +223,7 @@ class MapRegion(object): """:return: number of clients currently using this region""" from sys import getrefcount # -1: self on stack, -1 self in this method, -1 self in getrefcount - return getrefcount(self)-3 + return getrefcount(self) - 3 def usage_count(self): """:return: amount of usages so far""" @@ -245,17 +247,18 @@ class MapRegion(object): def includes_ofs(self, ofs): return self._b <= ofs < self._size - #END handle compat layer + # END handle compat layer #} END interface class MapRegionList(list): + """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( - '_path_or_fd', # path or file descriptor which is mapped by all our regions - '_file_size' # total size of the file we map - ) + '_path_or_fd', # path or file descriptor which is mapped by all our regions + '_file_size' # total size of the file we map + ) def __new__(cls, path): return super(MapRegionList, cls).__new__(cls) @@ -267,7 +270,7 @@ class MapRegionList(list): def client_count(self): """:return: amount of clients which hold a reference to this instance""" from sys import getrefcount - return getrefcount(self)-3 + return getrefcount(self) - 3 def path_or_fd(self): """:return: path or file descriptor we are attached to""" @@ -280,8 +283,8 @@ class MapRegionList(list): self._file_size = os.stat(self._path_or_fd).st_size else: self._file_size = os.fstat(self._path_or_fd).st_size - #END handle path type - #END update file size + # END handle path type + # END update file size return self._file_size #} END utility classes |