summaryrefslogtreecommitdiff
path: root/smmap
diff options
context:
space:
mode:
authorSebastian Thiel <byronimo@gmail.com>2015-01-04 11:19:57 +0100
committerSebastian Thiel <byronimo@gmail.com>2015-01-04 11:19:57 +0100
commit84929ed811142e366d6c5916125302c1419acad6 (patch)
tree3bf33015838b0fc3c10df5c8eb0e959ad8f2aebe /smmap
parenteb40b44ce4a6e646aabf7b7091d876738336c42f (diff)
downloadsmmap-84929ed811142e366d6c5916125302c1419acad6.tar.gz
Applied autopep8
autopep8 -v -j 8 --max-line-length 120 --in-place --recursive
Diffstat (limited to 'smmap')
-rw-r--r--smmap/buf.py23
-rw-r--r--smmap/exc.py2
-rw-r--r--smmap/mman.py109
-rw-r--r--smmap/test/lib.py8
-rw-r--r--smmap/test/test_buf.py20
-rw-r--r--smmap/test/test_mman.py59
-rw-r--r--smmap/test/test_tutorial.py8
-rw-r--r--smmap/test/test_util.py20
-rw-r--r--smmap/util.py61
9 files changed, 161 insertions, 149 deletions
diff --git a/smmap/buf.py b/smmap/buf.py
index 66029cb..17d2d36 100644
--- a/smmap/buf.py
+++ b/smmap/buf.py
@@ -10,6 +10,7 @@ except NameError:
class SlidingWindowMapBuffer(object):
+
"""A buffer like object which allows direct byte-wise object and slicing into
memory of a mapped file. The mapping is controlled by the provided cursor.
@@ -20,9 +21,9 @@ class SlidingWindowMapBuffer(object):
underneath, it can unfortunately not be used in any non-pure python method which
needs a buffer or string"""
__slots__ = (
- '_c', # our cursor
- '_size', # our supposed size
- )
+ '_c', # our cursor
+ '_size', # our supposed size
+ )
def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0):
"""Initalize the instance to operate on the given cursor.
@@ -57,7 +58,7 @@ class SlidingWindowMapBuffer(object):
if not c.includes_ofs(i):
c.use_region(i, 1)
# END handle region usage
- return c.buffer()[i-c.ofs_begin()]
+ return c.buffer()[i - c.ofs_begin()]
def __getslice__(self, i, j):
c = self._c
@@ -72,9 +73,9 @@ class SlidingWindowMapBuffer(object):
j = self._size + j
if (c.ofs_begin() <= i) and (j < c.ofs_end()):
b = c.ofs_begin()
- return c.buffer()[i-b:j-b]
+ return c.buffer()[i - b:j - b]
else:
- l = j-i # total length
+ l = j - i # total length
ofs = i
# It's fastest to keep tokens and join later, especially in py3, which was 7 times slower
# in the previous iteration of this code
@@ -86,7 +87,7 @@ class SlidingWindowMapBuffer(object):
ofs += len(d)
l -= len(d)
md.append(d)
- #END while there are bytes to read
+ # END while there are bytes to read
return bytes().join(md)
# END fast or slow path
#{ Interface
@@ -100,7 +101,7 @@ class SlidingWindowMapBuffer(object):
:return: True if the buffer can be used"""
if cursor:
self._c = cursor
- #END update our cursor
+ # END update our cursor
# reuse existing cursors if possible
if self._c is not None and self._c.is_associated():
@@ -112,9 +113,9 @@ class SlidingWindowMapBuffer(object):
# If not, the user is in trouble.
if size > self._c.file_size():
size = self._c.file_size() - offset
- #END handle size
+ # END handle size
self._size = size
- #END set size
+ # END set size
return res
# END use our cursor
return False
@@ -128,7 +129,7 @@ class SlidingWindowMapBuffer(object):
self._size = 0
if self._c is not None:
self._c.unuse_region()
- #END unuse region
+ # END unuse region
def cursor(self):
""":return: the currently set cursor which provides access to the data"""
diff --git a/smmap/exc.py b/smmap/exc.py
index 5e90cf7..1176645 100644
--- a/smmap/exc.py
+++ b/smmap/exc.py
@@ -2,8 +2,10 @@
class MemoryManagerError(Exception):
+
"""Base class for all exceptions thrown by the memory manager"""
class RegionCollectionError(MemoryManagerError):
+
"""Thrown if a memory region could not be collected, or if no region for collection was found"""
diff --git a/smmap/mman.py b/smmap/mman.py
index 6663687..c7a4595 100644
--- a/smmap/mman.py
+++ b/smmap/mman.py
@@ -1,12 +1,12 @@
"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files"""
from .util import (
- MapWindow,
- MapRegion,
- MapRegionList,
- is_64_bit,
- string_types,
- buffer,
- )
+ MapWindow,
+ MapRegion,
+ MapRegionList,
+ is_64_bit,
+ string_types,
+ buffer,
+)
from weakref import ref
import sys
@@ -19,6 +19,7 @@ __all__ = ["StaticWindowMapManager", "SlidingWindowMapManager", "WindowCursor"]
class WindowCursor(object):
+
"""
Pointer into the mapped region of the memory manager, keeping the map
alive until it is destroyed and no other client uses it.
@@ -29,12 +30,12 @@ class WindowCursor(object):
that it must be suited for the somewhat quite different sliding manager. It could be improved, but
I see no real need to do so."""
__slots__ = (
- '_manager', # the manger keeping all file regions
- '_rlist', # a regions list with regions for our file
+ '_manager', # the manger keeping all file regions
+ '_rlist', # a regions list with regions for our file
'_region', # our current region or None
'_ofs', # relative offset from the actually mapped area to our start area
'_size' # maximum size we should provide
- )
+ )
def __init__(self, manager=None, regions=None):
self._manager = manager
@@ -65,8 +66,8 @@ class WindowCursor(object):
# this python problem (for now).
# The next step is to get rid of the error prone getrefcount alltogether.
pass
- #END exception handling
- #END handle regions
+ # END exception handling
+ # END handle regions
def _copy_from(self, rhs):
"""Copy all data from rhs into this instance, handles usage count"""
@@ -121,11 +122,11 @@ class WindowCursor(object):
# offset too large ?
if offset >= fsize:
return self
- #END handle offset
+ # END handle offset
if need_region:
self._region = man._obtain_region(self._rlist, offset, size, flags, False)
- #END need region handling
+ # END need region handling
self._region.increment_usage_count()
self._ofs = offset - self._region._b
@@ -221,13 +222,14 @@ class WindowCursor(object):
:raise ValueError: if the mapping was not created by a file descriptor"""
if isinstance(self._rlist.path_or_fd(), string_types()):
raise ValueError("File descriptor queried although mapping was generated from path")
- #END handle type
+ # END handle type
return self._rlist.path_or_fd()
#} END interface
class StaticWindowMapManager(object):
+
"""Provides a manager which will produce single size cursors that are allowed
to always map the whole file.
@@ -240,13 +242,13 @@ class StaticWindowMapManager(object):
accommodate this fact"""
__slots__ = [
- '_fdict', # mapping of path -> StorageHelper (of some kind
- '_window_size', # maximum size of a window
- '_max_memory_size', # maximum amount of memory we may allocate
- '_max_handle_count', # maximum amount of handles to keep open
- '_memory_size', # currently allocated memory size
- '_handle_count', # amount of currently allocated file handles
- ]
+ '_fdict', # mapping of path -> StorageHelper (of some kind
+ '_window_size', # maximum size of a window
+ '_max_memory_size', # maximum amount of memory we may allocate
+ '_max_handle_count', # maximum amount of handles to keep open
+ '_memory_size', # currently allocated memory size
+ '_handle_count', # amount of currently allocated file handles
+ ]
#{ Configuration
MapRegionListCls = MapRegionList
@@ -279,7 +281,7 @@ class StaticWindowMapManager(object):
coeff = 64
if is_64_bit():
coeff = 1024
- #END handle arch
+ # END handle arch
self._window_size = coeff * self._MB_in_bytes
# END handle max window size
@@ -287,9 +289,9 @@ class StaticWindowMapManager(object):
coeff = 1024
if is_64_bit():
coeff = 8192
- #END handle arch
+ # END handle arch
self._max_memory_size = coeff * self._MB_in_bytes
- #END handle max memory size
+ # END handle max memory size
#{ Internal Methods
@@ -310,23 +312,23 @@ class StaticWindowMapManager(object):
for regions in self._fdict.values():
for region in regions:
# check client count - consider that we keep one reference ourselves !
- if (region.client_count()-2 == 0 and
- (lru_region is None or region._uc < lru_region._uc)):
+ if (region.client_count() - 2 == 0 and
+ (lru_region is None or region._uc < lru_region._uc)):
lru_region = region
lru_list = regions
# END update lru_region
- #END for each region
- #END for each regions list
+ # END for each region
+ # END for each regions list
if lru_region is None:
break
- #END handle region not found
+ # END handle region not found
num_found += 1
del(lru_list[lru_list.index(lru_region)])
self._memory_size -= lru_region.size()
self._handle_count -= 1
- #END while there is more memory to free
+ # END while there is more memory to free
return num_found
def _obtain_region(self, a, offset, size, flags, is_recursive):
@@ -336,7 +338,7 @@ class StaticWindowMapManager(object):
:return: The newly created region"""
if self._memory_size + size > self._max_memory_size:
self._collect_lru_region(size)
- #END handle collection
+ # END handle collection
r = None
if a:
@@ -354,10 +356,10 @@ class StaticWindowMapManager(object):
# we already tried this, and still have no success in obtaining
# a mapping. This is an exception, so we propagate it
raise
- #END handle existing recursion
+ # END handle existing recursion
self._collect_lru_region(0)
return self._obtain_region(a, offset, size, flags, True)
- #END handle exceptions
+ # END handle exceptions
self._handle_count += 1
self._memory_size += r.size()
@@ -404,7 +406,7 @@ class StaticWindowMapManager(object):
def num_open_files(self):
"""Amount of opened files in the system"""
- return reduce(lambda x, y: x+y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0)
+ return reduce(lambda x, y: x + y, (1 for rlist in self._fdict.values() if len(rlist) > 0), 0)
def window_size(self):
""":return: size of each window when allocating new regions"""
@@ -441,7 +443,7 @@ class StaticWindowMapManager(object):
**Note:** does nothing on non-windows platforms"""
if sys.platform != 'win32':
return
- #END early bailout
+ # END early bailout
num_closed = 0
for path, rlist in self._fdict.items():
@@ -449,13 +451,14 @@ class StaticWindowMapManager(object):
for region in rlist:
region._mf.close()
num_closed += 1
- #END path matches
- #END for each path
+ # END path matches
+ # END for each path
return num_closed
#} END special purpose interface
class SlidingWindowMapManager(StaticWindowMapManager):
+
"""Maintains a list of ranges of mapped memory regions in one or more files and allows to easily
obtain additional regions assuring there is no overlap.
Once a certain memory limit is reached globally, or if there cannot be more open file handles
@@ -482,18 +485,18 @@ class SlidingWindowMapManager(StaticWindowMapManager):
lo = 0
hi = len(a)
while lo < hi:
- mid = (lo+hi)//2
+ mid = (lo + hi) // 2
ofs = a[mid]._b
if ofs <= offset:
if a[mid].includes_ofs(offset):
r = a[mid]
break
- #END have region
- lo = mid+1
+ # END have region
+ lo = mid + 1
else:
hi = mid
- #END handle position
- #END while bisecting
+ # END handle position
+ # END while bisecting
if r is None:
window_size = self._window_size
@@ -506,7 +509,7 @@ class SlidingWindowMapManager(StaticWindowMapManager):
# Save calls !
if self._memory_size + window_size > self._max_memory_size:
self._collect_lru_region(window_size)
- #END handle collection
+ # END handle collection
# we assume the list remains sorted by offset
insert_pos = 0
@@ -514,7 +517,7 @@ class SlidingWindowMapManager(StaticWindowMapManager):
if len_regions == 1:
if a[0]._b <= offset:
insert_pos = 1
- #END maintain sort
+ # END maintain sort
else:
# find insert position
insert_pos = len_regions
@@ -522,8 +525,8 @@ class SlidingWindowMapManager(StaticWindowMapManager):
if region._b > offset:
insert_pos = i
break
- #END if insert position is correct
- #END for each region
+ # END if insert position is correct
+ # END for each region
# END obtain insert pos
# adjust the actual offset and size values to create the largest
@@ -531,13 +534,13 @@ class SlidingWindowMapManager(StaticWindowMapManager):
if insert_pos == 0:
if len_regions:
right = self.MapWindowCls.from_region(a[insert_pos])
- #END adjust right side
+ # END adjust right side
else:
if insert_pos != len_regions:
right = self.MapWindowCls.from_region(a[insert_pos])
# END adjust right window
left = self.MapWindowCls.from_region(a[insert_pos - 1])
- #END adjust surrounding windows
+ # END adjust surrounding windows
mid.extend_left_to(left, window_size)
mid.extend_right_to(right, window_size)
@@ -546,13 +549,13 @@ class SlidingWindowMapManager(StaticWindowMapManager):
# it can happen that we align beyond the end of the file
if mid.ofs_end() > right.ofs:
mid.size = right.ofs - mid.ofs
- #END readjust size
+ # END readjust size
# insert new region at the right offset to keep the order
try:
if self._handle_count >= self._max_handle_count:
raise Exception
- #END assert own imposed max file handles
+ # END assert own imposed max file handles
r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags)
except Exception:
# apparently we are out of system resources or hit a limit
@@ -563,10 +566,10 @@ class SlidingWindowMapManager(StaticWindowMapManager):
# we already tried this, and still have no success in obtaining
# a mapping. This is an exception, so we propagate it
raise
- #END handle existing recursion
+ # END handle existing recursion
self._collect_lru_region(0)
return self._obtain_region(a, offset, size, flags, True)
- #END handle exceptions
+ # END handle exceptions
self._handle_count += 1
self._memory_size += r.size()
diff --git a/smmap/test/lib.py b/smmap/test/lib.py
index 67aec63..93cb09a 100644
--- a/smmap/test/lib.py
+++ b/smmap/test/lib.py
@@ -9,6 +9,7 @@ __all__ = ['TestBase', 'FileCreator']
#{ Utilities
class FileCreator(object):
+
"""A instance which creates a temporary file with a prefix and a given size
and provides this info to the user.
Once it gets deleted, it will remove the temporary file as well."""
@@ -21,7 +22,7 @@ class FileCreator(object):
self._size = size
fp = open(self._path, "wb")
- fp.seek(size-1)
+ fp.seek(size - 1)
fp.write(b'1')
fp.close()
@@ -32,7 +33,7 @@ class FileCreator(object):
os.remove(self.path)
except OSError:
pass
- #END exception handling
+ # END exception handling
@property
def path(self):
@@ -46,6 +47,7 @@ class FileCreator(object):
class TestBase(TestCase):
+
"""Foundation used by all tests"""
#{ Configuration
@@ -58,7 +60,7 @@ class TestBase(TestCase):
# nothing for now
pass
- #END overrides
+ # END overrides
#{ Interface
diff --git a/smmap/test/test_buf.py b/smmap/test/test_buf.py
index d07b7f4..0337715 100644
--- a/smmap/test/test_buf.py
+++ b/smmap/test/test_buf.py
@@ -3,9 +3,9 @@ from __future__ import print_function
from .lib import TestBase, FileCreator
from smmap.mman import (
- SlidingWindowMapManager,
- StaticWindowMapManager
- )
+ SlidingWindowMapManager,
+ StaticWindowMapManager
+)
from smmap.buf import SlidingWindowMapBuffer
from random import randint
@@ -57,11 +57,11 @@ class TestBuf(TestBase):
with open(fc.path, 'rb') as fp:
data = fp.read()
assert data[offset] == buf[0]
- assert data[offset:offset*2] == buf[0:offset]
+ assert data[offset:offset * 2] == buf[0:offset]
# negative indices, partial slices
- assert buf[-1] == buf[len(buf)-1]
- assert buf[-10:] == buf[len(buf)-10:len(buf)]
+ assert buf[-1] == buf[len(buf) - 1]
+ assert buf[-10:] == buf[len(buf) - 10:len(buf)]
# end access makes its cursor invalid
buf.end_access()
@@ -97,7 +97,7 @@ class TestBuf(TestBase):
buf.begin_access()
while num_accesses_left:
num_accesses_left -= 1
- if access_mode: # multi
+ if access_mode: # multi
ofs_start = randint(0, fsize)
ofs_end = randint(ofs_start, fsize)
d = buf[ofs_start:ofs_end]
@@ -108,7 +108,7 @@ class TestBuf(TestBase):
pos = randint(0, fsize)
assert buf[pos] == data[pos]
num_bytes += 1
- #END handle mode
+ # END handle mode
# END handle num accesses
buf.end_access()
@@ -116,10 +116,10 @@ class TestBuf(TestBase):
assert manager.collect()
assert manager.num_file_handles() == 0
elapsed = max(time() - st, 0.001) # prevent zero division errors on windows
- mb = float(1000*1000)
+ mb = float(1000 * 1000)
mode_str = (access_mode and "slice") or "single byte"
print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)"
- % (man_id, max_num_accesses, mode_str, type(item), num_bytes/mb, elapsed, (num_bytes/mb)/elapsed),
+ % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed),
file=sys.stderr)
# END handle access mode
# END for each manager
diff --git a/smmap/test/test_mman.py b/smmap/test/test_mman.py
index d903af6..b718b06 100644
--- a/smmap/test/test_mman.py
+++ b/smmap/test/test_mman.py
@@ -3,10 +3,10 @@ from __future__ import print_function
from .lib import TestBase, FileCreator
from smmap.mman import (
- WindowCursor,
- SlidingWindowMapManager,
- StaticWindowMapManager
- )
+ WindowCursor,
+ SlidingWindowMapManager,
+ StaticWindowMapManager
+)
from smmap.util import align_to_mmap
from random import randint
@@ -29,7 +29,7 @@ class TestMMan(TestBase):
cv = man.make_cursor(fc.path)
assert not cv.is_valid() # no region mapped yet
- assert cv.is_associated()# but it know where to map it from
+ assert cv.is_associated() # but it know where to map it from
assert cv.file_size() == fc.size
assert cv.path() == fc.path
@@ -60,7 +60,7 @@ class TestMMan(TestBase):
winsize_cmp_val = 0
if isinstance(man, StaticWindowMapManager):
winsize_cmp_val = -1
- #END handle window size
+ # END handle window size
assert man.window_size() > winsize_cmp_val
assert man.mapped_memory_size() == 0
assert man.max_mapped_memory_size() > 0
@@ -89,8 +89,8 @@ class TestMMan(TestBase):
self.assertRaises(ValueError, c.path)
else:
self.assertRaises(ValueError, c.fd)
- #END handle value error
- #END for each input
+ # END handle value error
+ # END for each input
os.close(fd)
# END for each manager type
@@ -101,7 +101,7 @@ class TestMMan(TestBase):
data = fp.read()
fd = os.open(fc.path, os.O_RDONLY)
max_num_handles = 15
- #small_size =
+ # small_size =
for mtype, args in ((StaticWindowMapManager, (0, fc.size // 3, max_num_handles)),
(SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),):
for item in (fc.path, fd):
@@ -120,22 +120,23 @@ class TestMMan(TestBase):
size = man.window_size() // 2
assert c.use_region(base_offset, size).is_valid()
rr = c.region_ref()
- assert rr().client_count() == 2 # the manager and the cursor and us
+ assert rr().client_count() == 2 # the manager and the cursor and us
assert man.num_open_files() == 1
assert man.num_file_handles() == 1
assert man.mapped_memory_size() == rr().size()
- #assert c.size() == size # the cursor may overallocate in its static version
+ # assert c.size() == size # the cursor may overallocate in its static version
assert c.ofs_begin() == base_offset
assert rr().ofs_begin() == 0 # it was aligned and expanded
if man.window_size():
- assert rr().size() == align_to_mmap(man.window_size(), True) # but isn't larger than the max window (aligned)
+ # but isn't larger than the max window (aligned)
+ assert rr().size() == align_to_mmap(man.window_size(), True)
else:
assert rr().size() == fc.size
- #END ignore static managers which dont use windows and are aligned to file boundaries
+ # END ignore static managers which dont use windows and are aligned to file boundaries
- assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())]
+ assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())]
# obtain second window, which spans the first part of the file - it is a still the same window
nsize = (size or fc.size) - 10
@@ -153,16 +154,16 @@ class TestMMan(TestBase):
if man.window_size():
assert man.num_file_handles() == 2
assert c.size() < size
- assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore
- assert rr().client_count() == 1 # only held by manager
+ assert c.region_ref()() is not rr() # old region is still available, but has not curser ref anymore
+ assert rr().client_count() == 1 # only held by manager
else:
assert c.size() < fc.size
- #END ignore static managers which only have one handle per file
+ # END ignore static managers which only have one handle per file
rr = c.region_ref()
- assert rr().client_count() == 2 # manager + cursor
- assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left
- assert rr().ofs_end() <= fc.size # it cannot be larger than the file
- assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())]
+ assert rr().client_count() == 2 # manager + cursor
+ assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left
+ assert rr().ofs_end() <= fc.size # it cannot be larger than the file
+ assert c.buffer()[:] == data[base_offset:base_offset + (size or c.size())]
# unising a region makes the cursor invalid
c.unuse_region()
@@ -171,7 +172,7 @@ class TestMMan(TestBase):
# but doesn't change anything regarding the handle count - we cache it and only
# remove mapped regions if we have to
assert man.num_file_handles() == 2
- #END ignore this for static managers
+ # END ignore this for static managers
# iterate through the windows, verify data contents
# this will trigger map collection after a while
@@ -193,21 +194,21 @@ class TestMMan(TestBase):
# precondition
if man.window_size():
assert max_mapped_memory_size >= mapped_memory_size()
- #END statics will overshoot, which is fine
+ # END statics will overshoot, which is fine
assert max_file_handles >= num_file_handles()
assert c.use_region(base_offset, (size or c.size())).is_valid()
csize = c.size()
- assert c.buffer()[:] == data[base_offset:base_offset+csize]
+ assert c.buffer()[:] == data[base_offset:base_offset + csize]
memory_read += csize
assert includes_ofs(base_offset)
- assert includes_ofs(base_offset+csize-1)
- assert not includes_ofs(base_offset+csize)
+ assert includes_ofs(base_offset + csize - 1)
+ assert not includes_ofs(base_offset + csize)
# END while we should do an access
- elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows
+ elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows
mb = float(1000 * 1000)
print("%s: Read %i mb of memory with %i random on cursor initialized with %s accesses in %fs (%f mb/s)\n"
- % (mtype, memory_read/mb, max_random_accesses, type(item), elapsed, (memory_read/mb)/elapsed),
+ % (mtype, memory_read / mb, max_random_accesses, type(item), elapsed, (memory_read / mb) / elapsed),
file=sys.stderr)
# an offset as large as the size doesn't work !
@@ -217,6 +218,6 @@ class TestMMan(TestBase):
assert man.num_file_handles()
assert man.collect()
assert man.num_file_handles() == 0
- #END for each item
+ # END for each item
# END for each manager type
os.close(fd)
diff --git a/smmap/test/test_tutorial.py b/smmap/test/test_tutorial.py
index 5c931de..f7a2128 100644
--- a/smmap/test/test_tutorial.py
+++ b/smmap/test/test_tutorial.py
@@ -20,7 +20,7 @@ class TestTutorial(TestBase):
# Cursors
##########
import smmap.test.lib
- fc = smmap.test.lib.FileCreator(1024*1024*8, "test_file")
+ fc = smmap.test.lib.FileCreator(1024 * 1024 * 8, "test_file")
# obtain a cursor to access some file.
c = mman.make_cursor(fc.path)
@@ -40,7 +40,7 @@ class TestTutorial(TestBase):
assert c.size()
c.buffer()[0] # first byte
c.buffer()[1:10] # first 9 bytes
- c.buffer()[c.size()-1] # last byte
+ c.buffer()[c.size() - 1] # last byte
# its recommended not to create big slices when feeding the buffer
# into consumers (e.g. struct or zlib).
@@ -72,8 +72,8 @@ class TestTutorial(TestBase):
assert buf.cursor().is_valid()
buf[0] # access the first byte
- buf[-1] # access the last ten bytes on the file
- buf[-10:]# access the last ten bytes
+ buf[-1] # access the last ten bytes on the file
+ buf[-10:] # access the last ten bytes
# If you want to keep the instance between different accesses, use the
# dedicated methods
diff --git a/smmap/test/test_util.py b/smmap/test/test_util.py
index 745fedf..0bbf91b 100644
--- a/smmap/test/test_util.py
+++ b/smmap/test/test_util.py
@@ -1,13 +1,13 @@
from .lib import TestBase, FileCreator
from smmap.util import (
- MapWindow,
- MapRegion,
- MapRegionList,
- ALLOCATIONGRANULARITY,
- is_64_bit,
- align_to_mmap
- )
+ MapWindow,
+ MapRegion,
+ MapRegionList,
+ ALLOCATIONGRANULARITY,
+ is_64_bit,
+ align_to_mmap
+)
import os
import sys
@@ -74,7 +74,7 @@ class TestMMan(TestBase):
assert rhalfofs.ofs_begin() == rofs and rhalfofs.size() == fc.size - rofs
assert rhalfsize.ofs_begin() == 0 and rhalfsize.size() == half_size
- assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size-1) and rfull.includes_ofs(half_size)
+ assert rfull.includes_ofs(0) and rfull.includes_ofs(fc.size - 1) and rfull.includes_ofs(half_size)
assert not rfull.includes_ofs(-1) and not rfull.includes_ofs(sys.maxsize)
# with the values we have, this test only works on windows where an alignment
# size of 4096 is assumed.
@@ -83,7 +83,7 @@ class TestMMan(TestBase):
# argument of mmap.
if sys.platform != 'win32':
assert rhalfofs.includes_ofs(rofs) and not rhalfofs.includes_ofs(0)
- #END handle platforms
+ # END handle platforms
# auto-refcount
assert rfull.client_count() == 1
@@ -111,7 +111,7 @@ class TestMMan(TestBase):
assert len(ml) == 0
assert ml.path_or_fd() == item
assert ml.file_size() == fc.size
- #END handle input
+ # END handle input
os.close(fd)
def test_util(self):
diff --git a/smmap/util.py b/smmap/util.py
index 394e6b1..e079a52 100644
--- a/smmap/util.py
+++ b/smmap/util.py
@@ -10,10 +10,10 @@ except ImportError:
# in python pre 2.6, the ALLOCATIONGRANULARITY does not exist as it is mainly
# useful for aligning the offset. The offset argument doesn't exist there though
from mmap import PAGESIZE as ALLOCATIONGRANULARITY
-#END handle pythons missing quality assurance
+# END handle pythons missing quality assurance
__all__ = ["align_to_mmap", "is_64_bit", "buffer",
- "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"]
+ "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"]
#{ Utilities
@@ -25,7 +25,7 @@ except NameError:
def buffer(obj, offset, size):
# return memoryview(obj)[offset:offset+size]
# doing it directly is much faster !
- return obj[offset:offset+size]
+ return obj[offset:offset + size]
def string_types():
@@ -45,7 +45,7 @@ def align_to_mmap(num, round_up):
res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY
if round_up and (res != num):
res += ALLOCATIONGRANULARITY
- #END handle size
+ # END handle size
return res
@@ -59,11 +59,12 @@ def is_64_bit():
#{ Utility Classes
class MapWindow(object):
+
"""Utility type which is used to snap windows towards each other, and to adjust their size"""
__slots__ = (
- 'ofs', # offset into the file in bytes
- 'size' # size of the window in bytes
- )
+ 'ofs', # offset into the file in bytes
+ 'size' # size of the window in bytes
+ )
def __init__(self, offset, size):
self.ofs = offset
@@ -104,21 +105,22 @@ class MapWindow(object):
class MapRegion(object):
+
"""Defines a mapped region of memory, aligned to pagesizes
**Note:** deallocates used region automatically on destruction"""
__slots__ = [
- '_b', # beginning of mapping
- '_mf', # mapped memory chunk (as returned by mmap)
- '_uc', # total amount of usages
- '_size', # cached size of our memory map
- '__weakref__'
- ]
+ '_b', # beginning of mapping
+ '_mf', # mapped memory chunk (as returned by mmap)
+ '_uc', # total amount of usages
+ '_size', # cached size of our memory map
+ '__weakref__'
+ ]
_need_compat_layer = sys.version_info[0] < 3 and sys.version_info[1] < 6
if _need_compat_layer:
__slots__.append('_mfb') # mapped memory buffer to provide offset
- #END handle additional slot
+ # END handle additional slot
#{ Configuration
# Used for testing only. If True, all data will be loaded into memory at once.
@@ -142,7 +144,7 @@ class MapRegion(object):
fd = path_or_fd
else:
fd = os.open(path_or_fd, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags)
- #END handle fd
+ # END handle fd
try:
kwargs = dict(access=ACCESS_READ, offset=ofs)
@@ -162,18 +164,18 @@ class MapRegion(object):
self._mf = self._read_into_memory(fd, ofs, actual_size)
else:
self._mf = mmap(fd, actual_size, **kwargs)
- #END handle memory mode
+ # END handle memory mode
self._size = len(self._mf)
if self._need_compat_layer:
self._mfb = buffer(self._mf, ofs, self._size)
- #END handle buffer wrapping
+ # END handle buffer wrapping
finally:
if isinstance(path_or_fd, string_types()):
os.close(fd)
- #END only close it if we opened it
- #END close file handle
+ # END only close it if we opened it
+ # END close file handle
def _read_into_memory(self, fd, offset, size):
""":return: string data as read from the given file descriptor, offset and size """
@@ -181,11 +183,11 @@ class MapRegion(object):
mf = ''
bytes_todo = size
while bytes_todo:
- chunk = 1024*1024
+ chunk = 1024 * 1024
d = os.read(fd, chunk)
bytes_todo -= len(d)
mf += d
- #END loop copy items
+ # END loop copy items
return mf
def __repr__(self):
@@ -221,7 +223,7 @@ class MapRegion(object):
""":return: number of clients currently using this region"""
from sys import getrefcount
# -1: self on stack, -1 self in this method, -1 self in getrefcount
- return getrefcount(self)-3
+ return getrefcount(self) - 3
def usage_count(self):
""":return: amount of usages so far"""
@@ -245,17 +247,18 @@ class MapRegion(object):
def includes_ofs(self, ofs):
return self._b <= ofs < self._size
- #END handle compat layer
+ # END handle compat layer
#} END interface
class MapRegionList(list):
+
"""List of MapRegion instances associating a path with a list of regions."""
__slots__ = (
- '_path_or_fd', # path or file descriptor which is mapped by all our regions
- '_file_size' # total size of the file we map
- )
+ '_path_or_fd', # path or file descriptor which is mapped by all our regions
+ '_file_size' # total size of the file we map
+ )
def __new__(cls, path):
return super(MapRegionList, cls).__new__(cls)
@@ -267,7 +270,7 @@ class MapRegionList(list):
def client_count(self):
""":return: amount of clients which hold a reference to this instance"""
from sys import getrefcount
- return getrefcount(self)-3
+ return getrefcount(self) - 3
def path_or_fd(self):
""":return: path or file descriptor we are attached to"""
@@ -280,8 +283,8 @@ class MapRegionList(list):
self._file_size = os.stat(self._path_or_fd).st_size
else:
self._file_size = os.fstat(self._path_or_fd).st_size
- #END handle path type
- #END update file size
+ # END handle path type
+ # END update file size
return self._file_size
#} END utility classes