summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--doc/release/1.14.0-notes.rst11
-rw-r--r--numpy/lib/format.py51
-rw-r--r--numpy/lib/tests/test_format.py7
3 files changed, 47 insertions, 22 deletions
diff --git a/doc/release/1.14.0-notes.rst b/doc/release/1.14.0-notes.rst
index 0bcb3e4a2..98d58df97 100644
--- a/doc/release/1.14.0-notes.rst
+++ b/doc/release/1.14.0-notes.rst
@@ -217,6 +217,17 @@ selected via the ``--fcompiler`` and ``--compiler`` options to
supported; by default a gfortran-compatible static archive
``openblas.a`` is looked for.
+``numpy.save`` aligns data to 64 bytes instead of 16
+----------------------------------------------------
+Saving NumPy arrays in the ``npy`` format with ``numpy.save`` inserts
+padding before the array data to align it at 64 bytes. Previously
+this was only 16 bytes (and sometimes less due to a bug in the code
+for version 2). Now the alignment is 64 bytes, which matches the
+widest SIMD instruction set commonly available, and is also the most
+common cache line size. This makes ``npy`` files easier to use in
+programs which open them with ``mmap``, especially on Linux where an
+``mmap`` offset must be a multiple of the page size.
+
Changes
=======
diff --git a/numpy/lib/format.py b/numpy/lib/format.py
index 14dec01d5..84af2afc8 100644
--- a/numpy/lib/format.py
+++ b/numpy/lib/format.py
@@ -100,9 +100,9 @@ the header data HEADER_LEN.
The next HEADER_LEN bytes form the header data describing the array's
format. It is an ASCII string which contains a Python literal expression
of a dictionary. It is terminated by a newline (``\\n``) and padded with
-spaces (``\\x20``) to make the total length of
-``magic string + 4 + HEADER_LEN`` be evenly divisible by 16 for alignment
-purposes.
+spaces (``\\x20``) to make the total of
+``len(magic string) + 2 + len(length) + HEADER_LEN`` be evenly divisible
+by 64 for alignment purposes.
The dictionary contains three keys:
@@ -163,6 +163,7 @@ else:
MAGIC_PREFIX = b'\x93NUMPY'
MAGIC_LEN = len(MAGIC_PREFIX) + 2
+ARRAY_ALIGN = 64 # plausible values are powers of 2 between 16 and 4096
BUFFER_SIZE = 2**18 # size of buffer for reading npz files in bytes
# difference between version 1.0 and 2.0 is a 4 byte (I) header length
@@ -304,27 +305,33 @@ def _write_array_header(fp, d, version=None):
header.append("'%s': %s, " % (key, repr(value)))
header.append("}")
header = "".join(header)
- # Pad the header with spaces and a final newline such that the magic
- # string, the header-length short and the header are aligned on a
- # 16-byte boundary. Hopefully, some system, possibly memory-mapping,
- # can take advantage of our premature optimization.
- current_header_len = MAGIC_LEN + 2 + len(header) + 1 # 1 for the newline
- topad = 16 - (current_header_len % 16)
- header = header + ' '*topad + '\n'
header = asbytes(_filter_header(header))
- hlen = len(header)
- if hlen < 256*256 and version in (None, (1, 0)):
+ hlen = len(header) + 1 # 1 for newline
+ padlen_v1 = ARRAY_ALIGN - ((MAGIC_LEN + struct.calcsize('<H') + hlen) % ARRAY_ALIGN)
+ padlen_v2 = ARRAY_ALIGN - ((MAGIC_LEN + struct.calcsize('<I') + hlen) % ARRAY_ALIGN)
+
+ # Which version(s) we write depends on the total header size; v1 has a max of 65535
+ if hlen + padlen_v1 < 2**16 and version in (None, (1, 0)):
version = (1, 0)
- header_prefix = magic(1, 0) + struct.pack('<H', hlen)
- elif hlen < 2**32 and version in (None, (2, 0)):
+ header_prefix = magic(1, 0) + struct.pack('<H', hlen + padlen_v1)
+ topad = padlen_v1
+ elif hlen + padlen_v2 < 2**32 and version in (None, (2, 0)):
version = (2, 0)
- header_prefix = magic(2, 0) + struct.pack('<I', hlen)
+ header_prefix = magic(2, 0) + struct.pack('<I', hlen + padlen_v2)
+ topad = padlen_v2
else:
msg = "Header length %s too big for version=%s"
msg %= (hlen, version)
raise ValueError(msg)
+ # Pad the header with spaces and a final newline such that the magic
+ # string, the header-length short and the header are aligned on a
+ # ARRAY_ALIGN byte boundary. This supports memory mapping of dtypes
+ # aligned up to ARRAY_ALIGN on systems like Linux where mmap()
+ # offset must be page-aligned (i.e. the beginning of the file).
+ header = header + b' '*topad + b'\n'
+
fp.write(header_prefix)
fp.write(header)
return version
@@ -468,18 +475,18 @@ def _read_array_header(fp, version):
# header.
import struct
if version == (1, 0):
- hlength_str = _read_bytes(fp, 2, "array header length")
- header_length = struct.unpack('<H', hlength_str)[0]
- header = _read_bytes(fp, header_length, "array header")
+ hlength_type = '<H'
elif version == (2, 0):
- hlength_str = _read_bytes(fp, 4, "array header length")
- header_length = struct.unpack('<I', hlength_str)[0]
- header = _read_bytes(fp, header_length, "array header")
+ hlength_type = '<I'
else:
raise ValueError("Invalid version %r" % version)
+ hlength_str = _read_bytes(fp, struct.calcsize(hlength_type), "array header length")
+ header_length = struct.unpack(hlength_type, hlength_str)[0]
+ header = _read_bytes(fp, header_length, "array header")
+
# The header is a pretty-printed string representation of a literal
- # Python dictionary with trailing newlines padded to a 16-byte
+ # Python dictionary with trailing newlines padded to a ARRAY_ALIGN byte
# boundary. The keys are strings.
# "shape" : tuple of int
# "fortran_order" : bool
diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py
index 155732882..2d2b4cea2 100644
--- a/numpy/lib/tests/test_format.py
+++ b/numpy/lib/tests/test_format.py
@@ -615,6 +615,11 @@ def test_version_2_0():
format.write_array(f, d)
assert_(w[0].category is UserWarning)
+ # check alignment of data portion
+ f.seek(0)
+ header = f.readline()
+ assert_(len(header) % format.ARRAY_ALIGN == 0)
+
f.seek(0)
n = format.read_array(f)
assert_array_equal(d, n)
@@ -758,6 +763,7 @@ def test_read_array_header_1_0():
s.seek(format.MAGIC_LEN)
shape, fortran, dtype = format.read_array_header_1_0(s)
+ assert_(s.tell() % format.ARRAY_ALIGN == 0)
assert_((shape, fortran, dtype) == ((3, 6), False, float))
@@ -770,6 +776,7 @@ def test_read_array_header_2_0():
s.seek(format.MAGIC_LEN)
shape, fortran, dtype = format.read_array_header_2_0(s)
+ assert_(s.tell() % format.ARRAY_ALIGN == 0)
assert_((shape, fortran, dtype) == ((3, 6), False, float))