Merge branch 'master' into implement-numpy-implementation

7 files changed, 229 insertions, 59 deletions

diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py
index 2ebd483d5..4a3fbdf57 100644
--- a/numpy/lib/tests/test_format.py
+++ b/numpy/lib/tests/test_format.py
@@ -411,6 +411,7 @@ record_arrays = [
     np.array(NbufferT, dtype=np.dtype(Ndescr).newbyteorder('<')),
     np.array(PbufferT, dtype=np.dtype(Pdescr).newbyteorder('>')),
     np.array(NbufferT, dtype=np.dtype(Ndescr).newbyteorder('>')),
+    np.zeros(1, dtype=[('c', ('<f8', (5,)), (2,))])
 ]
 
 
@@ -628,6 +629,61 @@ def test_pickle_disallow():
     assert_raises(ValueError, np.save, path, np.array([None], dtype=object),
                   allow_pickle=False)
 
+@pytest.mark.parametrize('dt', [
+    np.dtype(np.dtype([('a', np.int8),
+                       ('b', np.int16),
+                       ('c', np.int32),
+                      ], align=True),
+             (3,)),
+    np.dtype([('x', np.dtype({'names':['a','b'],
+                              'formats':['i1','i1'],
+                              'offsets':[0,4],
+                              'itemsize':8,
+                             },
+                    (3,)),
+               (4,),
+             )]),
+    np.dtype([('x',
+                   ('<f8', (5,)),
+                   (2,),
+               )]),
+    np.dtype([('x', np.dtype((
+        np.dtype((
+            np.dtype({'names':['a','b'],
+                      'formats':['i1','i1'],
+                      'offsets':[0,4],
+                      'itemsize':8}),
+            (3,)
+            )),
+        (4,)
+        )))
+        ]),
+    np.dtype([
+        ('a', np.dtype((
+            np.dtype((
+                np.dtype((
+                    np.dtype([
+                        ('a', int),
+                        ('b', np.dtype({'names':['a','b'],
+                                        'formats':['i1','i1'],
+                                        'offsets':[0,4],
+                                        'itemsize':8})),
+                    ]),
+                    (3,),
+                )),
+                (4,),
+            )),
+            (5,),
+        )))
+        ]),
+    ])
+
+def test_descr_to_dtype(dt):
+    dt1 = format.descr_to_dtype(dt.descr)
+    assert_equal_(dt1, dt)
+    arr1 = np.zeros(3, dt)
+    arr2 = roundtrip(arr1)
+    assert_array_equal(arr1, arr2)
 
 def test_version_2_0():
     f = BytesIO()
diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py
index e2c24a123..a3d4c6efb 100644
--- a/numpy/lib/tests/test_function_base.py
+++ b/numpy/lib/tests/test_function_base.py
@@ -5,6 +5,7 @@ import warnings
 import sys
 import decimal
 import types
+from fractions import Fraction
 import pytest
 
 import numpy as np
@@ -695,6 +696,9 @@ class TestDiff(object):
         assert_raises(np.AxisError, diff, x, axis=3)
         assert_raises(np.AxisError, diff, x, axis=-4)
 
+        x = np.array(1.11111111111, np.float64)
+        assert_raises(ValueError, diff, x)
+
     def test_nd(self):
         x = 20 * rand(10, 20, 30)
         out1 = x[:, :, 1:] - x[:, :, :-1]
@@ -944,7 +948,7 @@ class TestGradient(object):
         assert_equal(type(out), type(x))
         # And make sure that the output and input don't have aliased mask
         # arrays
-        assert_(x.mask is not out.mask)
+        assert_(x._mask is not out._mask)
         # Also check that edge_order=2 doesn't alter the original mask
         x2 = np.ma.arange(5)
         x2[2] = np.ma.masked
@@ -2508,6 +2512,21 @@ class TestPercentile(object):
         assert_equal(np.percentile(x, 0), np.nan)
         assert_equal(np.percentile(x, 0, interpolation='nearest'), np.nan)
 
+    def test_fraction(self):
+        x = [Fraction(i, 2) for i in np.arange(8)]
+
+        p = np.percentile(x, Fraction(0))
+        assert_equal(p, Fraction(0))
+        assert_equal(type(p), Fraction)
+
+        p = np.percentile(x, Fraction(100))
+        assert_equal(p, Fraction(7, 2))
+        assert_equal(type(p), Fraction)
+
+        p = np.percentile(x, Fraction(50))
+        assert_equal(p, Fraction(7, 4))
+        assert_equal(type(p), Fraction)
+
     def test_api(self):
         d = np.ones(5)
         np.percentile(d, 5, None, None, False)
@@ -2912,6 +2931,26 @@ class TestQuantile(object):
         assert_equal(np.quantile(x, 1), 3.5)
         assert_equal(np.quantile(x, 0.5), 1.75)
 
+    def test_fraction(self):
+        # fractional input, integral quantile
+        x = [Fraction(i, 2) for i in np.arange(8)]
+
+        q = np.quantile(x, 0)
+        assert_equal(q, 0)
+        assert_equal(type(q), Fraction)
+
+        q = np.quantile(x, 1)
+        assert_equal(q, Fraction(7, 2))
+        assert_equal(type(q), Fraction)
+
+        q = np.quantile(x, Fraction(1, 2))
+        assert_equal(q, Fraction(7, 4))
+        assert_equal(type(q), Fraction)
+
+        # repeat with integral input but fractional quantile
+        x = np.arange(8)
+        assert_equal(np.quantile(x, Fraction(1, 2)), Fraction(7, 2))
+
     def test_no_p_overwrite(self):
         # this is worth retesting, because quantile does not make a copy
         p0 = np.array([0, 0.75, 0.25, 0.5, 1.0])
diff --git a/numpy/lib/tests/test_histograms.py b/numpy/lib/tests/test_histograms.py
index c96b01d42..afaa526af 100644
--- a/numpy/lib/tests/test_histograms.py
+++ b/numpy/lib/tests/test_histograms.py
@@ -554,15 +554,11 @@ class TestHistogramOptimBinNums(object):
             return a / (a + b)
 
         ll = [[nbins_ratio(seed, size) for size in np.geomspace(start=10, stop=100, num=4).round().astype(int)]
-              for seed in range(256)]
+              for seed in range(10)]
 
         # the average difference between the two methods decreases as the dataset size increases.
-        assert_almost_equal(abs(np.mean(ll, axis=0) - 0.5),
-                            [0.1065248,
-                             0.0968844,
-                             0.0331818,
-                             0.0178057],
-                            decimal=3)
+        avg = abs(np.mean(ll, axis=0) - 0.5)
+        assert_almost_equal(avg, [0.15, 0.09, 0.08, 0.03], decimal=2)
 
     def test_simple_range(self):
         """
diff --git a/numpy/lib/tests/test_index_tricks.py b/numpy/lib/tests/test_index_tricks.py
index 028bba37d..2f7e97831 100644
--- a/numpy/lib/tests/test_index_tricks.py
+++ b/numpy/lib/tests/test_index_tricks.py
@@ -106,6 +106,9 @@ class TestRavelUnravelIndex(object):
                 np.ravel_multi_index(arr, (41, 7, 120, 36, 2706, 8, 6)),
                 [5627771580, 117259570957])
 
+        # test unravel_index for big indices (issue #9538)
+        assert_raises(ValueError, np.unravel_index, 1, (2**32-1, 2**31+1))
+
         # test overflow checking for too big array (issue #7546)
         dummy_arr = ([0],[0])
         half_max = np.iinfo(np.intp).max // 2
@@ -292,11 +295,16 @@ class TestIndexExpression(object):
 
 class TestIx_(object):
     def test_regression_1(self):
-        # Test empty inputs create outputs of indexing type, gh-5804
-        # Test both lists and arrays
-        for func in (range, np.arange):
-            a, = np.ix_(func(0))
-            assert_equal(a.dtype, np.intp)
+        # Test empty untyped inputs create outputs of indexing type, gh-5804
+        a, = np.ix_(range(0))
+        assert_equal(a.dtype, np.intp)
+
+        a, = np.ix_([])
+        assert_equal(a.dtype, np.intp)
+
+        # but if the type is specified, don't change it
+        a, = np.ix_(np.array([], dtype=np.float32))
+        assert_equal(a.dtype, np.float32)
 
     def test_shape_and_dtype(self):
         sizes = (4, 5, 3, 2)
diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py
index 9d6111e37..c6193d79f 100644
--- a/numpy/lib/tests/test_io.py
+++ b/numpy/lib/tests/test_io.py
@@ -561,6 +561,19 @@ class TestSaveTxt(object):
         s.seek(0)
         assert_equal(s.read(), utf8 + '\n')
 
+    @pytest.mark.parametrize("fmt", [u"%f", b"%f"])
+    @pytest.mark.parametrize("iotype", [StringIO, BytesIO])
+    def test_unicode_and_bytes_fmt(self, fmt, iotype):
+        # string type of fmt should not matter, see also gh-4053
+        a = np.array([1.])
+        s = iotype()
+        np.savetxt(s, a, fmt=fmt)
+        s.seek(0)
+        if iotype is StringIO:
+            assert_equal(s.read(), u"%f\n" % 1.)
+        else:
+            assert_equal(s.read(), b"%f\n" % 1.)
+
 
 class LoadTxtBase(object):
     def check_compressed(self, fopen, suffixes):
diff --git a/numpy/lib/tests/test_packbits.py b/numpy/lib/tests/test_packbits.py
index 00d5ca827..95a465c36 100644
--- a/numpy/lib/tests/test_packbits.py
+++ b/numpy/lib/tests/test_packbits.py
@@ -2,7 +2,8 @@ from __future__ import division, absolute_import, print_function
 
 import numpy as np
 from numpy.testing import assert_array_equal, assert_equal, assert_raises
-
+import pytest
+from itertools import chain
 
 def test_packbits():
     # Copied from the docstring.
@@ -50,8 +51,8 @@ def test_packbits_empty_with_axis():
                 assert_equal(b.dtype, np.uint8)
                 assert_equal(b.shape, out_shape)
 
-
-def test_packbits_large():
+@pytest.mark.parametrize('bitorder', ('little', 'big'))
+def test_packbits_large(bitorder):
     # test data large enough for 16 byte vectorization
     a = np.array([1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0,
                   0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1,
@@ -71,7 +72,7 @@ def test_packbits_large():
     a = a.repeat(3)
     for dtype in '?bBhHiIlLqQ':
         arr = np.array(a, dtype=dtype)
-        b = np.packbits(arr, axis=None)
+        b = np.packbits(arr, axis=None, bitorder=bitorder)
         assert_equal(b.dtype, np.uint8)
         r = [252, 127, 192, 3, 254, 7, 252, 0, 7, 31, 240, 0, 28, 1, 255, 252,
              113, 248, 3, 255, 192, 28, 15, 192, 28, 126, 0, 224, 127, 255,
@@ -81,9 +82,10 @@ def test_packbits_large():
              255, 224, 1, 255, 252, 126, 63, 0, 1, 192, 252, 14, 63, 0, 15,
              199, 252, 113, 255, 3, 128, 56, 252, 14, 7, 0, 113, 255, 255, 142, 56, 227,
              129, 248, 227, 129, 199, 31, 128]
-        assert_array_equal(b, r)
+        if bitorder == 'big':
+            assert_array_equal(b, r)
         # equal for size being multiple of 8
-        assert_array_equal(np.unpackbits(b)[:-4], a)
+        assert_array_equal(np.unpackbits(b, bitorder=bitorder)[:-4], a)
 
         # check last byte of different remainders (16 byte vectorization)
         b = [np.packbits(arr[:-i], axis=None)[-1] for i in range(1, 16)]
@@ -229,6 +231,20 @@ def test_unpackbits():
                                     [0, 0, 0, 0, 0, 1, 1, 1],
                                     [0, 0, 0, 1, 0, 1, 1, 1]]))
 
+def test_pack_unpack_order():
+    a = np.array([[2], [7], [23]], dtype=np.uint8)
+    b = np.unpackbits(a, axis=1)
+    assert_equal(b.dtype, np.uint8)
+    b_little = np.unpackbits(a, axis=1, bitorder='little')
+    b_big = np.unpackbits(a, axis=1, bitorder='big')
+    assert_array_equal(b, b_big)
+    assert_array_equal(a, np.packbits(b_little, axis=1, bitorder='little'))
+    assert_array_equal(b[:,::-1], b_little)
+    assert_array_equal(a, np.packbits(b_big, axis=1, bitorder='big'))
+    assert_raises(ValueError, np.unpackbits, a, bitorder='r')
+    assert_raises(TypeError, np.unpackbits, a, bitorder=10)
+
+
 
 def test_unpackbits_empty():
     a = np.empty((0,), dtype=np.uint8)
@@ -268,8 +284,7 @@ def test_unpackbits_large():
     assert_array_equal(np.packbits(np.unpackbits(d, axis=0), axis=0), d)
 
 
-def test_unpackbits_count():
-    # test complete invertibility of packbits and unpackbits with count
+class TestCount():
     x = np.array([
         [1, 0, 1, 0, 0, 1, 0],
         [0, 1, 1, 1, 0, 0, 0],
@@ -279,53 +294,85 @@ def test_unpackbits_count():
         [0, 0, 1, 1, 1, 0, 0],
         [0, 1, 0, 1, 0, 1, 0],
     ], dtype=np.uint8)
-
     padded1 = np.zeros(57, dtype=np.uint8)
     padded1[:49] = x.ravel()
+    padded1b = np.zeros(57, dtype=np.uint8)
+    padded1b[:49] = x[::-1].copy().ravel()
+    padded2 = np.zeros((9, 9), dtype=np.uint8)
+    padded2[:7, :7] = x
 
-    packed = np.packbits(x)
-    for count in range(58):
-        unpacked = np.unpackbits(packed, count=count)
+    @pytest.mark.parametrize('bitorder', ('little', 'big'))
+    @pytest.mark.parametrize('count', chain(range(58), range(-1, -57, -1)))
+    def test_roundtrip(self, bitorder, count):
+        if count < 0:
+            # one extra zero of padding
+            cutoff = count - 1
+        else:
+            cutoff = count
+        # test complete invertibility of packbits and unpackbits with count
+        packed = np.packbits(self.x, bitorder=bitorder)
+        unpacked = np.unpackbits(packed, count=count, bitorder=bitorder)
         assert_equal(unpacked.dtype, np.uint8)
-        assert_array_equal(unpacked, padded1[:count])
-    for count in range(-1, -57, -1):
-        unpacked = np.unpackbits(packed, count=count)
-        assert_equal(unpacked.dtype, np.uint8)
-        # count -1 because padded1 has 57 instead of 56 elements
-        assert_array_equal(unpacked, padded1[:count-1])
-    for kwargs in [{}, {'count': None}]:
+        assert_array_equal(unpacked, self.padded1[:cutoff])
+
+    @pytest.mark.parametrize('kwargs', [
+                    {}, {'count': None},
+                    ])
+    def test_count(self, kwargs):
+        packed = np.packbits(self.x)
         unpacked = np.unpackbits(packed, **kwargs)
         assert_equal(unpacked.dtype, np.uint8)
-        assert_array_equal(unpacked, padded1[:-1])
-    assert_raises(ValueError, np.unpackbits, packed, count=-57)
-
-    padded2 = np.zeros((9, 9), dtype=np.uint8)
-    padded2[:7, :7] = x
-
-    packed0 = np.packbits(x, axis=0)
-    packed1 = np.packbits(x, axis=1)
-    for count in range(10):
-        unpacked0 = np.unpackbits(packed0, axis=0, count=count)
+        assert_array_equal(unpacked, self.padded1[:-1])
+
+    @pytest.mark.parametrize('bitorder', ('little', 'big'))
+    # delta==-1 when count<0 because one extra zero of padding
+    @pytest.mark.parametrize('count', chain(range(8), range(-1, -9, -1)))
+    def test_roundtrip_axis(self, bitorder, count):
+        if count < 0:
+            # one extra zero of padding
+            cutoff = count - 1
+        else:
+            cutoff = count
+        packed0 = np.packbits(self.x, axis=0, bitorder=bitorder)
+        unpacked0 = np.unpackbits(packed0, axis=0, count=count,
+                                  bitorder=bitorder)
         assert_equal(unpacked0.dtype, np.uint8)
-        assert_array_equal(unpacked0, padded2[:count, :x.shape[1]])
-        unpacked1 = np.unpackbits(packed1, axis=1, count=count)
-        assert_equal(unpacked1.dtype, np.uint8)
-        assert_array_equal(unpacked1, padded2[:x.shape[1], :count])
-    for count in range(-1, -9, -1):
-        unpacked0 = np.unpackbits(packed0, axis=0, count=count)
-        assert_equal(unpacked0.dtype, np.uint8)
-        # count -1 because one extra zero of padding
-        assert_array_equal(unpacked0, padded2[:count-1, :x.shape[1]])
-        unpacked1 = np.unpackbits(packed1, axis=1, count=count)
+        assert_array_equal(unpacked0, self.padded2[:cutoff, :self.x.shape[1]])
+
+        packed1 = np.packbits(self.x, axis=1, bitorder=bitorder)
+        unpacked1 = np.unpackbits(packed1, axis=1, count=count,
+                                  bitorder=bitorder)
         assert_equal(unpacked1.dtype, np.uint8)
-        assert_array_equal(unpacked1, padded2[:x.shape[0], :count-1])
-    for kwargs in [{}, {'count': None}]:
+        assert_array_equal(unpacked1, self.padded2[:self.x.shape[0], :cutoff])
+
+    @pytest.mark.parametrize('kwargs', [
+                    {}, {'count': None},
+                    {'bitorder' : 'little'},
+                    {'bitorder': 'little', 'count': None},
+                    {'bitorder' : 'big'},
+                    {'bitorder': 'big', 'count': None},
+                    ])
+    def test_axis_count(self, kwargs):
+        packed0 = np.packbits(self.x, axis=0)
         unpacked0 = np.unpackbits(packed0, axis=0, **kwargs)
         assert_equal(unpacked0.dtype, np.uint8)
-        assert_array_equal(unpacked0, padded2[:-1, :x.shape[1]])
+        if kwargs.get('bitorder', 'big') == 'big':
+            assert_array_equal(unpacked0, self.padded2[:-1, :self.x.shape[1]])
+        else:
+            assert_array_equal(unpacked0[::-1, :], self.padded2[:-1, :self.x.shape[1]])
+
+        packed1 = np.packbits(self.x, axis=1)
         unpacked1 = np.unpackbits(packed1, axis=1, **kwargs)
         assert_equal(unpacked1.dtype, np.uint8)
-        assert_array_equal(unpacked1, padded2[:x.shape[0], :-1])
-    assert_raises(ValueError, np.unpackbits, packed0, axis=0, count=-9)
-    assert_raises(ValueError, np.unpackbits, packed1, axis=1, count=-9)
-
+        if kwargs.get('bitorder', 'big') == 'big':
+            assert_array_equal(unpacked1, self.padded2[:self.x.shape[0], :-1])
+        else:
+            assert_array_equal(unpacked1[:, ::-1], self.padded2[:self.x.shape[0], :-1])
+
+    def test_bad_count(self):
+        packed0 = np.packbits(self.x, axis=0)
+        assert_raises(ValueError, np.unpackbits, packed0, axis=0, count=-9)
+        packed1 = np.packbits(self.x, axis=1)
+        assert_raises(ValueError, np.unpackbits, packed1, axis=1, count=-9)
+        packed = np.packbits(self.x)
+        assert_raises(ValueError, np.unpackbits, packed, count=-57)
diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py
index 069693613..f713fb64d 100644
--- a/numpy/lib/tests/test_recfunctions.py
+++ b/numpy/lib/tests/test_recfunctions.py
@@ -214,6 +214,8 @@ class TestRecFunctions(object):
                      dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')])
         out = np.mean(structured_to_unstructured(b[['x', 'z']]), axis=-1)
         assert_equal(out, np.array([ 3. ,  5.5,  9. , 11. ]))
+        out = np.mean(structured_to_unstructured(b[['x']]), axis=-1)
+        assert_equal(out, np.array([ 1. ,  4. ,  7. , 10. ]))
 
         c = np.arange(20).reshape((4,5))
         out = unstructured_to_structured(c, a.dtype)
@@ -241,6 +243,15 @@ class TestRecFunctions(object):
         assert_(dd.base is d)
         assert_(ddd.base is d)
 
+        # including uniform fields with subarrays unpacked
+        d = np.array([(1, [2,  3], [[ 4,  5], [ 6,  7]]),
+                      (8, [9, 10], [[11, 12], [13, 14]])],
+                     dtype=[('x0', 'i4'), ('x1', ('i4', 2)), ('x2', ('i4', (2, 2)))])
+        dd = structured_to_unstructured(d)
+        ddd = unstructured_to_structured(dd, d.dtype)
+        assert_(dd.base is d)
+        assert_(ddd.base is d)
+
         # test that nested fields with identical names don't break anything
         point = np.dtype([('x', int), ('y', int)])
         triangle = np.dtype([('a', point), ('b', point), ('c', point)])