summaryrefslogtreecommitdiff
path: root/numpy
diff options
context:
space:
mode:
Diffstat (limited to 'numpy')
-rw-r--r--numpy/core/_add_newdocs.py11
-rw-r--r--numpy/core/_dtype.py16
-rw-r--r--numpy/core/_internal.py57
-rw-r--r--numpy/core/src/multiarray/array_method.c2
-rw-r--r--numpy/core/src/multiarray/arrayobject.c147
-rw-r--r--numpy/core/src/multiarray/convert_datatype.c95
-rw-r--r--numpy/core/src/multiarray/dtypemeta.c79
-rw-r--r--numpy/core/tests/test_casting_unittests.py24
-rw-r--r--numpy/core/tests/test_deprecations.py8
-rw-r--r--numpy/core/tests/test_dtype.py17
-rw-r--r--numpy/core/tests/test_multiarray.py132
-rw-r--r--numpy/core/tests/test_nditer.py12
-rw-r--r--numpy/core/tests/test_numeric.py64
-rw-r--r--numpy/lib/tests/test_recfunctions.py1
-rw-r--r--numpy/testing/tests/test_utils.py7
15 files changed, 471 insertions, 201 deletions
diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py
index 021895b84..fb9c30d93 100644
--- a/numpy/core/_add_newdocs.py
+++ b/numpy/core/_add_newdocs.py
@@ -1826,7 +1826,8 @@ add_newdoc('numpy.core.multiarray', 'promote_types',
Returns the data type with the smallest size and smallest scalar
kind to which both ``type1`` and ``type2`` may be safely cast.
- The returned data type is always in native byte order.
+ The returned data type is always considered "canonical", this mainly
+ means that the promoted dtype will always be in native byte order.
This function is symmetric, but rarely associative.
@@ -1844,6 +1845,8 @@ add_newdoc('numpy.core.multiarray', 'promote_types',
Notes
-----
+ Please see `numpy.result_type` for additional information about promotion.
+
.. versionadded:: 1.6.0
Starting in NumPy 1.9, promote_types function now returns a valid string
@@ -1852,6 +1855,12 @@ add_newdoc('numpy.core.multiarray', 'promote_types',
dtype, even if it wasn't long enough to store the max integer/float value
converted to a string.
+ .. versionchanged:: 1.23.0
+
+ NumPy now supports promotion for more structured dtypes. It will now
+ remove unnecessary padding from a structure dtype and promote included
+ fields individually.
+
See Also
--------
result_type, dtype, can_cast
diff --git a/numpy/core/_dtype.py b/numpy/core/_dtype.py
index c3a22b1c6..3db80c17e 100644
--- a/numpy/core/_dtype.py
+++ b/numpy/core/_dtype.py
@@ -237,6 +237,11 @@ def _struct_dict_str(dtype, includealignedflag):
return ret
+def _aligned_offset(offset, alignment):
+ # round up offset:
+ return - (-offset // alignment) * alignment
+
+
def _is_packed(dtype):
"""
Checks whether the structured data type in 'dtype'
@@ -249,12 +254,23 @@ def _is_packed(dtype):
Duplicates the C `is_dtype_struct_simple_unaligned_layout` function.
"""
+ align = dtype.isalignedstruct
+ max_alignment = 1
total_offset = 0
for name in dtype.names:
fld_dtype, fld_offset, title = _unpack_field(*dtype.fields[name])
+
+ if align:
+ total_offset = _aligned_offset(total_offset, fld_dtype.alignment)
+ max_alignment = max(max_alignment, fld_dtype.alignment)
+
if fld_offset != total_offset:
return False
total_offset += fld_dtype.itemsize
+
+ if align:
+ total_offset = _aligned_offset(total_offset, max_alignment)
+
if total_offset != dtype.itemsize:
return False
return True
diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py
index 8942955f6..9a1787dde 100644
--- a/numpy/core/_internal.py
+++ b/numpy/core/_internal.py
@@ -10,7 +10,7 @@ import sys
import platform
import warnings
-from .multiarray import dtype, array, ndarray
+from .multiarray import dtype, array, ndarray, promote_types
try:
import ctypes
except ImportError:
@@ -433,6 +433,61 @@ def _copy_fields(ary):
'formats': [dt.fields[name][0] for name in dt.names]}
return array(ary, dtype=copy_dtype, copy=True)
+def _promote_fields(dt1, dt2):
+ """ Perform type promotion for two structured dtypes.
+
+ Parameters
+ ----------
+ dt1 : structured dtype
+ First dtype.
+ dt2 : structured dtype
+ Second dtype.
+
+ Returns
+ -------
+ out : dtype
+ The promoted dtype
+
+ Notes
+ -----
+ If one of the inputs is aligned, the result will be. The titles of
+ both descriptors must match (point to the same field).
+ """
+ # Both must be structured and have the same names in the same order
+ if (dt1.names is None or dt2.names is None) or dt1.names != dt2.names:
+ raise TypeError("invalid type promotion")
+
+ # if both are identical, we can (maybe!) just return the same dtype.
+ identical = dt1 is dt2
+ new_fields = []
+ for name in dt1.names:
+ field1 = dt1.fields[name]
+ field2 = dt2.fields[name]
+ new_descr = promote_types(field1[0], field2[0])
+ identical = identical and new_descr is field1[0]
+
+ # Check that the titles match (if given):
+ if field1[2:] != field2[2:]:
+ raise TypeError("invalid type promotion")
+ if len(field1) == 2:
+ new_fields.append((name, new_descr))
+ else:
+ new_fields.append(((field1[2], name), new_descr))
+
+ res = dtype(new_fields, align=dt1.isalignedstruct or dt2.isalignedstruct)
+
+ # Might as well preserve identity (and metadata) if the dtype is identical
+ # and the itemsize, offsets are also unmodified. This could probably be
+ # sped up, but also probably just be removed entirely.
+ if identical and res.itemsize == dt1.itemsize:
+ for name in dt1.names:
+ if dt1.fields[name][1] != res.fields[name][1]:
+ return res # the dtype changed.
+ return dt1
+
+ return res
+
+
def _getfield_is_safe(oldtype, newtype, offset):
""" Checks safety of getfield for object arrays.
diff --git a/numpy/core/src/multiarray/array_method.c b/numpy/core/src/multiarray/array_method.c
index c937ff4cc..3450273b1 100644
--- a/numpy/core/src/multiarray/array_method.c
+++ b/numpy/core/src/multiarray/array_method.c
@@ -588,7 +588,7 @@ boundarraymethod__resolve_descripors(
return NULL;
}
else if (casting < 0) {
- return Py_BuildValue("iO", casting, Py_None, Py_None);
+ return Py_BuildValue("iOO", casting, Py_None, Py_None);
}
PyObject *result_tuple = PyTuple_New(nin + nout);
diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c
index a773b3e0b..a1f0e2d5b 100644
--- a/numpy/core/src/multiarray/arrayobject.c
+++ b/numpy/core/src/multiarray/arrayobject.c
@@ -1025,35 +1025,85 @@ static PyObject *
_void_compare(PyArrayObject *self, PyArrayObject *other, int cmp_op)
{
if (!(cmp_op == Py_EQ || cmp_op == Py_NE)) {
- PyErr_SetString(PyExc_ValueError,
+ PyErr_SetString(PyExc_TypeError,
"Void-arrays can only be compared for equality.");
return NULL;
}
- if (PyArray_HASFIELDS(self)) {
- PyObject *res = NULL, *temp, *a, *b;
- PyObject *key, *value, *temp2;
- PyObject *op;
- Py_ssize_t pos = 0;
+ if (PyArray_TYPE(other) != NPY_VOID) {
+ PyErr_SetString(PyExc_TypeError,
+ "Cannot compare structured or void to non-void arrays.");
+ return NULL;
+ }
+ if (PyArray_HASFIELDS(self) && PyArray_HASFIELDS(other)) {
+ PyArray_Descr *self_descr = PyArray_DESCR(self);
+ PyArray_Descr *other_descr = PyArray_DESCR(other);
+
+ /* Use promotion to decide whether the comparison is valid */
+ PyArray_Descr *promoted = PyArray_PromoteTypes(self_descr, other_descr);
+ if (promoted == NULL) {
+ PyErr_SetString(PyExc_TypeError,
+ "Cannot compare structured arrays unless they have a "
+ "common dtype. I.e. `np.result_type(arr1, arr2)` must "
+ "be defined.");
+ return NULL;
+ }
+ Py_DECREF(promoted);
+
npy_intp result_ndim = PyArray_NDIM(self) > PyArray_NDIM(other) ?
PyArray_NDIM(self) : PyArray_NDIM(other);
- op = (cmp_op == Py_EQ ? n_ops.logical_and : n_ops.logical_or);
- while (PyDict_Next(PyArray_DESCR(self)->fields, &pos, &key, &value)) {
- if (NPY_TITLE_KEY(key, value)) {
- continue;
- }
- a = array_subscript_asarray(self, key);
+ int field_count = PyTuple_GET_SIZE(self_descr->names);
+ if (field_count != PyTuple_GET_SIZE(other_descr->names)) {
+ PyErr_SetString(PyExc_TypeError,
+ "Cannot compare structured dtypes with different number of "
+ "fields. (unreachable error please report to NumPy devs)");
+ return NULL;
+ }
+
+ PyObject *op = (cmp_op == Py_EQ ? n_ops.logical_and : n_ops.logical_or);
+ PyObject *res = NULL;
+ for (int i = 0; i < field_count; ++i) {
+ PyObject *fieldname, *temp, *temp2;
+
+ fieldname = PyTuple_GET_ITEM(self_descr->names, i);
+ PyArrayObject *a = (PyArrayObject *)array_subscript_asarray(
+ self, fieldname);
if (a == NULL) {
Py_XDECREF(res);
return NULL;
}
- b = array_subscript_asarray(other, key);
+ fieldname = PyTuple_GET_ITEM(other_descr->names, i);
+ PyArrayObject *b = (PyArrayObject *)array_subscript_asarray(
+ other, fieldname);
if (b == NULL) {
Py_XDECREF(res);
Py_DECREF(a);
return NULL;
}
- temp = array_richcompare((PyArrayObject *)a,b,cmp_op);
+ /*
+ * If the fields were subarrays, the dimensions may have changed.
+ * In that case, the new shape (subarray part) must match exactly.
+ * (If this is 0, there is no subarray.)
+ */
+ int field_dims_a = PyArray_NDIM(a) - PyArray_NDIM(self);
+ int field_dims_b = PyArray_NDIM(b) - PyArray_NDIM(other);
+ if (field_dims_a != field_dims_b || (
+ field_dims_a != 0 && /* neither is subarray */
+ /* Compare only the added (subarray) dimensions: */
+ !PyArray_CompareLists(
+ PyArray_DIMS(a) + PyArray_NDIM(self),
+ PyArray_DIMS(b) + PyArray_NDIM(other),
+ field_dims_a))) {
+ PyErr_SetString(PyExc_TypeError,
+ "Cannot compare subarrays with different shapes. "
+ "(unreachable error, please report to NumPy devs.)");
+ Py_DECREF(a);
+ Py_DECREF(b);
+ Py_XDECREF(res);
+ return NULL;
+ }
+
+ temp = array_richcompare(a, (PyObject *)b, cmp_op);
Py_DECREF(a);
Py_DECREF(b);
if (temp == NULL) {
@@ -1138,7 +1188,24 @@ _void_compare(PyArrayObject *self, PyArrayObject *other, int cmp_op)
}
return res;
}
+ else if (PyArray_HASFIELDS(self) || PyArray_HASFIELDS(other)) {
+ PyErr_SetString(PyExc_TypeError,
+ "Cannot compare structured with unstructured void arrays. "
+ "(unreachable error, please report to NumPy devs.)");
+ return NULL;
+ }
else {
+ /*
+ * Since arrays absorb subarray descriptors, this path can only be
+ * reached when both arrays have unstructured voids "V<len>" dtypes.
+ */
+ if (PyArray_ITEMSIZE(self) != PyArray_ITEMSIZE(other)) {
+ PyErr_SetString(PyExc_TypeError,
+ "cannot compare unstructured voids of different length. "
+ "Use bytes to compare. "
+ "(This may return array of False in the future.)");
+ return NULL;
+ }
/* compare as a string. Assumes self and other have same descr->type */
return _strings_richcompare(self, other, cmp_op, 0);
}
@@ -1323,8 +1390,6 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op)
*/
if (PyArray_TYPE(self) == NPY_VOID) {
- int _res;
-
array_other = (PyArrayObject *)PyArray_FROM_O(other);
/*
* If not successful, indicate that the items cannot be compared
@@ -1341,28 +1406,7 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op)
return Py_NotImplemented;
}
- _res = PyArray_CheckCastSafety(
- NPY_EQUIV_CASTING,
- PyArray_DESCR(self), PyArray_DESCR(array_other), NULL);
- if (_res < 0) {
- PyErr_Clear();
- _res = 0;
- }
- if (_res == 0) {
- /* 2015-05-07, 1.10 */
- Py_DECREF(array_other);
- if (DEPRECATE_FUTUREWARNING(
- "elementwise == comparison failed and returning scalar "
- "instead; this will raise an error or perform "
- "elementwise comparison in the future.") < 0) {
- return NULL;
- }
- Py_INCREF(Py_False);
- return Py_False;
- }
- else {
- result = _void_compare(self, array_other, cmp_op);
- }
+ result = _void_compare(self, array_other, cmp_op);
Py_DECREF(array_other);
return result;
}
@@ -1378,8 +1422,6 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op)
*/
if (PyArray_TYPE(self) == NPY_VOID) {
- int _res;
-
array_other = (PyArrayObject *)PyArray_FROM_O(other);
/*
* If not successful, indicate that the items cannot be compared
@@ -1396,29 +1438,8 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op)
return Py_NotImplemented;
}
- _res = PyArray_CheckCastSafety(
- NPY_EQUIV_CASTING,
- PyArray_DESCR(self), PyArray_DESCR(array_other), NULL);
- if (_res < 0) {
- PyErr_Clear();
- _res = 0;
- }
- if (_res == 0) {
- /* 2015-05-07, 1.10 */
- Py_DECREF(array_other);
- if (DEPRECATE_FUTUREWARNING(
- "elementwise != comparison failed and returning scalar "
- "instead; this will raise an error or perform "
- "elementwise comparison in the future.") < 0) {
- return NULL;
- }
- Py_INCREF(Py_True);
- return Py_True;
- }
- else {
- result = _void_compare(self, array_other, cmp_op);
- Py_DECREF(array_other);
- }
+ result = _void_compare(self, array_other, cmp_op);
+ Py_DECREF(array_other);
return result;
}
diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c
index 7fd436a9d..79f918d2a 100644
--- a/numpy/core/src/multiarray/convert_datatype.c
+++ b/numpy/core/src/multiarray/convert_datatype.c
@@ -1013,7 +1013,7 @@ PyArray_FindConcatenationDescriptor(
npy_intp n, PyArrayObject **arrays, PyObject *requested_dtype)
{
if (requested_dtype == NULL) {
- return PyArray_LegacyResultType(n, arrays, 0, NULL);
+ return PyArray_ResultType(n, arrays, 0, NULL);
}
PyArray_DTypeMeta *common_dtype;
@@ -1074,7 +1074,13 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2)
PyArray_Descr *res;
/* Fast path for identical inputs (NOTE: This path preserves metadata!) */
- if (type1 == type2 && PyArray_ISNBO(type1->byteorder)) {
+ if (type1 == type2
+ /*
+ * Short-cut for legacy/builtin dtypes except void, since void has
+ * no reliable byteorder. Note: This path preserves metadata!
+ */
+ && NPY_DT_is_legacy(NPY_DTYPE(type1))
+ && PyArray_ISNBO(type1->byteorder) && type1->type_num != NPY_VOID) {
Py_INCREF(type1);
return type1;
}
@@ -2973,7 +2979,7 @@ nonstructured_to_structured_resolve_descriptors(
*view_offset = field_view_off - to_off;
}
}
- if (PyTuple_Size(given_descrs[1]->names) != 1) {
+ if (PyTuple_Size(given_descrs[1]->names) != 1 || *view_offset < 0) {
/*
* Assume that a view is impossible when there is more than one
* field. (Fields could overlap, but that seems weird...)
@@ -3279,8 +3285,7 @@ can_cast_fields_safety(
{
Py_ssize_t field_count = PyTuple_Size(from->names);
if (field_count != PyTuple_Size(to->names)) {
- /* TODO: This should be rejected! */
- return NPY_UNSAFE_CASTING;
+ return -1;
}
NPY_CASTING casting = NPY_NO_CASTING;
@@ -3292,18 +3297,41 @@ can_cast_fields_safety(
if (from_tup == NULL) {
return give_bad_field_error(from_key);
}
- PyArray_Descr *from_base = (PyArray_Descr*)PyTuple_GET_ITEM(from_tup, 0);
+ PyArray_Descr *from_base = (PyArray_Descr *) PyTuple_GET_ITEM(from_tup, 0);
- /*
- * TODO: This should use to_key (order), compare gh-15509 by
- * by Allan Haldane. And raise an error on failure.
- * (Fixing that may also requires fixing/changing promotion.)
- */
- PyObject *to_tup = PyDict_GetItem(to->fields, from_key);
+ /* Check whether the field names match */
+ PyObject *to_key = PyTuple_GET_ITEM(to->names, i);
+ PyObject *to_tup = PyDict_GetItem(to->fields, to_key);
if (to_tup == NULL) {
- return NPY_UNSAFE_CASTING;
+ return give_bad_field_error(from_key);
+ }
+ PyArray_Descr *to_base = (PyArray_Descr *) PyTuple_GET_ITEM(to_tup, 0);
+
+ int cmp = PyUnicode_Compare(from_key, to_key);
+ if (error_converting(cmp)) {
+ return -1;
+ }
+ if (cmp != 0) {
+ /* Field name mismatch, consider this at most SAFE. */
+ casting = PyArray_MinCastSafety(casting, NPY_SAFE_CASTING);
+ }
+
+ /* Also check the title (denote mismatch as SAFE only) */
+ PyObject *from_title = from_key;
+ PyObject *to_title = to_key;
+ if (PyTuple_GET_SIZE(from_tup) > 2) {
+ from_title = PyTuple_GET_ITEM(from_tup, 2);
+ }
+ if (PyTuple_GET_SIZE(to_tup) > 2) {
+ to_title = PyTuple_GET_ITEM(to_tup, 2);
+ }
+ cmp = PyObject_RichCompareBool(from_title, to_title, Py_EQ);
+ if (error_converting(cmp)) {
+ return -1;
+ }
+ if (!cmp) {
+ casting = PyArray_MinCastSafety(casting, NPY_SAFE_CASTING);
}
- PyArray_Descr *to_base = (PyArray_Descr*)PyTuple_GET_ITEM(to_tup, 0);
NPY_CASTING field_casting = PyArray_GetCastInfo(
from_base, to_base, NULL, &field_view_off);
@@ -3336,39 +3364,22 @@ can_cast_fields_safety(
*view_offset = NPY_MIN_INTP;
}
}
- if (*view_offset != 0) {
- /* If the calculated `view_offset` is not 0, it can only be "equiv" */
- return PyArray_MinCastSafety(casting, NPY_EQUIV_CASTING);
- }
- /*
- * If the itemsize (includes padding at the end), fields, or names
- * do not match, this cannot be a view and also not a "no" cast
- * (identical dtypes).
- * It may be possible that this can be relaxed in some cases.
- */
- if (from->elsize != to->elsize) {
- /*
- * The itemsize may mismatch even if all fields and formats match
- * (due to additional padding).
- */
- return PyArray_MinCastSafety(casting, NPY_EQUIV_CASTING);
+ if (*view_offset != 0 || from->elsize != to->elsize) {
+ /* Can never be considered "no" casting. */
+ casting = PyArray_MinCastSafety(casting, NPY_EQUIV_CASTING);
}
- int cmp = PyObject_RichCompareBool(from->fields, to->fields, Py_EQ);
- if (cmp != 1) {
- if (cmp == -1) {
- PyErr_Clear();
- }
- return PyArray_MinCastSafety(casting, NPY_EQUIV_CASTING);
+ /* The new dtype may have access outside the old one due to padding: */
+ if (*view_offset < 0) {
+ /* negative offsets would give indirect access before original dtype */
+ *view_offset = NPY_MIN_INTP;
}
- cmp = PyObject_RichCompareBool(from->names, to->names, Py_EQ);
- if (cmp != 1) {
- if (cmp == -1) {
- PyErr_Clear();
- }
- return PyArray_MinCastSafety(casting, NPY_EQUIV_CASTING);
+ if (from->elsize < to->elsize + *view_offset) {
+ /* new dtype has indirect access outside of the original dtype */
+ *view_offset = NPY_MIN_INTP;
}
+
return casting;
}
diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c
index bee8e8cdf..577478d2a 100644
--- a/numpy/core/src/multiarray/dtypemeta.c
+++ b/numpy/core/src/multiarray/dtypemeta.c
@@ -401,26 +401,81 @@ void_ensure_canonical(PyArray_Descr *self)
static PyArray_Descr *
void_common_instance(PyArray_Descr *descr1, PyArray_Descr *descr2)
{
- /*
- * We currently do not support promotion of void types unless they
- * are equivalent.
- */
- if (!PyArray_CanCastTypeTo(descr1, descr2, NPY_EQUIV_CASTING)) {
- if (descr1->subarray == NULL && descr1->names == NULL &&
- descr2->subarray == NULL && descr2->names == NULL) {
+ if (descr1->subarray == NULL && descr1->names == NULL &&
+ descr2->subarray == NULL && descr2->names == NULL) {
+ if (descr1->elsize != descr2->elsize) {
PyErr_SetString(PyExc_TypeError,
"Invalid type promotion with void datatypes of different "
"lengths. Use the `np.bytes_` datatype instead to pad the "
"shorter value with trailing zero bytes.");
+ return NULL;
}
- else {
+ Py_INCREF(descr1);
+ return descr1;
+ }
+
+ if (descr1->names != NULL && descr2->names != NULL) {
+ /* If both have fields promoting individual fields may be possible */
+ static PyObject *promote_fields_func = NULL;
+ npy_cache_import("numpy.core._internal", "_promote_fields",
+ &promote_fields_func);
+ if (promote_fields_func == NULL) {
+ return NULL;
+ }
+ PyObject *result = PyObject_CallFunctionObjArgs(promote_fields_func,
+ descr1, descr2, NULL);
+ if (result == NULL) {
+ return NULL;
+ }
+ if (!PyObject_TypeCheck(result, Py_TYPE(descr1))) {
+ PyErr_SetString(PyExc_RuntimeError,
+ "Internal NumPy error: `_promote_fields` did not return "
+ "a valid descriptor object.");
+ Py_DECREF(result);
+ return NULL;
+ }
+ return (PyArray_Descr *)result;
+ }
+ else if (descr1->subarray != NULL && descr2->subarray != NULL) {
+ int cmp = PyObject_RichCompareBool(
+ descr1->subarray->shape, descr2->subarray->shape, Py_EQ);
+ if (error_converting(cmp)) {
+ return NULL;
+ }
+ if (!cmp) {
PyErr_SetString(PyExc_TypeError,
- "invalid type promotion with structured datatype(s).");
+ "invalid type promotion with subarray datatypes "
+ "(shape mismatch).");
+ return NULL;
}
- return NULL;
+ PyArray_Descr *new_base = PyArray_PromoteTypes(
+ descr1->subarray->base, descr2->subarray->base);
+ if (new_base == NULL) {
+ return NULL;
+ }
+ /*
+ * If it is the same dtype and the container did not change, we might
+ * as well preserve identity and metadata. This could probably be
+ * changed.
+ */
+ if (descr1 == descr2 && new_base == descr1->subarray->base) {
+ Py_DECREF(new_base);
+ Py_INCREF(descr1);
+ return descr1;
+ }
+
+ PyArray_Descr *new_descr = PyArray_DescrNew(descr1);
+ if (new_descr == NULL) {
+ Py_DECREF(new_base);
+ return NULL;
+ }
+ Py_SETREF(new_descr->subarray->base, new_base);
+ return new_descr;
}
- Py_INCREF(descr1);
- return descr1;
+
+ PyErr_SetString(PyExc_TypeError,
+ "invalid type promotion with structured datatype(s).");
+ return NULL;
}
NPY_NO_EXPORT int
diff --git a/numpy/core/tests/test_casting_unittests.py b/numpy/core/tests/test_casting_unittests.py
index a57e46fd0..5c5ff55b4 100644
--- a/numpy/core/tests/test_casting_unittests.py
+++ b/numpy/core/tests/test_casting_unittests.py
@@ -715,23 +715,25 @@ class TestCasting:
@pytest.mark.parametrize(["to_dt", "expected_off"],
[ # Same as `from_dt` but with both fields shifted:
(np.dtype({"names": ["a", "b"], "formats": ["i4", "f4"],
- "offsets": [2, 6]}), -2),
+ "offsets": [0, 4]}), 2),
# Additional change of the names
- # TODO: Tests will need changing for order vs. name based casting:
- (np.dtype({"names": ["b", "a"], "formats": ["f4", "i4"],
- "offsets": [6, 2]}), -2),
- # Incompatible field offset change (offsets -2 and 0)
- (np.dtype({"names": ["b", "a"], "formats": ["f4", "i4"],
- "offsets": [6, 0]}), None)])
+ (np.dtype({"names": ["b", "a"], "formats": ["i4", "f4"],
+ "offsets": [0, 4]}), 2),
+ # Incompatible field offset change
+ (np.dtype({"names": ["b", "a"], "formats": ["i4", "f4"],
+ "offsets": [0, 6]}), None)])
def test_structured_field_offsets(self, to_dt, expected_off):
# This checks the cast-safety and view offset for swapped and "shifted"
# fields which are viewable
from_dt = np.dtype({"names": ["a", "b"],
"formats": ["i4", "f4"],
- "offsets": [0, 4]})
+ "offsets": [2, 6]})
cast = get_castingimpl(type(from_dt), type(to_dt))
safety, _, view_off = cast._resolve_descriptors((from_dt, to_dt))
- assert safety == Casting.equiv
+ if from_dt.names == to_dt.names:
+ assert safety == Casting.equiv
+ else:
+ assert safety == Casting.safe
# Shifting the original data pointer by -2 will align both by
# effectively adding 2 bytes of spacing before `from_dt`.
assert view_off == expected_off
@@ -742,7 +744,9 @@ class TestCasting:
("(1,1)i", "i", 0),
("(2,1)i", "(2,1)i", 0),
# field cases (field to field is tested explicitly also):
- ("i", dict(names=["a"], formats=["i"], offsets=[2]), -2),
+ # Not considered viewable, because a negative offset would allow
+ # may structured dtype to indirectly access invalid memory.
+ ("i", dict(names=["a"], formats=["i"], offsets=[2]), None),
(dict(names=["a"], formats=["i"], offsets=[2]), "i", 2),
# Currently considered not viewable, due to multiple fields
# even though they overlap (maybe we should not allow that?)
diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py
index c46b294eb..3dd60de91 100644
--- a/numpy/core/tests/test_deprecations.py
+++ b/numpy/core/tests/test_deprecations.py
@@ -185,14 +185,6 @@ class TestComparisonDeprecations(_DeprecationTestCase):
self.assert_deprecated(lambda: np.arange(2) == NotArray())
self.assert_deprecated(lambda: np.arange(2) != NotArray())
- struct1 = np.zeros(2, dtype="i4,i4")
- struct2 = np.zeros(2, dtype="i4,i4,i4")
-
- assert_warns(FutureWarning, lambda: struct1 == 1)
- assert_warns(FutureWarning, lambda: struct1 == struct2)
- assert_warns(FutureWarning, lambda: struct1 != 1)
- assert_warns(FutureWarning, lambda: struct1 != struct2)
-
def test_array_richcompare_legacy_weirdness(self):
# It doesn't really work to use assert_deprecated here, b/c part of
# the point of assert_deprecated is to check that when warnings are
diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py
index ebc6acb22..356b53df9 100644
--- a/numpy/core/tests/test_dtype.py
+++ b/numpy/core/tests/test_dtype.py
@@ -180,11 +180,11 @@ class TestBuiltin:
'formats': ['i4', 'f4'],
'offsets': [0, 4]})
y = np.dtype({'names': ['B', 'A'],
- 'formats': ['f4', 'i4'],
+ 'formats': ['i4', 'f4'],
'offsets': [4, 0]})
assert_equal(x == y, False)
- # But it is currently an equivalent cast:
- assert np.can_cast(x, y, casting="equiv")
+ # This is an safe cast (not equiv) due to the different names:
+ assert np.can_cast(x, y, casting="safe")
class TestRecord:
@@ -1163,6 +1163,9 @@ class TestDTypeMakeCanonical:
def test_make_canonical_hypothesis(self, dtype):
canonical = np.result_type(dtype)
self.check_canonical(dtype, canonical)
+ # result_type with two arguments should always give identical results:
+ two_arg_result = np.result_type(dtype, dtype)
+ assert np.can_cast(two_arg_result, canonical, casting="no")
@pytest.mark.slow
@hypothesis.given(
@@ -1177,6 +1180,10 @@ class TestDTypeMakeCanonical:
assert dtype_with_empty_space.itemsize == dtype.itemsize
canonicalized = np.result_type(dtype_with_empty_space)
self.check_canonical(dtype_with_empty_space, canonicalized)
+ # promotion with two arguments should always give identical results:
+ two_arg_result = np.promote_types(
+ dtype_with_empty_space, dtype_with_empty_space)
+ assert np.can_cast(two_arg_result, canonicalized, casting="no")
# Ensure that we also check aligned struct (check the opposite, in
# case hypothesis grows support for `align`. Then repeat the test:
@@ -1185,6 +1192,10 @@ class TestDTypeMakeCanonical:
assert dtype_with_empty_space.itemsize == dtype_aligned.itemsize
canonicalized = np.result_type(dtype_with_empty_space)
self.check_canonical(dtype_with_empty_space, canonicalized)
+ # promotion with two arguments should always give identical results:
+ two_arg_result = np.promote_types(
+ dtype_with_empty_space, dtype_with_empty_space)
+ assert np.can_cast(two_arg_result, canonicalized, casting="no")
class TestPickling:
diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py
index 9ba365211..cf071a640 100644
--- a/numpy/core/tests/test_multiarray.py
+++ b/numpy/core/tests/test_multiarray.py
@@ -31,6 +31,7 @@ from numpy.testing import (
)
from numpy.testing._private.utils import _no_tracing
from numpy.core.tests._locales import CommaDecimalPointLocale
+from numpy.lib.recfunctions import repack_fields
# Need to test an object that does not fully implement math interface
from datetime import timedelta, datetime
@@ -1210,7 +1211,8 @@ class TestStructured:
assert_equal(a == b, [False, True])
assert_equal(a != b, [True, False])
- # Check that broadcasting with a subarray works
+ # Check that broadcasting with a subarray works, including cases that
+ # require promotion to work:
a = np.array([[(0,)], [(1,)]], dtype=[('a', 'f8')])
b = np.array([(0,), (0,), (1,)], dtype=[('a', 'f8')])
assert_equal(a == b, [[True, True, False], [False, False, True]])
@@ -1233,27 +1235,56 @@ class TestStructured:
# Check that incompatible sub-array shapes don't result to broadcasting
x = np.zeros((1,), dtype=[('a', ('f4', (1, 2))), ('b', 'i1')])
y = np.zeros((1,), dtype=[('a', ('f4', (2,))), ('b', 'i1')])
- # This comparison invokes deprecated behaviour, and will probably
- # start raising an error eventually. What we really care about in this
- # test is just that it doesn't return True.
- with suppress_warnings() as sup:
- sup.filter(FutureWarning, "elementwise == comparison failed")
- assert_equal(x == y, False)
+ # The main importance is that it does not return True:
+ with pytest.raises(TypeError):
+ x == y
x = np.zeros((1,), dtype=[('a', ('f4', (2, 1))), ('b', 'i1')])
y = np.zeros((1,), dtype=[('a', ('f4', (2,))), ('b', 'i1')])
- # This comparison invokes deprecated behaviour, and will probably
- # start raising an error eventually. What we really care about in this
- # test is just that it doesn't return True.
- with suppress_warnings() as sup:
- sup.filter(FutureWarning, "elementwise == comparison failed")
- assert_equal(x == y, False)
+ # The main importance is that it does not return True:
+ with pytest.raises(TypeError):
+ x == y
- # Check that structured arrays that are different only in
- # byte-order work
+ def test_structured_comparisons_with_promotion(self):
+ # Check that structured arrays can be compared so long as their
+ # dtypes promote fine:
a = np.array([(5, 42), (10, 1)], dtype=[('a', '>i8'), ('b', '<f8')])
b = np.array([(5, 43), (10, 1)], dtype=[('a', '<i8'), ('b', '>f8')])
assert_equal(a == b, [False, True])
+ assert_equal(a != b, [True, False])
+
+ a = np.array([(5, 42), (10, 1)], dtype=[('a', '>f8'), ('b', '<f8')])
+ b = np.array([(5, 43), (10, 1)], dtype=[('a', '<i8'), ('b', '>i8')])
+ assert_equal(a == b, [False, True])
+ assert_equal(a != b, [True, False])
+
+ # Including with embedded subarray dtype (although subarray comparison
+ # itself may still be a bit weird and compare the raw data)
+ a = np.array([(5, 42), (10, 1)], dtype=[('a', '10>f8'), ('b', '5<f8')])
+ b = np.array([(5, 43), (10, 1)], dtype=[('a', '10<i8'), ('b', '5>i8')])
+ assert_equal(a == b, [False, True])
+ assert_equal(a != b, [True, False])
+
+ def test_void_comparison_failures(self):
+ # In principle, one could decide to return an array of False for some
+ # if comparisons are impossible. But right now we return TypeError
+ # when "void" dtype are involved.
+ x = np.zeros(3, dtype=[('a', 'i1')])
+ y = np.zeros(3)
+ # Cannot compare non-structured to structured:
+ with pytest.raises(TypeError):
+ x == y
+
+ # Added title prevents promotion, but casts are OK:
+ y = np.zeros(3, dtype=[(('title', 'a'), 'i1')])
+ assert np.can_cast(y.dtype, x.dtype)
+ with pytest.raises(TypeError):
+ x == y
+
+ x = np.zeros(3, dtype="V7")
+ y = np.zeros(3, dtype="V8")
+ with pytest.raises(TypeError):
+ x == y
def test_casting(self):
# Check that casting a structured array to change its byte order
@@ -1428,7 +1459,7 @@ class TestStructured:
assert_equal(testassign(arr, v1), ans)
assert_equal(testassign(arr, v2), ans)
assert_equal(testassign(arr, v3), ans)
- assert_raises(ValueError, lambda: testassign(arr, v4))
+ assert_raises(TypeError, lambda: testassign(arr, v4))
assert_equal(testassign(arr, v5), ans)
w[:] = 4
assert_equal(arr, np.array([(1,4),(1,4)], dtype=dt))
@@ -1463,6 +1494,75 @@ class TestStructured:
assert_raises(ValueError, lambda : a[['b','b']]) # field exists, but repeated
a[['b','c']] # no exception
+ def test_structured_cast_promotion_fieldorder(self):
+ # gh-15494
+ # dtypes with different field names are not promotable
+ A = ("a", "<i8")
+ B = ("b", ">i8")
+ ab = np.array([(1, 2)], dtype=[A, B])
+ ba = np.array([(1, 2)], dtype=[B, A])
+ assert_raises(TypeError, np.concatenate, ab, ba)
+ assert_raises(TypeError, np.result_type, ab.dtype, ba.dtype)
+ assert_raises(TypeError, np.promote_types, ab.dtype, ba.dtype)
+
+ # dtypes with same field names/order but different memory offsets
+ # and byte-order are promotable to packed nbo.
+ assert_equal(np.promote_types(ab.dtype, ba[['a', 'b']].dtype),
+ repack_fields(ab.dtype.newbyteorder('N')))
+
+ # gh-13667
+ # dtypes with different fieldnames but castable field types are castable
+ assert_equal(np.can_cast(ab.dtype, ba.dtype), True)
+ assert_equal(ab.astype(ba.dtype).dtype, ba.dtype)
+ assert_equal(np.can_cast('f8,i8', [('f0', 'f8'), ('f1', 'i8')]), True)
+ assert_equal(np.can_cast('f8,i8', [('f1', 'f8'), ('f0', 'i8')]), True)
+ assert_equal(np.can_cast('f8,i8', [('f1', 'i8'), ('f0', 'f8')]), False)
+ assert_equal(np.can_cast('f8,i8', [('f1', 'i8'), ('f0', 'f8')],
+ casting='unsafe'), True)
+
+ ab[:] = ba # make sure assignment still works
+
+ # tests of type-promotion of corresponding fields
+ dt1 = np.dtype([("", "i4")])
+ dt2 = np.dtype([("", "i8")])
+ assert_equal(np.promote_types(dt1, dt2), np.dtype([('f0', 'i8')]))
+ assert_equal(np.promote_types(dt2, dt1), np.dtype([('f0', 'i8')]))
+ assert_raises(TypeError, np.promote_types, dt1, np.dtype([("", "V3")]))
+ assert_equal(np.promote_types('i4,f8', 'i8,f4'),
+ np.dtype([('f0', 'i8'), ('f1', 'f8')]))
+ # test nested case
+ dt1nest = np.dtype([("", dt1)])
+ dt2nest = np.dtype([("", dt2)])
+ assert_equal(np.promote_types(dt1nest, dt2nest),
+ np.dtype([('f0', np.dtype([('f0', 'i8')]))]))
+
+ # note that offsets are lost when promoting:
+ dt = np.dtype({'names': ['x'], 'formats': ['i4'], 'offsets': [8]})
+ a = np.ones(3, dtype=dt)
+ assert_equal(np.concatenate([a, a]).dtype, np.dtype([('x', 'i4')]))
+
+ @pytest.mark.parametrize("dtype_dict", [
+ dict(names=["a", "b"], formats=["i4", "f"], itemsize=100),
+ dict(names=["a", "b"], formats=["i4", "f"],
+ offsets=[0, 12])])
+ @pytest.mark.parametrize("align", [True, False])
+ def test_structured_promotion_packs(self, dtype_dict, align):
+ # Structured dtypes are packed when promoted (we consider the packed
+ # form to be "canonical"), so tere is no extra padding.
+ dtype = np.dtype(dtype_dict, align=align)
+ # Remove non "canonical" dtype options:
+ dtype_dict.pop("itemsize", None)
+ dtype_dict.pop("offsets", None)
+ expected = np.dtype(dtype_dict, align=align)
+
+ res = np.promote_types(dtype, dtype)
+ assert res.itemsize == expected.itemsize
+ assert res.fields == expected.fields
+
+ # But the "expected" one, should just be returned unchanged:
+ res = np.promote_types(expected, expected)
+ assert res is expected
+
def test_structured_asarray_is_view(self):
# A scalar viewing an array preserves its view even when creating a
# new array. This test documents behaviour, it may not be the best
diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py
index d96c14e54..b43bc50e9 100644
--- a/numpy/core/tests/test_nditer.py
+++ b/numpy/core/tests/test_nditer.py
@@ -1990,13 +1990,13 @@ def test_iter_buffered_cast_structured_type_failure_with_cleanup():
a = np.array([(1, 2, 3), (4, 5, 6)], dtype=sdt1)
for intent in ["readwrite", "readonly", "writeonly"]:
- # If the following assert fails, the place where the error is raised
- # within nditer may change. That is fine, but it may make sense for
- # a new (hard to design) test to replace it. The `simple_arr` is
- # designed to require a multi-step cast (due to having fields).
- assert np.can_cast(a.dtype, sdt2, casting="unsafe")
+ # This test was initially designed to test an error at a different
+ # place, but will now raise earlier to to the cast not being possible:
+ # `assert np.can_cast(a.dtype, sdt2, casting="unsafe")` fails.
+ # Without a faulty DType, there is probably no reliable
+ # way to get the initial tested behaviour.
simple_arr = np.array([1, 2], dtype="i,i") # requires clean up
- with pytest.raises(ValueError):
+ with pytest.raises(TypeError):
nditer((simple_arr, a), ['buffered', 'refs_ok'], [intent, intent],
casting='unsafe', op_dtypes=["f,f", sdt2])
diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py
index a7f5df472..0b03c6576 100644
--- a/numpy/core/tests/test_numeric.py
+++ b/numpy/core/tests/test_numeric.py
@@ -932,9 +932,28 @@ class TestTypes:
# Promote with object:
assert_equal(promote_types('O', S+'30'), np.dtype('O'))
+ @pytest.mark.parametrize(["dtype1", "dtype2"],
+ [[np.dtype("V6"), np.dtype("V10")], # mismatch shape
+ # Mismatching names:
+ [np.dtype([("name1", "i8")]), np.dtype([("name2", "i8")])],
+ ])
+ def test_invalid_void_promotion(self, dtype1, dtype2):
+ with pytest.raises(TypeError):
+ np.promote_types(dtype1, dtype2)
+
+ @pytest.mark.parametrize(["dtype1", "dtype2"],
+ [[np.dtype("V10"), np.dtype("V10")],
+ [np.dtype([("name1", "i8")]),
+ np.dtype([("name1", np.dtype("i8").newbyteorder())])],
+ [np.dtype("i8,i8"), np.dtype("i8,>i8")],
+ [np.dtype("i8,i8"), np.dtype("i4,i4")],
+ ])
+ def test_valid_void_promotion(self, dtype1, dtype2):
+ assert np.promote_types(dtype1, dtype2) == dtype1
+
@pytest.mark.parametrize("dtype",
- list(np.typecodes["All"]) +
- ["i,i", "S3", "S100", "U3", "U100", rational])
+ list(np.typecodes["All"]) +
+ ["i,i", "10i", "S3", "S100", "U3", "U100", rational])
def test_promote_identical_types_metadata(self, dtype):
# The same type passed in twice to promote types always
# preserves metadata
@@ -951,14 +970,14 @@ class TestTypes:
return
res = np.promote_types(dtype, dtype)
- if res.char in "?bhilqpBHILQPefdgFDGOmM" or dtype.type is rational:
- # Metadata is lost for simple promotions (they create a new dtype)
+
+ # Metadata is (currently) generally lost on byte-swapping (except for
+ # unicode.
+ if dtype.char != "U":
assert res.metadata is None
else:
assert res.metadata == metadata
- if dtype.kind != "V":
- # the result is native (except for structured void)
- assert res.isnative
+ assert res.isnative
@pytest.mark.slow
@pytest.mark.filterwarnings('ignore:Promotion of numbers:FutureWarning')
@@ -987,8 +1006,10 @@ class TestTypes:
# Promotion failed, this test only checks metadata
return
- if res.char in "?bhilqpBHILQPefdgFDGOmM" or res.type is rational:
- # All simple types lose metadata (due to using promotion table):
+ if res.char not in "USV" or res.names is not None or res.shape != ():
+ # All except string dtypes (and unstructured void) lose metadata
+ # on promotion (unless both dtypes are identical).
+ # At some point structured ones did not, but were restrictive.
assert res.metadata is None
elif res == dtype1:
# If one result is the result, it is usually returned unchanged:
@@ -1008,32 +1029,9 @@ class TestTypes:
dtype1 = dtype1.newbyteorder()
assert dtype1.metadata == metadata1
res_bs = np.promote_types(dtype1, dtype2)
- if res_bs.names is not None:
- # Structured promotion doesn't remove byteswap:
- assert res_bs.newbyteorder() == res
- else:
- assert res_bs == res
+ assert res_bs == res
assert res_bs.metadata == res.metadata
- @pytest.mark.parametrize(["dtype1", "dtype2"],
- [[np.dtype("V6"), np.dtype("V10")],
- [np.dtype([("name1", "i8")]), np.dtype([("name2", "i8")])],
- [np.dtype("i8,i8"), np.dtype("i4,i4")],
- ])
- def test_invalid_void_promotion(self, dtype1, dtype2):
- # Mainly test structured void promotion, which currently allows
- # byte-swapping, but nothing else:
- with pytest.raises(TypeError):
- np.promote_types(dtype1, dtype2)
-
- @pytest.mark.parametrize(["dtype1", "dtype2"],
- [[np.dtype("V10"), np.dtype("V10")],
- [np.dtype([("name1", "<i8")]), np.dtype([("name1", ">i8")])],
- [np.dtype("i8,i8"), np.dtype("i8,>i8")],
- ])
- def test_valid_void_promotion(self, dtype1, dtype2):
- assert np.promote_types(dtype1, dtype2) is dtype1
-
def test_can_cast(self):
assert_(np.can_cast(np.int32, np.int64))
assert_(np.can_cast(np.float64, complex))
diff --git a/numpy/lib/tests/test_recfunctions.py b/numpy/lib/tests/test_recfunctions.py
index 2f3c14df3..9b2506a7c 100644
--- a/numpy/lib/tests/test_recfunctions.py
+++ b/numpy/lib/tests/test_recfunctions.py
@@ -835,7 +835,6 @@ class TestJoinBy:
b = np.ones(3, dtype=[('c', 'u1'), ('b', 'f4'), ('a', 'i4')])
assert_raises(ValueError, join_by, ['a', 'b', 'b'], a, b)
- @pytest.mark.xfail(reason="See comment at gh-9343")
def test_same_name_different_dtypes_key(self):
a_dtype = np.dtype([('key', 'S5'), ('value', '<f4')])
b_dtype = np.dtype([('key', 'S10'), ('value', '<f4')])
diff --git a/numpy/testing/tests/test_utils.py b/numpy/testing/tests/test_utils.py
index 1aaa8f559..4026a7a14 100644
--- a/numpy/testing/tests/test_utils.py
+++ b/numpy/testing/tests/test_utils.py
@@ -151,14 +151,13 @@ class TestArrayEqual(_GenericTest):
self._test_equal(a, b)
- c = np.empty(2, [('floupipi', float), ('floupa', float)])
+ c = np.empty(2, [('floupipi', float),
+ ('floupi', float), ('floupa', float)])
c['floupipi'] = a['floupi'].copy()
c['floupa'] = a['floupa'].copy()
- with suppress_warnings() as sup:
- l = sup.record(FutureWarning, message="elementwise == ")
+ with pytest.raises(TypeError):
self._test_not_equal(c, b)
- assert_equal(len(l), 1)
def test_masked_nan_inf(self):
# Regression test for gh-11121
View Lorry Controller Status