diff options
Diffstat (limited to 'numpy/core')
64 files changed, 769 insertions, 10076 deletions
diff --git a/numpy/core/_methods.py b/numpy/core/_methods.py index 80e12298c..d3c150ac8 100644 --- a/numpy/core/_methods.py +++ b/numpy/core/_methods.py @@ -5,34 +5,51 @@ from numpy.core import multiarray as mu from numpy.core import umath as um from numpy.core.numeric import asanyarray -def _amax(a, axis=None, out=None, skipna=False, keepdims=False): +def _amax(a, axis=None, out=None, keepdims=False): return um.maximum.reduce(a, axis=axis, - out=out, skipna=skipna, keepdims=keepdims) + out=out, keepdims=keepdims) -def _amin(a, axis=None, out=None, skipna=False, keepdims=False): +def _amin(a, axis=None, out=None, keepdims=False): return um.minimum.reduce(a, axis=axis, - out=out, skipna=skipna, keepdims=keepdims) + out=out, keepdims=keepdims) -def _sum(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): +def _sum(a, axis=None, dtype=None, out=None, keepdims=False): return um.add.reduce(a, axis=axis, dtype=dtype, - out=out, skipna=skipna, keepdims=keepdims) + out=out, keepdims=keepdims) -def _prod(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): +def _prod(a, axis=None, dtype=None, out=None, keepdims=False): return um.multiply.reduce(a, axis=axis, dtype=dtype, - out=out, skipna=skipna, keepdims=keepdims) - -def _mean(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): + out=out, keepdims=keepdims) + +def _any(a, axis=None, dtype=None, out=None, keepdims=False): + return um.logical_or.reduce(a, axis=axis, dtype=dtype, out=out, + keepdims=keepdims) + +def _all(a, axis=None, dtype=None, out=None, keepdims=False): + return um.logical_and.reduce(a, axis=axis, dtype=dtype, out=out, + keepdims=keepdims) + +def _count_reduce_items(arr, axis): + if axis is None: + axis = tuple(xrange(arr.ndim)) + if not isinstance(axis, tuple): + axis = (axis,) + items = 1 + for ax in axis: + items *= arr.shape[ax] + return items + +def _mean(a, axis=None, dtype=None, out=None, keepdims=False): arr = asanyarray(a) # Upgrade bool, unsigned int, and int to float64 if dtype is None and arr.dtype.kind in ['b','u','i']: ret = um.add.reduce(arr, axis=axis, dtype='f8', - out=out, skipna=skipna, keepdims=keepdims) + out=out, keepdims=keepdims) else: ret = um.add.reduce(arr, axis=axis, dtype=dtype, - out=out, skipna=skipna, keepdims=keepdims) - rcount = mu.count_reduce_items(arr, axis=axis, - skipna=skipna, keepdims=keepdims) + out=out, keepdims=keepdims) + rcount = _count_reduce_items(arr, axis) if isinstance(ret, mu.ndarray): ret = um.true_divide(ret, rcount, out=ret, casting='unsafe', subok=False) @@ -41,18 +58,15 @@ def _mean(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): return ret def _var(a, axis=None, dtype=None, out=None, ddof=0, - skipna=False, keepdims=False): + keepdims=False): arr = asanyarray(a) # First compute the mean, saving 'rcount' for reuse later if dtype is None and arr.dtype.kind in ['b','u','i']: - arrmean = um.add.reduce(arr, axis=axis, dtype='f8', - skipna=skipna, keepdims=True) + arrmean = um.add.reduce(arr, axis=axis, dtype='f8', keepdims=True) else: - arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, - skipna=skipna, keepdims=True) - rcount = mu.count_reduce_items(arr, axis=axis, - skipna=skipna, keepdims=True) + arrmean = um.add.reduce(arr, axis=axis, dtype=dtype, keepdims=True) + rcount = _count_reduce_items(arr, axis) if isinstance(arrmean, mu.ndarray): arrmean = um.true_divide(arrmean, rcount, out=arrmean, casting='unsafe', subok=False) @@ -69,8 +83,7 @@ def _var(a, axis=None, dtype=None, out=None, ddof=0, x = um.multiply(x, x, out=x) # add.reduce((arr - arrmean) ** 2, axis) - ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, - skipna=skipna, keepdims=keepdims) + ret = um.add.reduce(x, axis=axis, dtype=dtype, out=out, keepdims=keepdims) # add.reduce((arr - arrmean) ** 2, axis) / (n - ddof) if not keepdims and isinstance(rcount, mu.ndarray): @@ -84,10 +97,9 @@ def _var(a, axis=None, dtype=None, out=None, ddof=0, return ret -def _std(a, axis=None, dtype=None, out=None, ddof=0, - skipna=False, keepdims=False): +def _std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False): ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof, - skipna=skipna, keepdims=keepdims) + keepdims=keepdims) if isinstance(ret, mu.ndarray): ret = um.sqrt(ret, out=ret) diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index 35c82a21d..19be452e5 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -15,7 +15,7 @@ __docformat__ = 'restructuredtext' import sys import numerictypes as _nt from umath import maximum, minimum, absolute, not_equal, isnan, isinf -from multiarray import format_longfloat, datetime_as_string, datetime_data, isna +from multiarray import format_longfloat, datetime_as_string, datetime_data from fromnumeric import ravel @@ -29,7 +29,6 @@ _float_output_suppress_small = False _line_width = 75 _nan_str = 'nan' _inf_str = 'inf' -_na_str = 'NA' _formatter = None # formatting function for array elements if sys.version_info[0] >= 3: @@ -37,7 +36,7 @@ if sys.version_info[0] >= 3: def set_printoptions(precision=None, threshold=None, edgeitems=None, linewidth=None, suppress=None, - nanstr=None, infstr=None, nastr=None, + nanstr=None, infstr=None, formatter=None): """ Set printing options. @@ -65,8 +64,6 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, String representation of floating point not-a-number (default nan). infstr : str, optional String representation of floating point infinity (default inf). - nastr : str, optional - String representation of NA missing value (default NA). formatter : dict of callables, optional If not None, the keys should indicate the type(s) that the respective formatting function applies to. Callables should return a string. @@ -144,7 +141,7 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, global _summaryThreshold, _summaryEdgeItems, _float_output_precision, \ _line_width, _float_output_suppress_small, _nan_str, _inf_str, \ - _na_str, _formatter + _formatter if linewidth is not None: _line_width = linewidth if threshold is not None: @@ -159,8 +156,6 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, _nan_str = nanstr if infstr is not None: _inf_str = infstr - if nastr is not None: - _na_str = nastr _formatter = formatter def get_printoptions(): @@ -195,7 +190,6 @@ def get_printoptions(): suppress=_float_output_suppress_small, nanstr=_nan_str, infstr=_inf_str, - nastr=_na_str, formatter=_formatter) return d @@ -219,19 +213,14 @@ def _leading_trailing(a): return b def _boolFormatter(x): - if isna(x): - return str(x).replace('NA', _na_str, 1) - elif x: + if x: return ' True' else: return 'False' def repr_format(x): - if isna(x): - return str(x).replace('NA', _na_str, 1) - else: - return repr(x) + return repr(x) def _array2string(a, max_line_width, precision, suppress_small, separator=' ', prefix="", formatter=None): @@ -437,20 +426,17 @@ def array2string(a, max_line_width=None, precision=None, if a.shape == (): x = a.item() - if isna(x): - lst = str(x).replace('NA', _na_str, 1) - else: - try: - lst = a._format(x) - msg = "The `_format` attribute is deprecated in Numpy " \ - "2.0 and will be removed in 2.1. Use the " \ - "`formatter` kw instead." - import warnings - warnings.warn(msg, DeprecationWarning) - except AttributeError: - if isinstance(x, tuple): - x = _convert_arrays(x) - lst = style(x) + try: + lst = a._format(x) + msg = "The `_format` attribute is deprecated in Numpy " \ + "2.0 and will be removed in 2.1. Use the " \ + "`formatter` kw instead." + import warnings + warnings.warn(msg, DeprecationWarning) + except AttributeError: + if isinstance(x, tuple): + x = _convert_arrays(x) + lst = style(x) elif reduce(product, a.shape) == 0: # treat as a null array if any of shape elements == 0 lst = "[]" @@ -553,17 +539,15 @@ class FloatFormat(object): import numeric as _nc errstate = _nc.seterr(all='ignore') try: - special = isnan(data) | isinf(data) | isna(data) - special[isna(data)] = False + special = isnan(data) | isinf(data) valid = not_equal(data, 0) & ~special - valid[isna(data)] = False non_zero = absolute(data.compress(valid)) if len(non_zero) == 0: max_val = 0. min_val = 0. else: - max_val = maximum.reduce(non_zero, skipna=True) - min_val = minimum.reduce(non_zero, skipna=True) + max_val = maximum.reduce(non_zero) + min_val = minimum.reduce(non_zero) if max_val >= 1.e8: self.exp_format = True if not self.suppress_small and (min_val < 0.0001 @@ -594,8 +578,7 @@ class FloatFormat(object): if _nc.any(special): self.max_str_len = max(self.max_str_len, len(_nan_str), - len(_inf_str)+1, - len(_na_str)) + len(_inf_str)+1) if self.sign: format = '%#+' else: @@ -609,9 +592,7 @@ class FloatFormat(object): import numeric as _nc err = _nc.seterr(invalid='ignore') try: - if isna(x): - return self.special_fmt % (str(x).replace('NA', _na_str, 1),) - elif isnan(x): + if isnan(x): if self.sign: return self.special_fmt % ('+' + _nan_str,) else: @@ -654,8 +635,8 @@ _MININT = -sys.maxint-1 class IntegerFormat(object): def __init__(self, data): try: - max_str_len = max(len(str(maximum.reduce(data, skipna=True))), - len(str(minimum.reduce(data, skipna=True)))) + max_str_len = max(len(str(maximum.reduce(data))), + len(str(minimum.reduce(data)))) self.format = '%' + str(max_str_len) + 'd' except (TypeError, NotImplementedError): # if reduce(data) fails, this instance will not be called, just @@ -666,9 +647,7 @@ class IntegerFormat(object): pass def __call__(self, x): - if isna(x): - return str(x).replace('NA', _na_str, 1) - elif _MININT < x < _MAXINT: + if _MININT < x < _MAXINT: return self.format % x else: return "%s" % x @@ -681,9 +660,7 @@ class LongFloatFormat(object): self.sign = sign def __call__(self, x): - if isna(x): - return str(x).replace('NA', _na_str, 1) - elif isnan(x): + if isnan(x): if self.sign: return '+' + _nan_str else: @@ -711,12 +688,9 @@ class LongComplexFormat(object): self.imag_format = LongFloatFormat(precision, sign=True) def __call__(self, x): - if isna(x): - return str(x).replace('NA', _na_str, 1) - else: - r = self.real_format(x.real) - i = self.imag_format(x.imag) - return r + i + 'j' + r = self.real_format(x.real) + i = self.imag_format(x.imag) + return r + i + 'j' class ComplexFormat(object): @@ -726,17 +700,14 @@ class ComplexFormat(object): sign=True) def __call__(self, x): - if isna(x): - return str(x).replace('NA', _na_str, 1) + r = self.real_format(x.real, strip_zeros=False) + i = self.imag_format(x.imag, strip_zeros=False) + if not self.imag_format.exp_format: + z = i.rstrip('0') + i = z + 'j' + ' '*(len(i)-len(z)) else: - r = self.real_format(x.real, strip_zeros=False) - i = self.imag_format(x.imag, strip_zeros=False) - if not self.imag_format.exp_format: - z = i.rstrip('0') - i = z + 'j' + ' '*(len(i)-len(z)) - else: - i = i + 'j' - return r + i + i = i + 'j' + return r + i class DatetimeFormat(object): def __init__(self, x, unit=None, @@ -761,13 +732,10 @@ class DatetimeFormat(object): self.casting = casting def __call__(self, x): - if isna(x): - return str(x).replace('NA', _na_str, 1) - else: - return "'%s'" % datetime_as_string(x, - unit=self.unit, - timezone=self.timezone, - casting=self.casting) + return "'%s'" % datetime_as_string(x, + unit=self.unit, + timezone=self.timezone, + casting=self.casting) class TimedeltaFormat(object): def __init__(self, data): @@ -778,8 +746,5 @@ class TimedeltaFormat(object): self.format = '%' + str(max_str_len) + 'd' def __call__(self, x): - if isna(x): - return str(x).replace('NA', _na_str, 1) - else: - return self.format % x.astype('i8') + return self.format % x.astype('i8') diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index 05f382240..7973fd373 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -43,8 +43,6 @@ API_FILES = [join('multiarray', 'array_assign_array.c'), join('multiarray', 'iterators.c'), join('multiarray', 'methods.c'), join('multiarray', 'multiarraymodule.c'), - join('multiarray', 'na_mask.c'), - join('multiarray', 'na_object.c'), join('multiarray', 'nditer_api.c'), join('multiarray', 'nditer_constr.c'), join('multiarray', 'nditer_pywrap.c'), diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index 15b868e23..befa099ae 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -14,12 +14,10 @@ exception, so it should hopefully not get unnoticed). multiarray_global_vars = { 'NPY_NUMUSERTYPES': 7, - 'Npy_NA': 299, } multiarray_global_vars_types = { 'NPY_NUMUSERTYPES': 'int', - 'Npy_NA': 'PyObject *', } multiarray_scalar_bool_values = { @@ -71,7 +69,6 @@ multiarray_types_api = { 'PyHalfArrType_Type': 217, 'NpyIter_Type': 218, # End 1.6 API - 'NpyNA_Type': 281, } #define NPY_NUMUSERTYPES (*(int *)PyArray_API[6]) @@ -320,32 +317,14 @@ multiarray_funcs_api = { 'PyArray_ConvertClipmodeSequence': 279, 'PyArray_MatrixProduct2': 280, # End 1.6 API - 'NpyIter_GetFirstMaskNAOp': 282, - 'NpyIter_GetMaskNAIndexArray': 283, - 'NpyIter_IsFirstVisit': 284, - 'PyArray_SetBaseObject': 285, - 'PyArray_HasNASupport': 286, - 'PyArray_ContainsNA': 287, - 'PyArray_AllocateMaskNA': 288, - 'PyArray_CreateSortedStridePerm': 289, - 'PyArray_AssignZero': 290, - 'PyArray_AssignOne': 291, - 'PyArray_AssignNA': 292, - 'PyArray_AssignMaskNA': 293, - 'PyArray_AssignRawScalar': 294, - 'PyArray_AssignArray': 295, - 'PyArray_ReduceWrapper': 296, - 'PyArray_RemoveAxesInPlace': 297, - 'PyArray_DebugPrint': 298, - 'NpyNA_GetDType': 300, - 'NpyNA_IsMultiNA': 301, - 'NpyNA_GetPayload': 302, - 'NpyNA_FromObject': 303, - 'NpyNA_FromDTypeAndPayload': 304, - 'PyArray_AllowNAConverter': 305, - 'PyArray_OutputAllowNAConverter': 306, - 'PyArray_FailUnlessWriteable': 307, - 'PyArray_SetUpdateIfCopyBase': 308, + 'NpyIter_IsFirstVisit': 281, + 'PyArray_SetBaseObject': 282, + 'PyArray_ReduceWrapper': 283, + 'PyArray_CreateSortedStridePerm': 284, + 'PyArray_RemoveAxesInPlace': 285, + 'PyArray_DebugPrint': 286, + 'PyArray_FailUnlessWriteable': 287, + 'PyArray_SetUpdateIfCopyBase': 288, } ufunc_types_api = { diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index 2b108bcf9..3e89079dd 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -1419,7 +1419,7 @@ def clip(a, a_min, a_max, out=None): return clip(a_min, a_max, out) -def sum(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): +def sum(a, axis=None, dtype=None, out=None, keepdims=False): """ Sum of array elements over a given axis. @@ -1449,9 +1449,6 @@ def sum(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): (the shape of `a` with `axis` removed, i.e., ``numpy.delete(a.shape, axis)``). Its type is preserved. See `doc.ufuncs` (Section "Output arguments") for more details. - skipna : bool, optional - If this is set to True, skips any NA values during summation - instead of propagating them. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, @@ -1510,14 +1507,14 @@ def sum(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): sum = a.sum except AttributeError: return _methods._sum(a, axis=axis, dtype=dtype, - out=out, skipna=skipna, keepdims=keepdims) - # NOTE: Dropping the skipna and keepdims parameters here... + out=out, keepdims=keepdims) + # NOTE: Dropping the keepdims parameters here... return sum(axis=axis, dtype=dtype, out=out) else: return _methods._sum(a, axis=axis, dtype=dtype, - out=out, skipna=skipna, keepdims=keepdims) + out=out, keepdims=keepdims) -def product (a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): +def product (a, axis=None, dtype=None, out=None, keepdims=False): """ Return the product of array elements over a given axis. @@ -1526,10 +1523,10 @@ def product (a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): prod : equivalent function; see for details. """ - return um.multiply.reduce(a, axis=axis, dtype=dtype, out=out, skipna=skipna, keepdims=keepdims) + return um.multiply.reduce(a, axis=axis, dtype=dtype, out=out, keepdims=keepdims) -def sometrue(a, axis=None, out=None, skipna=False, keepdims=False): +def sometrue(a, axis=None, out=None, keepdims=False): """ Check whether some values are true. @@ -1543,11 +1540,11 @@ def sometrue(a, axis=None, out=None, skipna=False, keepdims=False): arr = asanyarray(a) try: - return arr.any(axis=axis, out=out, skipna=skipna, keepdims=keepdims) + return arr.any(axis=axis, out=out, keepdims=keepdims) except TypeError: return arr.any(axis=axis, out=out) -def alltrue (a, axis=None, out=None, skipna=False, keepdims=False): +def alltrue (a, axis=None, out=None, keepdims=False): """ Check if all elements of input array are true. @@ -1559,11 +1556,11 @@ def alltrue (a, axis=None, out=None, skipna=False, keepdims=False): arr = asanyarray(a) try: - return arr.all(axis=axis, out=out, skipna=skipna, keepdims=keepdims) + return arr.all(axis=axis, out=out, keepdims=keepdims) except TypeError: return arr.all(axis=axis, out=out) -def any(a, axis=None, out=None, skipna=False, keepdims=False): +def any(a, axis=None, out=None, keepdims=False): """ Test whether any array element along a given axis evaluates to True. @@ -1589,9 +1586,6 @@ def any(a, axis=None, out=None, skipna=False, keepdims=False): (e.g., if it is of type float, then it will remain so, returning 1.0 for True and 0.0 for False, regardless of the type of `a`). See `doc.ufuncs` (Section "Output arguments") for details. - skipna : bool, optional - If this is set to True, skips any NA values during summation - instead of propagating them. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, @@ -1642,11 +1636,11 @@ def any(a, axis=None, out=None, skipna=False, keepdims=False): arr = asanyarray(a) try: - return arr.any(axis=axis, out=out, skipna=skipna, keepdims=keepdims) + return arr.any(axis=axis, out=out, keepdims=keepdims) except TypeError: return arr.any(axis=axis, out=out) -def all(a, axis=None, out=None, skipna=False, keepdims=False): +def all(a, axis=None, out=None, keepdims=False): """ Test whether all array elements along a given axis evaluate to True. @@ -1670,9 +1664,6 @@ def all(a, axis=None, out=None, skipna=False, keepdims=False): type is preserved (e.g., if ``dtype(out)`` is float, the result will consist of 0.0's and 1.0's). See `doc.ufuncs` (Section "Output arguments") for more details. - skipna : bool, optional - If this is set to True, skips any NA values during summation - instead of propagating them. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, @@ -1718,7 +1709,7 @@ def all(a, axis=None, out=None, skipna=False, keepdims=False): arr = asanyarray(a) try: - return arr.all(axis=axis, out=out, skipna=skipna, keepdims=keepdims) + return arr.all(axis=axis, out=out, keepdims=keepdims) except TypeError: return arr.all(axis=axis, out=out) @@ -1853,7 +1844,7 @@ def ptp(a, axis=None, out=None): return ptp(axis, out) -def amax(a, axis=None, out=None, skipna=False, keepdims=False): +def amax(a, axis=None, out=None, keepdims=False): """ Return the maximum of an array or maximum along an axis. @@ -1867,9 +1858,6 @@ def amax(a, axis=None, out=None, skipna=False, keepdims=False): Alternate output array in which to place the result. Must be of the same shape and buffer length as the expected output. See `doc.ufuncs` (Section "Output arguments") for more details. - skipna : bool, optional - If this is set to True, skips any NA values during reduction - instead of propagating them. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, @@ -1920,14 +1908,14 @@ def amax(a, axis=None, out=None, skipna=False, keepdims=False): amax = a.max except AttributeError: return _methods._amax(a, axis=axis, - out=out, skipna=skipna, keepdims=keepdims) - # NOTE: Dropping the skipna and keepdims parameters + out=out, keepdims=keepdims) + # NOTE: Dropping and keepdims parameter return amax(axis=axis, out=out) else: return _methods._amax(a, axis=axis, - out=out, skipna=skipna, keepdims=keepdims) + out=out, keepdims=keepdims) -def amin(a, axis=None, out=None, skipna=False, keepdims=False): +def amin(a, axis=None, out=None, keepdims=False): """ Return the minimum of an array or minimum along an axis. @@ -1941,9 +1929,6 @@ def amin(a, axis=None, out=None, skipna=False, keepdims=False): Alternative output array in which to place the result. Must be of the same shape and buffer length as the expected output. See `doc.ufuncs` (Section "Output arguments") for more details. - skipna : bool, optional - If this is set to True, skips any NA values during reduction - instead of propagating them. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, @@ -1994,12 +1979,12 @@ def amin(a, axis=None, out=None, skipna=False, keepdims=False): amin = a.min except AttributeError: return _methods._amin(a, axis=axis, - out=out, skipna=skipna, keepdims=keepdims) - # NOTE: Dropping the skipna and keepdims parameters + out=out, keepdims=keepdims) + # NOTE: Dropping the keepdims parameter return amin(axis=axis, out=out) else: return _methods._amin(a, axis=axis, - out=out, skipna=skipna, keepdims=keepdims) + out=out, keepdims=keepdims) def alen(a): """ @@ -2034,7 +2019,7 @@ def alen(a): return len(array(a,ndmin=1)) -def prod(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): +def prod(a, axis=None, dtype=None, out=None, keepdims=False): """ Return the product of array elements over a given axis. @@ -2062,9 +2047,6 @@ def prod(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): Alternative output array in which to place the result. It must have the same shape as the expected output, but the type of the output values will be cast if necessary. - skipna : bool, optional - If this is set to True, skips any NA values during reduction - instead of propagating them. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, @@ -2127,11 +2109,11 @@ def prod(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): prod = a.prod except AttributeError: return _methods._prod(a, axis=axis, dtype=dtype, - out=out, skipna=skipna, keepdims=keepdims) + out=out, keepdims=keepdims) return prod(axis=axis, dtype=dtype, out=out) else: return _methods._prod(a, axis=axis, dtype=dtype, - out=out, skipna=skipna, keepdims=keepdims) + out=out, keepdims=keepdims) def cumprod(a, axis=None, dtype=None, out=None): """ @@ -2420,7 +2402,7 @@ def round_(a, decimals=0, out=None): return round(decimals, out) -def mean(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): +def mean(a, axis=None, dtype=None, out=None, keepdims=False): """ Compute the arithmetic mean along the specified axis. @@ -2445,9 +2427,6 @@ def mean(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): is ``None``; if provided, it must have the same shape as the expected output, but the type will be cast if necessary. See `doc.ufuncs` for details. - skipna : bool, optional - If this is set to True, skips any NA values during calculation - instead of propagating them. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, @@ -2506,11 +2485,10 @@ def mean(a, axis=None, dtype=None, out=None, skipna=False, keepdims=False): pass return _methods._mean(a, axis=axis, dtype=dtype, - out=out, skipna=skipna, keepdims=keepdims) + out=out, keepdims=keepdims) -def std(a, axis=None, dtype=None, out=None, ddof=0, - skipna=False, keepdims=False): +def std(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False): """ Compute the standard deviation along the specified axis. @@ -2537,9 +2515,6 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, Means Delta Degrees of Freedom. The divisor used in calculations is ``N - ddof``, where ``N`` represents the number of elements. By default `ddof` is zero. - skipna : bool, optional - If this is set to True, skips any NA values during calculation - instead of propagating them. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, @@ -2612,10 +2587,10 @@ def std(a, axis=None, dtype=None, out=None, ddof=0, pass return _methods._std(a, axis=axis, dtype=dtype, out=out, ddof=ddof, - skipna=skipna, keepdims=keepdims) + keepdims=keepdims) def var(a, axis=None, dtype=None, out=None, ddof=0, - skipna=False, keepdims=False): + keepdims=False): """ Compute the variance along the specified axis. @@ -2643,9 +2618,6 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, "Delta Degrees of Freedom": the divisor used in the calculation is ``N - ddof``, where ``N`` represents the number of elements. By default `ddof` is zero. - skipna : bool, optional - If this is set to True, skips any NA values during calculation - instead of propagating them. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, @@ -2718,4 +2690,4 @@ def var(a, axis=None, dtype=None, out=None, ddof=0, pass return _methods._var(a, axis=axis, dtype=dtype, out=out, ddof=ddof, - skipna=skipna, keepdims=keepdims) + keepdims=keepdims) diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index 3e919c761..b2f9dc70c 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -3,7 +3,7 @@ __all__ = ['logspace', 'linspace'] import numeric as _nx from numeric import array -def linspace(start, stop, num=50, endpoint=True, retstep=False, maskna=False): +def linspace(start, stop, num=50, endpoint=True, retstep=False): """ Return evenly spaced numbers over a specified interval. @@ -29,8 +29,6 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, maskna=False): retstep : bool, optional If True, return (`samples`, `step`), where `step` is the spacing between samples. - maskna : boolean - If this is true, the returned array will have an NA mask. Returns ------- @@ -75,22 +73,22 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, maskna=False): """ num = int(num) if num <= 0: - return array([], float, maskna=maskna) + return array([], float) if endpoint: if num == 1: - return array([float(start)], maskna=maskna) + return array([float(start)]) step = (stop-start)/float((num-1)) - y = _nx.arange(0, num, maskna=maskna) * step + start + y = _nx.arange(0, num) * step + start y[-1] = stop else: step = (stop-start)/float(num) - y = _nx.arange(0, num, maskna=maskna) * step + start + y = _nx.arange(0, num) * step + start if retstep: return y, step else: return y -def logspace(start,stop,num=50,endpoint=True,base=10.0, maskna=False): +def logspace(start,stop,num=50,endpoint=True,base=10.0): """ Return numbers spaced evenly on a log scale. @@ -116,8 +114,6 @@ def logspace(start,stop,num=50,endpoint=True,base=10.0, maskna=False): The base of the log space. The step size between the elements in ``ln(samples) / ln(base)`` (or ``log_base(samples)``) is uniform. Default is 10.0. - maskna : boolean - If this is true, the returned array will have an NA mask. Returns ------- @@ -166,6 +162,6 @@ def logspace(start,stop,num=50,endpoint=True,base=10.0, maskna=False): >>> plt.show() """ - y = linspace(start,stop,num=num,endpoint=endpoint,maskna=maskna) + y = linspace(start,stop,num=num,endpoint=endpoint) return _nx.power(base,y) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index d77c5c90f..3862865ae 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -316,41 +316,6 @@ struct NpyAuxData_tag { #define NPY_AUXDATA_CLONE(auxdata) \ ((auxdata)->clone(auxdata)) -/********************************************************************* - * NumPy functions for dealing with masks, such as in masked iteration - *********************************************************************/ - -typedef npy_uint8 npy_mask; -#define NPY_MASK NPY_UINT8 - -/* - * Bit 0 of the mask indicates whether a value is exposed - * or hidden. This is compatible with a 'where=' boolean - * mask, because NumPy booleans are 1 byte, and contain - * either the value 0 or 1. - */ -static NPY_INLINE npy_bool -NpyMaskValue_IsExposed(npy_mask mask) -{ - return (mask & 0x01) != 0; -} - -/* - * Bits 1 through 7 of the mask contain the payload. - */ -static NPY_INLINE npy_uint8 -NpyMaskValue_GetPayload(npy_mask mask) -{ - return ((npy_uint8)mask) >> 1; -} - -static NPY_INLINE npy_mask -NpyMaskValue_Create(npy_bool exposed, npy_uint8 payload) -{ - return (npy_mask)(exposed != 0) | (npy_mask)(payload << 1); -} - - #define NPY_ERR(str) fprintf(stderr, #str); fflush(stderr); #define NPY_ERR2(str) fprintf(stderr, str); fflush(stderr); @@ -707,28 +672,6 @@ typedef struct tagPyArrayObject_fields { int flags; /* For weak references */ PyObject *weakreflist; - - /* New fields added as of NumPy 1.7 */ - - /* - * Descriptor for the mask dtype. - * If no mask: NULL - * If mask : bool/uint8/structured dtype of mask dtypes - */ - PyArray_Descr *maskna_dtype; - /* - * Raw data buffer for mask. If the array has the flag - * NPY_ARRAY_OWNMASKNA enabled, it owns this memory and - * must call PyArray_free on it when destroyed. - */ - char *maskna_data; - /* - * Just like dimensions and strides point into the same memory - * buffer, we now just make that buffer 3x the nd instead of 2x - * and use the same buffer. This is always allocated, regardless - * of whether there is an NA mask or not. - */ - npy_intp *maskna_strides; } PyArrayObject_fields; /* @@ -895,34 +838,6 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); #define NPY_ARRAY_UPDATEIFCOPY 0x1000 /* - * If this flag is set, then the array has an NA mask corresponding - * to the array data. If the flag NPY_ARRAY_OWNMASKNA is requested - * in a constructor, this flag is also implied even if it is not set. - * - * This flag may be requested in constructor functions. - * This flag may be tested for in PyArray_FLAGS(arr). - */ -#define NPY_ARRAY_MASKNA 0x2000 - -/* - * If this flag is set, then the array owns the memory for the - * missing values NA mask. - * - * This flag may be requested in constructor functions. - * This flag may be tested for in PyArray_FLAGS(arr). - */ -#define NPY_ARRAY_OWNMASKNA 0x4000 - -/* - * If this flag is set, then arrays which have an NA mask, or arrays - * which have an NA dtype are permitted to pass through. If not, - * an array with NA support causes an error to be thrown. - * - * This flag may be requested in constructor functions. - */ -#define NPY_ARRAY_ALLOWNA 0x8000 - -/* * NOTE: there are also internal flags defined in multiarray/arrayobject.h, * which start at bit 31 and work down. */ @@ -1027,17 +942,6 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); #define NPY_DISABLE_C_API #endif -/***************************** - * NA object, added in 1.7 - *****************************/ - -/* Direct access to the fields of the NA object is just internal to NumPy. */ -typedef struct tagNpyNA { - PyObject_HEAD -} NpyNA; - -#define NpyNA_Check(op) PyObject_TypeCheck(op, &NpyNA_Type) - /********************************** * The nditer object, added in 1.6 **********************************/ @@ -1109,10 +1013,6 @@ typedef void (NpyIter_GetMultiIndexFunc)(NpyIter *iter, #define NPY_ITER_WRITEMASKED 0x10000000 /* This array is the mask for all WRITEMASKED operands */ #define NPY_ITER_ARRAYMASK 0x20000000 -/* Split this operand up into data and an NA mask */ -#define NPY_ITER_USE_MASKNA 0x40000000 -/* Iterate over the data, even if it has an NA mask and without USE_MASKNA */ -#define NPY_ITER_IGNORE_MASKNA 0x80000000 #define NPY_ITER_GLOBAL_FLAGS 0x0000ffff #define NPY_ITER_PER_OP_FLAGS 0xffff0000 @@ -1633,34 +1533,6 @@ PyArray_CLEARFLAGS(PyArrayObject *arr, int flags) ((PyArrayObject_fields *)arr)->flags &= ~flags; } -/* Access to the missing values NA mask, added in 1.7 */ - -static NPY_INLINE PyArray_Descr * -PyArray_MASKNA_DTYPE(PyArrayObject *arr) -{ - return ((PyArrayObject_fields *)arr)->maskna_dtype; -} - -static NPY_INLINE char * -PyArray_MASKNA_DATA(PyArrayObject *arr) -{ - return ((PyArrayObject_fields *)arr)->maskna_data; -} - -/* For the corresponding DIMS, use PyArray_DIMS(arr) */ -static NPY_INLINE npy_intp * -PyArray_MASKNA_STRIDES(PyArrayObject *arr) -{ - return ((PyArrayObject_fields *)arr)->maskna_strides; -} - -static NPY_INLINE npy_bool -PyArray_HASMASKNA(PyArrayObject *arr) -{ - return (((PyArrayObject_fields *)arr)->flags & NPY_ARRAY_MASKNA) != 0; -} - - #define PyTypeNum_ISBOOL(type) ((type) == NPY_BOOL) #define PyTypeNum_ISUNSIGNED(type) (((type) == NPY_UBYTE) || \ @@ -1803,21 +1675,19 @@ typedef struct { /* * This is a function for assigning a reduction identity to the result, - * before doing the reduction computation. If 'preservena' is True, - * any masked NA values in 'result' should not be overwritten. The + * before doing the reduction computation. The * value in 'data' is passed through from PyArray_ReduceWrapper. * * This function could, for example, simply be a call like - * return PyArray_AssignZero(result, NULL, preservena, NULL); + * return PyArray_AssignZero(result, NULL, NULL); * * It should return -1 on failure, or 0 on success. */ typedef int (PyArray_AssignReduceIdentityFunc)(PyArrayObject *result, - int preservena, void *data); + void *data); /* - * This is a function for the reduce loop. Both the unmasked and - * masked variants have the same prototype, but should behave differently. + * This is a function for the reduce loop. * * The needs_api parameter indicates whether it's ok to release the GIL during * the loop, such as when the iternext() function never calls @@ -1828,7 +1698,7 @@ typedef int (PyArray_AssignReduceIdentityFunc)(PyArrayObject *result, * when the 'assign_identity' parameter was NULL when calling * PyArray_ReduceWrapper. * - * The unmasked loop gets two data pointers and two strides, and should + * The loop gets two data pointers and two strides, and should * look roughly like this: * { * NPY_BEGIN_THREADS_DEF; @@ -1848,7 +1718,6 @@ typedef int (PyArray_AssignReduceIdentityFunc)(PyArrayObject *result, * --count; * --skip_first_count; * data1 += stride1; - * //data2 += stride2; // In masked loop * } * else { * skip_first_count -= count; @@ -1893,31 +1762,6 @@ typedef int (PyArray_AssignReduceIdentityFunc)(PyArrayObject *result, * return (needs_api && PyErr_Occurred()) ? -1 : 0; * } * - * The masked loop gets three data pointers and three strides, and - * looks identical except for the iteration loops which should be - * like this: - * do { - * char *data0 = dataptr[0], *data1 = dataptr[1], *data2 = dataptr[2]; - * npy_intp stride0 = strideptr[0], stride1 = strideptr[1], - * stride2 = strideptr[2]; - * npy_intp count = *countptr; - * - * // Skipping first visits would go here - * - * while (count--) { - * if (NpyMaskValue_IsExposed((npy_mask)*data2)) { - * *(result_t *)data0 = my_reduce_op(*(result_t *)data0, - * *(operand_t *)data1); - * } - * data0 += stride0; - * data1 += stride1; - * data2 += stride2; - * } - * - * // Jumping to the faster loop would go here - * - * } while (iternext(iter)); - * * If needs_api is True, this function should call PyErr_Occurred() * to check if an error occurred during processing, and return -1 for * error, 0 for success. diff --git a/numpy/core/include/numpy/ufuncobject.h b/numpy/core/include/numpy/ufuncobject.h index dab5fd6e4..7f717a2c1 100644 --- a/numpy/core/include/numpy/ufuncobject.h +++ b/numpy/core/include/numpy/ufuncobject.h @@ -30,7 +30,9 @@ typedef void (PyUFunc_StridedInnerLoopFunc)( /* * The most generic one-dimensional inner loop for - * a masked standard element-wise ufunc. + * a masked standard element-wise ufunc. "Masked" here means that it skips + * doing calculations on any items for which the maskptr array has a true + * value. */ typedef void (PyUFunc_MaskedStridedInnerLoopFunc)( char **dataptrs, npy_intp *strides, diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index aa7d2c29b..a104db19c 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -1,11 +1,11 @@ __all__ = ['newaxis', 'ndarray', 'flatiter', 'nditer', 'nested_iters', 'ufunc', - 'arange', 'array', 'zeros', 'count_nonzero', 'count_reduce_items', + 'arange', 'array', 'zeros', 'count_nonzero', 'empty', 'broadcast', 'dtype', 'fromstring', 'fromfile', 'frombuffer', 'int_asbuffer', 'where', 'argwhere', 'copyto', 'concatenate', 'fastCopyAndTranspose', 'lexsort', 'set_numeric_ops', 'can_cast', 'promote_types', 'min_scalar_type', 'result_type', 'asarray', 'asanyarray', 'ascontiguousarray', 'asfortranarray', - 'isfortran', 'isna', 'empty_like', 'zeros_like', 'ones_like', + 'isfortran', 'empty_like', 'zeros_like', 'ones_like', 'correlate', 'convolve', 'inner', 'dot', 'einsum', 'outer', 'vdot', 'alterdot', 'restoredot', 'roll', 'rollaxis', 'cross', 'tensordot', 'array2string', 'get_printoptions', 'set_printoptions', @@ -63,7 +63,7 @@ copyto = multiarray.copyto ufunc = type(sin) -def zeros_like(a, dtype=None, order='K', subok=True, maskna=False): +def zeros_like(a, dtype=None, order='K', subok=True): """ Return an array of zeros with the same shape and type as a given array. @@ -81,8 +81,6 @@ def zeros_like(a, dtype=None, order='K', subok=True, maskna=False): 'F' means F-order, 'A' means 'F' if `a` is Fortran contiguous, 'C' otherwise. 'K' means match the layout of `a` as closely as possible. - maskna : boolean - If this is true, the returned array will have an NA mask. Returns ------- @@ -115,11 +113,11 @@ def zeros_like(a, dtype=None, order='K', subok=True, maskna=False): array([ 0., 0., 0.]) """ - res = empty_like(a, dtype=dtype, order=order, subok=subok, maskna=maskna) + res = empty_like(a, dtype=dtype, order=order, subok=subok) multiarray.copyto(res, 0, casting='unsafe') return res -def ones(shape, dtype=None, order='C', maskna=False): +def ones(shape, dtype=None, order='C'): """ Return a new array of given shape and type, filled with ones. @@ -147,11 +145,11 @@ def ones(shape, dtype=None, order='C', maskna=False): [ 1., 1.]]) """ - a = empty(shape, dtype, order, maskna) + a = empty(shape, dtype, order) multiarray.copyto(a, 1, casting='unsafe') return a -def ones_like(a, dtype=None, order='K', subok=True, maskna=False): +def ones_like(a, dtype=None, order='K', subok=True): """ Return an array of ones with the same shape and type as a given array. @@ -169,8 +167,6 @@ def ones_like(a, dtype=None, order='K', subok=True, maskna=False): 'F' means F-order, 'A' means 'F' if `a` is Fortran contiguous, 'C' otherwise. 'K' means match the layout of `a` as closely as possible. - maskna : boolean - If this is true, the returned array will have an NA mask. Returns ------- @@ -203,7 +199,7 @@ def ones_like(a, dtype=None, order='K', subok=True, maskna=False): array([ 1., 1., 1.]) """ - res = empty_like(a, dtype=dtype, order=order, subok=subok, maskna=maskna) + res = empty_like(a, dtype=dtype, order=order, subok=subok) multiarray.copyto(res, 1, casting='unsafe') return res @@ -230,7 +226,6 @@ arange = multiarray.arange array = multiarray.array zeros = multiarray.zeros count_nonzero = multiarray.count_nonzero -count_reduce_items = multiarray.count_reduce_items empty = multiarray.empty empty_like = multiarray.empty_like fromstring = multiarray.fromstring @@ -253,9 +248,8 @@ lexsort = multiarray.lexsort compare_chararrays = multiarray.compare_chararrays putmask = multiarray.putmask einsum = multiarray.einsum -isna = multiarray.isna -def asarray(a, dtype=None, order=None, maskna=None, ownmaskna=False): +def asarray(a, dtype=None, order=None): """ Convert the input to an array. @@ -270,13 +264,6 @@ def asarray(a, dtype=None, order=None, maskna=None, ownmaskna=False): order : {'C', 'F'}, optional Whether to use row-major ('C') or column-major ('F' for FORTRAN) memory representation. Defaults to 'C'. - maskna : bool or None, optional - If this is set to True, it forces the array to have an NA mask. - If this is set to False, it forces the array to not have an NA - mask. - ownmaskna : bool, optional - If this is set to True, forces the array to have a mask which - it owns. Returns ------- @@ -330,10 +317,9 @@ def asarray(a, dtype=None, order=None, maskna=None, ownmaskna=False): True """ - return array(a, dtype, copy=False, order=order, - maskna=maskna, ownmaskna=ownmaskna) + return array(a, dtype, copy=False, order=order) -def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False): +def asanyarray(a, dtype=None, order=None): """ Convert the input to an ndarray, but pass ndarray subclasses through. @@ -348,13 +334,6 @@ def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False): order : {'C', 'F'}, optional Whether to use row-major ('C') or column-major ('F') memory representation. Defaults to 'C'. - maskna : bool or None, optional - If this is set to True, it forces the array to have an NA mask. - If this is set to False, it forces the array to not have an NA - mask. - ownmaskna : bool, optional - If this is set to True, forces the array to have a mask which - it owns. Returns ------- @@ -390,10 +369,9 @@ def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False): True """ - return array(a, dtype, copy=False, order=order, subok=True, - maskna=maskna, ownmaskna=ownmaskna) + return array(a, dtype, copy=False, order=order, subok=True) -def ascontiguousarray(a, dtype=None, maskna=None, ownmaskna=False): +def ascontiguousarray(a, dtype=None): """ Return a contiguous array in memory (C order). @@ -403,13 +381,6 @@ def ascontiguousarray(a, dtype=None, maskna=None, ownmaskna=False): Input array. dtype : str or dtype object, optional Data-type of returned array. - maskna : bool or None, optional - If this is set to True, it forces the array to have an NA mask. - If this is set to False, it forces the array to not have an NA - mask. - ownmaskna : bool, optional - If this is set to True, forces the array to have a mask which - it owns. Returns ------- @@ -434,10 +405,9 @@ def ascontiguousarray(a, dtype=None, maskna=None, ownmaskna=False): True """ - return array(a, dtype, copy=False, order='C', ndmin=1, - maskna=maskna, ownmaskna=ownmaskna) + return array(a, dtype, copy=False, order='C', ndmin=1) -def asfortranarray(a, dtype=None, maskna=None, ownmaskna=False): +def asfortranarray(a, dtype=None): """ Return an array laid out in Fortran order in memory. @@ -447,13 +417,6 @@ def asfortranarray(a, dtype=None, maskna=None, ownmaskna=False): Input array. dtype : str or dtype object, optional By default, the data-type is inferred from the input data. - maskna : bool or None, optional - If this is set to True, it forces the array to have an NA mask. - If this is set to False, it forces the array to not have an NA - mask. - ownmaskna : bool, optional - If this is set to True, forces the array to have a mask which - it owns. Returns ------- @@ -478,8 +441,7 @@ def asfortranarray(a, dtype=None, maskna=None, ownmaskna=False): True """ - return array(a, dtype, copy=False, order='F', ndmin=1, - maskna=maskna, ownmaskna=ownmaskna) + return array(a, dtype, copy=False, order='F', ndmin=1) def require(a, dtype=None, requirements=None): """ @@ -1454,15 +1416,6 @@ def array_repr(arr, max_line_width=None, precision=None, suppress_small=None): skipdtype = (arr.dtype.type in _typelessdata) and arr.size > 0 - if arr.flags.maskna: - whichna = isna(arr) - # If nothing is NA, explicitly signal the NA-mask - if not any(whichna): - lst += ", maskna=True" - # If everything is NA, can't skip the dtype - if skipdtype and all(whichna): - skipdtype = False - if skipdtype: return "%s(%s)" % (cName, lst) else: @@ -1821,6 +1774,7 @@ def binary_repr(num, width=None): '1101' """ + # ' <-- unbreak Emacs fontification sign = '' if num < 0: if width is None: @@ -1926,7 +1880,7 @@ def _maketup(descr, val): res = [_maketup(fields[name][0],val) for name in dt.names] return tuple(res) -def identity(n, dtype=None, maskna=False): +def identity(n, dtype=None): """ Return the identity array. @@ -1939,8 +1893,6 @@ def identity(n, dtype=None, maskna=False): Number of rows (and columns) in `n` x `n` output. dtype : data-type, optional Data-type of the output. Defaults to ``float``. - maskna : bool, optional - If this is true, the returned array will have an NA mask. Returns ------- @@ -1957,7 +1909,7 @@ def identity(n, dtype=None, maskna=False): """ from numpy import eye - return eye(n, dtype=dtype, maskna=maskna) + return eye(n, dtype=dtype) def allclose(a, b, rtol=1.e-5, atol=1.e-8): """ @@ -2017,9 +1969,6 @@ def allclose(a, b, rtol=1.e-5, atol=1.e-8): x = array(a, copy=False, ndmin=1) y = array(b, copy=False, ndmin=1) - if any(isnan(x)) or any(isnan(y)): - return False - xinf = isinf(x) yinf = isinf(y) if any(xinf) or any(yinf): @@ -2108,18 +2057,17 @@ def isclose(a, b, rtol=1.e-5, atol=1.e-8, equal_nan=False): y = array(b, copy=False, subok=True, ndmin=1) xfin = isfinite(x) yfin = isfinite(y) - if all(xfin, skipna=True) and all(yfin, skipna=True): + if all(xfin) and all(yfin): return within_tol(x, y, atol, rtol) else: finite = xfin & yfin - cond = zeros_like(finite, subok=True, maskna=finite.flags.maskna) + cond = zeros_like(finite, subok=True) # Because we're using boolean indexing, x & y must be the same shape. # Ideally, we'd just do x, y = broadcast_arrays(x, y). It's in # lib.stride_tricks, though, so we can't import it here. x = x * ones_like(cond) y = y * ones_like(cond) - # Avoid subtraction with infinite/nan values and indexing with na... - finite[isna(finite)] = False + # Avoid subtraction with infinite/nan values... cond[finite] = within_tol(x[finite], y[finite], atol, rtol) # Check for equality of infinite values... cond[~finite] = (x[~finite] == y[~finite]) diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index 6dc82f706..e39eb2232 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -87,12 +87,11 @@ __all__ = ['sctypeDict', 'sctypeNA', 'typeDict', 'typeNA', 'sctypes', 'maximum_sctype', 'issctype', 'typecodes', 'find_common_type', 'issubdtype', 'datetime_data','datetime_as_string', 'busday_offset', 'busday_count', 'is_busday', 'busdaycalendar', - 'NA', 'NAType'] + ] from numpy.core.multiarray import typeinfo, ndarray, array, \ empty, dtype, datetime_data, datetime_as_string, \ - busday_offset, busday_count, is_busday, busdaycalendar, \ - NA, NAType + busday_offset, busday_count, is_busday, busdaycalendar import types as _types import sys diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 235705dff..96b1576a3 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -688,7 +688,6 @@ def configuration(parent_package='',top_path=None): join(local_dir, subpath, 'arraytypes.c.src'), join(local_dir, subpath, 'nditer_templ.c.src'), join(local_dir, subpath, 'lowlevel_strided_loops.c.src'), - join(local_dir, subpath, 'boolean_ops.c.src'), join(local_dir, subpath, 'einsum.c.src')] # numpy.distutils generate .c from .c.src in weird directories, we have @@ -727,8 +726,6 @@ def configuration(parent_package='',top_path=None): join('src', 'multiarray', 'shape.h'), join('src', 'multiarray', 'ucsnarrow.h'), join('src', 'multiarray', 'usertypes.h'), - join('src', 'multiarray', 'na_mask.h'), - join('src', 'multiarray', 'na_object.h'), join('src', 'private', 'lowlevel_strided_loops.h'), join('include', 'numpy', 'arrayobject.h'), join('include', 'numpy', '_neighborhood_iterator_imp.h'), @@ -757,7 +754,6 @@ def configuration(parent_package='',top_path=None): join('src', 'multiarray', 'array_assign.c'), join('src', 'multiarray', 'array_assign_scalar.c'), join('src', 'multiarray', 'array_assign_array.c'), - join('src', 'multiarray', 'boolean_ops.c.src'), join('src', 'multiarray', 'buffer.c'), join('src', 'multiarray', 'calculation.c'), join('src', 'multiarray', 'common.c'), @@ -781,8 +777,6 @@ def configuration(parent_package='',top_path=None): join('src', 'multiarray', 'mapping.c'), join('src', 'multiarray', 'methods.c'), join('src', 'multiarray', 'multiarraymodule.c'), - join('src', 'multiarray', 'na_mask.c'), - join('src', 'multiarray', 'na_object.c'), join('src', 'multiarray', 'nditer_templ.c.src'), join('src', 'multiarray', 'nditer_api.c'), join('src', 'multiarray', 'nditer_constr.c'), diff --git a/numpy/core/src/multiarray/array_assign.h b/numpy/core/src/multiarray/array_assign.h index c58a63d22..3fecff007 100644 --- a/numpy/core/src/multiarray/array_assign.h +++ b/numpy/core/src/multiarray/array_assign.h @@ -14,11 +14,6 @@ * src_order: The rule for how 'src' is to be made flat. * casting: An exception is raised if the copy violates this * casting rule. - * preservena: If 0, overwrites everything in 'dst', if 1, it - * preserves elements in 'dst' which are NA. - * preservewhichna: Must be NULL. When multi-NA support is implemented, - * this will be an array of flags for 'preservena=True', - * indicating which NA payload values to preserve. * * Returns 0 on success, -1 on failure. */ @@ -30,7 +25,16 @@ PyArray_AssignArrayAsFlat(PyArrayObject *dst, NPY_ORDER dst_order, npy_bool preservena, npy_bool *preservewhichna); */ +NPY_NO_EXPORT int +PyArray_AssignArray(PyArrayObject *dst, PyArrayObject *src, + PyArrayObject *wheremask, + NPY_CASTING casting); +NPY_NO_EXPORT int +PyArray_AssignRawScalar(PyArrayObject *dst, + PyArray_Descr *src_dtype, char *src_data, + PyArrayObject *wheremask, + NPY_CASTING casting); /******** LOW-LEVEL SCALAR TO ARRAY ASSIGNMENT ********/ diff --git a/numpy/core/src/multiarray/array_assign_array.c b/numpy/core/src/multiarray/array_assign_array.c index a9f3a35fe..98c2b19ff 100644 --- a/numpy/core/src/multiarray/array_assign_array.c +++ b/numpy/core/src/multiarray/array_assign_array.c @@ -21,8 +21,6 @@ #include "methods.h" #include "shape.h" #include "lowlevel_strided_loops.h" -#include "na_object.h" -#include "na_mask.h" #include "array_assign.h" @@ -189,7 +187,7 @@ raw_array_wheremasked_assign_array(int ndim, npy_intp *shape, NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { /* Process the innermost dimension */ stransfer(dst_data, dst_strides_it[0], src_data, src_strides_it[0], - (npy_mask *)wheremask_data, wheremask_strides_it[0], + (npy_bool *)wheremask_data, wheremask_strides_it[0], shape_it[0], src_itemsize, transferdata); } NPY_RAW_ITER_THREE_NEXT(idim, ndim, coord, shape_it, dst_data, dst_strides_it, @@ -206,166 +204,6 @@ raw_array_wheremasked_assign_array(int ndim, npy_intp *shape, } /* - * Assigns the elements of 'src' to 'dst' where the 'wheremask' - * is True, except for those which are masked as NA according - * to 'maskna'. - * - * Returns 0 on success, -1 on failure. - */ -NPY_NO_EXPORT int -raw_array_wheremasked_assign_array_preservena(int ndim, npy_intp *shape, - PyArray_Descr *dst_dtype, char *dst_data, npy_intp *dst_strides, - PyArray_Descr *src_dtype, char *src_data, npy_intp *src_strides, - PyArray_Descr *maskna_dtype, char *maskna_data, - npy_intp *maskna_strides, - PyArray_Descr *wheremask_dtype, char *wheremask_data, - npy_intp *wheremask_strides) -{ - int idim; - npy_intp shape_it[NPY_MAXDIMS]; - npy_intp dst_strides_it[NPY_MAXDIMS]; - npy_intp src_strides_it[NPY_MAXDIMS]; - npy_intp maskna_strides_it[NPY_MAXDIMS]; - npy_intp wheremask_strides_it[NPY_MAXDIMS]; - npy_intp coord[NPY_MAXDIMS]; - - PyArray_MaskedStridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; - int aligned, needs_api = 0; - npy_intp src_itemsize = src_dtype->elsize; - - PyArray_StridedBinaryOp *maskand_stransfer = NULL; - NpyAuxData *maskand_transferdata = NULL; - - char *maskna_buffer; - npy_intp maskna_itemsize; - - NPY_BEGIN_THREADS_DEF; - - /* Check alignment */ - aligned = raw_array_is_aligned(ndim, - dst_data, dst_strides, dst_dtype->alignment) && - raw_array_is_aligned(ndim, - src_data, src_strides, src_dtype->alignment); - - /* Use raw iteration with no heap allocation */ - if (PyArray_PrepareFourRawArrayIter( - ndim, shape, - dst_data, dst_strides, - src_data, src_strides, - maskna_data, maskna_strides, - wheremask_data, wheremask_strides, - &ndim, shape_it, - &dst_data, dst_strides_it, - &src_data, src_strides_it, - &maskna_data, maskna_strides_it, - &wheremask_data, wheremask_strides_it) < 0) { - return -1; - } - - /* Allocate a buffer for inverting/anding the mask */ - maskna_itemsize = maskna_dtype->elsize; - maskna_buffer = PyArray_malloc(NPY_ARRAY_ASSIGN_BUFFERSIZE * - maskna_itemsize); - if (maskna_buffer == NULL) { - PyErr_NoMemory(); - return -1; - } - - /* - * Overlap check for the 1D case. Higher dimensional arrays cause - * a temporary copy before getting here. - */ - if (ndim == 1 && src_data < dst_data && - src_data + shape_it[0] * src_strides_it[0] > dst_data) { - src_data += (shape_it[0] - 1) * src_strides_it[0]; - dst_data += (shape_it[0] - 1) * dst_strides_it[0]; - maskna_data += (shape_it[0] - 1) * maskna_strides_it[0]; - wheremask_data += (shape_it[0] - 1) * wheremask_strides_it[0]; - src_strides_it[0] = -src_strides_it[0]; - dst_strides_it[0] = -dst_strides_it[0]; - maskna_strides_it[0] = -maskna_strides_it[0]; - wheremask_strides_it[0] = -wheremask_strides_it[0]; - } - - /* Get the function to do the casting */ - if (PyArray_GetMaskedDTypeTransferFunction(aligned, - src_strides[0], dst_strides_it[0], maskna_itemsize, - src_dtype, dst_dtype, maskna_dtype, - 0, - &stransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { - PyArray_free(maskna_buffer); - return -1; - } - - /* - * Get the function to invert the mask. The output - * of the binary operation is the dtype 'maskna_dtype' - */ - if (PyArray_GetMaskAndFunction( - maskna_strides_it[0], maskna_dtype, 0, - wheremask_strides_it[0], wheremask_dtype, 0, - &maskand_stransfer, &maskand_transferdata) < 0) { - PyArray_free(maskna_buffer); - NPY_AUXDATA_FREE(transferdata); - return -1; - } - - if (!needs_api) { - NPY_BEGIN_THREADS; - } - - NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { - npy_intp buffered_count, count; - char *dst_d, *src_d, *maskna_d, *wheremask_d; - /* Process the innermost dimension a buffer size at a time */ - count = shape_it[0]; - dst_d = dst_data; - src_d = src_data; - maskna_d = maskna_data; - wheremask_d = wheremask_data; - do { - buffered_count = count < NPY_ARRAY_ASSIGN_BUFFERSIZE - ? count - : NPY_ARRAY_ASSIGN_BUFFERSIZE; - - /* Prepare the mask into the buffer */ - maskand_stransfer(maskna_buffer, maskna_itemsize, - maskna_d, maskna_strides_it[0], - wheremask_d, wheremask_strides_it[0], - buffered_count, maskand_transferdata); - - /* Transfer the data based on the buffered mask */ - stransfer(dst_d, dst_strides_it[0], src_d, src_strides_it[0], - (npy_mask *)maskna_buffer, maskna_itemsize, - buffered_count, src_itemsize, transferdata); - - dst_d += buffered_count * dst_strides_it[0]; - src_d += buffered_count * src_strides_it[0]; - maskna_d += buffered_count * maskna_strides_it[0]; - wheremask_d += buffered_count * wheremask_strides_it[0]; - count -= buffered_count; - } while (count > 0); - } NPY_RAW_ITER_FOUR_NEXT(idim, ndim, coord, shape_it, - dst_data, dst_strides_it, - src_data, src_strides_it, - maskna_data, maskna_strides_it, - wheremask_data, wheremask_strides_it); - - if (!needs_api) { - NPY_END_THREADS; - } - - PyArray_free(maskna_buffer); - NPY_AUXDATA_FREE(transferdata); - NPY_AUXDATA_FREE(maskand_transferdata); - - return (needs_api && PyErr_Occurred()) ? -1 : 0; -} - -/*NUMPY_API - * * An array assignment function for copying arrays, broadcasting 'src' into * 'dst'. This function makes a temporary copy of 'src' if 'src' and * 'dst' overlap, to be able to handle views of the same data with @@ -376,44 +214,23 @@ raw_array_wheremasked_assign_array_preservena(int ndim, npy_intp *shape, * wheremask: If non-NULL, a boolean mask specifying where to copy. * casting: An exception is raised if the copy violates this * casting rule. - * preservena: If 0, overwrites everything in 'dst', if 1, it - * preserves elements in 'dst' which are NA. - * preservewhichna: Must be NULL. When multi-NA support is implemented, - * this will be an array of flags for 'preservena=True', - * indicating which NA payload values to preserve. * * Returns 0 on success, -1 on failure. */ NPY_NO_EXPORT int PyArray_AssignArray(PyArrayObject *dst, PyArrayObject *src, PyArrayObject *wheremask, - NPY_CASTING casting, - npy_bool preservena, npy_bool *preservewhichna) + NPY_CASTING casting) { - int dst_has_maskna = PyArray_HASMASKNA(dst); - int src_has_maskna = PyArray_HASMASKNA(src); int copied_src = 0; - npy_intp src_strides[NPY_MAXDIMS], src_maskna_strides[NPY_MAXDIMS]; + npy_intp src_strides[NPY_MAXDIMS]; /* Use array_assign_scalar if 'src' NDIM is 0 */ if (PyArray_NDIM(src) == 0) { - /* If the array is masked, assign to the NA mask */ - if (PyArray_HASMASKNA(src)) { - NpyNA *na = NpyNA_FromObject((PyObject *)src, 1); - - if (na != NULL) { - /* TODO: With multi-NA, preservena must also be followed */ - int retcode = PyArray_AssignNA(dst, na, wheremask, - preservena, preservewhichna); - Py_DECREF(na); - return retcode; - } - } - return PyArray_AssignRawScalar( dst, PyArray_DESCR(src), PyArray_DATA(src), - wheremask, casting, preservena, preservewhichna); + wheremask, casting); } /* @@ -436,7 +253,6 @@ PyArray_AssignArray(PyArrayObject *dst, PyArrayObject *src, * try and reject this with as little work as possible. */ if (PyArray_DATA(src) == PyArray_DATA(dst) && - PyArray_MASKNA_DATA(src) == PyArray_MASKNA_DATA(dst) && PyArray_DESCR(src) == PyArray_DESCR(dst) && PyArray_NDIM(src) == PyArray_NDIM(dst) && PyArray_CompareLists(PyArray_DIMS(src), @@ -471,28 +287,6 @@ PyArray_AssignArray(PyArrayObject *dst, PyArrayObject *src, goto fail; } - if (preservewhichna != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "multi-NA support is not yet implemented"); - goto fail; - } - - if (src_has_maskna && !dst_has_maskna) { - int containsna = PyArray_ContainsNA(src, wheremask, NULL); - if (containsna == -1) { - goto fail; - } - else if (containsna) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which " - "does not support NAs"); - goto fail; - } - else { - src_has_maskna = 0; - } - } - /* * When ndim is 1 and the strides point in the same direction, * the lower-level inner loop handles copying @@ -514,16 +308,7 @@ PyArray_AssignArray(PyArrayObject *dst, PyArrayObject *src, goto fail; } - /* Make the temporary copy have an NA mask if necessary */ - if (PyArray_HASMASKNA(src)) { - if (PyArray_AllocateMaskNA(tmp, 1, 0, 1) < 0) { - Py_DECREF(tmp); - goto fail; - } - } - - if (PyArray_AssignArray(tmp, src, - NULL, NPY_UNSAFE_CASTING, 0, NULL) < 0) { + if (PyArray_AssignArray(tmp, src, NULL, NPY_UNSAFE_CASTING) < 0) { Py_DECREF(tmp); goto fail; } @@ -563,120 +348,17 @@ PyArray_AssignArray(PyArrayObject *dst, PyArrayObject *src, } } - if (src_has_maskna) { - if (broadcast_strides(PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_NDIM(src), PyArray_DIMS(src), - PyArray_MASKNA_STRIDES(src), "input array", - src_maskna_strides) < 0) { - goto fail; - } - } - if (wheremask == NULL) { /* A straightforward value assignment */ - if (!preservena || !dst_has_maskna) { - /* If assigning to an array with an NA mask, set to all exposed */ - if (dst_has_maskna) { - if (src_has_maskna) { - /* Assign the NA mask */ - if (raw_array_assign_array( - PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_MASKNA_DTYPE(dst), - PyArray_MASKNA_DATA(dst), - PyArray_MASKNA_STRIDES(dst), - PyArray_MASKNA_DTYPE(src), - PyArray_MASKNA_DATA(src), - src_maskna_strides) < 0) { - goto fail; - } - - /* Assign the values based on the 'src' NA mask */ - if (raw_array_wheremasked_assign_array( - PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_DESCR(dst), PyArray_DATA(dst), - PyArray_STRIDES(dst), - PyArray_DESCR(src), PyArray_DATA(src), - src_strides, - PyArray_MASKNA_DTYPE(src), - PyArray_MASKNA_DATA(src), - src_maskna_strides) < 0) { - goto fail; - } - - goto finish; - } - else { - if (PyArray_AssignMaskNA(dst, 1, NULL, - preservena, preservewhichna) < 0) { - goto fail; - } - } - } - - /* Do the assignment with raw array iteration */ - if (raw_array_assign_array(PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_DESCR(dst), PyArray_DATA(dst), PyArray_STRIDES(dst), - PyArray_DESCR(src), PyArray_DATA(src), src_strides) < 0) { - goto fail; - } - } - /* A value assignment without overwriting NA values */ - else { - if (src_has_maskna) { - /* Assign the NA mask, wheremasked with the 'dst' NA mask */ - if (raw_array_wheremasked_assign_array( - PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_MASKNA_DTYPE(dst), - PyArray_MASKNA_DATA(dst), - PyArray_MASKNA_STRIDES(dst), - PyArray_MASKNA_DTYPE(src), - PyArray_MASKNA_DATA(src), - src_maskna_strides, - PyArray_MASKNA_DTYPE(dst), - PyArray_MASKNA_DATA(dst), - PyArray_MASKNA_STRIDES(dst)) < 0) { - goto fail; - } - } - - /* - * The 'dst' NA mask now has exposed precisely the values we - * want to assign, so use it for this assignment. - */ - if (raw_array_wheremasked_assign_array( - PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_DESCR(dst), PyArray_DATA(dst), - PyArray_STRIDES(dst), - PyArray_DESCR(src), PyArray_DATA(src), - PyArray_STRIDES(src), - PyArray_MASKNA_DTYPE(dst), PyArray_MASKNA_DATA(dst), - PyArray_MASKNA_STRIDES(dst)) < 0) { - goto fail; - } + /* Do the assignment with raw array iteration */ + if (raw_array_assign_array(PyArray_NDIM(dst), PyArray_DIMS(dst), + PyArray_DESCR(dst), PyArray_DATA(dst), PyArray_STRIDES(dst), + PyArray_DESCR(src), PyArray_DATA(src), src_strides) < 0) { + goto fail; } } else { npy_intp wheremask_strides[NPY_MAXDIMS]; - int containsna = PyArray_ContainsNA(wheremask, NULL, NULL); - - if (containsna == -1) { - goto fail; - } - else if (containsna) { - if (!dst_has_maskna) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which " - "does not support NAs"); - goto fail; - } - else { - /* TODO: add support for this */ - PyErr_SetString(PyExc_ValueError, - "A where mask with NA values is not supported " - "yet"); - goto fail; - } - } /* Broadcast the wheremask to 'dst' for raw iteration */ if (broadcast_strides(PyArray_NDIM(dst), PyArray_DIMS(dst), @@ -687,111 +369,15 @@ PyArray_AssignArray(PyArrayObject *dst, PyArrayObject *src, } /* A straightforward where-masked assignment */ - if (!preservena || !dst_has_maskna) { - /* If assigning to an array with an NA mask, set to all exposed */ - if (dst_has_maskna) { - /* - * TODO: If the where mask has NA values, this part - * changes too. - */ - if (src_has_maskna) { - /* Assign the NA mask */ - if (raw_array_wheremasked_assign_array( - PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_MASKNA_DTYPE(dst), - PyArray_MASKNA_DATA(dst), - PyArray_MASKNA_STRIDES(dst), - PyArray_MASKNA_DTYPE(src), - PyArray_MASKNA_DATA(src), - src_maskna_strides, - PyArray_DESCR(wheremask), - PyArray_DATA(wheremask), - wheremask_strides) < 0) { - goto fail; - } - - /* - * Assign the values based on the wheremask, not - * overwriting values also masked by the 'src' NA mask - */ - if (raw_array_wheremasked_assign_array_preservena( - PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_DESCR(dst), PyArray_DATA(dst), - PyArray_STRIDES(dst), - PyArray_DESCR(src), PyArray_DATA(src), - src_strides, - PyArray_MASKNA_DTYPE(src), - PyArray_MASKNA_DATA(src), - src_maskna_strides, - PyArray_DESCR(wheremask), - PyArray_DATA(wheremask), - wheremask_strides)) { - goto fail; - } - - goto finish; - } - else { - if (PyArray_AssignMaskNA(dst, 1, wheremask, - preservena, preservewhichna) < 0) { - goto fail; - } - } - } - - /* Do the masked assignment with raw array iteration */ - if (raw_array_wheremasked_assign_array( - PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_DESCR(dst), PyArray_DATA(dst), PyArray_STRIDES(dst), - PyArray_DESCR(src), PyArray_DATA(src), src_strides, - PyArray_DESCR(wheremask), PyArray_DATA(wheremask), - wheremask_strides) < 0) { - goto fail; - } - } - /* A masked value assignment without overwriting NA values */ - else { - if (src_has_maskna) { - /* - * Assign the NA mask, wheremasked with the 'dst' NA mask - * and the parameter 'wheremask' - */ - if (raw_array_wheremasked_assign_array_preservena( - PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_MASKNA_DTYPE(dst), - PyArray_MASKNA_DATA(dst), - PyArray_MASKNA_STRIDES(dst), - PyArray_MASKNA_DTYPE(src), - PyArray_MASKNA_DATA(src), - src_maskna_strides, - PyArray_MASKNA_DTYPE(dst), - PyArray_MASKNA_DATA(dst), - PyArray_MASKNA_STRIDES(dst), - PyArray_DESCR(wheremask), - PyArray_DATA(wheremask), - wheremask_strides) < 0) { - goto fail; - } - } - - /* - * The 'dst' NA mask together with the 'wheremask' now have - * exposed precisely the values we want to assign, so use - * it's another wheremasked preservena assignment. - */ - if (raw_array_wheremasked_assign_array_preservena( - PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_DESCR(dst), PyArray_DATA(dst), - PyArray_STRIDES(dst), - PyArray_DESCR(src), PyArray_DATA(src), - PyArray_STRIDES(src), - PyArray_MASKNA_DTYPE(dst), PyArray_MASKNA_DATA(dst), - PyArray_MASKNA_STRIDES(dst), - PyArray_DESCR(wheremask), PyArray_DATA(wheremask), - wheremask_strides) < 0) { - goto fail; - } - } + /* Do the masked assignment with raw array iteration */ + if (raw_array_wheremasked_assign_array( + PyArray_NDIM(dst), PyArray_DIMS(dst), + PyArray_DESCR(dst), PyArray_DATA(dst), PyArray_STRIDES(dst), + PyArray_DESCR(src), PyArray_DATA(src), src_strides, + PyArray_DESCR(wheremask), PyArray_DATA(wheremask), + wheremask_strides) < 0) { + goto fail; + } } finish: diff --git a/numpy/core/src/multiarray/array_assign_scalar.c b/numpy/core/src/multiarray/array_assign_scalar.c index 60658ed4d..c59e6752d 100644 --- a/numpy/core/src/multiarray/array_assign_scalar.c +++ b/numpy/core/src/multiarray/array_assign_scalar.c @@ -21,7 +21,6 @@ #include "methods.h" #include "shape.h" #include "lowlevel_strided_loops.h" -#include "na_mask.h" #include "array_assign.h" @@ -152,7 +151,7 @@ raw_array_wheremasked_assign_scalar(int ndim, npy_intp *shape, NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { /* Process the innermost dimension */ stransfer(dst_data, dst_strides_it[0], src_data, 0, - (npy_mask *)wheremask_data, wheremask_strides_it[0], + (npy_bool *)wheremask_data, wheremask_strides_it[0], shape_it[0], src_itemsize, transferdata); } NPY_RAW_ITER_TWO_NEXT(idim, ndim, coord, shape_it, dst_data, dst_strides_it, @@ -168,145 +167,6 @@ raw_array_wheremasked_assign_scalar(int ndim, npy_intp *shape, } /* - * Assigns the scalar value to every element of the destination raw array - * where the 'wheremask' is True, except for those which are masked as NA - * according to 'maskna'. - * - * Returns 0 on success, -1 on failure. - */ -NPY_NO_EXPORT int -raw_array_wheremasked_assign_scalar_preservena(int ndim, npy_intp *shape, - PyArray_Descr *dst_dtype, char *dst_data, npy_intp *dst_strides, - PyArray_Descr *src_dtype, char *src_data, - PyArray_Descr *maskna_dtype, char *maskna_data, - npy_intp *maskna_strides, - PyArray_Descr *wheremask_dtype, char *wheremask_data, - npy_intp *wheremask_strides) -{ - int idim; - npy_intp shape_it[NPY_MAXDIMS], dst_strides_it[NPY_MAXDIMS]; - npy_intp maskna_strides_it[NPY_MAXDIMS]; - npy_intp wheremask_strides_it[NPY_MAXDIMS]; - npy_intp coord[NPY_MAXDIMS]; - - PyArray_MaskedStridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; - int aligned, needs_api = 0; - npy_intp src_itemsize = src_dtype->elsize; - - PyArray_StridedBinaryOp *maskand_stransfer = NULL; - NpyAuxData *maskand_transferdata = NULL; - - char *maskna_buffer; - npy_intp maskna_itemsize; - - NPY_BEGIN_THREADS_DEF; - - /* Check alignment */ - aligned = raw_array_is_aligned(ndim, dst_data, dst_strides, - dst_dtype->alignment); - if (((npy_intp)src_data & (src_dtype->alignment - 1)) != 0) { - aligned = 0; - } - - /* Use raw iteration with no heap allocation */ - if (PyArray_PrepareThreeRawArrayIter( - ndim, shape, - dst_data, dst_strides, - maskna_data, maskna_strides, - wheremask_data, wheremask_strides, - &ndim, shape_it, - &dst_data, dst_strides_it, - &maskna_data, maskna_strides_it, - &wheremask_data, wheremask_strides_it) < 0) { - return -1; - } - - /* Allocate a buffer for inverting/anding the mask */ - maskna_itemsize = maskna_dtype->elsize; - maskna_buffer = PyArray_malloc(NPY_ARRAY_ASSIGN_BUFFERSIZE * - maskna_itemsize); - if (maskna_buffer == NULL) { - PyErr_NoMemory(); - return -1; - } - - /* Get the function to do the casting */ - if (PyArray_GetMaskedDTypeTransferFunction(aligned, - 0, dst_strides_it[0], maskna_itemsize, - src_dtype, dst_dtype, maskna_dtype, - 0, - &stransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { - PyArray_free(maskna_buffer); - return -1; - } - - /* - * Get the function to invert the mask. The output - * of the binary operation is the dtype 'maskna_dtype' - */ - if (PyArray_GetMaskAndFunction( - maskna_strides_it[0], maskna_dtype, 0, - wheremask_strides_it[0], wheremask_dtype, 0, - &maskand_stransfer, &maskand_transferdata) < 0) { - PyArray_free(maskna_buffer); - NPY_AUXDATA_FREE(transferdata); - return -1; - } - - if (!needs_api) { - NPY_BEGIN_THREADS; - } - - NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { - npy_intp buffered_count, count; - char *dst_d, *maskna_d, *wheremask_d; - /* Process the innermost dimension a buffer size at a time */ - count = shape_it[0]; - dst_d = dst_data; - maskna_d = maskna_data; - wheremask_d = wheremask_data; - do { - buffered_count = count < NPY_ARRAY_ASSIGN_BUFFERSIZE - ? count - : NPY_ARRAY_ASSIGN_BUFFERSIZE; - - /* Prepare the mask into the buffer */ - maskand_stransfer(maskna_buffer, maskna_itemsize, - maskna_d, maskna_strides_it[0], - wheremask_d, wheremask_strides_it[0], - buffered_count, maskand_transferdata); - - /* Transfer the data based on the buffered mask */ - stransfer(dst_d, dst_strides_it[0], src_data, 0, - (npy_mask *)maskna_buffer, maskna_itemsize, - buffered_count, src_itemsize, transferdata); - - dst_d += buffered_count * dst_strides_it[0]; - maskna_d += buffered_count * maskna_strides_it[0]; - wheremask_d += buffered_count * wheremask_strides_it[0]; - count -= buffered_count; - } while (count > 0); - } NPY_RAW_ITER_THREE_NEXT(idim, ndim, coord, shape_it, - dst_data, dst_strides_it, - maskna_data, maskna_strides_it, - wheremask_data, wheremask_strides_it); - - if (!needs_api) { - NPY_END_THREADS; - } - - PyArray_free(maskna_buffer); - NPY_AUXDATA_FREE(transferdata); - NPY_AUXDATA_FREE(maskand_transferdata); - - return (needs_api && PyErr_Occurred()) ? -1 : 0; -} - - -/*NUMPY_API - * * Assigns a scalar value specified by 'src_dtype' and 'src_data' * to elements of 'dst'. * @@ -316,11 +176,6 @@ raw_array_wheremasked_assign_scalar_preservena(int ndim, npy_intp *shape, * wheremask: If non-NULL, a boolean mask specifying where to copy. * casting: An exception is raised if the assignment violates this * casting rule. - * preservena: If 0, overwrites everything in 'dst', if 1, it - * preserves elements in 'dst' which are NA. - * preservewhichna: Must be NULL. When multi-NA support is implemented, - * this will be an array of flags for 'preservena=True', - * indicating which NA payload values to preserve. * * This function is implemented in array_assign_scalar.c. * @@ -328,12 +183,11 @@ raw_array_wheremasked_assign_scalar_preservena(int ndim, npy_intp *shape, */ NPY_NO_EXPORT int PyArray_AssignRawScalar(PyArrayObject *dst, - PyArray_Descr *src_dtype, char *src_data, - PyArrayObject *wheremask, - NPY_CASTING casting, - npy_bool preservena, npy_bool *preservewhichna) + PyArray_Descr *src_dtype, char *src_data, + PyArrayObject *wheremask, + NPY_CASTING casting) { - int allocated_src_data = 0, dst_has_maskna = PyArray_HASMASKNA(dst); + int allocated_src_data = 0; npy_longlong scalarbuffer[4]; if (PyArray_FailUnlessWriteable(dst, "assignment destination") < 0) { @@ -358,12 +212,6 @@ PyArray_AssignRawScalar(PyArrayObject *dst, return -1; } - if (preservewhichna != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "multi-NA support is not yet implemented"); - return -1; - } - /* * Make a copy of the src data if it's a different dtype than 'dst' * or isn't aligned, and the destination we're copying to has @@ -399,56 +247,15 @@ PyArray_AssignRawScalar(PyArrayObject *dst, if (wheremask == NULL) { /* A straightforward value assignment */ - if (!preservena || !dst_has_maskna) { - /* If assigning to an array with an NA mask, set to all exposed */ - if (dst_has_maskna) { - if (PyArray_AssignMaskNA(dst, 1, NULL, - preservena, preservewhichna) < 0) { - goto fail; - } - } - - /* Do the assignment with raw array iteration */ - if (raw_array_assign_scalar(PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_DESCR(dst), PyArray_DATA(dst), PyArray_STRIDES(dst), - src_dtype, src_data) < 0) { - goto fail; - } - } - /* A value assignment without overwriting NA values */ - else { - if (raw_array_wheremasked_assign_scalar( - PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_DESCR(dst), PyArray_DATA(dst), PyArray_STRIDES(dst), - src_dtype, src_data, - PyArray_MASKNA_DTYPE(dst), PyArray_MASKNA_DATA(dst), - PyArray_MASKNA_STRIDES(dst)) < 0) { - goto fail; - } + /* Do the assignment with raw array iteration */ + if (raw_array_assign_scalar(PyArray_NDIM(dst), PyArray_DIMS(dst), + PyArray_DESCR(dst), PyArray_DATA(dst), PyArray_STRIDES(dst), + src_dtype, src_data) < 0) { + goto fail; } } else { npy_intp wheremask_strides[NPY_MAXDIMS]; - int containsna = PyArray_ContainsNA(wheremask, NULL, NULL); - - if (containsna == -1) { - goto fail; - } - else if (containsna) { - if (!dst_has_maskna) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which " - "does not support NAs"); - goto fail; - } - else { - /* TODO: add support for this */ - PyErr_SetString(PyExc_ValueError, - "A where mask with NA values is not supported " - "yet"); - goto fail; - } - } /* Broadcast the wheremask to 'dst' for raw iteration */ if (broadcast_strides(PyArray_NDIM(dst), PyArray_DIMS(dst), @@ -458,42 +265,14 @@ PyArray_AssignRawScalar(PyArrayObject *dst, goto fail; } - /* A straightforward where-masked assignment */ - if (!preservena || !dst_has_maskna) { - /* If assigning to an array with an NA mask, set to all exposed */ - if (dst_has_maskna) { - /* - * TODO: If the where mask has NA values, this part - * changes too. - */ - if (PyArray_AssignMaskNA(dst, 1, wheremask, - preservena, preservewhichna) < 0) { - goto fail; - } - } - - /* Do the masked assignment with raw array iteration */ - if (raw_array_wheremasked_assign_scalar( - PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_DESCR(dst), PyArray_DATA(dst), PyArray_STRIDES(dst), - src_dtype, src_data, - PyArray_DESCR(wheremask), PyArray_DATA(wheremask), - wheremask_strides) < 0) { - goto fail; - } - } - /* A masked value assignment without overwriting NA values */ - else { - if (raw_array_wheremasked_assign_scalar_preservena( - PyArray_NDIM(dst), PyArray_DIMS(dst), - PyArray_DESCR(dst), PyArray_DATA(dst), PyArray_STRIDES(dst), - src_dtype, src_data, - PyArray_MASKNA_DTYPE(dst), PyArray_MASKNA_DATA(dst), - PyArray_MASKNA_STRIDES(dst), - PyArray_DESCR(wheremask), PyArray_DATA(wheremask), - wheremask_strides) < 0) { - goto fail; - } + /* Do the masked assignment with raw array iteration */ + if (raw_array_wheremasked_assign_scalar( + PyArray_NDIM(dst), PyArray_DIMS(dst), + PyArray_DESCR(dst), PyArray_DATA(dst), PyArray_STRIDES(dst), + src_dtype, src_data, + PyArray_DESCR(wheremask), PyArray_DATA(wheremask), + wheremask_strides) < 0) { + goto fail; } } diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 4f0181b4a..3cba466dd 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -49,8 +49,6 @@ maintainer email: oliphant.travis@ieee.org #include "getset.h" #include "sequence.h" #include "buffer.h" -#include "na_object.h" -#include "na_mask.h" /*NUMPY_API Compute the size of an array (in number of items) @@ -147,8 +145,7 @@ PyArray_SetBaseObject(PyArrayObject *arr, PyObject *obj) /* * Don't allow chains of views, always set the base * to the owner of the data. That is, either the first object - * which isn't an array, the first object with an NA mask - * which owns that NA mask, or the first object which owns + * which isn't an array, or the first object which owns * its own data. */ while (PyArray_Check(obj) && (PyObject *)arr != obj) { @@ -164,14 +161,6 @@ PyArray_SetBaseObject(PyArrayObject *arr, PyObject *obj) if (PyArray_CHKFLAGS(obj_arr, NPY_ARRAY_OWNDATA)) { break; } - /* - * If 'arr' doesn't own its NA mask, then if - * 'obj' is NA masked and owns the mask, stop collapsing - */ - if (!PyArray_CHKFLAGS(arr, NPY_ARRAY_OWNMASKNA) && - PyArray_CHKFLAGS(obj_arr, NPY_ARRAY_OWNMASKNA)) { - break; - } /* If there's no base, stop collapsing */ tmp = PyArray_BASE(obj_arr); if (tmp == NULL) { @@ -201,7 +190,7 @@ PyArray_SetBaseObject(PyArrayObject *arr, PyObject *obj) NPY_NO_EXPORT int PyArray_CopyObject(PyArrayObject *dest, PyObject *src_object) { - int ret, contains_na = 0; + int ret = 0; PyArrayObject *src; PyArray_Descr *dtype = NULL; int ndim = 0; @@ -236,17 +225,8 @@ PyArray_CopyObject(PyArrayObject *dest, PyObject *src_object) * Get either an array object we can copy from, or its parameters * if there isn't a convenient array available. */ - if (PyArray_GetArrayParamsFromObjectEx(src_object, PyArray_DESCR(dest), - 0, &dtype, &ndim, dims, &contains_na, &src, NULL) < 0) { - Py_DECREF(src_object); - return -1; - } - - if (contains_na && !(PyArray_HasNASupport(dest) || - PyArray_DESCR(dest)->type_num == NPY_OBJECT)) { - PyErr_SetString(PyExc_ValueError, - "Cannot set NumPy array values to NA values without first " - "enabling NA support in the array"); + if (PyArray_GetArrayParamsFromObject(src_object, PyArray_DESCR(dest), + 0, &dtype, &ndim, dims, &src, NULL) < 0) { Py_DECREF(src_object); return -1; } @@ -255,13 +235,6 @@ PyArray_CopyObject(PyArrayObject *dest, PyObject *src_object) if (src == NULL) { /* If the input is scalar */ if (ndim == 0) { - NpyNA *na = NULL; - - /* Get an NpyNA if src is NA, but don't raise an error */ - if (PyArray_HasNASupport(dest)) { - na = NpyNA_FromObject(src_object, 1); - } - /* If there's one dest element and src is a Python scalar */ if (PyArray_IsScalar(src_object, Generic)) { char *value; @@ -276,24 +249,11 @@ PyArray_CopyObject(PyArrayObject *dest, PyObject *src_object) /* TODO: switch to SAME_KIND casting */ retcode = PyArray_AssignRawScalar(dest, dtype, value, - NULL, NPY_UNSAFE_CASTING, 0, NULL); + NULL, NPY_UNSAFE_CASTING); Py_DECREF(dtype); Py_DECREF(src_object); return retcode; } - /* Assigning NA affects the mask if it exists */ - else if (na != NULL) { - if (PyArray_AssignNA(dest, na, NULL, 0, NULL) < 0) { - Py_DECREF(dtype); - Py_DECREF(na); - Py_DECREF(src_object); - return -1; - } - - Py_DECREF(na); - Py_DECREF(src_object); - return 0; - } /* Otherwise use the dtype's setitem function */ else { if (PyArray_SIZE(dest) == 1) { @@ -301,10 +261,6 @@ PyArray_CopyObject(PyArrayObject *dest, PyObject *src_object) Py_DECREF(src_object); ret = PyArray_DESCR(dest)->f->setitem(src_object, PyArray_DATA(dest), dest); - /* Unmask the value if necessary */ - if (ret == 0 && PyArray_HASMASKNA(dest)) { - PyArray_MASKNA_DATA(dest)[0] = 1; - } return ret; } else { @@ -344,12 +300,6 @@ PyArray_CopyObject(PyArrayObject *dest, PyObject *src_object) Py_DECREF(src_object); return -1; } - if (PyArray_HASMASKNA(dest)) { - if (PyArray_AllocateMaskNA(dest, 1, 0, 1) < 0) { - Py_DECREF(src_object); - return -1; - } - } if (PyArray_AssignFromSequence(src, src_object) < 0) { Py_DECREF(src); Py_DECREF(src_object); @@ -457,14 +407,6 @@ array_dealloc(PyArrayObject *self) PyDataMem_FREE(fa->data); } - /* If the array has an NA mask, free its associated data */ - if (fa->flags & NPY_ARRAY_MASKNA) { - Py_XDECREF(fa->maskna_dtype); - if (fa->flags & NPY_ARRAY_OWNMASKNA) { - PyDataMem_FREE(fa->maskna_data); - } - } - PyDimMem_FREE(fa->dimensions); Py_DECREF(fa->descr); Py_TYPE(self)->tp_free((PyObject *)self); @@ -579,24 +521,8 @@ PyArray_DebugPrint(PyArrayObject *obj) printf(" NPY_WRITEABLE"); if (fobj->flags & NPY_ARRAY_UPDATEIFCOPY) printf(" NPY_UPDATEIFCOPY"); - if (fobj->flags & NPY_ARRAY_MASKNA) - printf(" MASKNA"); - if (fobj->flags & NPY_ARRAY_OWNMASKNA) - printf(" OWNMASKNA"); printf("\n"); - if (fobj->flags & NPY_ARRAY_MASKNA) { - printf(" maskna dtype : "); - PyObject_Print((PyObject *)fobj->maskna_dtype, stdout, 0); - printf("\n"); - printf(" maskna data : %p\n", fobj->maskna_data); - printf(" maskna strides:"); - for (i = 0; i < fobj->nd; ++i) { - printf(" %d", (int)fobj->maskna_strides[i]); - } - printf("\n"); - } - if (fobj->base != NULL && PyArray_Check(fobj->base)) { printf("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"); printf("Dump of array's BASE:\n"); @@ -1348,8 +1274,8 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) dtype = PyArray_DTYPE(self); Py_INCREF(dtype); } - array_other = (PyArrayObject *)PyArray_FromAny(other, dtype, 0, 0, - NPY_ARRAY_ALLOWNA, NULL); + array_other = (PyArrayObject *)PyArray_FromAny(other, dtype, 0, 0, 0, + NULL); /* * If not successful, indicate that the items cannot be compared * this way. @@ -1405,8 +1331,8 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) dtype = PyArray_DTYPE(self); Py_INCREF(dtype); } - array_other = (PyArrayObject *)PyArray_FromAny(other, dtype, 0, 0, - NPY_ARRAY_ALLOWNA, NULL); + array_other = (PyArrayObject *)PyArray_FromAny(other, dtype, 0, 0, 0, + NULL); /* * If not successful, indicate that the items cannot be compared * this way. diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index 1846d8114..c446619b6 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -17,7 +17,6 @@ #include "ctors.h" #include "usertypes.h" #include "_datetime.h" -#include "na_object.h" #include "arrayobject.h" #include "numpyos.h" @@ -154,11 +153,6 @@ static int if (PyArray_IsScalar(op, @kind@)) { temp = ((Py@kind@ScalarObject *)op)->obval; } - else if (NpyNA_Check(op) || NpyNA_IsZeroDimArrayNA(op)) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which does not support NAs"); - return -1; - } else { temp = (@type@)@func2@(op); } @@ -226,11 +220,6 @@ static int if (PyArray_IsScalar(op, @kind@)){ temp = ((Py@kind@ScalarObject *)op)->obval; } - else if (NpyNA_Check(op) || NpyNA_IsZeroDimArrayNA(op)) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which does not support NAs"); - return -1; - } else { if (PyArray_Check(op) && (PyArray_NDIM((PyArrayObject *)op) == 0)) { op2 = PyArray_DESCR((PyArrayObject *)op)->f->getitem( @@ -285,11 +274,6 @@ LONGDOUBLE_setitem(PyObject *op, char *ov, PyArrayObject *ap) { if (PyArray_IsScalar(op, LongDouble)) { temp = ((PyLongDoubleScalarObject *)op)->obval; } - else if (NpyNA_Check(op) || NpyNA_IsZeroDimArrayNA(op)) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which does not support NAs"); - return -1; - } else { temp = (npy_longdouble) MyPyFloat_AsDouble(op); } @@ -339,11 +323,6 @@ UNICODE_setitem(PyObject *op, char *ov, PyArrayObject *ap) "setting an array element with a sequence"); return -1; } - if (NpyNA_Check(op) || NpyNA_IsZeroDimArrayNA(op)) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which does not support NAs"); - return -1; - } /* Sequence_Size might have returned an error */ if (PyErr_Occurred()) { PyErr_Clear(); @@ -448,12 +427,6 @@ STRING_setitem(PyObject *op, char *ov, PyArrayObject *ap) "cannot set an array element with a sequence"); return -1; } - if (NpyNA_Check(op) || NpyNA_IsZeroDimArrayNA(op)) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which " - "does not support NAs"); - return -1; - } #if defined(NPY_PY3K) if (PyUnicode_Check(op)) { /* Assume ASCII codec -- function similarly as Python 2 */ @@ -755,12 +728,6 @@ VOID_setitem(PyObject *op, char *ip, PyArrayObject *ap) return res; } - if (NpyNA_Check(op) || NpyNA_IsZeroDimArrayNA(op)) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which does not support NAs"); - return -1; - } - /* Default is to use buffer interface to set item */ { const void *buffer; diff --git a/numpy/core/src/multiarray/boolean_ops.c.src b/numpy/core/src/multiarray/boolean_ops.c.src deleted file mode 100644 index 6e60aac2b..000000000 --- a/numpy/core/src/multiarray/boolean_ops.c.src +++ /dev/null @@ -1,405 +0,0 @@ -/* - * This file implements some boolean methods which have special NA - * interactions, including np.any and np.all. Could later gain - * np.logical_or and np.logical_and, as well as np.bitwise_or and - * np.bitwise_and because they are often used for boolean operations. - * - * NOTE: These functions assume that the input boolean data is valid, - * i.e. that each boolean is either 0 or 1. Any deviation from this - * may produce incorrect answers. - * - * Written by Mark Wiebe (mwwiebe@gmail.com) - * Copyright (c) 2011 by Enthought, Inc. - * - * See LICENSE.txt for the license. - */ - -#define PY_SSIZE_T_CLEAN -#include <Python.h> - -#define NPY_NO_DEPRECATED_API NPY_API_VERSION -#define _MULTIARRAYMODULE -#include <numpy/arrayobject.h> - -#include "npy_config.h" -#include "numpy/npy_3kcompat.h" - -#include "reduction.h" - -/* Typedef for the reduction inner loop */ -typedef void (reduce_inner_loop)(char **, npy_intp *, npy_intp); - -#define ANY_COMBINE_MASK(val, mask) ((val) & (mask)) -#define ALL_COMBINE_MASK(val, mask) ((val) | ((mask) ^ 0x01)) - -/**begin repeat - * #oper = any, all# - * #Oper = Any, All# - * #Identity = Zero, One# - * #combineop = |=, &=# - * #combinemask = ANY_COMBINE_MASK, ALL_COMBINE_MASK# - * #shortcircuit = !value, value# - * #idval = 0, 1# - */ - -static int -assign_identity_@oper@(PyArrayObject *result, int preservena, void *data) -{ - return PyArray_Assign@Identity@(result, NULL, preservena, NULL); -} - -static void -@oper@_inner_gen_gen(char **dataptr, npy_intp *strides, npy_intp count) -{ - char *data0 = dataptr[0], *data1 = dataptr[1]; - npy_intp stride0 = strides[0], stride1 = strides[1]; - npy_intp i; - - for (i = 0; i < count; ++i) { - *data0 @combineop@ *data1; - - data0 += stride0; - data1 += stride1; - } -} - -static void -@oper@_inner_0stride_gen(char **dataptr, npy_intp *strides, npy_intp count) -{ - char *data0 = dataptr[0], *data1 = dataptr[1]; - npy_intp stride1 = strides[1]; - npy_intp i; - char value = *data0; - - for (i = 0; i < count && @shortcircuit@; ++i) { - value @combineop@ *data1; - - data1 += stride1; - } - - *(npy_bool *)data0 = value; -} - -static void -@oper@_inner_0stride_contig(char **dataptr, npy_intp *strides, npy_intp count) -{ - char *data0 = dataptr[0], *data1 = dataptr[1]; - char value = *data0; - - if (@shortcircuit@ && memchr(data1, 1-@idval@, count) != NULL) { - *(npy_bool *)data0 = 1-@idval@; - } -} - -static void -@oper@_inner_contig_contig(char **dataptr, npy_intp *strides, npy_intp count) -{ - char *data0 = dataptr[0], *data1 = dataptr[1]; - npy_intp i; - - for (i = 0; i < count; ++i) { - *data0 @combineop@ *data1; - - ++data0; - ++data1; - } -} - -static int -reduce_@oper@_loop(NpyIter *iter, - char **dataptr, - npy_intp *strides, - npy_intp *countptr, - NpyIter_IterNextFunc *iternext, - int needs_api, - npy_intp skip_first_count, - void *data) -{ - npy_intp fixed_strides[2]; - reduce_inner_loop *inner_loop; - NPY_BEGIN_THREADS_DEF; - - NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); - - if (!needs_api) { - NPY_BEGIN_THREADS; - } - - /* Choose a loop specialized based on the strides */ - if (fixed_strides[0] == 0) { - if (fixed_strides[1] == 1) { - inner_loop = &@oper@_inner_0stride_contig; - } - else { - inner_loop = &@oper@_inner_0stride_gen; - } - } - else { - if (fixed_strides[0] == 1 && fixed_strides[1] == 1) { - inner_loop = &@oper@_inner_contig_contig; - } - else { - inner_loop = &@oper@_inner_gen_gen; - } - } - - /* - * 'skip_first_count' will always be 0 because we are doing a reduction - * with an identity. - */ - - do { - inner_loop(dataptr, strides, *countptr); - } while (iternext(iter)); - - if (!needs_api) { - NPY_END_THREADS; - } - - return (needs_api && PyErr_Occurred()) ? -1 : 0; -} - -static void -@oper@_masked_inner_gen_gen_gen(char **dataptr, - npy_intp *strides, npy_intp count) -{ - char *data0 = dataptr[0], *data1 = dataptr[1], *data2 = dataptr[2]; - npy_intp stride0 = strides[0], stride1 = strides[1], stride2 = strides[2]; - npy_intp i; - - for (i = 0; i < count; ++i) { - *data0 @combineop@ @combinemask@(*data1, *data2); - - data0 += stride0; - data1 += stride1; - data2 += stride2; - } -} - -static void -@oper@_masked_inner_0stride_gen_gen(char **dataptr, - npy_intp *strides, npy_intp count) -{ - char *data0 = dataptr[0], *data1 = dataptr[1], *data2 = dataptr[2]; - npy_intp stride1 = strides[1], stride2 = strides[2]; - npy_intp i; - char value = *data0; - - for (i = 0; i < count && @shortcircuit@; ++i) { - value @combineop@ @combinemask@(*data1, *data2); - - data1 += stride1; - data2 += stride2; - } - - *(npy_bool *)data0 = value; -} - -static void -@oper@_masked_inner_0stride_gen_0stride(char **dataptr, - npy_intp *strides, npy_intp count) -{ - char *data0 = dataptr[0], *data1 = dataptr[1], *data2 = dataptr[2]; - npy_intp stride1 = strides[1]; - npy_intp i; - char maskvalue = *data2; - - if (maskvalue) { - char value = *data0; - - for (i = 0; i < count && @shortcircuit@; ++i) { - value @combineop@ *data1; - - data1 += stride1; - } - - *(npy_bool *)data0 = value; - } -} - -static int -reduce_@oper@_masked_loop(NpyIter *iter, - char **dataptr, - npy_intp *strides, - npy_intp *countptr, - NpyIter_IterNextFunc *iternext, - int needs_api, - npy_intp skip_first_count, - void *data) -{ - npy_intp fixed_strides[3]; - reduce_inner_loop *inner_loop; - NPY_BEGIN_THREADS_DEF; - - NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); - - if (!needs_api) { - NPY_BEGIN_THREADS; - } - - /* Choose a loop specialized based on the strides */ - if (fixed_strides[0] == 0) { - if (fixed_strides[2] == 0) { - inner_loop = &@oper@_masked_inner_0stride_gen_0stride; - } - else { - inner_loop = &@oper@_masked_inner_0stride_gen_gen; - } - } - else { - inner_loop = &@oper@_masked_inner_gen_gen_gen; - } - - /* - * 'skip_first_count' will always be 0 because we are doing a reduction - * with an identity. - */ - - do { - inner_loop(dataptr, strides, *countptr); - } while (iternext(iter)); - - if (!needs_api) { - NPY_END_THREADS; - } - - return (needs_api && PyErr_Occurred()) ? -1 : 0; -} - -static void -@oper@_adv_masked_inner_gen_gen_gen_gen(char **dataptr, - npy_intp *strides, npy_intp count) -{ - char *data0 = dataptr[0], *data1 = dataptr[1]; - char *data2 = dataptr[2], *data3 = dataptr[3]; - npy_intp stride0 = strides[0], stride1 = strides[1]; - npy_intp stride2 = strides[2], stride3 = strides[3]; - npy_intp i; - - for (i = 0; i < count; ++i) { - /* Normal case */ - if (*data2) { - *data0 @combineop@ *data1; - } - /* - * If the value is an exposed True (for any) or False (for all), - * expose the result as well - */ - else if (@combinemask@(*data1, *data3) == 1-@idval@) { - *data0 = 1-@idval@; - *data2 = 1; - } - - data0 += stride0; - data1 += stride1; - data2 += stride2; - data3 += stride3; - } -} - -static void -@oper@_adv_masked_inner_0stride_gen_0stride_gen(char **dataptr, - npy_intp *strides, npy_intp count) -{ - char *data0 = dataptr[0], *data1 = dataptr[1]; - char *data2 = dataptr[2], *data3 = dataptr[3]; - npy_intp stride1 = strides[1], stride3 = strides[3]; - npy_intp i; - char maskvalue = *data2; - char value = maskvalue ? *data0 : @idval@; - - for (i = 0; i < count && @shortcircuit@; ++i) { - /* Normal case */ - if (maskvalue) { - value @combineop@ *data1; - } - /* - * If the value is an exposed True (for any) or False (for all), - * expose the result as well - */ - else if (@combinemask@(*data1, *data3) == 1-@idval@) { - value = 1-@idval@; - maskvalue = 1; - break; - } - - data1 += stride1; - data3 += stride3; - } - - if (maskvalue) { - *data0 = value; - *data2 = maskvalue; - } -} - -static int -reduce_@oper@_advanced_masked_loop(NpyIter *iter, - char **dataptr, - npy_intp *strides, - npy_intp *countptr, - NpyIter_IterNextFunc *iternext, - int needs_api, - npy_intp skip_first_count, - void *data) -{ - npy_intp fixed_strides[4]; - reduce_inner_loop *inner_loop; - NPY_BEGIN_THREADS_DEF; - - NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); - - if (!needs_api) { - NPY_BEGIN_THREADS; - } - - /* Choose a loop specialized based on the strides */ - if (fixed_strides[0] == 0 && fixed_strides[2] == 0) { - inner_loop = &@oper@_adv_masked_inner_0stride_gen_0stride_gen; - } - else { - inner_loop = &@oper@_adv_masked_inner_gen_gen_gen_gen; - } - - /* - * 'skip_first_count' will always be 0 because we are doing a reduction - * with an identity. - */ - - do { - inner_loop(dataptr, strides, *countptr); - } while (iternext(iter)); - - if (!needs_api) { - NPY_END_THREADS; - } - - return (needs_api && PyErr_Occurred()) ? -1 : 0; -} - -NPY_NO_EXPORT PyArrayObject * -PyArray_Reduce@Oper@(PyArrayObject *arr, PyArrayObject *out, - npy_bool *axis_flags, int skipna, int keepdims) -{ - PyArrayObject *result; - PyArray_Descr *bool_dtype; - - bool_dtype = PyArray_DescrFromType(NPY_BOOL); - if (bool_dtype == NULL) { - return NULL; - } - - result = PyArray_ReduceWrapper(arr, out, NULL, - bool_dtype, bool_dtype, - NPY_UNSAFE_CASTING, - axis_flags, 1, skipna, NULL, keepdims, 1, - &assign_identity_@oper@, - &reduce_@oper@_loop, - &reduce_@oper@_masked_loop, - &reduce_@oper@_advanced_masked_loop, - NULL, 0, "@oper@"); - Py_DECREF(bool_dtype); - return result; -} - -/**end repeat**/ diff --git a/numpy/core/src/multiarray/boolean_ops.h b/numpy/core/src/multiarray/boolean_ops.h deleted file mode 100644 index a6674e2aa..000000000 --- a/numpy/core/src/multiarray/boolean_ops.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef _NPY_PRIVATE__BOOLEAN_OPS_H_ -#define _NPY_PRIVATE__BOOLEAN_OPS_H_ - -NPY_NO_EXPORT PyArrayObject * -PyArray_ReduceAny(PyArrayObject *arr, PyArrayObject *out, - npy_bool *axis_flags, int skipna, int keepdims); - -NPY_NO_EXPORT PyArrayObject * -PyArray_ReduceAll(PyArrayObject *arr, PyArrayObject *out, - npy_bool *axis_flags, int skipna, int keepdims); - - -#endif diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index a1ad23911..f9a7dddc0 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -603,10 +603,6 @@ array_getbuffer(PyObject *obj, Py_buffer *view, int flags) self = (PyArrayObject*)obj; /* Check whether we can provide the wanted properties */ - if (PyArray_HASMASKNA(obj)) { - PyErr_SetString(PyExc_TypeError, "NA-masked arrays are not supported by the Python buffer protocol"); - goto fail; - } if ((flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS && !PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)) { PyErr_SetString(PyExc_ValueError, "ndarray is not C-contiguous"); diff --git a/numpy/core/src/multiarray/calculation.c b/numpy/core/src/multiarray/calculation.c index 623c70eea..d8d02ef0c 100644 --- a/numpy/core/src/multiarray/calculation.c +++ b/numpy/core/src/multiarray/calculation.c @@ -267,7 +267,7 @@ PyArray_Max(PyArrayObject *ap, int axis, PyArrayObject *out) PyArrayObject *arr; PyObject *ret; - arr = (PyArrayObject *)PyArray_CheckAxis(ap, &axis, NPY_ARRAY_ALLOWNA); + arr = (PyArrayObject *)PyArray_CheckAxis(ap, &axis, 0); if (arr == NULL) { return NULL; } @@ -286,7 +286,7 @@ PyArray_Min(PyArrayObject *ap, int axis, PyArrayObject *out) PyArrayObject *arr; PyObject *ret; - arr=(PyArrayObject *)PyArray_CheckAxis(ap, &axis, NPY_ARRAY_ALLOWNA); + arr=(PyArrayObject *)PyArray_CheckAxis(ap, &axis, 0); if (arr == NULL) { return NULL; } @@ -306,7 +306,7 @@ PyArray_Ptp(PyArrayObject *ap, int axis, PyArrayObject *out) PyObject *ret; PyObject *obj1 = NULL, *obj2 = NULL; - arr=(PyArrayObject *)PyArray_CheckAxis(ap, &axis, NPY_ARRAY_ALLOWNA); + arr=(PyArrayObject *)PyArray_CheckAxis(ap, &axis, 0); if (arr == NULL) { return NULL; } @@ -359,7 +359,7 @@ __New_PyArray_Std(PyArrayObject *self, int axis, int rtype, PyArrayObject *out, int i, n; npy_intp val; - arrnew = (PyArrayObject *)PyArray_CheckAxis(self, &axis, NPY_ARRAY_ALLOWNA); + arrnew = (PyArrayObject *)PyArray_CheckAxis(self, &axis, 0); if (arrnew == NULL) { return NULL; } @@ -493,8 +493,7 @@ __New_PyArray_Std(PyArrayObject *self, int axis, int rtype, PyArrayObject *out, finish: if (out) { if (PyArray_AssignArray(out, (PyArrayObject *)ret, - NULL, NPY_DEFAULT_ASSIGN_CASTING, - 0, NULL) < 0) { + NULL, NPY_DEFAULT_ASSIGN_CASTING) < 0) { Py_DECREF(ret); return NULL; } @@ -514,7 +513,7 @@ PyArray_Sum(PyArrayObject *self, int axis, int rtype, PyArrayObject *out) { PyObject *arr, *ret; - arr = PyArray_CheckAxis(self, &axis, NPY_ARRAY_ALLOWNA); + arr = PyArray_CheckAxis(self, &axis, 0); if (arr == NULL) { return NULL; } @@ -532,7 +531,7 @@ PyArray_Prod(PyArrayObject *self, int axis, int rtype, PyArrayObject *out) { PyObject *arr, *ret; - arr = PyArray_CheckAxis(self, &axis, NPY_ARRAY_ALLOWNA); + arr = PyArray_CheckAxis(self, &axis, 0); if (arr == NULL) { return NULL; } @@ -551,7 +550,7 @@ PyArray_CumSum(PyArrayObject *self, int axis, int rtype, PyArrayObject *out) { PyObject *arr, *ret; - arr = PyArray_CheckAxis(self, &axis, NPY_ARRAY_ALLOWNA); + arr = PyArray_CheckAxis(self, &axis, 0); if (arr == NULL) { return NULL; } @@ -570,7 +569,7 @@ PyArray_CumProd(PyArrayObject *self, int axis, int rtype, PyArrayObject *out) { PyObject *arr, *ret; - arr = PyArray_CheckAxis(self, &axis, NPY_ARRAY_ALLOWNA); + arr = PyArray_CheckAxis(self, &axis, 0); if (arr == NULL) { return NULL; } @@ -661,8 +660,7 @@ PyArray_Round(PyArrayObject *a, int decimals, PyArrayObject *out) if (PyArray_ISINTEGER(a)) { if (out) { if (PyArray_AssignArray(out, a, - NULL, NPY_DEFAULT_ASSIGN_CASTING, - 0, NULL) < 0) { + NULL, NPY_DEFAULT_ASSIGN_CASTING) < 0) { return NULL; } Py_INCREF(out); @@ -752,7 +750,7 @@ PyArray_Mean(PyArrayObject *self, int axis, int rtype, PyArrayObject *out) PyObject *obj1 = NULL, *obj2 = NULL, *ret; PyArrayObject *arr; - arr = (PyArrayObject *)PyArray_CheckAxis(self, &axis, NPY_ARRAY_ALLOWNA); + arr = (PyArrayObject *)PyArray_CheckAxis(self, &axis, 0); if (arr == NULL) { return NULL; } @@ -788,7 +786,7 @@ PyArray_Any(PyArrayObject *self, int axis, PyArrayObject *out) { PyObject *arr, *ret; - arr = PyArray_CheckAxis(self, &axis, NPY_ARRAY_ALLOWNA); + arr = PyArray_CheckAxis(self, &axis, 0); if (arr == NULL) { return NULL; } @@ -807,7 +805,7 @@ PyArray_All(PyArrayObject *self, int axis, PyArrayObject *out) { PyObject *arr, *ret; - arr = PyArray_CheckAxis(self, &axis, NPY_ARRAY_ALLOWNA); + arr = PyArray_CheckAxis(self, &axis, 0); if (arr == NULL) { return NULL; } @@ -888,11 +886,6 @@ _slow_array_clip(PyArrayObject *self, PyObject *min, PyObject *max, PyArrayObjec /*NUMPY_API * Clip - * - * TODO: For adding NA support, a Clip UFunc should be created, then - * this should call that ufunc. 'min' and 'max' can default to - * the -inf/+inf or the smallest/largest representable values - * of the dtype respectively. */ NPY_NO_EXPORT PyObject * PyArray_Clip(PyArrayObject *self, PyObject *min, PyObject *max, PyArrayObject *out) @@ -921,17 +914,9 @@ PyArray_Clip(PyArrayObject *self, PyObject *min, PyObject *max, PyArrayObject *o } func = PyArray_DESCR(self)->f->fastclip; - /* Trigger the slow array clip for NA support as well */ - if (func == NULL || - PyArray_HASMASKNA(self) || - (min != NULL && - (!PyArray_CheckAnyScalar(min) || - (PyArray_Check(min) && - PyArray_HASMASKNA((PyArrayObject *)min)))) || - (max != NULL && - (!PyArray_CheckAnyScalar(max) || - (PyArray_Check(max) && - PyArray_HASMASKNA((PyArrayObject *)max))))) { + if (func == NULL + || (min != NULL && !PyArray_CheckAnyScalar(min)) + || (max != NULL && !PyArray_CheckAnyScalar(max))) { return _slow_array_clip(self, min, max, out); } /* Use the fast scalar clip function */ @@ -1102,12 +1087,6 @@ PyArray_Clip(PyArrayObject *self, PyObject *min, PyObject *max, PyArrayObject *o goto fail; } - if ((maxa != NULL && PyArray_HASMASKNA(maxa)) || - (mina != NULL && PyArray_HASMASKNA(mina))) { - if (PyArray_AllocateMaskNA(out, 1, 0, 1) < 0) { - goto fail; - } - } outgood = 1; } else Py_INCREF(out); @@ -1152,8 +1131,7 @@ PyArray_Clip(PyArrayObject *self, PyObject *min, PyObject *max, PyArrayObject *o } if (PyArray_DATA(newout) != PyArray_DATA(newin)) { if (PyArray_AssignArray(newout, newin, - NULL, NPY_DEFAULT_ASSIGN_CASTING, - 0, NULL) < 0) { + NULL, NPY_DEFAULT_ASSIGN_CASTING) < 0) { goto fail; } } @@ -1206,8 +1184,7 @@ PyArray_Conjugate(PyArrayObject *self, PyArrayObject *out) PyArrayObject *ret; if (out) { if (PyArray_AssignArray(out, self, - NULL, NPY_DEFAULT_ASSIGN_CASTING, - 0, NULL) < 0) { + NULL, NPY_DEFAULT_ASSIGN_CASTING) < 0) { return NULL; } ret = out; diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 4be3afb4d..9281a6d84 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -84,12 +84,6 @@ _use_default_type(PyObject *op) * * 'maxdims' is the maximum recursion depth. * - * 'out_contains_na' gets set to 1 if an np.NA object is encountered. - * The NA does not affect the dtype produced, so if this is set to 1 - * and the result is for an array without NA support, the dtype should - * be switched to NPY_OBJECT. When adding multi-NA support, this should - * also signal whether just regular NAs or NAs with payloads were seen. - * * 'out_dtype' should be either NULL or a minimal starting dtype when * the function is called. It is updated with the results of type * promotion. This dtype does not get updated when processing NA objects. @@ -98,30 +92,28 @@ _use_default_type(PyObject *op) * Returns 0 on success, -1 on failure. */ NPY_NO_EXPORT int -PyArray_DTypeFromObject(PyObject *obj, int maxdims, int *out_contains_na, - PyArray_Descr **out_dtype) +PyArray_DTypeFromObject(PyObject *obj, int maxdims, PyArray_Descr **out_dtype) { int res; - res = PyArray_DTypeFromObjectHelper(obj, maxdims, out_contains_na, - out_dtype, 0); + res = PyArray_DTypeFromObjectHelper(obj, maxdims, out_dtype, 0); if (res == RETRY_WITH_STRING) { - res = PyArray_DTypeFromObjectHelper(obj, maxdims, out_contains_na, - out_dtype, NPY_STRING); + res = PyArray_DTypeFromObjectHelper(obj, maxdims, + out_dtype, NPY_STRING); if (res == RETRY_WITH_UNICODE) { res = PyArray_DTypeFromObjectHelper(obj, maxdims, - out_contains_na, out_dtype, NPY_UNICODE); + out_dtype, NPY_UNICODE); } } else if (res == RETRY_WITH_UNICODE) { - res = PyArray_DTypeFromObjectHelper(obj, maxdims, out_contains_na, - out_dtype, NPY_UNICODE); + res = PyArray_DTypeFromObjectHelper(obj, maxdims, + out_dtype, NPY_UNICODE); } return res; } NPY_NO_EXPORT int -PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, int *out_contains_na, +PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, PyArray_Descr **out_dtype, int string_type) { int i, size; @@ -133,15 +125,6 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, int *out_contains_na, /* Check if it's an ndarray */ if (PyArray_Check(obj)) { - /* Check for any NAs in the array */ - int containsna = PyArray_ContainsNA((PyArrayObject *)obj, NULL, NULL); - - if (containsna == -1) { - goto fail; - } - else if (containsna) { - *out_contains_na = 1; - } dtype = PyArray_DESCR((PyArrayObject *)obj); Py_INCREF(dtype); goto promote_types; @@ -240,12 +223,6 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, int *out_contains_na, goto promote_types; } - /* Check if it's an NA */ - if (NpyNA_Check(obj)) { - *out_contains_na = 1; - return 0; - } - /* Check if it's an ASCII string */ if (PyBytes_Check(obj)) { int itemsize = PyString_GET_SIZE(obj); @@ -430,7 +407,7 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, int *out_contains_na, goto fail; } res = PyArray_DTypeFromObjectHelper(ip, maxdims - 1, - out_contains_na, out_dtype, string_type); + out_dtype, string_type); if (res < 0) { Py_DECREF(ip); goto fail; diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index a682a4a9d..b68d4286b 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -11,12 +11,6 @@ * * 'maxdims' is the maximum recursion depth. * - * 'out_contains_na' gets set to 1 if an np.NA object is encountered. - * The NA does not affect the dtype produced, so if this is set to 1 - * and the result is for an array without NA support, the dtype should - * be switched to NPY_OBJECT. When adding multi-NA support, this should - * also signal whether just regular NAs or NAs with payloads were seen. - * * 'out_dtype' should be either NULL or a minimal starting dtype when * the function is called. It is updated with the results of type * promotion. This dtype does not get updated when processing NA objects. @@ -24,11 +18,11 @@ * Returns 0 on success, -1 on failure. */ NPY_NO_EXPORT int -PyArray_DTypeFromObject(PyObject *obj, int maxdims, int *out_contains_na, +PyArray_DTypeFromObject(PyObject *obj, int maxdims, PyArray_Descr **out_dtype); NPY_NO_EXPORT int -PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, int *out_contains_na, +PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims, PyArray_Descr **out_dtype, int string_status); /* diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index b883eefdc..ec759b85b 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -28,46 +28,12 @@ * PyArg_ParseTuple. It will immediately return an object of array type * or will convert to a NPY_ARRAY_CARRAY any other object. * - * This function will not allow an array which supports NA through, - * to allow code which doesn't support NA to continue working as is. - * * If you use PyArray_Converter, you must DECREF the array when finished * as you get a new reference to it. */ NPY_NO_EXPORT int PyArray_Converter(PyObject *object, PyObject **address) { - if (PyArray_Check(object) && !PyArray_HASMASKNA((PyArrayObject *)object)) { - *address = object; - Py_INCREF(object); - return NPY_SUCCEED; - } - else { - *address = PyArray_FromAny(object, NULL, 0, 0, - NPY_ARRAY_CARRAY, NULL); - if (*address == NULL) { - return NPY_FAIL; - } - return NPY_SUCCEED; - } -} - -/*NUMPY_API - * - * Useful to pass as converter function for O& processing in PyArgs_ParseTuple. - * - * This conversion function can be used with the "O&" argument for - * PyArg_ParseTuple. It will immediately return an object of array type - * or will convert to a NPY_ARRAY_CARRAY any other object. - * - * This function allows NA-arrays through. - * - * If you use PyArray_AllowNAConverter, you must DECREF the array when finished - * as you get a new reference to it. - */ -NPY_NO_EXPORT int -PyArray_AllowNAConverter(PyObject *object, PyObject **address) -{ if (PyArray_Check(object)) { *address = object; Py_INCREF(object); @@ -75,7 +41,7 @@ PyArray_AllowNAConverter(PyObject *object, PyObject **address) } else { *address = PyArray_FromAny(object, NULL, 0, 0, - NPY_ARRAY_CARRAY | NPY_ARRAY_ALLOWNA, NULL); + NPY_ARRAY_CARRAY, NULL); if (*address == NULL) { return NPY_FAIL; } @@ -95,36 +61,6 @@ PyArray_OutputConverter(PyObject *object, PyArrayObject **address) return NPY_SUCCEED; } if (PyArray_Check(object)) { - if (PyArray_HASMASKNA((PyArrayObject *)object)) { - PyErr_SetString(PyExc_TypeError, - "this operation does not yet support output " - "arrays with NA support"); - *address = NULL; - return NPY_FAIL; - } - *address = (PyArrayObject *)object; - return NPY_SUCCEED; - } - else { - PyErr_SetString(PyExc_TypeError, - "output must be an array"); - *address = NULL; - return NPY_FAIL; - } -} - -/*NUMPY_API - * Useful to pass as converter function for O& processing in - * PyArgs_ParseTuple for output arrays - */ -NPY_NO_EXPORT int -PyArray_OutputAllowNAConverter(PyObject *object, PyArrayObject **address) -{ - if (object == NULL || object == Py_None) { - *address = NULL; - return NPY_SUCCEED; - } - if (PyArray_Check(object)) { *address = (PyArrayObject *)object; return NPY_SUCCEED; } diff --git a/numpy/core/src/multiarray/convert.c b/numpy/core/src/multiarray/convert.c index 4b698f6e5..042aea07b 100644 --- a/numpy/core/src/multiarray/convert.c +++ b/numpy/core/src/multiarray/convert.c @@ -15,6 +15,7 @@ #include "mapping.h" #include "lowlevel_strided_loops.h" #include "scalartypes.h" +#include "array_assign.h" #include "convert.h" @@ -404,7 +405,7 @@ PyArray_FillWithScalar(PyArrayObject *arr, PyObject *obj) if (value != NULL) { /* TODO: switch to SAME_KIND casting */ retcode = PyArray_AssignRawScalar(arr, dtype, value, - NULL, NPY_UNSAFE_CASTING, 0, NULL); + NULL, NPY_UNSAFE_CASTING); Py_DECREF(dtype); return retcode; } @@ -412,8 +413,7 @@ PyArray_FillWithScalar(PyArrayObject *arr, PyObject *obj) else { PyArrayObject *src_arr; - src_arr = (PyArrayObject *)PyArray_FromAny(obj, NULL, 0, 0, - NPY_ARRAY_ALLOWNA, NULL); + src_arr = (PyArrayObject *)PyArray_FromAny(obj, NULL, 0, 0, 0, NULL); if (src_arr == NULL) { return -1; } @@ -432,24 +432,17 @@ PyArray_FillWithScalar(PyArrayObject *arr, PyObject *obj) } } -/*NUMPY_API - * +/* * Fills an array with zeros. * * dst: The destination array. * wheremask: If non-NULL, a boolean mask specifying where to set the values. - * preservena: If 0, overwrites everything in 'dst', if 1, it - * preserves elements in 'dst' which are NA. - * preservewhichna: Must be NULL. When multi-NA support is implemented, - * this will be an array of flags for 'preservena=True', - * indicating which NA payload values to preserve. * * Returns 0 on success, -1 on failure. */ NPY_NO_EXPORT int PyArray_AssignZero(PyArrayObject *dst, - PyArrayObject *wheremask, - npy_bool preservena, npy_bool *preservewhichna) + PyArrayObject *wheremask) { npy_bool value; PyArray_Descr *bool_dtype; @@ -463,31 +456,23 @@ PyArray_AssignZero(PyArrayObject *dst, value = 0; retcode = PyArray_AssignRawScalar(dst, bool_dtype, (char *)&value, - wheremask, NPY_SAFE_CASTING, - preservena, preservewhichna); + wheremask, NPY_SAFE_CASTING); Py_DECREF(bool_dtype); return retcode; } -/*NUMPY_API - * +/* * Fills an array with ones. * * dst: The destination array. * wheremask: If non-NULL, a boolean mask specifying where to set the values. - * preservena: If 0, overwrites everything in 'dst', if 1, it - * preserves elements in 'dst' which are NA. - * preservewhichna: Must be NULL. When multi-NA support is implemented, - * this will be an array of flags for 'preservena=True', - * indicating which NA payload values to preserve. * * Returns 0 on success, -1 on failure. */ NPY_NO_EXPORT int PyArray_AssignOne(PyArrayObject *dst, - PyArrayObject *wheremask, - npy_bool preservena, npy_bool *preservewhichna) + PyArrayObject *wheremask) { npy_bool value; PyArray_Descr *bool_dtype; @@ -501,8 +486,7 @@ PyArray_AssignOne(PyArrayObject *dst, value = 1; retcode = PyArray_AssignRawScalar(dst, bool_dtype, (char *)&value, - wheremask, NPY_SAFE_CASTING, - preservena, preservewhichna); + wheremask, NPY_SAFE_CASTING); Py_DECREF(bool_dtype); return retcode; @@ -521,14 +505,7 @@ PyArray_NewCopy(PyArrayObject *obj, NPY_ORDER order) return NULL; } - if (PyArray_HASMASKNA(obj)) { - if (PyArray_AllocateMaskNA(ret, 1, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - } - - if (PyArray_AssignArray(ret, obj, NULL, NPY_UNSAFE_CASTING, 0, NULL) < 0) { + if (PyArray_AssignArray(ret, obj, NULL, NPY_UNSAFE_CASTING) < 0) { Py_DECREF(ret); return NULL; } @@ -556,7 +533,6 @@ PyArray_View(PyArrayObject *self, PyArray_Descr *type, PyTypeObject *pytype) } flags = PyArray_FLAGS(self); - flags &= ~(NPY_ARRAY_MASKNA|NPY_ARRAY_OWNMASKNA); dtype = PyArray_DESCR(self); Py_INCREF(dtype); @@ -571,28 +547,6 @@ PyArray_View(PyArrayObject *self, PyArray_Descr *type, PyTypeObject *pytype) return NULL; } - /* Take a view of the mask if it exists */ - if (PyArray_HASMASKNA(self)) { - PyArrayObject_fields *fa = (PyArrayObject_fields *)ret; - - if (PyArray_HASFIELDS(self)) { - PyErr_SetString(PyExc_RuntimeError, - "NA masks with fields are not supported yet"); - Py_DECREF(ret); - Py_DECREF(type); - return NULL; - } - - fa->maskna_dtype = PyArray_MASKNA_DTYPE(self); - Py_INCREF(fa->maskna_dtype); - fa->maskna_data = PyArray_MASKNA_DATA(self); - if (fa->nd > 0) { - memcpy(fa->maskna_strides, PyArray_MASKNA_STRIDES(self), - fa->nd * sizeof(npy_intp)); - } - fa->flags |= NPY_ARRAY_MASKNA; - } - /* Set the base object */ Py_INCREF(self); if (PyArray_SetBaseObject(ret, (PyObject *)self) < 0) { diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index aa834c4cf..6fdb10bb2 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -709,11 +709,7 @@ PyArray_CanCastArrayTo(PyArrayObject *arr, PyArray_Descr *to, /* If it's a scalar, check the value */ if (PyArray_NDIM(arr) == 0 && !PyArray_HASFIELDS(arr)) { - /* Only check the value if it's not masked */ - if (!PyArray_HASMASKNA(arr) || - NpyMaskValue_IsExposed((npy_mask)*PyArray_MASKNA_DATA(arr))) { - return can_cast_scalar_to(from, PyArray_DATA(arr), to, casting); - } + return can_cast_scalar_to(from, PyArray_DATA(arr), to, casting); } /* Otherwise, use the standard rules */ @@ -1332,12 +1328,9 @@ PyArray_MinScalarType(PyArrayObject *arr) { PyArray_Descr *dtype = PyArray_DESCR(arr); /* - * If the array isn't a numeric scalar or is a scalar but with - * its value masked out, just return the array's dtype. + * If the array isn't a numeric scalar, just return the array's dtype. */ - if (PyArray_NDIM(arr) > 0 || !PyTypeNum_ISNUMBER(dtype->type_num) || - (PyArray_HASMASKNA(arr) && !NpyMaskValue_IsExposed( - (npy_mask)*PyArray_MASKNA_DATA(arr)))) { + if (PyArray_NDIM(arr) > 0 || !PyTypeNum_ISNUMBER(dtype->type_num)) { Py_INCREF(dtype); return dtype; } @@ -1739,7 +1732,7 @@ NPY_NO_EXPORT int PyArray_ObjectType(PyObject *op, int minimum_type) { PyArray_Descr *dtype = NULL; - int ret, contains_na = 0; + int ret; if (minimum_type != NPY_NOTYPE && minimum_type >= 0) { dtype = PyArray_DescrFromType(minimum_type); @@ -1748,14 +1741,11 @@ PyArray_ObjectType(PyObject *op, int minimum_type) } } - if (PyArray_DTypeFromObject(op, NPY_MAXDIMS, &contains_na, &dtype) < 0) { + if (PyArray_DTypeFromObject(op, NPY_MAXDIMS, &dtype) < 0) { return NPY_NOTYPE; } - if (contains_na) { - ret = NPY_OBJECT; - } - else if (dtype == NULL) { + if (dtype == NULL) { ret = NPY_DEFAULT_TYPE; } else { @@ -1869,7 +1859,7 @@ PyArray_ConvertToCommonType(PyObject *op, int *retn) /* Make sure all arrays are actual array objects. */ for (i = 0; i < n; i++) { - int flags = NPY_ARRAY_CARRAY | NPY_ARRAY_ALLOWNA; + int flags = NPY_ARRAY_CARRAY; if ((otmp = PySequence_GetItem(op, i)) == NULL) { goto fail; diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 311e2f863..c9cdffb23 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -23,7 +23,7 @@ #include "methods.h" #include "_datetime.h" #include "datetime_strings.h" -#include "na_object.h" +#include "array_assign.h" /* * Reading from a file or a string. @@ -380,11 +380,10 @@ copy_and_swap(void *dst, void *src, int itemsize, npy_intp numitems, /* adapted from Numarray */ static int setArrayFromSequence(PyArrayObject *a, PyObject *s, - int dim, npy_intp offset, npy_intp maskoffset) + int dim, npy_intp offset) { Py_ssize_t i, slen; int res = 0; - int a_has_maskna = PyArray_HASMASKNA(a); /* * This code is to ensure that the sequence access below will @@ -431,8 +430,6 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s, /* Broadcast the one element from the sequence to all the outputs */ if (slen == 1) { PyObject *o; - NpyNA *na = NULL; - char maskvalue = 0; npy_intp alen = PyArray_DIM(a, dim); o = PySequence_GetItem(s, 0); @@ -440,41 +437,20 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s, goto fail; } - /* Check if the value being assigned is NA */ - if (a_has_maskna) { - na = NpyNA_FromObject(o, 1); - if (na != NULL) { - maskvalue = (char)NpyNA_AsMaskValue(na); - } - else { - maskvalue = 1; - } - } - for (i = 0; i < alen; i++) { if ((PyArray_NDIM(a) - dim) > 1) { - res = setArrayFromSequence(a, o, dim+1, offset, maskoffset); + res = setArrayFromSequence(a, o, dim+1, offset); } else { - /* Assign a value if it isn't NA */ - if (na == NULL) { - res = PyArray_DESCR(a)->f->setitem(o, + res = PyArray_DESCR(a)->f->setitem(o, (PyArray_DATA(a) + offset), a); - } - /* Assign to the mask if a supports MASKNA */ - if (a_has_maskna) { - *(PyArray_MASKNA_DATA(a) + maskoffset) = maskvalue; - } } if (res < 0) { Py_DECREF(o); - Py_XDECREF(na); goto fail; } offset += PyArray_STRIDES(a)[dim]; - maskoffset += PyArray_MASKNA_STRIDES(a)[dim]; } - Py_XDECREF(na); Py_DECREF(o); } /* Copy element by element */ @@ -485,38 +461,17 @@ setArrayFromSequence(PyArrayObject *a, PyObject *s, goto fail; } if ((PyArray_NDIM(a) - dim) > 1) { - res = setArrayFromSequence(a, o, dim+1, offset, maskoffset); + res = setArrayFromSequence(a, o, dim+1, offset); } else { - - /* Assignment without an NA mask */ - if (!a_has_maskna) { - res = PyArray_DESCR(a)->f->setitem(o, - (PyArray_DATA(a) + offset), a); - } - /* Assignment with an NA mask */ - else { - NpyNA *na = NpyNA_FromObject(o, 1); - char maskvalue; - if (na != NULL) { - maskvalue = (char)NpyNA_AsMaskValue(na); - res = 0; - } - else { - maskvalue = 1; - res = PyArray_DESCR(a)->f->setitem(o, - (PyArray_DATA(a) + offset), a); - } - - *(PyArray_MASKNA_DATA(a) + maskoffset) = maskvalue; - } + res = PyArray_DESCR(a)->f->setitem(o, + (PyArray_DATA(a) + offset), a); } Py_DECREF(o); if (res < 0) { goto fail; } offset += PyArray_STRIDES(a)[dim]; - maskoffset += PyArray_MASKNA_STRIDES(a)[dim]; } } @@ -541,7 +496,7 @@ PyArray_AssignFromSequence(PyArrayObject *self, PyObject *v) "assignment to 0-d array"); return -1; } - return setArrayFromSequence(self, v, 0, 0, 0); + return setArrayFromSequence(self, v, 0, 0); } /* @@ -635,12 +590,6 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it, return 0; } - /* obj is an NA */ - if (NpyNA_Check(obj)) { - *maxndim = 0; - return 0; - } - /* obj is not a Sequence */ if (!PySequence_Check(obj) || #if defined(NPY_PY3K) @@ -981,8 +930,6 @@ PyArray_NewFromDescr(PyTypeObject *subtype, PyArray_Descr *descr, int nd, fa->descr = descr; fa->base = (PyObject *)NULL; fa->weakreflist = (PyObject *)NULL; - fa->maskna_dtype = NULL; - fa->maskna_data = NULL; if (nd > 0) { fa->dimensions = PyDimMem_NEW(3*nd); @@ -991,7 +938,6 @@ PyArray_NewFromDescr(PyTypeObject *subtype, PyArray_Descr *descr, int nd, goto fail; } fa->strides = fa->dimensions + nd; - fa->maskna_strides = fa->dimensions + 2 * nd; memcpy(fa->dimensions, dims, sizeof(npy_intp)*nd); if (strides == NULL) { /* fill it in */ sd = _array_fill_strides(fa->strides, dims, nd, sd, @@ -1042,14 +988,6 @@ PyArray_NewFromDescr(PyTypeObject *subtype, PyArray_Descr *descr, int nd, * Caller must arrange for this to be reset if truly desired */ fa->flags &= ~NPY_ARRAY_OWNDATA; - - /* Flagging MASKNA is incompatible with providing the data pointer */ - if (fa->flags & NPY_ARRAY_MASKNA) { - PyErr_SetString(PyExc_ValueError, - "Cannot construct a view of data together with the " - "NPY_ARRAY_MASKNA flag, the NA mask must be added later"); - goto fail; - } } fa->data = data; @@ -1328,26 +1266,72 @@ fail: #endif } -/* - * A slight generalization of PyArray_GetArrayParamsFromObject, - * which also returns whether the input data contains any numpy.NA - * values. +/*NUMPY_API + * Retrieves the array parameters for viewing/converting an arbitrary + * PyObject* to a NumPy array. This allows the "innate type and shape" + * of Python list-of-lists to be discovered without + * actually converting to an array. + * + * In some cases, such as structured arrays and the __array__ interface, + * a data type needs to be used to make sense of the object. When + * this is needed, provide a Descr for 'requested_dtype', otherwise + * provide NULL. This reference is not stolen. Also, if the requested + * dtype doesn't modify the interpretation of the input, out_dtype will + * still get the "innate" dtype of the object, not the dtype passed + * in 'requested_dtype'. + * + * If writing to the value in 'op' is desired, set the boolean + * 'writeable' to 1. This raises an error when 'op' is a scalar, list + * of lists, or other non-writeable 'op'. + * + * Result: When success (0 return value) is returned, either out_arr + * is filled with a non-NULL PyArrayObject and + * the rest of the parameters are untouched, or out_arr is + * filled with NULL, and the rest of the parameters are + * filled. + * + * Typical usage: + * + * PyArrayObject *arr = NULL; + * PyArray_Descr *dtype = NULL; + * int ndim = 0; + * npy_intp dims[NPY_MAXDIMS]; * - * This isn't exposed in the public API. + * if (PyArray_GetArrayParamsFromObject(op, NULL, 1, &dtype, + * &ndim, &dims, &arr, NULL) < 0) { + * return NULL; + * } + * if (arr == NULL) { + * ... validate/change dtype, validate flags, ndim, etc ... + * // Could make custom strides here too + * arr = PyArray_NewFromDescr(&PyArray_Type, dtype, ndim, + * dims, NULL, + * is_f_order ? NPY_ARRAY_F_CONTIGUOUS : 0, + * NULL); + * if (arr == NULL) { + * return NULL; + * } + * if (PyArray_CopyObject(arr, op) < 0) { + * Py_DECREF(arr); + * return NULL; + * } + * } + * else { + * ... in this case the other parameters weren't filled, just + * validate and possibly copy arr itself ... + * } + * ... use arr ... */ NPY_NO_EXPORT int -PyArray_GetArrayParamsFromObjectEx(PyObject *op, +PyArray_GetArrayParamsFromObject(PyObject *op, PyArray_Descr *requested_dtype, npy_bool writeable, PyArray_Descr **out_dtype, int *out_ndim, npy_intp *out_dims, - int *out_contains_na, PyArrayObject **out_arr, PyObject *context) { PyObject *tmp; - *out_contains_na = 0; - /* If op is an array */ if (PyArray_Check(op)) { if (writeable @@ -1389,34 +1373,6 @@ PyArray_GetArrayParamsFromObjectEx(PyObject *op, return 0; } - /* If op is a numpy.NA */ - if (NpyNA_Check(op)) { - NpyNA_fields *fna = (NpyNA_fields *)op; - - if (writeable) { - PyErr_SetString(PyExc_RuntimeError, - "cannot write to numpy.NA"); - return -1; - } - /* Use the NA's dtype if available */ - if (fna->dtype != NULL) { - *out_dtype = fna->dtype; - Py_INCREF(*out_dtype); - } - /* Otherwise use the default NumPy dtype */ - else { - *out_dtype = PyArray_DescrFromType(NPY_DEFAULT_TYPE); - if (*out_dtype == NULL) { - return -1; - } - } - *out_ndim = 0; - *out_arr = NULL; - *out_contains_na = 1; - return 0; - - } - /* If op supports the PEP 3118 buffer interface */ if (!PyBytes_Check(op) && !PyUnicode_Check(op) && _array_from_buffer_3118(op, (PyObject **)out_arr) == 0) { @@ -1494,8 +1450,7 @@ PyArray_GetArrayParamsFromObjectEx(PyObject *op, } else { *out_dtype = NULL; - if (PyArray_DTypeFromObject(op, NPY_MAXDIMS, - out_contains_na, out_dtype) < 0) { + if (PyArray_DTypeFromObject(op, NPY_MAXDIMS, out_dtype) < 0) { if (PyErr_ExceptionMatches(PyExc_MemoryError)) { return -1; } @@ -1605,89 +1560,6 @@ PyArray_GetArrayParamsFromObjectEx(PyObject *op, } /*NUMPY_API - * Retrieves the array parameters for viewing/converting an arbitrary - * PyObject* to a NumPy array. This allows the "innate type and shape" - * of Python list-of-lists to be discovered without - * actually converting to an array. - * - * In some cases, such as structured arrays and the __array__ interface, - * a data type needs to be used to make sense of the object. When - * this is needed, provide a Descr for 'requested_dtype', otherwise - * provide NULL. This reference is not stolen. Also, if the requested - * dtype doesn't modify the interpretation of the input, out_dtype will - * still get the "innate" dtype of the object, not the dtype passed - * in 'requested_dtype'. - * - * If writing to the value in 'op' is desired, set the boolean - * 'writeable' to 1. This raises an error when 'op' is a scalar, list - * of lists, or other non-writeable 'op'. - * - * Result: When success (0 return value) is returned, either out_arr - * is filled with a non-NULL PyArrayObject and - * the rest of the parameters are untouched, or out_arr is - * filled with NULL, and the rest of the parameters are - * filled. - * - * Typical usage: - * - * PyArrayObject *arr = NULL; - * PyArray_Descr *dtype = NULL; - * int ndim = 0; - * npy_intp dims[NPY_MAXDIMS]; - * - * if (PyArray_GetArrayParamsFromObject(op, NULL, 1, &dtype, - * &ndim, &dims, &arr, NULL) < 0) { - * return NULL; - * } - * if (arr == NULL) { - * ... validate/change dtype, validate flags, ndim, etc ... - * // Could make custom strides here too - * arr = PyArray_NewFromDescr(&PyArray_Type, dtype, ndim, - * dims, NULL, - * is_f_order ? NPY_ARRAY_F_CONTIGUOUS : 0, - * NULL); - * if (arr == NULL) { - * return NULL; - * } - * if (PyArray_CopyObject(arr, op) < 0) { - * Py_DECREF(arr); - * return NULL; - * } - * } - * else { - * ... in this case the other parameters weren't filled, just - * validate and possibly copy arr itself ... - * } - * ... use arr ... - */ -NPY_NO_EXPORT int -PyArray_GetArrayParamsFromObject(PyObject *op, - PyArray_Descr *requested_dtype, - npy_bool writeable, - PyArray_Descr **out_dtype, - int *out_ndim, npy_intp *out_dims, - PyArrayObject **out_arr, PyObject *context) -{ - int contains_na = 0, retcode; - retcode = PyArray_GetArrayParamsFromObjectEx(op, requested_dtype, - writeable, out_dtype, out_ndim, out_dims, - &contains_na, out_arr, context); - - /* If NAs were detected, switch to an NPY_OBJECT dtype */ - if (retcode == 0 && *out_arr == NULL && contains_na) { - if ((*out_dtype)->type_num != NPY_OBJECT) { - Py_DECREF(*out_dtype); - *out_dtype = PyArray_DescrFromType(NPY_OBJECT); - if (*out_dtype == NULL) { - retcode = -1; - } - } - } - - return retcode; -} - -/*NUMPY_API * Does not check for NPY_ARRAY_ENSURECOPY and NPY_ARRAY_NOTSWAPPED in flags * Steals a reference to newtype --- which can be NULL */ @@ -1701,13 +1573,13 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, */ PyArrayObject *arr = NULL, *ret; PyArray_Descr *dtype = NULL; - int ndim = 0, contains_na = 0; + int ndim = 0; npy_intp dims[NPY_MAXDIMS]; /* Get either the array or its parameters if it isn't an array */ - if (PyArray_GetArrayParamsFromObjectEx(op, newtype, + if (PyArray_GetArrayParamsFromObject(op, newtype, 0, &dtype, - &ndim, dims, &contains_na, &arr, context) < 0) { + &ndim, dims, &arr, context) < 0) { Py_XDECREF(newtype); return NULL; } @@ -1721,14 +1593,6 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, /* If we got dimensions and dtype instead of an array */ if (arr == NULL) { - /* - * If the input data contains any NAs, and the ALLOWNA flag is - * enabled, produce an array with an NA mask. - */ - if (contains_na && (flags & NPY_ARRAY_ALLOWNA) != 0) { - flags |= NPY_ARRAY_MASKNA; - } - if (flags & NPY_ARRAY_UPDATEIFCOPY) { Py_XDECREF(newtype); PyErr_SetString(PyExc_TypeError, @@ -1782,19 +1646,6 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, Py_DECREF(dtype); } - /* - * If there are NAs, but no requested NA support, - * switch to NPY_OBJECT. Alternatively - raise an error? - */ - if (contains_na && - (flags & (NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA)) == 0) { - Py_DECREF(newtype); - newtype = PyArray_DescrFromType(NPY_OBJECT); - if (newtype == NULL) { - return NULL; - } - } - /* Create an array and copy the data */ ret = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, newtype, ndim, dims, @@ -1804,28 +1655,6 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, return NULL; } - /* - * Add an NA mask if requested, or if allowed and the data - * has NAs - */ - if ((flags & (NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA)) != 0) { - if (PyArray_AllocateMaskNA(ret, - (flags&NPY_ARRAY_OWNMASKNA) != 0, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - - /* Special case assigning a single NA */ - if (ndim == 0) { - NpyNA *na = NpyNA_FromObject(op, 1); - if (na != NULL) { - PyArray_MASKNA_DATA(ret)[0] = - (char)NpyNA_AsMaskValue(na); - return (PyObject *)ret; - } - } - } - if (ndim > 0) { if (PyArray_AssignFromSequence(ret, op) < 0) { Py_DECREF(ret); @@ -2013,11 +1842,6 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) /* If a writeable array was requested, and arr is not */ ((flags & NPY_ARRAY_WRITEABLE) && (!(arrflags & NPY_ARRAY_WRITEABLE))) || - /* If an array with no NA mask was requested, and arr has one */ - ((flags & (NPY_ARRAY_ALLOWNA | - NPY_ARRAY_MASKNA | - NPY_ARRAY_OWNMASKNA)) == 0 && - (arrflags & NPY_ARRAY_MASKNA)) || !PyArray_EquivTypes(oldtype, newtype); if (copy) { @@ -2041,35 +1865,11 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) return NULL; } - /* - * Allocate an NA mask if necessary from the input, - * is NAs are being allowed. - */ - if ((arrflags & NPY_ARRAY_MASKNA) && (flags & NPY_ARRAY_ALLOWNA)) { - if (PyArray_AllocateMaskNA(ret, 1, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - } - - /* - * If a ALLOWNA was not enabled, and 'arr' has an NA mask, - * this will raise an error if 'arr' contains any NA values. - */ if (PyArray_CopyInto(ret, arr) < 0) { Py_DECREF(ret); return NULL; } - /* Allocate an NA mask if requested but wasn't from the input */ - if ((flags & (NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA)) != 0 && - !PyArray_HASMASKNA(ret)) { - if (PyArray_AllocateMaskNA(ret, 1, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - } - if (flags & NPY_ARRAY_UPDATEIFCOPY) { Py_INCREF(arr); if (PyArray_SetUpdateIfCopyBase(ret, arr) < 0) { @@ -2084,11 +1884,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) */ else { int needview = ((flags & NPY_ARRAY_ENSUREARRAY) && - !PyArray_CheckExact(arr)) || - ((flags & NPY_ARRAY_MASKNA) && - !(arrflags & NPY_ARRAY_MASKNA)) || - ((flags & NPY_ARRAY_OWNMASKNA) && - !(arrflags & NPY_ARRAY_OWNMASKNA)); + !PyArray_CheckExact(arr)); Py_DECREF(newtype); if (needview) { @@ -2104,14 +1900,6 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) if (ret == NULL) { return NULL; } - - if (flags & (NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA)) { - int ownmaskna = (flags & NPY_ARRAY_OWNMASKNA) != 0; - if (PyArray_AllocateMaskNA(ret, ownmaskna, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - } } else { Py_INCREF(arr); @@ -2439,20 +2227,15 @@ NPY_NO_EXPORT PyArray_Descr * PyArray_DescrFromObject(PyObject *op, PyArray_Descr *mintype) { PyArray_Descr *dtype; - int contains_na = 0; dtype = mintype; Py_XINCREF(dtype); - if (PyArray_DTypeFromObject(op, NPY_MAXDIMS, &contains_na, &dtype) < 0) { + if (PyArray_DTypeFromObject(op, NPY_MAXDIMS, &dtype) < 0) { return NULL; } - if (contains_na) { - Py_XDECREF(dtype); - return PyArray_DescrFromType(NPY_OBJECT); - } - else if (dtype == NULL) { + if (dtype == NULL) { return PyArray_DescrFromType(NPY_DEFAULT_TYPE); } else { @@ -2567,25 +2350,20 @@ NPY_NO_EXPORT int PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) { PyArray_StridedUnaryOp *stransfer = NULL; - PyArray_MaskedStridedUnaryOp *maskedstransfer = NULL; NpyAuxData *transferdata = NULL; - PyArray_StridedUnaryOp *maskna_stransfer = NULL; - NpyAuxData *maskna_transferdata = NULL; NpyIter *dst_iter, *src_iter; NpyIter_IterNextFunc *dst_iternext, *src_iternext; char **dst_dataptr, **src_dataptr; npy_intp dst_stride, src_stride; - npy_intp maskna_src_stride = 0, maskna_dst_stride = 0; npy_intp *dst_countptr, *src_countptr; npy_uint32 baseflags; char *dst_data, *src_data; - char *maskna_dst_data = NULL, *maskna_src_data = NULL; npy_intp dst_count, src_count, count; - npy_intp src_itemsize, maskna_src_itemsize = 0; + npy_intp src_itemsize; npy_intp dst_size, src_size; - int needs_api, use_maskna = 0; + int needs_api; NPY_BEGIN_THREADS_DEF; @@ -2623,41 +2401,6 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) NPY_ITER_REFS_OK; /* - * If 'src' has a mask, and 'dst' doesn't, need to validate that - * 'src' has everything exposed. Otherwise, the mask needs to - * be copied as well. - */ - if (PyArray_HASMASKNA(src)) { - if (PyArray_HASMASKNA(dst)) { - use_maskna = 1; - baseflags |= NPY_ITER_USE_MASKNA; - } - else { - int containsna = PyArray_ContainsNA(src, NULL, NULL); - if (containsna == -1) { - return -1; - } - else if (containsna) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which " - "does not support NAs"); - return -1; - } - baseflags |= NPY_ITER_IGNORE_MASKNA; - } - } - /* - * If 'dst' has a mask but 'src' doesn't, set all of 'dst' - * to be exposed, then proceed without worrying about the mask. - */ - else if (PyArray_HASMASKNA(dst)) { - if (PyArray_AssignMaskNA(dst, 1, NULL, 0, NULL) < 0) { - return -1; - } - baseflags |= NPY_ITER_IGNORE_MASKNA; - } - - /* * This copy is based on matching C-order traversals of src and dst. * By using two iterators, we can find maximal sub-chunks that * can be processed at once. @@ -2692,12 +2435,6 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) src_countptr = NpyIter_GetInnerLoopSizePtr(src_iter); src_itemsize = PyArray_DESCR(src)->elsize; - if (use_maskna) { - maskna_src_stride = NpyIter_GetInnerStrideArray(src_iter)[1]; - maskna_dst_stride = NpyIter_GetInnerStrideArray(dst_iter)[1]; - maskna_src_itemsize = PyArray_MASKNA_DTYPE(src)->elsize; - } - if (dst_iternext == NULL || src_iternext == NULL) { NpyIter_Deallocate(dst_iter); NpyIter_Deallocate(src_iter); @@ -2713,48 +2450,16 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) * we can pass them to this function to take advantage of * contiguous strides, etc. */ - if (!use_maskna) { - if (PyArray_GetDTypeTransferFunction( - PyArray_ISALIGNED(src) && PyArray_ISALIGNED(dst), - src_stride, dst_stride, - PyArray_DESCR(src), PyArray_DESCR(dst), - 0, - &stransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { - NpyIter_Deallocate(dst_iter); - NpyIter_Deallocate(src_iter); - return -1; - } - } - else { - if (PyArray_GetMaskedDTypeTransferFunction( - PyArray_ISALIGNED(src) && PyArray_ISALIGNED(dst), - src_stride, - dst_stride, - maskna_src_stride, - PyArray_DESCR(src), - PyArray_DESCR(dst), - PyArray_MASKNA_DTYPE(src), - 0, - &maskedstransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { - NpyIter_Deallocate(dst_iter); - NpyIter_Deallocate(src_iter); - return -1; - } - - /* Also need a transfer function for the mask itself */ - if (PyArray_GetDTypeTransferFunction(1, - maskna_src_stride, maskna_dst_stride, - PyArray_MASKNA_DTYPE(src), PyArray_MASKNA_DTYPE(dst), - 0, - &maskna_stransfer, &maskna_transferdata, - &needs_api) != NPY_SUCCEED) { - NPY_AUXDATA_FREE(transferdata); - NpyIter_Deallocate(dst_iter); - NpyIter_Deallocate(src_iter); - return -1; - } + if (PyArray_GetDTypeTransferFunction( + PyArray_ISALIGNED(src) && PyArray_ISALIGNED(dst), + src_stride, dst_stride, + PyArray_DESCR(src), PyArray_DESCR(dst), + 0, + &stransfer, &transferdata, + &needs_api) != NPY_SUCCEED) { + NpyIter_Deallocate(dst_iter); + NpyIter_Deallocate(src_iter); + return -1; } if (!needs_api) { @@ -2765,88 +2470,37 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) src_count = *src_countptr; dst_data = dst_dataptr[0]; src_data = src_dataptr[0]; - /* - * The tests did not trigger this code, so added a new function - * ndarray.setasflat to the Python exposure in order to test it. - */ - if (!use_maskna) { - for(;;) { - /* Transfer the biggest amount that fits both */ - count = (src_count < dst_count) ? src_count : dst_count; - stransfer(dst_data, dst_stride, - src_data, src_stride, - count, src_itemsize, transferdata); - - /* If we exhausted the dst block, refresh it */ - if (dst_count == count) { - if (!dst_iternext(dst_iter)) { - break; - } - dst_count = *dst_countptr; - dst_data = dst_dataptr[0]; - } - else { - dst_count -= count; - dst_data += count*dst_stride; - } - - /* If we exhausted the src block, refresh it */ - if (src_count == count) { - if (!src_iternext(src_iter)) { - break; - } - src_count = *src_countptr; - src_data = src_dataptr[0]; - } - else { - src_count -= count; - src_data += count*src_stride; + for(;;) { + /* Transfer the biggest amount that fits both */ + count = (src_count < dst_count) ? src_count : dst_count; + stransfer(dst_data, dst_stride, + src_data, src_stride, + count, src_itemsize, transferdata); + + /* If we exhausted the dst block, refresh it */ + if (dst_count == count) { + if (!dst_iternext(dst_iter)) { + break; } + dst_count = *dst_countptr; + dst_data = dst_dataptr[0]; + } + else { + dst_count -= count; + dst_data += count*dst_stride; } - } - else { - maskna_src_data = src_dataptr[1]; - maskna_dst_data = dst_dataptr[1]; - for(;;) { - /* Transfer the biggest amount that fits both */ - count = (src_count < dst_count) ? src_count : dst_count; - maskedstransfer(dst_data, dst_stride, - src_data, src_stride, - (npy_mask *)maskna_src_data, maskna_src_stride, - count, src_itemsize, transferdata); - maskna_stransfer(maskna_dst_data, maskna_dst_stride, - maskna_src_data, maskna_src_stride, - count, maskna_src_itemsize, maskna_transferdata); - - /* If we exhausted the dst block, refresh it */ - if (dst_count == count) { - if (!dst_iternext(dst_iter)) { - break; - } - dst_count = *dst_countptr; - dst_data = dst_dataptr[0]; - maskna_dst_data = dst_dataptr[1]; - } - else { - dst_count -= count; - dst_data += count*dst_stride; - maskna_dst_data += count*maskna_dst_stride; - } - /* If we exhausted the src block, refresh it */ - if (src_count == count) { - if (!src_iternext(src_iter)) { - break; - } - src_count = *src_countptr; - src_data = src_dataptr[0]; - maskna_src_data = src_dataptr[1]; - } - else { - src_count -= count; - src_data += count*src_stride; - maskna_src_data += count*maskna_src_stride; + /* If we exhausted the src block, refresh it */ + if (src_count == count) { + if (!src_iternext(src_iter)) { + break; } + src_count = *src_countptr; + src_data = src_dataptr[0]; + } + else { + src_count -= count; + src_data += count*src_stride; } } @@ -2855,7 +2509,6 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) } NPY_AUXDATA_FREE(transferdata); - NPY_AUXDATA_FREE(maskna_transferdata); NpyIter_Deallocate(dst_iter); NpyIter_Deallocate(src_iter); @@ -2888,7 +2541,7 @@ PyArray_CopyAnyInto(PyArrayObject *dst, PyArrayObject *src) NPY_NO_EXPORT int PyArray_CopyInto(PyArrayObject *dst, PyArrayObject *src) { - return PyArray_AssignArray(dst, src, NULL, NPY_UNSAFE_CASTING, 0, NULL); + return PyArray_AssignArray(dst, src, NULL, NPY_UNSAFE_CASTING); } /*NUMPY_API @@ -2899,7 +2552,7 @@ PyArray_CopyInto(PyArrayObject *dst, PyArrayObject *src) NPY_NO_EXPORT int PyArray_MoveInto(PyArrayObject *dst, PyArrayObject *src) { - return PyArray_AssignArray(dst, src, NULL, NPY_UNSAFE_CASTING, 0, NULL); + return PyArray_AssignArray(dst, src, NULL, NPY_UNSAFE_CASTING); } /*NUMPY_API diff --git a/numpy/core/src/multiarray/ctors.h b/numpy/core/src/multiarray/ctors.h index e02ca8e11..757362fb8 100644 --- a/numpy/core/src/multiarray/ctors.h +++ b/numpy/core/src/multiarray/ctors.h @@ -73,22 +73,6 @@ NPY_NO_EXPORT int PyArray_AssignFromSequence(PyArrayObject *self, PyObject *v); /* - * A slight generalization of PyArray_GetArrayParamsFromObject, - * which also returns whether the input data contains any numpy.NA - * values. - * - * This isn't exposed in the public API. - */ -NPY_NO_EXPORT int -PyArray_GetArrayParamsFromObjectEx(PyObject *op, - PyArray_Descr *requested_dtype, - npy_bool writeable, - PyArray_Descr **out_dtype, - int *out_ndim, npy_intp *out_dims, - int *out_contains_na, - PyArrayObject **out_arr, PyObject *context); - -/* * Calls arr_of_subclass.__array_wrap__(towrap), in order to make 'towrap' * have the same ndarray subclass as 'arr_of_subclass'. */ diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 8fe968e7e..53fee1612 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -24,7 +24,6 @@ #include "methods.h" #include "_datetime.h" #include "datetime_strings.h" -#include "na_object.h" /* * Imports the PyDateTime functions so we can create these objects. @@ -2510,13 +2509,6 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, *out = NPY_DATETIME_NAT; return 0; } - /* Check for NA */ - else if (NpyNA_Check(obj) || NpyNA_IsZeroDimArrayNA(obj)) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which " - "does not support NAs"); - return -1; - } else { PyErr_SetString(PyExc_ValueError, "Could not convert object to NumPy datetime"); @@ -2778,13 +2770,6 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, *out = NPY_DATETIME_NAT; return 0; } - /* Check for NA */ - else if (NpyNA_Check(obj) || NpyNA_IsZeroDimArrayNA(obj)) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which " - "does not support NAs"); - return -1; - } else { PyErr_SetString(PyExc_ValueError, "Could not convert object to NumPy timedelta"); diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index 5bb24ad6b..b99dd5ff4 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -3060,7 +3060,7 @@ NpyAuxData *_masked_wrapper_transfer_data_clone(NpyAuxData *data) void _strided_masked_wrapper_decsrcref_transfer_function( char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, - npy_mask *mask, npy_intp mask_stride, + npy_bool *mask, npy_intp mask_stride, npy_intp N, npy_intp src_itemsize, NpyAuxData *transferdata) { @@ -3078,7 +3078,7 @@ void _strided_masked_wrapper_decsrcref_transfer_function( while (N > 0) { /* Skip masked values, still calling decsrcref for move_references */ subloopsize = 0; - while (subloopsize < N && !NpyMaskValue_IsExposed(*mask)) { + while (subloopsize < N && !*mask) { ++subloopsize; mask += mask_stride; } @@ -3089,7 +3089,7 @@ void _strided_masked_wrapper_decsrcref_transfer_function( N -= subloopsize; /* Process unmasked values */ subloopsize = 0; - while (subloopsize < N && NpyMaskValue_IsExposed(*mask)) { + while (subloopsize < N && *mask) { ++subloopsize; mask += mask_stride; } @@ -3104,7 +3104,7 @@ void _strided_masked_wrapper_decsrcref_transfer_function( void _strided_masked_wrapper_transfer_function( char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, - npy_mask *mask, npy_intp mask_stride, + npy_bool *mask, npy_intp mask_stride, npy_intp N, npy_intp src_itemsize, NpyAuxData *transferdata) { @@ -3121,7 +3121,7 @@ void _strided_masked_wrapper_transfer_function( while (N > 0) { /* Skip masked values */ subloopsize = 0; - while (subloopsize < N && !NpyMaskValue_IsExposed(*mask)) { + while (subloopsize < N && !*mask) { ++subloopsize; mask += mask_stride; } @@ -3130,7 +3130,7 @@ void _strided_masked_wrapper_transfer_function( N -= subloopsize; /* Process unmasked values */ subloopsize = 0; - while (subloopsize < N && NpyMaskValue_IsExposed(*mask)) { + while (subloopsize < N && *mask) { ++subloopsize; mask += mask_stride; } @@ -3765,7 +3765,7 @@ PyArray_GetMaskedDTypeTransferFunction(int aligned, /* TODO: Add struct-based mask_dtype support later */ if (mask_dtype->type_num != NPY_BOOL && - mask_dtype->type_num != NPY_MASK) { + mask_dtype->type_num != NPY_UINT8) { PyErr_SetString(PyExc_TypeError, "Only bool and uint8 masks are supported at the moment, " "structs of bool/uint8 is planned for the future"); @@ -3880,8 +3880,7 @@ PyArray_CastRawArrays(npy_intp count, * * This is intended for simple, lightweight iteration over arrays * where no buffering of any kind is needed, and the array may - * not be stored as a PyArrayObject. For example, to iterate over - * the NA mask of an array. + * not be stored as a PyArrayObject. * * The arrays shape, out_shape, strides, and out_strides must all * point to different data. @@ -4261,220 +4260,3 @@ PyArray_PrepareThreeRawArrayIter(int ndim, npy_intp *shape, return 0; } -/* See lowlevel_strided_loops.h for parameter docs. */ -NPY_NO_EXPORT int -PyArray_PrepareFourRawArrayIter(int ndim, npy_intp *shape, - char *dataA, npy_intp *stridesA, - char *dataB, npy_intp *stridesB, - char *dataC, npy_intp *stridesC, - char *dataD, npy_intp *stridesD, - int *out_ndim, npy_intp *out_shape, - char **out_dataA, npy_intp *out_stridesA, - char **out_dataB, npy_intp *out_stridesB, - char **out_dataC, npy_intp *out_stridesC, - char **out_dataD, npy_intp *out_stridesD) -{ - npy_stride_sort_item strideperm[NPY_MAXDIMS]; - int i, j; - - /* Special case 0 and 1 dimensions */ - if (ndim == 0) { - *out_ndim = 1; - *out_dataA = dataA; - *out_dataB = dataB; - *out_dataC = dataC; - *out_dataD = dataD; - out_shape[0] = 1; - out_stridesA[0] = 0; - out_stridesB[0] = 0; - out_stridesC[0] = 0; - out_stridesD[0] = 0; - return 0; - } - else if (ndim == 1) { - npy_intp stride_entryA = stridesA[0]; - npy_intp stride_entryB = stridesB[0]; - npy_intp stride_entryC = stridesC[0]; - npy_intp stride_entryD = stridesD[0]; - npy_intp shape_entry = shape[0]; - *out_ndim = 1; - out_shape[0] = shape[0]; - /* Always make a positive stride for the first operand */ - if (stride_entryA >= 0) { - *out_dataA = dataA; - *out_dataB = dataB; - *out_dataC = dataC; - *out_dataD = dataD; - out_stridesA[0] = stride_entryA; - out_stridesB[0] = stride_entryB; - out_stridesC[0] = stride_entryC; - out_stridesD[0] = stride_entryD; - } - else { - *out_dataA = dataA + stride_entryA * (shape_entry - 1); - *out_dataB = dataB + stride_entryB * (shape_entry - 1); - *out_dataC = dataC + stride_entryC * (shape_entry - 1); - *out_dataD = dataD + stride_entryD * (shape_entry - 1); - out_stridesA[0] = -stride_entryA; - out_stridesB[0] = -stride_entryB; - out_stridesC[0] = -stride_entryC; - out_stridesD[0] = -stride_entryD; - } - return 0; - } - - /* Sort the axes based on the destination strides */ - PyArray_CreateSortedStridePerm(ndim, shape, stridesA, strideperm); - for (i = 0; i < ndim; ++i) { - int iperm = strideperm[ndim - i - 1].perm; - out_shape[i] = shape[iperm]; - out_stridesA[i] = stridesA[iperm]; - out_stridesB[i] = stridesB[iperm]; - out_stridesC[i] = stridesC[iperm]; - out_stridesD[i] = stridesD[iperm]; - } - - /* Reverse any negative strides of operand A */ - for (i = 0; i < ndim; ++i) { - npy_intp stride_entryA = out_stridesA[i]; - npy_intp stride_entryB = out_stridesB[i]; - npy_intp stride_entryC = out_stridesC[i]; - npy_intp stride_entryD = out_stridesD[i]; - npy_intp shape_entry = out_shape[i]; - - if (stride_entryA < 0) { - dataA += stride_entryA * (shape_entry - 1); - dataB += stride_entryB * (shape_entry - 1); - dataC += stride_entryC * (shape_entry - 1); - dataD += stride_entryD * (shape_entry - 1); - out_stridesA[i] = -stride_entryA; - out_stridesB[i] = -stride_entryB; - out_stridesC[i] = -stride_entryC; - out_stridesD[i] = -stride_entryD; - } - /* Detect 0-size arrays here */ - if (shape_entry == 0) { - *out_ndim = 1; - *out_dataA = dataA; - *out_dataB = dataB; - *out_dataC = dataC; - *out_dataD = dataD; - out_shape[0] = 0; - out_stridesA[0] = 0; - out_stridesB[0] = 0; - out_stridesC[0] = 0; - out_stridesD[0] = 0; - return 0; - } - } - - /* Coalesce any dimensions where possible */ - i = 0; - for (j = 1; j < ndim; ++j) { - if (out_shape[i] == 1) { - /* Drop axis i */ - out_shape[i] = out_shape[j]; - out_stridesA[i] = out_stridesA[j]; - out_stridesB[i] = out_stridesB[j]; - out_stridesC[i] = out_stridesC[j]; - out_stridesD[i] = out_stridesD[j]; - } - else if (out_shape[j] == 1) { - /* Drop axis j */ - } - else if (out_stridesA[i] * out_shape[i] == out_stridesA[j] && - out_stridesB[i] * out_shape[i] == out_stridesB[j] && - out_stridesC[i] * out_shape[i] == out_stridesC[j] && - out_stridesD[i] * out_shape[i] == out_stridesD[j]) { - /* Coalesce axes i and j */ - out_shape[i] *= out_shape[j]; - } - else { - /* Can't coalesce, go to next i */ - ++i; - out_shape[i] = out_shape[j]; - out_stridesA[i] = out_stridesA[j]; - out_stridesB[i] = out_stridesB[j]; - out_stridesC[i] = out_stridesC[j]; - out_stridesD[i] = out_stridesD[j]; - } - } - ndim = i+1; - - *out_dataA = dataA; - *out_dataB = dataB; - *out_dataC = dataC; - *out_dataD = dataD; - *out_ndim = ndim; - return 0; -} - -/* - * Casts the elements from one n-dimensional array to another n-dimensional - * array with identical shape but possibly different strides and dtypes. - * Does not account for overlap. - * - * Returns NPY_SUCCEED or NPY_FAIL. - */ -NPY_NO_EXPORT int -PyArray_CastRawNDimArrays(int ndim, npy_intp *shape, - char *src, char *dst, - npy_intp *src_strides, npy_intp *dst_strides, - PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - int move_references) -{ - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; - int idim; - npy_intp src_align, dst_align; - int aligned, needs_api = 0; - npy_intp coord[NPY_MAXDIMS]; - npy_intp shape_it[NPY_MAXDIMS]; - npy_intp src_strides_it[NPY_MAXDIMS]; - npy_intp dst_strides_it[NPY_MAXDIMS]; - - /* Determine data alignment */ - src_align = (npy_intp)src; - for (idim = 0; idim < ndim; ++idim) { - src_align |= src_strides[idim]; - } - dst_align = (npy_intp)dst; - for (idim = 0; idim < ndim; ++idim) { - dst_align |= dst_strides[idim]; - } - aligned = (src_align & (src_dtype->alignment - 1)) == 0 && - (dst_align & (dst_dtype->alignment - 1)) == 0; - - if (PyArray_PrepareTwoRawArrayIter(ndim, shape, - dst, dst_strides, - src, src_strides, - &ndim, shape_it, - &dst, dst_strides_it, - &src, src_strides_it) < 0) { - return NPY_FAIL; - } - - /* Get the function to do the casting */ - if (PyArray_GetDTypeTransferFunction(aligned, - src_strides[0], dst_strides[0], - src_dtype, dst_dtype, - move_references, - &stransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { - return NPY_FAIL; - } - - NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { - stransfer(dst, dst_strides_it[0], - src, src_strides_it[0], shape_it[0], - src_dtype->elsize, transferdata); - } NPY_RAW_ITER_TWO_NEXT(idim, ndim, coord, shape_it, - src, src_strides_it, - dst, dst_strides_it); - - /* Cleanup */ - NPY_AUXDATA_FREE(transferdata); - - /* If needs_api was set to 1, it may have raised a Python exception */ - return (needs_api && PyErr_Occurred()) ? NPY_FAIL : NPY_SUCCEED; -} diff --git a/numpy/core/src/multiarray/einsum.c.src b/numpy/core/src/multiarray/einsum.c.src index 9910205d4..03def17b4 100644 --- a/numpy/core/src/multiarray/einsum.c.src +++ b/numpy/core/src/multiarray/einsum.c.src @@ -3018,7 +3018,7 @@ PyArray_EinsteinSum(char *subscripts, npy_intp nop, /* Initialize the output to all zeros and reset the iterator */ ret = NpyIter_GetOperandArray(iter)[nop]; Py_INCREF(ret); - PyArray_AssignZero(ret, NULL, 0, NULL); + PyArray_AssignZero(ret, NULL); /***************************/ diff --git a/numpy/core/src/multiarray/flagsobject.c b/numpy/core/src/multiarray/flagsobject.c index 5bc41b814..6e5455c01 100644 --- a/numpy/core/src/multiarray/flagsobject.c +++ b/numpy/core/src/multiarray/flagsobject.c @@ -259,92 +259,6 @@ arrayflags_num_get(PyArrayFlagsObject *self) return PyInt_FromLong(self->flags); } -static PyObject * -arrayflags_maskna_get(PyArrayFlagsObject *self) -{ - PyObject *item; - if (self->flags & NPY_ARRAY_MASKNA) { - item = Py_True; - } - else { - item = Py_False; - } - Py_INCREF(item); - return item; -} - -static int -arrayflags_maskna_set(PyArrayFlagsObject *self, PyObject *obj) -{ - if (obj == NULL) { - PyErr_SetString(PyExc_AttributeError, - "Cannot delete flags maskna attribute"); - return -1; - } - if (self->arr == NULL) { - PyErr_SetString(PyExc_ValueError, - "Cannot set flags on array scalars."); - return -1; - } - - if (PyObject_IsTrue(obj)) { - return PyArray_AllocateMaskNA((PyArrayObject *)self->arr, 0, 0, 1); - } - else { - if (self->flags & NPY_ARRAY_MASKNA) { - PyErr_SetString(PyExc_ValueError, - "Cannot remove a NumPy array's NA mask"); - return -1; - } - else { - return 0; - } - } -} - -static PyObject * -arrayflags_ownmaskna_get(PyArrayFlagsObject *self) -{ - PyObject *item; - if (self->flags & NPY_ARRAY_OWNMASKNA) { - item = Py_True; - } - else { - item = Py_False; - } - Py_INCREF(item); - return item; -} - -static int -arrayflags_ownmaskna_set(PyArrayFlagsObject *self, PyObject *obj) -{ - if (obj == NULL) { - PyErr_SetString(PyExc_AttributeError, - "Cannot delete flags ownmaskna attributes"); - return -1; - } - if (self->arr == NULL) { - PyErr_SetString(PyExc_ValueError, - "Cannot set flags on array scalars."); - return -1; - } - - if (PyObject_IsTrue(obj)) { - return PyArray_AllocateMaskNA((PyArrayObject *)self->arr, 1, 0, 1); - } - else { - if (self->flags & NPY_ARRAY_OWNMASKNA) { - PyErr_SetString(PyExc_ValueError, - "Cannot remove a NumPy array's NA mask"); - return -1; - } - else { - return 0; - } - } -} - /* relies on setflags order being write, align, uic */ static int arrayflags_updateifcopy_set(PyArrayFlagsObject *self, PyObject *obj) @@ -454,14 +368,6 @@ static PyGetSetDef arrayflags_getsets[] = { (getter)arrayflags_writeable_get, (setter)arrayflags_writeable_set, NULL, NULL}, - {"maskna", - (getter)arrayflags_maskna_get, - (setter)arrayflags_maskna_set, - NULL, NULL}, - {"ownmaskna", - (getter)arrayflags_ownmaskna_get, - (setter)arrayflags_ownmaskna_set, - NULL, NULL}, {"fnc", (getter)arrayflags_fnc_get, NULL, @@ -564,9 +470,6 @@ arrayflags_getitem(PyArrayFlagsObject *self, PyObject *ind) if (strncmp(key, "FARRAY", n) == 0) { return arrayflags_farray_get(self); } - if (strncmp(key, "MASKNA", n) == 0) { - return arrayflags_maskna_get(self); - } break; case 7: if (strncmp(key,"FORTRAN",n) == 0) { @@ -586,9 +489,6 @@ arrayflags_getitem(PyArrayFlagsObject *self, PyObject *ind) if (strncmp(key,"WRITEABLE",n) == 0) { return arrayflags_writeable_get(self); } - if (strncmp(key, "OWNMASKNA", n) == 0) { - return arrayflags_ownmaskna_get(self); - } break; case 10: if (strncmp(key,"CONTIGUOUS",n) == 0) { @@ -648,12 +548,6 @@ arrayflags_setitem(PyArrayFlagsObject *self, PyObject *ind, PyObject *item) ((n==1) && (strncmp(key, "U", n) == 0))) { return arrayflags_updateifcopy_set(self, item); } - else if ((n==6) && (strncmp(key, "MASKNA", n) == 0)) { - return arrayflags_maskna_set(self, item); - } - else if ((n==9) && (strncmp(key, "OWNMASKNA", n) == 0)) { - return arrayflags_ownmaskna_set(self, item); - } fail: PyErr_SetString(PyExc_KeyError, "Unknown flag"); @@ -679,13 +573,10 @@ arrayflags_print(PyArrayFlagsObject *self) return PyUString_FromFormat( " %s : %s\n %s : %s\n" " %s : %s\n %s : %s\n" - " %s : %s\n %s : %s\n" " %s : %s\n %s : %s", "C_CONTIGUOUS", _torf_(fl, NPY_ARRAY_C_CONTIGUOUS), "F_CONTIGUOUS", _torf_(fl, NPY_ARRAY_F_CONTIGUOUS), "OWNDATA", _torf_(fl, NPY_ARRAY_OWNDATA), - "MASKNA", _torf_(fl, NPY_ARRAY_MASKNA), - "OWNMASKNA", _torf_(fl, NPY_ARRAY_OWNMASKNA), "WRITEABLE", _torf_(fl, NPY_ARRAY_WRITEABLE), "ALIGNED", _torf_(fl, NPY_ARRAY_ALIGNED), "UPDATEIFCOPY", _torf_(fl, NPY_ARRAY_UPDATEIFCOPY)); diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c index 065a207d2..51caba240 100644 --- a/numpy/core/src/multiarray/getset.c +++ b/numpy/core/src/multiarray/getset.c @@ -76,10 +76,8 @@ array_shape_set(PyArrayObject *self, PyObject *val) return -1; } ((PyArrayObject_fields *)self)->strides = PyArray_DIMS(self) + nd; - ((PyArrayObject_fields *)self)->maskna_strides = PyArray_DIMS(self) + 2*nd; memcpy(PyArray_DIMS(self), PyArray_DIMS(ret), nd*sizeof(npy_intp)); memcpy(PyArray_STRIDES(self), PyArray_STRIDES(ret), nd*sizeof(npy_intp)); - memcpy(PyArray_MASKNA_STRIDES(self), PyArray_MASKNA_STRIDES(ret), nd*sizeof(npy_intp)); } else { ((PyArrayObject_fields *)self)->dimensions = NULL; @@ -443,13 +441,6 @@ array_descr_set(PyArrayObject *self, PyObject *arg) return -1; } - if (PyArray_HASMASKNA(self) && - newtype->elsize != PyArray_DESCR(self)->elsize) { - PyErr_SetString(PyExc_TypeError, - "view cannot change element size of NA-masked array"); - return -1; - } - if (PyDataType_FLAGCHK(newtype, NPY_ITEM_HASOBJECT) || PyDataType_FLAGCHK(newtype, NPY_ITEM_IS_POINTER) || PyDataType_FLAGCHK(PyArray_DESCR(self), NPY_ITEM_HASOBJECT) || diff --git a/numpy/core/src/multiarray/item_selection.c b/numpy/core/src/multiarray/item_selection.c index 3cbd5391c..49c304990 100644 --- a/numpy/core/src/multiarray/item_selection.c +++ b/numpy/core/src/multiarray/item_selection.c @@ -17,7 +17,6 @@ #include "arrayobject.h" #include "ctors.h" #include "lowlevel_strided_loops.h" -#include "na_object.h" #include "reduction.h" #include "item_selection.h" @@ -35,11 +34,11 @@ PyArray_TakeFrom(PyArrayObject *self0, PyObject *indices0, int axis, npy_intp nd, i, j, n, m, max_item, tmp, chunk, nelem; npy_intp shape[NPY_MAXDIMS]; char *src, *dest; - int err, use_maskna = 0; + int err; indices = NULL; self = (PyArrayObject *)PyArray_CheckAxis(self0, &axis, - NPY_ARRAY_CARRAY | NPY_ARRAY_ALLOWNA); + NPY_ARRAY_CARRAY); if (self == NULL) { return NULL; } @@ -83,18 +82,10 @@ PyArray_TakeFrom(PyArrayObject *self0, PyObject *indices0, int axis, goto fail; } - /* Allocate an NA mask if necessary */ - if (PyArray_HASMASKNA(self)) { - if (PyArray_AllocateMaskNA(obj, 1, 0, 1) < 0) { - goto fail; - } - use_maskna = 1; - } } else { int flags = NPY_ARRAY_CARRAY | - NPY_ARRAY_UPDATEIFCOPY | - NPY_ARRAY_ALLOWNA; + NPY_ARRAY_UPDATEIFCOPY; if ((PyArray_NDIM(out) != nd) || !PyArray_CompareLists(PyArray_DIMS(out), shape, nd)) { @@ -117,24 +108,6 @@ PyArray_TakeFrom(PyArrayObject *self0, PyObject *indices0, int axis, if (obj == NULL) { goto fail; } - - if (PyArray_HASMASKNA(self)) { - if (PyArray_HASMASKNA(obj)) { - use_maskna = 1; - } - else { - int containsna = PyArray_ContainsNA(self, NULL, NULL); - if (containsna == -1) { - goto fail; - } - else if (containsna) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which " - "does not support NAs"); - goto fail; - } - } - } } max_item = PyArray_DIMS(self)[axis]; @@ -144,104 +117,7 @@ PyArray_TakeFrom(PyArrayObject *self0, PyObject *indices0, int axis, dest = PyArray_DATA(obj); func = PyArray_DESCR(self)->f->fasttake; - if (use_maskna) { - char *dst_maskna = NULL, *src_maskna = NULL; - npy_intp itemsize = PyArray_DESCR(obj)->elsize; - PyArray_MaskedStridedUnaryOp *maskedstransfer = NULL; - NpyAuxData *transferdata = NULL; - int needs_api = 0; - - if (PyArray_GetMaskedDTypeTransferFunction( - 1, - itemsize, - itemsize, - 1, - PyArray_DESCR(obj), - PyArray_DESCR(obj), - PyArray_MASKNA_DTYPE(obj), - 0, - &maskedstransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { - goto fail; - } - - - src_maskna = PyArray_MASKNA_DATA(self); - dst_maskna = PyArray_MASKNA_DATA(obj); - - switch(clipmode) { - case NPY_RAISE: - for (i = 0; i < n; i++) { - for (j = 0; j < m; j++) { - tmp = ((npy_intp *)(PyArray_DATA(indices)))[j]; - if (check_and_adjust_index(&tmp, max_item, axis) < 0) { - NPY_AUXDATA_FREE(transferdata); - goto fail; - } - maskedstransfer(dest, itemsize, - src + tmp*chunk, itemsize, - (npy_mask *)(src_maskna + tmp*nelem), 1, - nelem, itemsize, transferdata); - dest += chunk; - memmove(dst_maskna, src_maskna + tmp*nelem, nelem); - dst_maskna += nelem; - } - src += chunk*max_item; - src_maskna += nelem*max_item; - } - break; - case NPY_WRAP: - for (i = 0; i < n; i++) { - for (j = 0; j < m; j++) { - tmp = ((npy_intp *)(PyArray_DATA(indices)))[j]; - if (tmp < 0) { - while (tmp < 0) { - tmp += max_item; - } - } - else if (tmp >= max_item) { - while (tmp >= max_item) { - tmp -= max_item; - } - } - maskedstransfer(dest, itemsize, - src + tmp*chunk, itemsize, - (npy_mask *)(src_maskna + tmp*nelem), 1, - nelem, itemsize, transferdata); - dest += chunk; - memmove(dst_maskna, src_maskna + tmp*nelem, nelem); - dst_maskna += nelem; - } - src += chunk*max_item; - src_maskna += nelem*max_item; - } - break; - case NPY_CLIP: - for (i = 0; i < n; i++) { - for (j = 0; j < m; j++) { - tmp = ((npy_intp *)(PyArray_DATA(indices)))[j]; - if (tmp < 0) { - tmp = 0; - } - else if (tmp >= max_item) { - tmp = max_item - 1; - } - maskedstransfer(dest, itemsize, - src + tmp*chunk, itemsize, - (npy_mask *)(src_maskna + tmp*nelem), 1, - nelem, itemsize, transferdata); - dest += chunk; - memmove(dst_maskna, src_maskna + tmp*nelem, nelem); - dst_maskna += nelem; - } - src += chunk*max_item; - src_maskna += nelem*max_item; - } - break; - } - NPY_AUXDATA_FREE(transferdata); - } - else if (func == NULL) { + if (func == NULL) { switch(clipmode) { case NPY_RAISE: for (i = 0; i < n; i++) { @@ -1829,12 +1705,11 @@ NPY_NO_EXPORT PyObject * PyArray_Diagonal(PyArrayObject *self, int offset, int axis1, int axis2) { int i, idim, ndim = PyArray_NDIM(self); - npy_intp *strides, *maskna_strides = NULL; - npy_intp stride1, stride2, maskna_stride1 = 0, maskna_stride2 = 0; + npy_intp *strides; + npy_intp stride1, stride2; npy_intp *shape, dim1, dim2; - int self_has_maskna = PyArray_HASMASKNA(self); - char *data, *maskna_data; + char *data; npy_intp diag_size; PyArrayObject *ret; PyArray_Descr *dtype; @@ -1876,22 +1751,15 @@ PyArray_Diagonal(PyArrayObject *self, int offset, int axis1, int axis2) strides = PyArray_STRIDES(self); stride1 = strides[axis1]; stride2 = strides[axis2]; - if (self_has_maskna) { - maskna_strides = PyArray_MASKNA_STRIDES(self); - maskna_stride1 = maskna_strides[axis1]; - maskna_stride2 = maskna_strides[axis2]; - } /* Compute the data pointers and diag_size for the view */ data = PyArray_DATA(self); - maskna_data = PyArray_MASKNA_DATA(self); if (offset > 0) { if (offset >= dim2) { diag_size = 0; } else { data += offset * stride2; - maskna_data += offset * maskna_stride2; diag_size = dim2 - offset; if (dim1 < diag_size) { @@ -1906,7 +1774,6 @@ PyArray_Diagonal(PyArrayObject *self, int offset, int axis1, int axis2) } else { data += offset * stride1; - maskna_data += offset * maskna_stride1; diag_size = dim1 - offset; if (dim2 < diag_size) { @@ -1938,7 +1805,7 @@ PyArray_Diagonal(PyArrayObject *self, int offset, int axis1, int axis2) ndim-1, ret_shape, ret_strides, data, - PyArray_FLAGS(self) & ~(NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA), + PyArray_FLAGS(self), (PyObject *)self); if (ret == NULL) { return NULL; @@ -1949,27 +1816,6 @@ PyArray_Diagonal(PyArrayObject *self, int offset, int axis1, int axis2) return NULL; } - /* Take a view of the mask if it exists */ - if (self_has_maskna) { - PyArrayObject_fields *fret = (PyArrayObject_fields *)ret; - - maskna_strides = PyArray_MASKNA_STRIDES(self); - - fret->maskna_dtype = PyArray_MASKNA_DTYPE(self); - Py_INCREF(fret->maskna_dtype); - fret->maskna_data = maskna_data; - /* Build the strides for the mask */ - i = 0; - for (idim = 0; idim < ndim; ++idim) { - if (idim != axis1 && idim != axis2) { - fret->maskna_strides[i] = maskna_strides[idim]; - ++i; - } - } - fret->maskna_strides[ndim-2] = maskna_stride1 + maskna_stride2; - fret->flags |= NPY_ARRAY_MASKNA; - } - /* For backwards compatibility, during the deprecation period: */ copy = PyArray_NewCopy(ret, NPY_KEEPORDER); if (!copy) { @@ -1999,7 +1845,7 @@ PyArray_Compress(PyArrayObject *self, PyObject *condition, int axis, return NULL; } cond = (PyArrayObject *)PyArray_FromAny(condition, dtype, - 0, 0, NPY_ARRAY_ALLOWNA, NULL); + 0, 0, 0, NULL); if (cond == NULL) { return NULL; } @@ -2076,9 +1922,9 @@ count_boolean_trues(int ndim, char *data, npy_intp *ashape, npy_intp *astrides) } static int -assign_reduce_identity_zero(PyArrayObject *result, int preservena, void *data) +assign_reduce_identity_zero(PyArrayObject *result, void *data) { - return PyArray_AssignZero(result, NULL, preservena, NULL); + return PyArray_AssignZero(result, NULL); } static int @@ -2126,64 +1972,14 @@ reduce_count_nonzero_loop(NpyIter *iter, return (needs_api && PyErr_Occurred()) ? -1 : 0; } -static int -reduce_count_nonzero_masked_loop(NpyIter *iter, - char **dataptr, - npy_intp *strides, - npy_intp *countptr, - NpyIter_IterNextFunc *iternext, - int needs_api, - npy_intp skip_first_count, - void *data) -{ - PyArray_NonzeroFunc *nonzero = (PyArray_NonzeroFunc *)data; - PyArrayObject *arr = NpyIter_GetOperandArray(iter)[1]; - - NPY_BEGIN_THREADS_DEF; - - if (!needs_api) { - NPY_BEGIN_THREADS; - } - - /* - * 'skip_first_count' will always be 0 because we are doing a reduction - * with an identity. - */ - - do { - char *data0 = dataptr[0], *data1 = dataptr[1], *data2 = dataptr[2]; - npy_intp stride0 = strides[0], stride1 = strides[1], - stride2 = strides[2]; - npy_intp count = *countptr; - - while (count--) { - if (NpyMaskValue_IsExposed((npy_mask)*data2) && - nonzero(data1, arr)) { - ++(*(npy_intp *)data0); - } - data0 += stride0; - data1 += stride1; - data2 += stride2; - } - } while (iternext(iter)); - - if (!needs_api) { - NPY_END_THREADS; - } - - return (needs_api && PyErr_Occurred()) ? -1 : 0; -} - - /* * A full reduction version of PyArray_CountNonzero, supporting * an 'out' parameter and doing the count as a reduction along - * selected axes. It also supports a 'skipna' parameter, which - * skips over any NA masked values in arr. + * selected axes. */ NPY_NO_EXPORT PyObject * PyArray_ReduceCountNonzero(PyArrayObject *arr, PyArrayObject *out, - npy_bool *axis_flags, int skipna, int keepdims) + npy_bool *axis_flags, int keepdims) { PyArray_NonzeroFunc *nonzero; PyArrayObject *result; @@ -2205,11 +2001,9 @@ PyArray_ReduceCountNonzero(PyArrayObject *arr, PyArrayObject *out, result = PyArray_ReduceWrapper(arr, out, NULL, PyArray_DESCR(arr), dtype, NPY_SAME_KIND_CASTING, - axis_flags, 1, skipna, NULL, keepdims, 0, + axis_flags, 1, keepdims, 0, &assign_reduce_identity_zero, &reduce_count_nonzero_loop, - &reduce_count_nonzero_masked_loop, - NULL, nonzero, 0, "count_nonzero"); Py_DECREF(dtype); if (out == NULL && result != NULL) { @@ -2221,8 +2015,7 @@ PyArray_ReduceCountNonzero(PyArrayObject *arr, PyArrayObject *out, } /*NUMPY_API - * Counts the number of non-zero elements in the array. Raises - * an error if the array contains an NA. + * Counts the number of non-zero elements in the array. * * Returns -1 on error. */ @@ -2239,20 +2032,6 @@ PyArray_CountNonzero(PyArrayObject *self) char **dataptr; npy_intp *strideptr, *innersizeptr; - /* If 'self' has an NA mask, make sure it has no NA values */ - if (PyArray_HASMASKNA(self)) { - int containsna = PyArray_ContainsNA(self, NULL, NULL); - if (containsna == -1) { - return -1; - } - else if (containsna) { - PyErr_SetString(PyExc_ValueError, - "Cannot count the number of nonzeros in an array " - "which contains an NA"); - return -1; - } - } - /* Special low-overhead version specific to the boolean type */ if (PyArray_DESCR(self)->type_num == NPY_BOOL) { return count_boolean_trues(PyArray_NDIM(self), PyArray_DATA(self), @@ -2285,11 +2064,8 @@ PyArray_CountNonzero(PyArrayObject *self) /* * Otherwise create and use an iterator to count the nonzeros. - * Can ignore any NAs because we already checked PyArray_ContainsNA - * earlier. */ iter = NpyIter_New(self, NPY_ITER_READONLY | - NPY_ITER_IGNORE_MASKNA | NPY_ITER_EXTERNAL_LOOP | NPY_ITER_REFS_OK, NPY_KEEPORDER, NPY_NO_CASTING, @@ -2352,9 +2128,7 @@ PyArray_Nonzero(PyArrayObject *self) char **dataptr; /* - * First count the number of non-zeros in 'self'. If 'self' contains - * an NA value, this will raise an error, so after this call - * we can assume 'self' contains no NAs. + * First count the number of non-zeros in 'self'. */ nonzero_count = PyArray_CountNonzero(self); if (nonzero_count < 0) { @@ -2391,12 +2165,9 @@ PyArray_Nonzero(PyArrayObject *self) } /* - * Build an iterator tracking a multi-index, in C order. We - * can ignore NAs because the PyArray_CountNonzero call checked - * that there were no NAs already. + * Build an iterator tracking a multi-index, in C order. */ iter = NpyIter_New(self, NPY_ITER_READONLY | - NPY_ITER_IGNORE_MASKNA | NPY_ITER_MULTI_INDEX | NPY_ITER_ZEROSIZE_OK | NPY_ITER_REFS_OK, @@ -2496,56 +2267,18 @@ PyArray_MultiIndexGetItem(PyArrayObject *self, npy_intp *multi_index) npy_intp *shape = PyArray_SHAPE(self); npy_intp *strides = PyArray_STRIDES(self); - /* Case with an NA mask */ - if (PyArray_HASMASKNA(self)) { - char *maskdata = PyArray_MASKNA_DATA(self); - npy_mask maskvalue; - npy_intp *maskstrides = PyArray_MASKNA_STRIDES(self); - - if (PyArray_HASFIELDS(self)) { - PyErr_SetString(PyExc_RuntimeError, - "field-NA is not supported yet in MultiIndexGetItem"); - return NULL; - } - - /* Get the data and maskdata pointer */ - for (idim = 0; idim < ndim; ++idim) { - npy_intp shapevalue = shape[idim]; - npy_intp ind = multi_index[idim]; - - if (check_and_adjust_index(&ind, shapevalue, idim) < 0) { - return NULL; - } - - data += ind * strides[idim]; - maskdata += ind * maskstrides[idim]; - } + /* Get the data pointer */ + for (idim = 0; idim < ndim; ++idim) { + npy_intp shapevalue = shape[idim]; + npy_intp ind = multi_index[idim]; - maskvalue = (npy_mask)*maskdata; - if (NpyMaskValue_IsExposed(maskvalue)) { - return PyArray_DESCR(self)->f->getitem(data, self); - } - else { - return (PyObject *)NpyNA_FromDTypeAndPayload( - PyArray_DTYPE(self), 0, 0); + if (check_and_adjust_index(&ind, shapevalue, idim) < 0) { + return NULL; } + data += ind * strides[idim]; } - /* Case without an NA mask */ - else { - /* Get the data pointer */ - for (idim = 0; idim < ndim; ++idim) { - npy_intp shapevalue = shape[idim]; - npy_intp ind = multi_index[idim]; - if (check_and_adjust_index(&ind, shapevalue, idim) < 0) { - return NULL; - } - - data += ind * strides[idim]; - } - - return PyArray_DESCR(self)->f->getitem(data, self); - } + return PyArray_DESCR(self)->f->getitem(data, self); } /* @@ -2563,67 +2296,16 @@ PyArray_MultiIndexSetItem(PyArrayObject *self, npy_intp *multi_index, npy_intp *shape = PyArray_SHAPE(self); npy_intp *strides = PyArray_STRIDES(self); - /* Case with an NA mask */ - if (PyArray_HASMASKNA(self)) { - char *maskdata = PyArray_MASKNA_DATA(self); - npy_intp *maskstrides = PyArray_MASKNA_STRIDES(self); - NpyNA *na = NpyNA_FromObject(obj, 1); + /* Get the data pointer */ + for (idim = 0; idim < ndim; ++idim) { + npy_intp shapevalue = shape[idim]; + npy_intp ind = multi_index[idim]; - if (PyArray_HASFIELDS(self)) { - PyErr_SetString(PyExc_RuntimeError, - "field-NA is not supported yet in MultiIndexSetItem"); + if (check_and_adjust_index(&ind, shapevalue, idim) < 0) { return -1; } - - /* Get the data and maskdata pointer */ - for (idim = 0; idim < ndim; ++idim) { - npy_intp shapevalue = shape[idim]; - npy_intp ind = multi_index[idim]; - - if (check_and_adjust_index(&ind, shapevalue, idim) < 0) { - return -1; - } - - data += ind * strides[idim]; - maskdata += ind * maskstrides[idim]; - } - - if (na == NULL) { - *maskdata = 1; - return PyArray_DESCR(self)->f->setitem(obj, data, self); - } - else { - char maskvalue = (char)NpyNA_AsMaskValue(na); - - if (maskvalue != 0 && - PyArray_MASKNA_DTYPE(self)->type_num != NPY_MASK) { - /* TODO: also handle struct-NA mask dtypes */ - PyErr_SetString(PyExc_ValueError, - "Cannot assign an NA with a payload to an " - "NA-array with a boolean mask, requires a " - "multi-NA mask"); - return -1; - } - - *maskdata = maskvalue; - - return 0; - } + data += ind * strides[idim]; } - /* Case without an NA mask */ - else { - /* Get the data pointer */ - for (idim = 0; idim < ndim; ++idim) { - npy_intp shapevalue = shape[idim]; - npy_intp ind = multi_index[idim]; - - if (check_and_adjust_index(&ind, shapevalue, idim) < 0) { - return -1; - } - data += ind * strides[idim]; - } - - return PyArray_DESCR(self)->f->setitem(obj, data, self); - } + return PyArray_DESCR(self)->f->setitem(obj, data, self); } diff --git a/numpy/core/src/multiarray/item_selection.h b/numpy/core/src/multiarray/item_selection.h index 5c1741aaf..5f4ba1faa 100644 --- a/numpy/core/src/multiarray/item_selection.h +++ b/numpy/core/src/multiarray/item_selection.h @@ -30,12 +30,11 @@ PyArray_MultiIndexSetItem(PyArrayObject *self, npy_intp *multi_index, /* * A full reduction version of PyArray_CountNonzero, supporting * an 'out' parameter and doing the count as a reduction along - * selected axes. It also supports a 'skipna' parameter, which - * skips over any NA masked values in arr. + * selected axes. */ NPY_NO_EXPORT PyObject * PyArray_ReduceCountNonzero(PyArrayObject *arr, PyArrayObject *out, - npy_bool *axis_flags, int skipna, int keepdims); + npy_bool *axis_flags, int keepdims); diff --git a/numpy/core/src/multiarray/iterators.c b/numpy/core/src/multiarray/iterators.c index 1933b145d..af0d0d3c2 100644 --- a/numpy/core/src/multiarray/iterators.c +++ b/numpy/core/src/multiarray/iterators.c @@ -85,21 +85,17 @@ parse_index_entry(PyObject *op, npy_intp *step_size, /* * Parses an index that has no fancy indexing. Populates - * out_dimensions, out_strides, and out_offset. If out_maskna_strides - * and out_maskoffset aren't NULL, then 'self' must have an NA mask - * which is used to populate those variables as well. + * out_dimensions, out_strides, and out_offset. */ NPY_NO_EXPORT int parse_index(PyArrayObject *self, PyObject *op, npy_intp *out_dimensions, npy_intp *out_strides, - npy_intp *out_offset, - npy_intp *out_maskna_strides, - npy_intp *out_maskna_offset) + npy_intp *out_offset) { int i, j, n; int nd_old, nd_new, n_add, n_ellipsis; - npy_intp n_steps, start, offset, maskna_offset, step_size; + npy_intp n_steps, start, offset, step_size; PyObject *op1 = NULL; int is_slice; @@ -124,7 +120,6 @@ parse_index(PyArrayObject *self, PyObject *op, nd_old = nd_new = 0; offset = 0; - maskna_offset = 0; for (i = 0; i < n; i++) { if (!is_slice) { op1 = PySequence_GetItem(op, i); @@ -143,9 +138,6 @@ parse_index(PyArrayObject *self, PyObject *op, if (n_steps == NEWAXIS_INDEX) { out_dimensions[nd_new] = 1; out_strides[nd_new] = 0; - if (out_maskna_strides != NULL) { - out_maskna_strides[nd_new] = 0; - } nd_new++; } else if (n_steps == ELLIPSIS_INDEX) { @@ -164,10 +156,6 @@ parse_index(PyArrayObject *self, PyObject *op, for (j = 0; j < n_add; j++) { out_dimensions[nd_new] = PyArray_DIMS(self)[nd_old]; out_strides[nd_new] = PyArray_STRIDES(self)[nd_old]; - if (out_maskna_strides != NULL) { - out_maskna_strides[nd_new] = - PyArray_MASKNA_STRIDES(self)[nd_old]; - } nd_new++; nd_old++; } } @@ -177,18 +165,11 @@ parse_index(PyArrayObject *self, PyObject *op, return -1; } offset += PyArray_STRIDES(self)[nd_old]*start; - if (out_maskna_offset != NULL) { - maskna_offset += PyArray_MASKNA_STRIDES(self)[nd_old]*start; - } nd_old++; if (n_steps != SINGLE_INDEX) { out_dimensions[nd_new] = n_steps; out_strides[nd_new] = step_size * PyArray_STRIDES(self)[nd_old-1]; - if (out_maskna_strides != NULL) { - out_maskna_strides[nd_new] = step_size * - PyArray_MASKNA_STRIDES(self)[nd_old-1]; - } nd_new++; } } @@ -200,16 +181,10 @@ parse_index(PyArrayObject *self, PyObject *op, for (j = 0; j < n_add; j++) { out_dimensions[nd_new] = PyArray_DIMS(self)[nd_old]; out_strides[nd_new] = PyArray_STRIDES(self)[nd_old]; - if (out_maskna_strides != NULL) { - out_maskna_strides[nd_new] = PyArray_MASKNA_STRIDES(self)[nd_old]; - } nd_new++; nd_old++; } *out_offset = offset; - if (out_maskna_offset != NULL) { - *out_maskna_offset = maskna_offset; - } return nd_new; } @@ -407,13 +382,6 @@ PyArray_IterNew(PyObject *obj) } ao = (PyArrayObject *)obj; - if (PyArray_HASMASKNA(ao)) { - PyErr_SetString(PyExc_ValueError, - "Old-style NumPy iterators do not support NA masks, " - "use numpy.nditer instead"); - return NULL; - } - it = (PyArrayIterObject *)PyArray_malloc(sizeof(PyArrayIterObject)); PyObject_Init((PyObject *)it, &PyArrayIter_Type); /* it = PyObject_New(PyArrayIterObject, &PyArrayIter_Type);*/ @@ -435,13 +403,6 @@ PyArray_BroadcastToShape(PyObject *obj, npy_intp *dims, int nd) int i, diff, j, compat, k; PyArrayObject *ao = (PyArrayObject *)obj; - if (PyArray_HASMASKNA(ao)) { - PyErr_SetString(PyExc_ValueError, - "Old-style NumPy iterators do not support NA masks, " - "use numpy.nditer instead"); - return NULL; - } - if (PyArray_NDIM(ao) > nd) { goto err; } diff --git a/numpy/core/src/multiarray/iterators.h b/numpy/core/src/multiarray/iterators.h index 6d648810e..8276a3ceb 100644 --- a/numpy/core/src/multiarray/iterators.h +++ b/numpy/core/src/multiarray/iterators.h @@ -3,17 +3,13 @@ /* * Parses an index that has no fancy indexing. Populates - * out_dimensions, out_strides, and out_offset. If out_maskstrides - * and out_maskoffset aren't NULL, then 'self' must have an NA mask - * which is used to populate those variables as well. + * out_dimensions, out_strides, and out_offset. */ NPY_NO_EXPORT int parse_index(PyArrayObject *self, PyObject *op, npy_intp *out_dimensions, npy_intp *out_strides, - npy_intp *out_offset, - npy_intp *out_maskna_strides, - npy_intp *out_maskna_offset); + npy_intp *out_offset); NPY_NO_EXPORT PyObject *iter_subscript(PyArrayIterObject *, PyObject *); diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index 7034f5797..769fb653e 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -14,7 +14,6 @@ #include "common.h" #include "iterators.h" #include "mapping.h" -#include "na_object.h" #include "lowlevel_strided_loops.h" #include "item_selection.h" @@ -69,27 +68,12 @@ array_big_item(PyArrayObject *self, npy_intp i) PyArray_NDIM(self)-1, PyArray_DIMS(self)+1, PyArray_STRIDES(self)+1, item, - PyArray_FLAGS(self) & ~(NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA), + PyArray_FLAGS(self), (PyObject *)self); if (ret == NULL) { return NULL; } - /* Take a view of the NA mask if it exists */ - if (PyArray_HASMASKNA(self)) { - PyArrayObject_fields *fa = (PyArrayObject_fields *)ret; - - fa->maskna_dtype = PyArray_MASKNA_DTYPE(self); - Py_INCREF(fa->maskna_dtype); - fa->maskna_data = PyArray_MASKNA_DATA(self) + - i * PyArray_MASKNA_STRIDES(self)[0]; - if (fa->nd > 0) { - memcpy(fa->maskna_strides, PyArray_MASKNA_STRIDES(self)+1, - fa->nd * sizeof(npy_intp)); - } - fa->flags |= NPY_ARRAY_MASKNA; - } - /* Set the base object */ Py_INCREF(self); if (PyArray_SetBaseObject(ret, (PyObject *)self) < 0) { @@ -125,30 +109,6 @@ array_item_nice(PyArrayObject *self, Py_ssize_t _i) } item = PyArray_DATA(self) + i * PyArray_STRIDE(self, 0); - if (PyArray_HASMASKNA(self)) { - npy_mask maskvalue; - - maskvalue = (npy_mask)*(PyArray_MASKNA_DATA(self) + - i * PyArray_MASKNA_STRIDES(self)[0]); - if (!NpyMaskValue_IsExposed(maskvalue)) { - NpyNA_fields *fna; - - fna = (NpyNA_fields *)NpyNA_Type.tp_new(&NpyNA_Type, NULL, NULL); - if (fna == NULL) { - return NULL; - } - - fna->dtype = PyArray_DESCR(self); - Py_INCREF(fna->dtype); - - if (PyArray_MASKNA_DTYPE(self)->type_num == NPY_MASK) { - fna->payload = NpyMaskValue_GetPayload(maskvalue); - } - - return (PyObject *)fna; - } - } - return PyArray_Scalar(item, PyArray_DESCR(self), (PyObject *)self); } else { @@ -182,8 +142,8 @@ array_ass_big_item(PyArrayObject *self, npy_intp i, PyObject *v) } - /* For multi-dimensional arrays and NA masked arrays, use CopyObject */ - if (PyArray_NDIM(self) > 1 || PyArray_HASMASKNA(self)) { + /* For multi-dimensional arrays, use CopyObject */ + if (PyArray_NDIM(self) > 1) { tmp = (PyArrayObject *)array_big_item(self, i); if(tmp == NULL) { return -1; @@ -468,7 +428,7 @@ add_new_axes_0d(PyArrayObject *arr, int newaxis_count) PyArray_DESCR(arr), newaxis_count, dimensions, NULL, PyArray_DATA(arr), - PyArray_FLAGS(arr) & ~(NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA), + PyArray_FLAGS(arr), (PyObject *)arr); if (ret == NULL) { return NULL; @@ -480,24 +440,6 @@ add_new_axes_0d(PyArrayObject *arr, int newaxis_count) return NULL; } - /* Take a view of the NA mask if it exists */ - if (PyArray_HASMASKNA(arr)) { - PyArrayObject_fields *fret = (PyArrayObject_fields *)ret; - - fret->maskna_dtype = PyArray_MASKNA_DTYPE(arr); - Py_INCREF(fret->maskna_dtype); - - fret->maskna_data = PyArray_MASKNA_DATA(arr); - - for (i = 0; i < newaxis_count; ++i) { - fret->maskna_strides[i] = fret->maskna_dtype->elsize; - } - - /* This view doesn't own the mask */ - fret->flags |= NPY_ARRAY_MASKNA; - fret->flags &= ~NPY_ARRAY_OWNMASKNA; - } - return (PyObject *)ret; } @@ -603,8 +545,7 @@ NPY_NO_EXPORT PyObject * array_subscript_simple(PyArrayObject *self, PyObject *op) { npy_intp dimensions[NPY_MAXDIMS], strides[NPY_MAXDIMS]; - npy_intp maskna_strides[NPY_MAXDIMS]; - npy_intp offset, maskna_offset; + npy_intp offset; int nd; PyArrayObject *ret; npy_intp value; @@ -637,14 +578,8 @@ array_subscript_simple(PyArrayObject *self, PyObject *op) } /* Standard (view-based) Indexing */ - if (PyArray_HASMASKNA(self)) { - nd = parse_index(self, op, dimensions, - strides, &offset, maskna_strides, &maskna_offset); - } - else { - nd = parse_index(self, op, dimensions, - strides, &offset, NULL, NULL); - } + nd = parse_index(self, op, dimensions, + strides, &offset); if (nd == -1) { return NULL; } @@ -655,7 +590,7 @@ array_subscript_simple(PyArrayObject *self, PyObject *op) PyArray_DESCR(self), nd, dimensions, strides, PyArray_DATA(self) + offset, - PyArray_FLAGS(self) & ~(NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA), + PyArray_FLAGS(self), (PyObject *)self); if (ret == NULL) { return NULL; @@ -667,25 +602,6 @@ array_subscript_simple(PyArrayObject *self, PyObject *op) } PyArray_UpdateFlags(ret, NPY_ARRAY_UPDATE_ALL); - /* Take a view of the NA mask if it exists */ - if (PyArray_HASMASKNA(self)) { - PyArrayObject_fields *fret = (PyArrayObject_fields *)ret; - - fret->maskna_dtype = PyArray_MASKNA_DTYPE(self); - Py_INCREF(fret->maskna_dtype); - - fret->maskna_data = PyArray_MASKNA_DATA(self) + maskna_offset; - - if (nd > 0) { - memcpy(fret->maskna_strides, maskna_strides, - nd * sizeof(npy_intp)); - } - - /* This view doesn't own the mask */ - fret->flags |= NPY_ARRAY_MASKNA; - fret->flags &= ~NPY_ARRAY_OWNMASKNA; - } - return (PyObject *)ret; } @@ -701,13 +617,13 @@ array_subscript_simple(PyArrayObject *self, PyObject *op) */ NPY_NO_EXPORT PyArrayObject * array_boolean_subscript(PyArrayObject *self, - PyArrayObject *bmask, NPY_ORDER order) + PyArrayObject *bmask, NPY_ORDER order) { npy_intp size, itemsize; - char *ret_data, *ret_maskna_data = NULL; + char *ret_data; PyArray_Descr *dtype; PyArrayObject *ret; - int self_has_maskna = PyArray_HASMASKNA(self), needs_api = 0, containsna; + int needs_api = 0; npy_intp bmask_size; if (PyArray_DESCR(bmask)->type_num != NPY_BOOL) { @@ -716,20 +632,6 @@ array_boolean_subscript(PyArrayObject *self, return NULL; } - /* - * See the Boolean Indexing section of the missing data NEP. - */ - containsna = PyArray_ContainsNA(bmask, NULL, NULL); - if (containsna == -1) { - return NULL; - } - else if (containsna) { - PyErr_SetString(PyExc_ValueError, - "The boolean mask indexing array " - "may not contain any NA values"); - return NULL; - } - if (PyArray_NDIM(bmask) != PyArray_NDIM(self)) { PyErr_SetString(PyExc_ValueError, "The boolean mask assignment indexing array " @@ -739,11 +641,6 @@ array_boolean_subscript(PyArrayObject *self, } - /* - * Since we've checked that the mask contains no NAs, we - * can do a straightforward count of the boolean True values - * in the raw mask data array. - */ size = count_boolean_trues(PyArray_NDIM(bmask), PyArray_DATA(bmask), PyArray_DIMS(bmask), PyArray_STRIDES(bmask)); /* Correction factor for broadcasting 'bmask' to 'self' */ @@ -761,15 +658,6 @@ array_boolean_subscript(PyArrayObject *self, return NULL; } - /* Allocate an NA mask for ret if required */ - if (self_has_maskna) { - if (PyArray_AllocateMaskNA(ret, 1, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - ret_maskna_data = PyArray_MASKNA_DATA(ret); - } - itemsize = dtype->elsize; ret_data = PyArray_DATA(ret); @@ -786,22 +674,14 @@ array_boolean_subscript(PyArrayObject *self, npy_intp innersize, *innerstrides; char **dataptrs; + npy_intp self_stride, bmask_stride, subloopsize; + char *self_data; + char *bmask_data; + /* Set up the iterator */ flags = NPY_ITER_EXTERNAL_LOOP | NPY_ITER_REFS_OK; - if (self_has_maskna) { - op_flags[0] = NPY_ITER_READONLY | - NPY_ITER_NO_BROADCAST | - NPY_ITER_USE_MASKNA; - } - else { - op_flags[0] = NPY_ITER_READONLY | - NPY_ITER_NO_BROADCAST; - } - /* - * Since we already checked PyArray_ContainsNA(bmask), can - * ignore any MASKNA of bmask. - */ - op_flags[1] = NPY_ITER_READONLY | NPY_ITER_IGNORE_MASKNA; + op_flags[0] = NPY_ITER_READONLY | NPY_ITER_NO_BROADCAST; + op_flags[1] = NPY_ITER_READONLY; iter = NpyIter_MultiNew(2, op, flags, order, NPY_NO_CASTING, op_flags, NULL); @@ -834,93 +714,35 @@ array_boolean_subscript(PyArrayObject *self, innerstrides = NpyIter_GetInnerStrideArray(iter); dataptrs = NpyIter_GetDataPtrArray(iter); - /* Regular inner loop */ - if (!self_has_maskna) { - npy_intp self_stride = innerstrides[0]; - npy_intp bmask_stride = innerstrides[1]; - npy_intp subloopsize; - char *self_data; - char *bmask_data; - do { - innersize = *NpyIter_GetInnerLoopSizePtr(iter); - self_data = dataptrs[0]; - bmask_data = dataptrs[1]; - - while (innersize > 0) { - /* Skip masked values */ - subloopsize = 0; - while (subloopsize < innersize && *bmask_data == 0) { - ++subloopsize; - bmask_data += bmask_stride; - } - innersize -= subloopsize; - self_data += subloopsize * self_stride; - /* Process unmasked values */ - subloopsize = 0; - while (subloopsize < innersize && *bmask_data != 0) { - ++subloopsize; - bmask_data += bmask_stride; - } - stransfer(ret_data, itemsize, self_data, self_stride, - subloopsize, itemsize, transferdata); - innersize -= subloopsize; - self_data += subloopsize * self_stride; - ret_data += subloopsize * itemsize; + self_stride = innerstrides[0]; + bmask_stride = innerstrides[1]; + do { + innersize = *NpyIter_GetInnerLoopSizePtr(iter); + self_data = dataptrs[0]; + bmask_data = dataptrs[1]; + + while (innersize > 0) { + /* Skip masked values */ + subloopsize = 0; + while (subloopsize < innersize && *bmask_data == 0) { + ++subloopsize; + bmask_data += bmask_stride; } - } while (iternext(iter)); - } - /* NA masked inner loop */ - else { - npy_intp i; - npy_intp self_stride = innerstrides[0]; - npy_intp bmask_stride = innerstrides[1]; - npy_intp maskna_stride = innerstrides[2]; - npy_intp subloopsize; - char *self_data; - char *bmask_data; - char *maskna_data; - do { - innersize = *NpyIter_GetInnerLoopSizePtr(iter); - self_data = dataptrs[0]; - bmask_data = dataptrs[1]; - maskna_data = dataptrs[2]; - - while (innersize > 0) { - /* Skip masked values */ - subloopsize = 0; - while (subloopsize < innersize && *bmask_data == 0) { - ++subloopsize; - bmask_data += bmask_stride; - } - innersize -= subloopsize; - self_data += subloopsize * self_stride; - maskna_data += subloopsize * maskna_stride; - /* Process unmasked values */ - subloopsize = 0; - while (subloopsize < innersize && *bmask_data != 0) { - ++subloopsize; - bmask_data += bmask_stride; - } - /* - * Because it's a newly allocated array, we - * don't have to be careful about not overwriting - * NA masked values. If 'ret' were an output parameter, - * we would have to avoid that. - */ - stransfer(ret_data, itemsize, self_data, self_stride, - subloopsize, itemsize, transferdata); - /* Copy the mask as well */ - for (i = 0; i < subloopsize; ++i) { - *ret_maskna_data = *maskna_data; - ++ret_maskna_data; - maskna_data += maskna_stride; - } - innersize -= subloopsize; - self_data += subloopsize * self_stride; - ret_data += subloopsize * itemsize; + innersize -= subloopsize; + self_data += subloopsize * self_stride; + /* Process unmasked values */ + subloopsize = 0; + while (subloopsize < innersize && *bmask_data != 0) { + ++subloopsize; + bmask_data += bmask_stride; } - } while (iternext(iter)); - } + stransfer(ret_data, itemsize, self_data, self_stride, + subloopsize, itemsize, transferdata); + innersize -= subloopsize; + self_data += subloopsize * self_stride; + ret_data += subloopsize * itemsize; + } + } while (iternext(iter)); NpyIter_Deallocate(iter); NPY_AUXDATA_FREE(transferdata); @@ -945,13 +767,10 @@ NPY_NO_EXPORT int array_ass_boolean_subscript(PyArrayObject *self, PyArrayObject *bmask, PyArrayObject *v, NPY_ORDER order) { - npy_intp size, src_itemsize, v_stride, v_maskna_stride = 0; - char *v_data, *v_maskna_data = NULL; - int self_has_maskna = PyArray_HASMASKNA(self); - int v_has_maskna = PyArray_HASMASKNA(v); - int needs_api = 0, containsna; + npy_intp size, src_itemsize, v_stride; + char *v_data; + int needs_api = 0; npy_intp bmask_size; - char constant_valid_mask = 1; if (PyArray_DESCR(bmask)->type_num != NPY_BOOL) { PyErr_SetString(PyExc_TypeError, @@ -976,41 +795,6 @@ array_ass_boolean_subscript(PyArrayObject *self, return -1; } - /* See the Boolean Indexing section of the missing data NEP */ - containsna = PyArray_ContainsNA(bmask, NULL, NULL); - if (containsna == -1) { - return -1; - } - else if (containsna) { - PyErr_SetString(PyExc_ValueError, - "The boolean mask assignment indexing array " - "may not contain any NA values"); - return -1; - } - - /* Can't assign an NA to an array which doesn't support it */ - if (v_has_maskna && !self_has_maskna) { - containsna = PyArray_ContainsNA(v, NULL, NULL); - if (containsna == -1) { - return -1; - } - else if (containsna) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which " - "does not support NAs"); - return -1; - } - /* If there are no actual NAs, allow the assignment */ - else { - v_has_maskna = 0; - } - } - - /* - * Since we've checked that the mask contains no NAs, we - * can do a straightforward count of the boolean True values - * in the raw mask data array. - */ size = count_boolean_trues(PyArray_NDIM(bmask), PyArray_DATA(bmask), PyArray_DIMS(bmask), PyArray_STRIDES(bmask)); /* Correction factor for broadcasting 'bmask' to 'self' */ @@ -1022,13 +806,6 @@ array_ass_boolean_subscript(PyArrayObject *self, /* Tweak the strides for 0-dim and broadcasting cases */ if (PyArray_NDIM(v) > 0 && PyArray_DIMS(v)[0] > 1) { v_stride = PyArray_STRIDES(v)[0]; - if (v_has_maskna) { - v_maskna_stride = PyArray_MASKNA_STRIDES(v)[0]; - } - else { - v_maskna_stride = 0; - } - if (size != PyArray_DIMS(v)[0]) { PyErr_Format(PyExc_ValueError, "NumPy boolean array indexing assignment " @@ -1040,18 +817,10 @@ array_ass_boolean_subscript(PyArrayObject *self, } else { v_stride = 0; - v_maskna_stride = 0; } src_itemsize = PyArray_DESCR(v)->elsize; v_data = PyArray_DATA(v); - if (v_has_maskna) { - v_maskna_data = PyArray_MASKNA_DATA(v); - } - /* If assigning unmasked to masked, use a 0-stride all valid mask */ - else if (self_has_maskna) { - v_maskna_data = &constant_valid_mask; - } /* Create an iterator for the data */ if (size > 0) { @@ -1064,22 +833,16 @@ array_ass_boolean_subscript(PyArrayObject *self, npy_intp innersize, *innerstrides; char **dataptrs; + PyArray_StridedUnaryOp *stransfer = NULL; + NpyAuxData *transferdata = NULL; + npy_intp self_stride, bmask_stride, subloopsize; + char *self_data; + char *bmask_data; + /* Set up the iterator */ flags = NPY_ITER_EXTERNAL_LOOP | NPY_ITER_REFS_OK; - if (self_has_maskna) { - op_flags[0] = NPY_ITER_WRITEONLY | - NPY_ITER_NO_BROADCAST | - NPY_ITER_USE_MASKNA; - } - else { - op_flags[0] = NPY_ITER_WRITEONLY | - NPY_ITER_NO_BROADCAST; - } - /* - * Since we already checked PyArray_ContainsNA(bmask), can - * ignore any MASKNA of bmask. - */ - op_flags[1] = NPY_ITER_READONLY | NPY_ITER_IGNORE_MASKNA; + op_flags[0] = NPY_ITER_WRITEONLY | NPY_ITER_NO_BROADCAST; + op_flags[1] = NPY_ITER_READONLY; iter = NpyIter_MultiNew(2, op, flags, order, NPY_NO_CASTING, op_flags, NULL); @@ -1096,147 +859,51 @@ array_ass_boolean_subscript(PyArrayObject *self, innerstrides = NpyIter_GetInnerStrideArray(iter); dataptrs = NpyIter_GetDataPtrArray(iter); - /* Regular inner loop */ - if (!self_has_maskna) { - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; - npy_intp self_stride = innerstrides[0]; - npy_intp bmask_stride = innerstrides[1]; - npy_intp subloopsize; - char *self_data; - char *bmask_data; - - /* Get a dtype transfer function */ - NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); - if (PyArray_GetDTypeTransferFunction( - PyArray_ISALIGNED(self) && PyArray_ISALIGNED(v), - v_stride, fixed_strides[0], - PyArray_DESCR(v), PyArray_DESCR(self), - 0, - &stransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { - NpyIter_Deallocate(iter); - return -1; - } + self_stride = innerstrides[0]; + bmask_stride = innerstrides[1]; - do { - innersize = *NpyIter_GetInnerLoopSizePtr(iter); - self_data = dataptrs[0]; - bmask_data = dataptrs[1]; - - while (innersize > 0) { - /* Skip masked values */ - subloopsize = 0; - while (subloopsize < innersize && *bmask_data == 0) { - ++subloopsize; - bmask_data += bmask_stride; - } - innersize -= subloopsize; - self_data += subloopsize * self_stride; - /* Process unmasked values */ - subloopsize = 0; - while (subloopsize < innersize && *bmask_data != 0) { - ++subloopsize; - bmask_data += bmask_stride; - } - stransfer(self_data, self_stride, v_data, v_stride, - subloopsize, src_itemsize, transferdata); - innersize -= subloopsize; - self_data += subloopsize * self_stride; - v_data += subloopsize * v_stride; - } - } while (iternext(iter)); - - NPY_AUXDATA_FREE(transferdata); + /* Get a dtype transfer function */ + NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); + if (PyArray_GetDTypeTransferFunction( + PyArray_ISALIGNED(self) && PyArray_ISALIGNED(v), + v_stride, fixed_strides[0], + PyArray_DESCR(v), PyArray_DESCR(self), + 0, + &stransfer, &transferdata, + &needs_api) != NPY_SUCCEED) { + NpyIter_Deallocate(iter); + return -1; } - /* NA masked inner loop */ - else { - PyArray_MaskedStridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; - npy_intp i; - npy_intp self_stride = innerstrides[0]; - npy_intp bmask_stride = innerstrides[1]; - npy_intp self_maskna_stride = innerstrides[2]; - npy_intp subloopsize; - PyArray_Descr *v_maskna_dtype; - char *self_data; - char *bmask_data; - char *self_maskna_data; - - if (PyArray_HASMASKNA(v)) { - v_maskna_dtype = PyArray_MASKNA_DTYPE(v); - Py_INCREF(v_maskna_dtype); - } - else { - v_maskna_dtype = PyArray_DescrFromType(NPY_BOOL); - if (v_maskna_dtype == NULL) { - NpyIter_Deallocate(iter); - return -1; - } - } - /* Get a dtype transfer function */ - NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); - if (PyArray_GetMaskedDTypeTransferFunction( - PyArray_ISALIGNED(self) && PyArray_ISALIGNED(v), - v_stride, fixed_strides[0], v_maskna_stride, - PyArray_DESCR(v), - PyArray_DESCR(self), - v_maskna_dtype, - 0, - &stransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { - Py_DECREF(v_maskna_dtype); - NpyIter_Deallocate(iter); - return -1; - } - Py_DECREF(v_maskna_dtype); - - do { - innersize = *NpyIter_GetInnerLoopSizePtr(iter); - self_data = dataptrs[0]; - bmask_data = dataptrs[1]; - self_maskna_data = dataptrs[2]; - - while (innersize > 0) { - /* Skip masked values */ - subloopsize = 0; - while (subloopsize < innersize && *bmask_data == 0) { - ++subloopsize; - bmask_data += bmask_stride; - } - innersize -= subloopsize; - self_data += subloopsize * self_stride; - self_maskna_data += subloopsize * self_maskna_stride; - /* Process unmasked values */ - subloopsize = 0; - while (subloopsize < innersize && *bmask_data != 0) { - ++subloopsize; - bmask_data += bmask_stride; - } - /* - * Because we're assigning to an existing array, - * we have to be careful about not overwriting - * NA masked values. - */ - stransfer(self_data, self_stride, v_data, v_stride, - (npy_mask *)v_maskna_data, v_maskna_stride, - subloopsize, src_itemsize, transferdata); - /* Copy the mask as well */ - for (i = 0; i < subloopsize; ++i) { - *self_maskna_data = *v_maskna_data; - self_maskna_data += self_maskna_stride; - v_maskna_data += v_maskna_stride; - } - innersize -= subloopsize; - self_data += subloopsize * self_stride; - v_data += subloopsize * v_stride; - } - } while (iternext(iter)); + do { + innersize = *NpyIter_GetInnerLoopSizePtr(iter); + self_data = dataptrs[0]; + bmask_data = dataptrs[1]; - NPY_AUXDATA_FREE(transferdata); - } + while (innersize > 0) { + /* Skip masked values */ + subloopsize = 0; + while (subloopsize < innersize && *bmask_data == 0) { + ++subloopsize; + bmask_data += bmask_stride; + } + innersize -= subloopsize; + self_data += subloopsize * self_stride; + /* Process unmasked values */ + subloopsize = 0; + while (subloopsize < innersize && *bmask_data != 0) { + ++subloopsize; + bmask_data += bmask_stride; + } + stransfer(self_data, self_stride, v_data, v_stride, + subloopsize, src_itemsize, transferdata); + innersize -= subloopsize; + self_data += subloopsize * self_stride; + v_data += subloopsize * v_stride; + } + } while (iternext(iter)); + NPY_AUXDATA_FREE(transferdata); NpyIter_Deallocate(iter); } @@ -1355,7 +1022,7 @@ array_subscript(PyArrayObject *self, PyObject *op) return NULL; } - /* Boolean indexing special case which supports mask NA */ + /* Boolean indexing special case */ if (PyArray_Check(op) && (PyArray_TYPE((PyArrayObject *)op) == NPY_BOOL) && (PyArray_NDIM(self) == PyArray_NDIM((PyArrayObject *)op))) { return (PyObject *)array_boolean_subscript(self, @@ -1588,44 +1255,18 @@ array_ass_sub(PyArrayObject *self, PyObject *ind, PyObject *op) npy_intp *strides = PyArray_STRIDES(self); char *item = PyArray_DATA(self); - if (!PyArray_HASMASKNA(self)) { - for (idim = 0; idim < ndim; idim++) { - npy_intp v = vals[idim]; - if (check_and_adjust_index(&v, shape[idim], idim) < 0) { - return -1; - } - item += v * strides[idim]; - } - return PyArray_DESCR(self)->f->setitem(op, item, self); - } - else { - char *maskna_item = PyArray_MASKNA_DATA(self); - npy_intp *maskna_strides = PyArray_MASKNA_STRIDES(self); - NpyNA *na; - - for (idim = 0; idim < ndim; idim++) { - npy_intp v = vals[idim]; - if (check_and_adjust_index(&v, shape[idim], idim) < 0) { - return -1; - } - item += v * strides[idim]; - maskna_item += v * maskna_strides[idim]; - } - na = NpyNA_FromObject(op, 1); - if (na == NULL) { - *maskna_item = 1; - return PyArray_DESCR(self)->f->setitem(op, item, self); - } - else { - *maskna_item = NpyNA_AsMaskValue(na); - Py_DECREF(na); - return 0; + for (idim = 0; idim < ndim; idim++) { + npy_intp v = vals[idim]; + if (check_and_adjust_index(&v, shape[idim], idim) < 0) { + return -1; } + item += v * strides[idim]; } + return PyArray_DESCR(self)->f->setitem(op, item, self); } PyErr_Clear(); - /* Boolean indexing special case with NA mask support */ + /* Boolean indexing special case */ if (PyArray_Check(ind) && (PyArray_TYPE((PyArrayObject *)ind) == NPY_BOOL) && (PyArray_NDIM(self) == PyArray_NDIM((PyArrayObject *)ind))) { @@ -1633,13 +1274,7 @@ array_ass_sub(PyArrayObject *self, PyObject *ind, PyObject *op) PyArrayObject *op_arr; PyArray_Descr *dtype = NULL; - /* If it's an NA with no dtype, specify the dtype explicitly */ - if (NpyNA_Check(op) && ((NpyNA_fields *)op)->dtype == NULL) { - dtype = PyArray_DESCR(self); - Py_INCREF(dtype); - } - op_arr = (PyArrayObject *)PyArray_FromAny(op, dtype, 0, 0, - NPY_ARRAY_ALLOWNA, NULL); + op_arr = (PyArrayObject *)PyArray_FromAny(op, dtype, 0, 0, 0, NULL); if (op_arr == NULL) { return -1; } @@ -1728,37 +1363,14 @@ array_subscript_nice(PyArrayObject *self, PyObject *op) npy_intp *strides = PyArray_STRIDES(self); char *item = PyArray_DATA(self); - if (!PyArray_HASMASKNA(self)) { - for (idim = 0; idim < ndim; idim++) { - npy_intp v = vals[idim]; - if (check_and_adjust_index(&v, shape[idim], idim) < 0) { - return NULL; - } - item += v * strides[idim]; - } - return PyArray_Scalar(item, PyArray_DESCR(self), (PyObject *)self); - } - else { - char *maskna_item = PyArray_MASKNA_DATA(self); - npy_intp *maskna_strides = PyArray_MASKNA_STRIDES(self); - - for (idim = 0; idim < ndim; idim++) { - npy_intp v = vals[idim]; - if (check_and_adjust_index(&v, shape[idim], idim) < 0) { - return NULL; - } - item += v * strides[idim]; - maskna_item += v * maskna_strides[idim]; - } - if (NpyMaskValue_IsExposed((npy_mask)*maskna_item)) { - return PyArray_Scalar(item, PyArray_DESCR(self), - (PyObject *)self); - } - else { - return (PyObject *)NpyNA_FromDTypeAndPayload( - PyArray_DESCR(self), 0, 0); + for (idim = 0; idim < ndim; idim++) { + npy_intp v = vals[idim]; + if (check_and_adjust_index(&v, shape[idim], idim) < 0) { + return NULL; } + item += v * strides[idim]; } + return PyArray_Scalar(item, PyArray_DESCR(self), (PyObject *)self); } PyErr_Clear(); diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index dacfabcbd..93d246671 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -19,7 +19,6 @@ #include "item_selection.h" #include "conversion_utils.h" #include "shape.h" -#include "boolean_ops.h" #include "methods.h" @@ -132,7 +131,7 @@ array_take(PyArrayObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O&O&O&", kwlist, &indices, PyArray_AxisConverter, &dimension, - PyArray_OutputAllowNAConverter, &out, + PyArray_OutputConverter, &out, PyArray_ClipmodeConverter, &mode)) return NULL; @@ -241,39 +240,14 @@ array_view(PyArrayObject *self, PyObject *args, PyObject *kwds) PyObject *out_dtype = NULL; PyObject *out_type = NULL; PyArray_Descr *dtype = NULL; - PyObject *ret; - int maskna = -1, ownmaskna = 0; - PyObject *maskna_in = Py_None; - static char *kwlist[] = {"dtype", "type", "maskna", "ownmaskna", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOOi", kwlist, + static char *kwlist[] = {"dtype", "type", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OO", kwlist, &out_dtype, - &out_type, - &maskna_in, - &ownmaskna)) { + &out_type)) { return NULL; } - /* Treat None the same as not providing the parameter */ - if (maskna_in != Py_None) { - maskna = PyObject_IsTrue(maskna_in); - if (maskna == -1) { - return NULL; - } - } - - /* 'ownmaskna' forces 'maskna' to be True */ - if (ownmaskna) { - if (maskna == 0) { - PyErr_SetString(PyExc_ValueError, - "cannot specify maskna=False and ownmaskna=True"); - return NULL; - } - else { - maskna = 1; - } - } - /* If user specified a positional argument, guess whether it represents a type or a dtype for backward compatibility. */ if (out_dtype) { @@ -304,30 +278,7 @@ array_view(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } - ret = PyArray_View(self, dtype, (PyTypeObject*)out_type); - if (ret == NULL) { - return NULL; - } - - if (maskna == 1) { - /* Ensure there is an NA mask if requested */ - if (PyArray_AllocateMaskNA((PyArrayObject *)ret, - ownmaskna, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - return ret; - } - else if (maskna == 0 && PyArray_HASMASKNA((PyArrayObject *)ret)) { - PyErr_SetString(PyExc_ValueError, - "Cannot take a view of an NA-masked array " - "with maskna=False"); - Py_DECREF(ret); - return NULL; - } - else { - return ret; - } + return PyArray_View(self, dtype, (PyTypeObject*)out_type); } static PyObject * @@ -381,7 +332,7 @@ array_ptp(PyArrayObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O&O&", kwlist, PyArray_AxisConverter, &axis, - PyArray_OutputAllowNAConverter, &out)) + PyArray_OutputConverter, &out)) return NULL; return PyArray_Ptp(self, axis, out); @@ -808,32 +759,6 @@ array_setscalar(PyArrayObject *self, PyObject *args) } } -/* Sets the array values from another array as if they were flat */ -static PyObject * -array_setasflat(PyArrayObject *self, PyObject *args) -{ - PyObject *arr_in; - PyArrayObject *arr; - - if (!PyArg_ParseTuple(args, "O", &arr_in)) { - return NULL; - } - - arr = (PyArrayObject *)PyArray_FromAny(arr_in, NULL, - 0, 0, NPY_ARRAY_ALLOWNA, NULL); - if (arr == NULL) { - return NULL; - } - - if (PyArray_CopyAnyInto(self, arr) != 0) { - Py_DECREF(arr); - return NULL; - } - - Py_DECREF(arr); - Py_RETURN_NONE; -} - NPY_NO_EXPORT const char * npy_casting_to_string(NPY_CASTING casting) { @@ -913,14 +838,6 @@ array_astype(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } - /* Keep maskna property */ - if (PyArray_HASMASKNA(self)) { - if (PyArray_AllocateMaskNA(ret, 1, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - } - if (PyArray_CopyInto(ret, self) < 0) { Py_DECREF(ret); return NULL; @@ -1098,53 +1015,14 @@ static PyObject * array_copy(PyArrayObject *self, PyObject *args, PyObject *kwds) { NPY_ORDER order = NPY_CORDER; - PyObject *maskna_in = Py_None; - int maskna = -1; - static char *kwlist[] = {"order", "maskna", NULL}; - PyArrayObject *ret; + static char *kwlist[] = {"order", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O&O", kwlist, - PyArray_OrderConverter, &order, - &maskna_in)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O&", kwlist, + PyArray_OrderConverter, &order)) { return NULL; } - /* Treat None the same as not providing the parameter */ - if (maskna_in != Py_None) { - maskna = PyObject_IsTrue(maskna_in); - if (maskna == -1) { - return NULL; - } - } - - /* If maskna=False was passed and self has an NA mask, strip it away */ - if (maskna == 0 && PyArray_HASMASKNA(self)) { - /* An array with no NA mask */ - ret = (PyArrayObject *)PyArray_NewLikeArray(self, order, NULL, 1); - if (ret == NULL) { - return NULL; - } - - /* AssignArray validates that 'self' contains no NA values */ - if (PyArray_AssignArray(ret, self, NULL, NPY_UNSAFE_CASTING, - 0, NULL) < 0) { - Py_DECREF(ret); - return NULL; - } - } - else { - ret = (PyArrayObject *)PyArray_NewCopy(self, order); - - /* Add the NA mask if requested */ - if (ret != NULL && maskna == 1) { - if (PyArray_AllocateMaskNA(ret, 1, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - } - } - - return (PyObject *)ret; + return PyArray_NewCopy(self, order); } #include <stdio.h> @@ -1694,7 +1572,6 @@ array_setstate(PyArrayObject *self, PyObject *args) if (nd > 0) { fa->dimensions = PyDimMem_NEW(3*nd); fa->strides = PyArray_DIMS(self) + nd; - fa->maskna_strides = PyArray_DIMS(self) + 2*nd; memcpy(PyArray_DIMS(self), dimensions, sizeof(npy_intp)*nd); _array_fill_strides(PyArray_STRIDES(self), dimensions, nd, PyArray_DESCR(self)->elsize, @@ -1989,74 +1866,16 @@ array_dot(PyArrayObject *self, PyObject *args, PyObject *kwds) static PyObject * -array_any(PyArrayObject *array, PyObject *args, PyObject *kwds) +array_any(PyArrayObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"axis", "out", "skipna", "keepdims", NULL}; - - PyObject *axis_in = NULL; - PyArrayObject *out = NULL; - PyArrayObject *ret = NULL; - npy_bool axis_flags[NPY_MAXDIMS]; - int skipna = 0, keepdims = 0; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "|OO&ii:any", kwlist, - &axis_in, - &PyArray_OutputAllowNAConverter, &out, - &skipna, - &keepdims)) { - return NULL; - } - - if (PyArray_ConvertMultiAxis(axis_in, PyArray_NDIM(array), - axis_flags) != NPY_SUCCEED) { - return NULL; - } - - ret = PyArray_ReduceAny(array, out, axis_flags, skipna, keepdims); - - if (out == NULL) { - return PyArray_Return(ret); - } - else { - return (PyObject *)ret; - } + NPY_FORWARD_NDARRAY_METHOD("_any"); } static PyObject * -array_all(PyArrayObject *array, PyObject *args, PyObject *kwds) +array_all(PyArrayObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"axis", "out", "skipna", "keepdims", NULL}; - - PyObject *axis_in = NULL; - PyArrayObject *out = NULL; - PyArrayObject *ret = NULL; - npy_bool axis_flags[NPY_MAXDIMS]; - int skipna = 0, keepdims = 0; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "|OO&ii:all", kwlist, - &axis_in, - &PyArray_OutputAllowNAConverter, &out, - &skipna, - &keepdims)) { - return NULL; - } - - if (PyArray_ConvertMultiAxis(axis_in, PyArray_NDIM(array), - axis_flags) != NPY_SUCCEED) { - return NULL; - } - - ret = PyArray_ReduceAll(array, out, axis_flags, skipna, keepdims); - - if (out == NULL) { - return PyArray_Return(ret); - } - else { - return (PyObject *)ret; - } + NPY_FORWARD_NDARRAY_METHOD("_all"); } static PyObject * @@ -2082,7 +1901,7 @@ array_compress(PyArrayObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O&O&", kwlist, &condition, PyArray_AxisConverter, &axis, - PyArray_OutputAllowNAConverter, &out)) { + PyArray_OutputConverter, &out)) { return NULL; } return PyArray_Return( @@ -2114,7 +1933,7 @@ array_trace(PyArrayObject *self, PyObject *args, PyObject *kwds) &axis1, &axis2, PyArray_DescrConverter2, &dtype, - PyArray_OutputAllowNAConverter, &out)) { + PyArray_OutputConverter, &out)) { Py_XDECREF(dtype); return NULL; } @@ -2137,7 +1956,7 @@ array_clip(PyArrayObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO&", kwlist, &min, &max, - PyArray_OutputAllowNAConverter, &out)) { + PyArray_OutputConverter, &out)) { return NULL; } if (max == NULL && min == NULL) { @@ -2154,7 +1973,7 @@ array_conjugate(PyArrayObject *self, PyObject *args) PyArrayObject *out = NULL; if (!PyArg_ParseTuple(args, "|O&", - PyArray_OutputAllowNAConverter, + PyArray_OutputConverter, &out)) { return NULL; } @@ -2218,7 +2037,7 @@ array_round(PyArrayObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "|iO&", kwlist, &decimals, - PyArray_OutputAllowNAConverter, &out)) { + PyArray_OutputConverter, &out)) { return NULL; } return PyArray_Return((PyArrayObject *)PyArray_Round(self, decimals, out)); @@ -2417,9 +2236,6 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { {"itemset", (PyCFunction) array_setscalar, METH_VARARGS, NULL}, - {"setasflat", - (PyCFunction) array_setasflat, - METH_VARARGS, NULL}, {"max", (PyCFunction)array_max, METH_VARARGS | METH_KEYWORDS, NULL}, diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 75929198d..fd4183c66 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -51,8 +51,6 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; #include "item_selection.h" #include "shape.h" #include "ctors.h" -#include "na_object.h" -#include "na_mask.h" #include "reduction.h" /* Only here for API compatibility */ @@ -319,7 +317,6 @@ PyArray_ConcatenateArrays(int narrays, PyArrayObject **arrays, int axis) PyArray_Descr *dtype = NULL; PyArrayObject *ret = NULL; PyArrayObject_fields *sliding_view = NULL; - int has_maskna; int orig_axis = axis; if (narrays <= 0) { @@ -349,11 +346,9 @@ PyArray_ConcatenateArrays(int narrays, PyArrayObject **arrays, int axis) /* * Figure out the final concatenated shape starting from the first - * array's shape. Also check whether any of the inputs have an - * NA mask. + * array's shape. */ memcpy(shape, PyArray_SHAPE(arrays[0]), ndim * sizeof(shape[0])); - has_maskna = PyArray_HASMASKNA(arrays[0]); for (iarrays = 1; iarrays < narrays; ++iarrays) { npy_intp *arr_shape; @@ -379,8 +374,6 @@ PyArray_ConcatenateArrays(int narrays, PyArrayObject **arrays, int axis) return NULL; } } - - has_maskna = has_maskna || PyArray_HASMASKNA(arrays[iarrays]); } /* Get the priority subtype for the array */ @@ -426,14 +419,6 @@ PyArray_ConcatenateArrays(int narrays, PyArrayObject **arrays, int axis) return NULL; } - /* Add an NA mask if required */ - if (has_maskna) { - if (PyArray_AllocateMaskNA(ret, 1, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - } - /* * Create a view which slides through ret for assigning the * successive input arrays. @@ -450,7 +435,7 @@ PyArray_ConcatenateArrays(int narrays, PyArrayObject **arrays, int axis) /* Copy the data for this array */ if (PyArray_AssignArray((PyArrayObject *)sliding_view, arrays[iarrays], - NULL, NPY_SAME_KIND_CASTING, 0, NULL) < 0) { + NULL, NPY_SAME_KIND_CASTING) < 0) { Py_DECREF(sliding_view); Py_DECREF(ret); return NULL; @@ -459,10 +444,6 @@ PyArray_ConcatenateArrays(int narrays, PyArrayObject **arrays, int axis) /* Slide to the start of the next window */ sliding_view->data += sliding_view->dimensions[axis] * sliding_view->strides[axis]; - if (has_maskna) { - sliding_view->maskna_data += sliding_view->dimensions[axis] * - sliding_view->maskna_strides[axis]; - } } Py_DECREF(sliding_view); @@ -483,7 +464,6 @@ PyArray_ConcatenateFlattenedArrays(int narrays, PyArrayObject **arrays, PyArray_Descr *dtype = NULL; PyArrayObject *ret = NULL; PyArrayObject_fields *sliding_view = NULL; - int has_maskna; if (narrays <= 0) { PyErr_SetString(PyExc_ValueError, @@ -497,10 +477,8 @@ PyArray_ConcatenateFlattenedArrays(int narrays, PyArrayObject **arrays, /* * Figure out the final concatenated shape starting from the first - * array's shape. Also check whether any of the inputs have an - * NA mask. + * array's shape. */ - has_maskna = PyArray_HASMASKNA(arrays[0]); for (iarrays = 1; iarrays < narrays; ++iarrays) { if (PyArray_SIZE(arrays[iarrays]) != shape[1]) { PyErr_SetString(PyExc_ValueError, @@ -508,8 +486,6 @@ PyArray_ConcatenateFlattenedArrays(int narrays, PyArrayObject **arrays, "number of elements"); return NULL; } - - has_maskna = has_maskna || PyArray_HASMASKNA(arrays[iarrays]); } /* Get the priority subtype for the array */ @@ -545,14 +521,6 @@ PyArray_ConcatenateFlattenedArrays(int narrays, PyArrayObject **arrays, return NULL; } - /* Add an NA mask if required */ - if (has_maskna) { - if (PyArray_AllocateMaskNA(ret, 1, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - } - /* * Create a view which slides through ret for assigning the * successive input arrays. @@ -577,9 +545,6 @@ PyArray_ConcatenateFlattenedArrays(int narrays, PyArrayObject **arrays, /* Slide to the start of the next window */ sliding_view->data += sliding_view->strides[0]; - if (has_maskna) { - sliding_view->maskna_data += sliding_view->maskna_strides[0]; - } } Py_DECREF(sliding_view); @@ -620,7 +585,7 @@ PyArray_Concatenate(PyObject *op, int axis) goto fail; } arrays[iarrays] = (PyArrayObject *)PyArray_FromAny(item, NULL, - 0, 0, NPY_ARRAY_ALLOWNA, NULL); + 0, 0, 0, NULL); Py_DECREF(item); if (arrays[iarrays] == NULL) { narrays = iarrays; @@ -1565,7 +1530,7 @@ _prepend_ones(PyArrayObject *arr, int nd, int ndmin) ret = (PyArrayObject *)PyArray_NewFromDescr(Py_TYPE(arr), dtype, ndmin, newdims, newstrides, PyArray_DATA(arr), - PyArray_FLAGS(arr) & ~(NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA), + PyArray_FLAGS(arr), (PyObject *)arr); if (ret == NULL) { return NULL; @@ -1576,24 +1541,6 @@ _prepend_ones(PyArrayObject *arr, int nd, int ndmin) return NULL; } - /* Take a view of the NA mask as well if necessary */ - if (PyArray_HASMASKNA(arr)) { - PyArrayObject_fields *fret = (PyArrayObject_fields *)ret; - - fret->maskna_dtype = PyArray_MASKNA_DTYPE(arr); - Py_INCREF(fret->maskna_dtype); - fret->maskna_data = PyArray_MASKNA_DATA(arr); - - for (i = 0; i < num; ++i) { - fret->maskna_strides[i] = 0; - } - for (i = num; i < ndmin; ++i) { - fret->maskna_strides[i] = PyArray_MASKNA_STRIDES(arr)[i - num]; - } - fret->flags |= NPY_ARRAY_MASKNA; - } - - return (PyObject *)ret; } @@ -1615,52 +1562,26 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) PyArray_Descr *type = NULL; PyArray_Descr *oldtype = NULL; NPY_ORDER order = NPY_KEEPORDER; - int flags = 0, maskna = -1, ownmaskna = 0; - PyObject *maskna_in = Py_None; + int flags = 0; static char *kwd[]= {"object", "dtype", "copy", "order", "subok", - "ndmin", "maskna", "ownmaskna", NULL}; + "ndmin", NULL}; if (PyTuple_GET_SIZE(args) > 2) { PyErr_SetString(PyExc_ValueError, "only 2 non-keyword arguments accepted"); return NULL; } - if(!PyArg_ParseTupleAndKeywords(args, kws, "O|O&O&O&O&iOi", kwd, + if(!PyArg_ParseTupleAndKeywords(args, kws, "O|O&O&O&O&i", kwd, &op, PyArray_DescrConverter2, &type, PyArray_BoolConverter, ©, PyArray_OrderConverter, &order, PyArray_BoolConverter, &subok, - &ndmin, - &maskna_in, - &ownmaskna)) { + &ndmin)) { goto clean_type; } - /* - * Treat None the same as not providing the parameter, set - * maskna to -1 (unprovided), 0 (False), or 1 (True). - */ - if (maskna_in != Py_None) { - maskna = PyObject_IsTrue(maskna_in); - if (maskna == -1) { - return NULL; - } - } - - /* 'ownmaskna' forces 'maskna' to be True */ - if (ownmaskna) { - if (maskna == 0) { - PyErr_SetString(PyExc_ValueError, - "cannot specify maskna=False and ownmaskna=True"); - return NULL; - } - else { - maskna = 1; - } - } - if (ndmin > NPY_MAXDIMS) { PyErr_Format(PyExc_ValueError, "ndmin bigger than allowable number of dimensions " @@ -1668,34 +1589,13 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) goto clean_type; } /* fast exit if simple call */ - if (((subok && PyArray_Check(op)) || - (!subok && PyArray_CheckExact(op))) && - ((maskna == -1) || - (maskna == 1 && PyArray_HASMASKNA((PyArrayObject *)op)) || - (maskna == 0 && !PyArray_HASMASKNA((PyArrayObject *)op)))) { + if ((subok && PyArray_Check(op)) || + (!subok && PyArray_CheckExact(op))) { oparr = (PyArrayObject *)op; if (type == NULL) { if (!copy && STRIDING_OK(oparr, order)) { - /* - * If mask ownership is requested and the array doesn't - * already own its own mask, make a view which owns its - * own mask. - */ - if (ownmaskna && - !(PyArray_FLAGS(oparr) & NPY_ARRAY_OWNMASKNA)) { - ret = (PyArrayObject *)PyArray_View(oparr, NULL, NULL); - if (ret == NULL) { - return NULL; - } - if (PyArray_AllocateMaskNA(ret, 1, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - } - else { - ret = oparr; - Py_INCREF(ret); - } + ret = oparr; + Py_INCREF(ret); goto finish; } else { @@ -1706,7 +1606,7 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) /* One more chance */ oldtype = PyArray_DESCR(oparr); if (PyArray_EquivTypes(oldtype, type)) { - if (!copy && !ownmaskna && STRIDING_OK(oparr, order)) { + if (!copy && STRIDING_OK(oparr, order)) { Py_INCREF(op); ret = oparr; goto finish; @@ -1740,23 +1640,6 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) flags |= NPY_ARRAY_ENSUREARRAY; } - /* If maskna is the default, allow NA to pass through */ - if (maskna == -1) { - flags |= NPY_ARRAY_ALLOWNA; - } - /* If maskna is True, force there to be an NA mask */ - else if (maskna == 1) { - flags |= NPY_ARRAY_MASKNA | NPY_ARRAY_ALLOWNA; - if (ownmaskna) { - flags |= NPY_ARRAY_OWNMASKNA; - } - } - /* - * Otherwise maskna is False, so we don't specify NPY_ARRAY_ALLOWNA. - * An array with an NA mask will cause a copy into an array - * without an NA mask - */ - flags |= NPY_ARRAY_FORCECAST; Py_XINCREF(type); ret = (PyArrayObject *)PyArray_CheckFromAny(op, type, @@ -1787,18 +1670,16 @@ static PyObject * array_copyto(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"dst","src","casting","where","preservena",NULL}; + static char *kwlist[] = {"dst","src","casting","where",NULL}; PyObject *wheremask_in = NULL; PyArrayObject *dst = NULL, *src = NULL, *wheremask = NULL; NPY_CASTING casting = NPY_SAME_KIND_CASTING; - int preservena = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O&|O&Oi", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O&|O&O", kwlist, &PyArray_Type, &dst, - &PyArray_AllowNAConverter, &src, + &PyArray_Converter, &src, &PyArray_CastingConverter, &casting, - &wheremask_in, - &preservena)) { + &wheremask_in)) { goto fail; } @@ -1809,14 +1690,13 @@ array_copyto(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) goto fail; } wheremask = (PyArrayObject *)PyArray_FromAny(wheremask_in, - dtype, 0, 0, NPY_ARRAY_ALLOWNA, NULL); + dtype, 0, 0, 0, NULL); if (wheremask == NULL) { goto fail; } } - if (PyArray_AssignArray(dst, src, - wheremask, casting, preservena, NULL) < 0) { + if (PyArray_AssignArray(dst, src, wheremask, casting) < 0) { goto fail; } @@ -1836,19 +1716,17 @@ static PyObject * array_empty(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"shape","dtype","order","maskna",NULL}; + static char *kwlist[] = {"shape","dtype","order",NULL}; PyArray_Descr *typecode = NULL; PyArray_Dims shape = {NULL, 0}; NPY_ORDER order = NPY_CORDER; npy_bool is_f_order; PyArrayObject *ret = NULL; - int maskna = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&i", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&", kwlist, PyArray_IntpConverter, &shape, PyArray_DescrConverter, &typecode, - PyArray_OrderConverter, &order, - &maskna)) { + PyArray_OrderConverter, &order)) { goto fail; } @@ -1868,17 +1746,6 @@ array_empty(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) ret = (PyArrayObject *)PyArray_Empty(shape.len, shape.ptr, typecode, is_f_order); - if (maskna) { - /* - * Default the mask to all NA values, because the data is - * not initialized - */ - if (PyArray_AllocateMaskNA(ret, 1, 0, 0) < 0) { - Py_DECREF(ret); - return NULL; - } - } - PyDimMem_FREE(shape.ptr); return (PyObject *)ret; @@ -1892,19 +1759,18 @@ static PyObject * array_empty_like(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"prototype","dtype","order","subok","maskna",NULL}; + static char *kwlist[] = {"prototype","dtype","order","subok",NULL}; PyArrayObject *prototype = NULL; PyArray_Descr *dtype = NULL; NPY_ORDER order = NPY_KEEPORDER; PyArrayObject *ret = NULL; - int subok = 1, maskna = 0; + int subok = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&ii", kwlist, - &PyArray_AllowNAConverter, &prototype, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&i", kwlist, + &PyArray_Converter, &prototype, &PyArray_DescrConverter2, &dtype, &PyArray_OrderConverter, &order, - &subok, - &maskna)) { + &subok)) { goto fail; } /* steals the reference to dtype if it's not NULL */ @@ -1912,17 +1778,6 @@ array_empty_like(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) order, dtype, subok); Py_DECREF(prototype); - if (maskna) { - /* - * Default the mask to all NA values, because the data is - * not initialized - */ - if (PyArray_AllocateMaskNA(ret, 1, 0, 0) < 0) { - Py_DECREF(ret); - return NULL; - } - } - return (PyObject *)ret; fail: @@ -1998,19 +1853,17 @@ array_scalar(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) static PyObject * array_zeros(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"shape","dtype","order","maskna",NULL}; + static char *kwlist[] = {"shape","dtype","order",NULL}; PyArray_Descr *typecode = NULL; PyArray_Dims shape = {NULL, 0}; NPY_ORDER order = NPY_CORDER; npy_bool is_f_order = NPY_FALSE; PyArrayObject *ret = NULL; - int maskna = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&i", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&", kwlist, PyArray_IntpConverter, &shape, PyArray_DescrConverter, &typecode, - PyArray_OrderConverter, &order, - &maskna)) { + PyArray_OrderConverter, &order)) { goto fail; } @@ -2030,13 +1883,6 @@ array_zeros(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) ret = (PyArrayObject *)PyArray_Zeros(shape.len, shape.ptr, typecode, (int) is_f_order); - if (maskna) { - if (PyArray_AllocateMaskNA(ret, 1, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - } - PyDimMem_FREE(shape.ptr); return (PyObject *)ret; @@ -2049,26 +1895,24 @@ array_zeros(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) static PyObject * array_count_nonzero(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"arr", "axis", "out", "skipna", "keepdims", NULL}; + static char *kwlist[] = {"arr", "axis", "out", "keepdims", NULL}; PyObject *array_in, *axis_in = NULL, *out_in = NULL; PyObject *ret = NULL; PyArrayObject *array, *out = NULL; npy_bool axis_flags[NPY_MAXDIMS]; - int skipna = 0, keepdims = 0; + int keepdims = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|OOii:count_nonzero", kwlist, + "O|OOi:count_nonzero", kwlist, &array_in, &axis_in, &out_in, - &skipna, &keepdims)) { return NULL; } - array = (PyArrayObject *)PyArray_FromAny(array_in, NULL, - 0, 0, NPY_ARRAY_ALLOWNA, NULL); + array = (PyArrayObject *)PyArray_FromAny(array_in, NULL, 0, 0, 0, NULL); if (array == NULL) { return NULL; } @@ -2089,46 +1933,7 @@ array_count_nonzero(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) } } - ret = PyArray_ReduceCountNonzero(array, out, axis_flags, skipna, keepdims); - - Py_DECREF(array); - - return ret; -} - -static PyObject * -array_count_reduce_items(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"arr", "axis", "skipna", "keepdims", NULL}; - - PyObject *array_in, *axis_in = NULL; - PyObject *ret = NULL; - PyArrayObject *array; - npy_bool axis_flags[NPY_MAXDIMS]; - int skipna = 0, keepdims = 0; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|Oii:count_reduce_items", kwlist, - &array_in, - &axis_in, - &skipna, - &keepdims)) { - return NULL; - } - - array = (PyArrayObject *)PyArray_FromAny(array_in, NULL, - 0, 0, NPY_ARRAY_ALLOWNA, NULL); - if (array == NULL) { - return NULL; - } - - if (PyArray_ConvertMultiAxis(axis_in, PyArray_NDIM(array), - axis_flags) != NPY_SUCCEED) { - Py_DECREF(array); - return NULL; - } - - ret = PyArray_CountReduceItems(array, axis_flags, skipna, keepdims); + ret = PyArray_ReduceCountNonzero(array, out, axis_flags, keepdims); Py_DECREF(array); @@ -2647,18 +2452,6 @@ finish: } static PyObject * -array_isna(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) -{ - PyObject *a; - static char *kwlist[] = {"a", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:isna", kwlist, &a)) { - return NULL; - } - return PyArray_IsNA(a); -} - -static PyObject * array_fastCopyAndTranspose(PyObject *NPY_UNUSED(dummy), PyObject *args) { PyObject *a0; @@ -2700,30 +2493,20 @@ array_correlate2(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) static PyObject * array_arange(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) { PyObject *o_start = NULL, *o_stop = NULL, *o_step = NULL, *range=NULL; - static char *kwd[]= {"start", "stop", "step", "dtype", "maskna", NULL}; + static char *kwd[]= {"start", "stop", "step", "dtype", NULL}; PyArray_Descr *typecode = NULL; - int maskna = 0; - if(!PyArg_ParseTupleAndKeywords(args, kws, "O|OOO&i", kwd, + if(!PyArg_ParseTupleAndKeywords(args, kws, "O|OOO&", kwd, &o_start, &o_stop, &o_step, - PyArray_DescrConverter2, &typecode, - &maskna)) { + PyArray_DescrConverter2, &typecode)) { Py_XDECREF(typecode); return NULL; } range = PyArray_ArangeObj(o_start, o_stop, o_step, typecode); Py_XDECREF(typecode); - /* Allocate an NA mask if requested */ - if (maskna) { - if (PyArray_AllocateMaskNA((PyArrayObject *)range, 1, 0, 1) < 0) { - Py_DECREF(range); - return NULL; - } - } - return range; } @@ -2979,7 +2762,7 @@ array_can_cast_safely(PyObject *NPY_UNUSED(self), PyObject *args, PyArray_IsPythonNumber(from_obj)) { PyArrayObject *arr; arr = (PyArrayObject *)PyArray_FromAny(from_obj, - NULL, 0, 0, NPY_ARRAY_ALLOWNA, NULL); + NULL, 0, 0, 0, NULL); if (arr == NULL) { goto finish; } @@ -3041,8 +2824,7 @@ array_min_scalar_type(PyObject *NPY_UNUSED(dummy), PyObject *args) return NULL; } - array = (PyArrayObject *)PyArray_FromAny(array_in, - NULL, 0, 0, NPY_ARRAY_ALLOWNA, NULL); + array = (PyArrayObject *)PyArray_FromAny(array_in, NULL, 0, 0, 0, NULL); if (array == NULL) { return NULL; } @@ -3087,7 +2869,7 @@ array_result_type(PyObject *NPY_UNUSED(dummy), PyObject *args) goto finish; } arr[narr] = (PyArrayObject *)PyArray_FromAny(obj, - NULL, 0, 0, NPY_ARRAY_ALLOWNA, NULL); + NULL, 0, 0, 0, NULL); if (arr[narr] == NULL) { goto finish; } @@ -3697,9 +3479,6 @@ static struct PyMethodDef array_module_methods[] = { {"count_nonzero", (PyCFunction)array_count_nonzero, METH_VARARGS|METH_KEYWORDS, NULL}, - {"count_reduce_items", - (PyCFunction)array_count_reduce_items, - METH_VARARGS|METH_KEYWORDS, NULL}, {"empty", (PyCFunction)array_empty, METH_VARARGS|METH_KEYWORDS, NULL}, @@ -3736,9 +3515,6 @@ static struct PyMethodDef array_module_methods[] = { {"einsum", (PyCFunction)array_einsum, METH_VARARGS|METH_KEYWORDS, NULL}, - {"isna", - (PyCFunction)array_isna, - METH_VARARGS|METH_KEYWORDS, NULL}, {"_fastCopyAndTranspose", (PyCFunction)array_fastCopyAndTranspose, METH_VARARGS, NULL}, @@ -4088,9 +3864,6 @@ PyMODINIT_FUNC initmultiarray(void) { if (PyType_Ready(&NpyIter_Type) < 0) { return RETVAL; } - if (PyType_Ready(&NpyNA_Type) < 0) { - return RETVAL; - } PyArrayDescr_Type.tp_hash = PyArray_DescrHash; if (PyType_Ready(&PyArrayDescr_Type) < 0) { @@ -4182,12 +3955,6 @@ PyMODINIT_FUNC initmultiarray(void) { Py_INCREF(&NpyBusDayCalendar_Type); PyDict_SetItemString(d, "busdaycalendar", (PyObject *)&NpyBusDayCalendar_Type); - /* NA Type */ - PyDict_SetItemString(d, "NAType", (PyObject *)&NpyNA_Type); - Py_INCREF(&NpyNA_Type); - /* NA Singleton */ - Py_INCREF(Npy_NA); - PyDict_SetItemString(d, "NA", Npy_NA); set_flaginfo(d); diff --git a/numpy/core/src/multiarray/multiarraymodule_onefile.c b/numpy/core/src/multiarray/multiarraymodule_onefile.c index bf2a90af0..9a4956cfa 100644 --- a/numpy/core/src/multiarray/multiarraymodule_onefile.c +++ b/numpy/core/src/multiarray/multiarraymodule_onefile.c @@ -49,9 +49,6 @@ #include "array_assign_scalar.c" #include "array_assign_array.c" #include "reduction.c" -#include "na_mask.c" -#include "na_object.c" -#include "boolean_ops.c" #include "ucsnarrow.c" #include "arrayobject.c" diff --git a/numpy/core/src/multiarray/na_mask.c b/numpy/core/src/multiarray/na_mask.c deleted file mode 100644 index 8fbe8ea60..000000000 --- a/numpy/core/src/multiarray/na_mask.c +++ /dev/null @@ -1,955 +0,0 @@ -/* - * This file implements missing value NA mask support for the NumPy array. - * - * Written by Mark Wiebe (mwwiebe@gmail.com) - * Copyright (c) 2011 by Enthought, Inc. - * - * See LICENSE.txt for the license. - */ - -#define PY_SSIZE_T_CLEAN -#include <Python.h> - -#define NPY_NO_DEPRECATED_API NPY_API_VERSION -#define _MULTIARRAYMODULE -#include <numpy/arrayobject.h> - -#include "npy_config.h" -#include "numpy/npy_3kcompat.h" - -#include "shape.h" -#include "lowlevel_strided_loops.h" -#include "array_assign.h" -#include "na_object.h" - -/*NUMPY_API - * - * Returns true if the array has an NA mask. When - * NA dtypes are implemented, will also return true - * if the array's dtype has NA support. - */ -NPY_NO_EXPORT npy_bool -PyArray_HasNASupport(PyArrayObject *arr) -{ - return PyArray_HASMASKNA(arr); -} - -/*NUMPY_API - * - * Returns false if the array has no NA support. Returns - * true if the array has NA support AND there is an - * NA anywhere in the array. - * - * If 'wheremask' is non-NULL, only positions with True - * in 'wheremask' are checked for NA. - * - * The parameter 'whichna' is not yet supported, but is - * provided for future multi-NA support. It should be set - * to NULL. - * - * Returns -1 on failure, otherwise 0 for False and 1 for True. - */ -NPY_NO_EXPORT int -PyArray_ContainsNA(PyArrayObject *arr, PyArrayObject *wheremask, - npy_bool *whichna) -{ - /* Validate that the parameter for future expansion is NULL */ - if (whichna != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "multi-NA is not yet supported in PyArray_ContainsNA"); - return -1; - } - - if (wheremask == NULL) { - /* Need NA support to contain NA */ - if (PyArray_HASMASKNA(arr)) { - int idim, ndim; - char *data; - npy_intp shape[NPY_MAXDIMS], strides[NPY_MAXDIMS]; - npy_intp i, coord[NPY_MAXDIMS]; - - if (PyArray_HASFIELDS(arr)) { - PyErr_SetString(PyExc_RuntimeError, - "field-NA is not yet supported"); - return -1; - } - - /* Use raw iteration with no heap memory allocation */ - if (PyArray_PrepareOneRawArrayIter( - PyArray_NDIM(arr), PyArray_DIMS(arr), - PyArray_MASKNA_DATA(arr), PyArray_MASKNA_STRIDES(arr), - &ndim, shape, - &data, strides) < 0) { - return -1; - } - - /* Do the iteration */ - NPY_RAW_ITER_START(idim, ndim, coord, shape) { - char *d = data; - /* Process the innermost dimension */ - for (i = 0; i < shape[0]; ++i, d += strides[0]) { - if (!NpyMaskValue_IsExposed((npy_mask)(*d))) { - return 1; - } - } - } NPY_RAW_ITER_ONE_NEXT(idim, ndim, coord, shape, data, strides); - } - } - else { - npy_intp wheremask_strides_bcast[NPY_MAXDIMS]; - int containsna; - - containsna = PyArray_ContainsNA(wheremask, NULL, NULL); - if (containsna != 0) { - if (containsna == -1) { - return -1; - } - else { - PyErr_SetString(PyExc_ValueError, - "the where mask may not contain any NA values"); - return -1; - } - } - - /* - * Broadcast the where-mask onto arr. Note that this - * is before checking if 'arr' has an NA mask, to - * catch any broadcasting errors. - */ - if (broadcast_strides(PyArray_NDIM(arr), PyArray_DIMS(arr), - PyArray_NDIM(wheremask), PyArray_DIMS(wheremask), - PyArray_STRIDES(wheremask), "where mask", - wheremask_strides_bcast) < 0) { - return -1; - } - - if (PyArray_DTYPE(wheremask)->type_num != NPY_BOOL) { - PyErr_SetString(PyExc_ValueError, - "the where mask must have a 'bool' dtype"); - return -1; - } - - if (PyArray_HASMASKNA(arr)) { - int idim, ndim; - char *data, *wheremask_data; - npy_intp shape[NPY_MAXDIMS], strides[NPY_MAXDIMS]; - npy_intp wheremask_strides[NPY_MAXDIMS]; - npy_intp i, coord[NPY_MAXDIMS]; - - if (PyArray_HASFIELDS(arr)) { - PyErr_SetString(PyExc_RuntimeError, - "field-NA is not yet supported"); - return -1; - } - - /* Use raw iteration with no heap memory allocation */ - if (PyArray_PrepareTwoRawArrayIter( - PyArray_NDIM(arr), PyArray_DIMS(arr), - PyArray_MASKNA_DATA(arr), PyArray_MASKNA_STRIDES(arr), - PyArray_DATA(wheremask), wheremask_strides_bcast, - &ndim, shape, - &data, strides, - &wheremask_data, wheremask_strides) < 0) { - return -1; - } - - /* Do the iteration */ - NPY_RAW_ITER_START(idim, ndim, coord, shape) { - char *d = data, *where_d = wheremask_data; - /* Process the innermost dimension */ - for (i = 0; i < shape[0]; ++i) { - if (*where_d && !NpyMaskValue_IsExposed((npy_mask)(*d))) { - return 1; - } - - d += strides[0]; - where_d += wheremask_strides[0]; - } - } NPY_RAW_ITER_TWO_NEXT(idim, ndim, coord, shape, - data, strides, - wheremask_data, wheremask_strides); - } - } - - return 0; -} - -/* - * Fills a raw array whose dtype has size one with the specified byte - * - * Returns 0 on success, -1 on failure. - */ -static int -fill_raw_byte_array(int ndim, npy_intp *shape, - char *data, npy_intp *strides, char fillvalue) -{ - int idim; - npy_intp shape_it[NPY_MAXDIMS], strides_it[NPY_MAXDIMS]; - npy_intp i, coord[NPY_MAXDIMS]; - - /* Use raw iteration with no heap memory allocation */ - if (PyArray_PrepareOneRawArrayIter( - ndim, shape, - data, strides, - &ndim, shape_it, - &data, strides_it) < 0) { - PyErr_Clear(); - return 1; - } - - /* Special case contiguous inner stride */ - if (strides_it[0] == 1) { - NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { - /* Process the innermost dimension */ - memset(data, fillvalue, shape_it[0]); - } NPY_RAW_ITER_ONE_NEXT(idim, ndim, coord, shape_it, data, strides_it); - } - /* General inner stride */ - else { - NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { - char *d = data; - /* Process the innermost dimension */ - for (i = 0; i < shape_it[0]; ++i, d += strides_it[0]) { - *d = fillvalue; - } - } NPY_RAW_ITER_ONE_NEXT(idim, ndim, coord, shape_it, data, strides_it); - } - - return 0; -} - -/*NUMPY_API - * - * Assigns the mask value to all the NA mask elements of - * the array. This routine is intended to be used to mask - * all the elments of an array, or if you will also be assigning - * values to everything at the same time, to unmask all the elements. - * - * If 'wheremask' isn't NULL, it should be a boolean mask which - * specifies where to do the assignment. - * - * The parameters 'preservena' and 'preservewhichna' are NOT YET - * SUPPORTED, but are in place to allow for future expansion to - * multi-NA. 'preservewhichna' should be set to NULL, while - * preservena has no effect for straight NPY_BOOL NA masks, because - * different NAs are indistinguishable. - * - * Returns 0 on success, -1 on failure. - */ -NPY_NO_EXPORT int -PyArray_AssignMaskNA(PyArrayObject *arr, npy_mask maskvalue, - PyArrayObject *wheremask, - npy_bool preservena, npy_bool *preservewhichna) -{ - PyArray_Descr *maskvalue_dtype; - int retcode = 0; - - /* Need NA support to fill the NA mask */ - if (!PyArray_HASMASKNA(arr)) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign to the NA mask of an " - "array which has no NA mask"); - return -1; - } - - if (preservewhichna != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "multi-NA support is not yet implemented"); - return -1; - } - - /* - * If the mask given has no payload, assign from boolean type, otherwise - * assign from the mask type. - */ - if ((maskvalue & (~0x01)) == 0) { - maskvalue_dtype = PyArray_DescrFromType(NPY_BOOL); - } - else { - maskvalue_dtype = PyArray_DescrFromType(NPY_MASK); - } - if (maskvalue_dtype == NULL) { - return -1; - } - - if (wheremask == NULL) { - retcode = raw_array_assign_scalar( - PyArray_NDIM(arr), PyArray_DIMS(arr), - PyArray_MASKNA_DTYPE(arr), - PyArray_MASKNA_DATA(arr), - PyArray_MASKNA_STRIDES(arr), - maskvalue_dtype, (char *)&maskvalue); - } - else { - npy_intp wheremask_strides[NPY_MAXDIMS]; - - /* Broadcast the wheremask to 'arr' */ - if (broadcast_strides(PyArray_NDIM(arr), PyArray_DIMS(arr), - PyArray_NDIM(wheremask), PyArray_DIMS(wheremask), - PyArray_STRIDES(wheremask), "where mask", - wheremask_strides) < 0) { - Py_DECREF(maskvalue_dtype); - return -1; - } - - retcode = raw_array_wheremasked_assign_scalar( - PyArray_NDIM(arr), PyArray_DIMS(arr), - PyArray_MASKNA_DTYPE(arr), - PyArray_MASKNA_DATA(arr), - PyArray_MASKNA_STRIDES(arr), - maskvalue_dtype, (char *)&maskvalue, - PyArray_DESCR(wheremask), PyArray_DATA(wheremask), - wheremask_strides); - } - - Py_DECREF(maskvalue_dtype); - return retcode; -} - -/*NUMPY_API - * - * If the array does not have an NA mask already, allocates one for it. - * - * If 'ownmaskna' is True, it also allocates one for it if the array does - * not already own its own mask, then copies the data from the old mask - * to the new mask. - * - * If 'multina' is True, the mask is allocated with an NPY_MASK dtype - * instead of NPY_BOOL. - * - * If a new mask is allocated, and no mask was there to copy from, - * the mask is filled with the 'defaultmask' value. Normally you - * set this to 1, so all the values are exposed. - * - * Returns -1 on failure, 0 on success. - */ -NPY_NO_EXPORT int -PyArray_AllocateMaskNA(PyArrayObject *arr, - npy_bool ownmaskna, - npy_bool multina, - npy_mask defaultmask) -{ - PyArrayObject_fields *fa = (PyArrayObject_fields *)arr; - PyArray_Descr *maskna_dtype = NULL; - char *maskna_data = NULL; - npy_intp size; - - /* If the array already owns a mask, done */ - if (fa->flags & NPY_ARRAY_OWNMASKNA) { - return 0; - } - - /* If ownership wasn't requested, and there's already a mask, done */ - if (!ownmaskna && (fa->flags & NPY_ARRAY_MASKNA)) { - return 0; - } - - size = PyArray_SIZE(arr); - - /* Create the mask dtype */ - if (PyArray_HASFIELDS(arr)) { - PyErr_SetString(PyExc_RuntimeError, - "NumPy field-NA isn't supported yet"); - return -1; - } - else { - maskna_dtype = PyArray_DescrFromType(multina ? NPY_MASK - : NPY_BOOL); - if (maskna_dtype == NULL) { - return -1; - } - } - - /* Allocate the mask memory */ - maskna_data = PyDataMem_NEW(size * maskna_dtype->elsize); - if (maskna_data == NULL) { - Py_DECREF(maskna_dtype); - PyErr_NoMemory(); - return -1; - } - - /* Copy the data and fill in the strides */ - if (fa->nd == 1) { - /* If there already was a mask copy it, otherwise set it to all ones */ - if (fa->flags & NPY_ARRAY_MASKNA) { - if (fa->maskna_strides[0] == 1) { - memcpy(maskna_data, fa->maskna_data, - size * maskna_dtype->elsize); - } - else { - if (PyArray_CastRawArrays(fa->dimensions[0], - (char *)fa->maskna_data, maskna_data, - fa->maskna_strides[0], maskna_dtype->elsize, - fa->maskna_dtype, maskna_dtype, 0) < 0) { - Py_DECREF(maskna_dtype); - PyArray_free(maskna_data); - return -1; - } - } - } - else { - memset(maskna_data, defaultmask, size * maskna_dtype->elsize); - } - - fa->maskna_strides[0] = maskna_dtype->elsize; - } - else if (fa->nd > 1) { - npy_stride_sort_item strideperm[NPY_MAXDIMS]; - npy_intp stride, maskna_strides[NPY_MAXDIMS], *shape; - int i; - - shape = fa->dimensions; - - /* This causes the NA mask and data memory orderings to match */ - PyArray_CreateSortedStridePerm(fa->nd, fa->dimensions, - fa->strides, strideperm); - stride = maskna_dtype->elsize; - for (i = fa->nd-1; i >= 0; --i) { - npy_intp i_perm = strideperm[i].perm; - maskna_strides[i_perm] = stride; - stride *= shape[i_perm]; - } - - /* If there already was a mask copy it, otherwise set it to all ones */ - if (fa->flags & NPY_ARRAY_MASKNA) { - if (PyArray_CastRawNDimArrays(fa->nd, fa->dimensions, - (char *)fa->maskna_data, maskna_data, - fa->maskna_strides, maskna_strides, - fa->maskna_dtype, maskna_dtype, 0) < 0) { - Py_DECREF(maskna_dtype); - PyArray_free(maskna_data); - return -1; - } - } - else { - memset(maskna_data, defaultmask, size * maskna_dtype->elsize); - } - - memcpy(fa->maskna_strides, maskna_strides, fa->nd * sizeof(npy_intp)); - } - else { - /* If there already was a mask copy it, otherwise set it to all ones */ - if (fa->flags & NPY_ARRAY_MASKNA) { - maskna_data[0] = fa->maskna_data[0]; - } - else { - maskna_data[0] = defaultmask; - } - } - - /* Set the NA mask data in the array */ - fa->maskna_dtype = maskna_dtype; - fa->maskna_data = maskna_data; - fa->flags |= (NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA); - - return 0; -} - -/*NUMPY_API - * - * Assigns the given NA value to all the elements in the array. If - * 'arr' has a mask, masks all the elements of the array. - * - * In the future, when 'arr' has an NA dtype, will assign the - * appropriate NA bitpatterns to the elements. - * - * The parameters 'preservena' and 'preservewhichna' are NOT YET - * SUPPORTED, but are in place to allow for future expansion to - * multi-NA. 'preservewhichna' should be set to NULL, while - * preservena has no effect for straight NPY_BOOL NA masks, because - * different NAs are indistinguishable. - * - * Returns -1 on failure, 0 on success. - */ -NPY_NO_EXPORT int -PyArray_AssignNA(PyArrayObject *arr, NpyNA *na, - PyArrayObject *wheremask, - npy_bool preservena, npy_bool *preservewhichna) -{ - NpyNA_fields *fna = (NpyNA_fields *)na; - char maskvalue; - - /* Turn the payload into a mask value */ - if (fna->payload == NPY_NA_NOPAYLOAD) { - maskvalue = 0; - } - else if (PyArray_MASKNA_DTYPE(arr)->type_num != - NPY_MASK) { - /* TODO: also handle struct-NA mask dtypes */ - PyErr_SetString(PyExc_ValueError, - "Cannot assign an NA with a payload to an " - "NA-array with a boolean mask, requires a multi-NA mask"); - return -1; - } - else { - maskvalue = (char)NpyMaskValue_Create(0, fna->payload); - } - - return PyArray_AssignMaskNA(arr, maskvalue, - wheremask, preservena, preservewhichna); -} - -/* - * A ufunc-like function, which returns a boolean or an array - * of booleans indicating which values are NA. - */ -NPY_NO_EXPORT PyObject * -PyArray_IsNA(PyObject *obj) -{ - /* NA objects are NA */ - if (NpyNA_Check(obj)) { - Py_INCREF(Py_True); - return Py_True; - } - /* Otherwise non-array objects are not NA */ - else if (!PyArray_Check(obj)) { - Py_INCREF(Py_False); - return Py_False; - } - /* Create a boolean array based on the mask */ - else { - PyArrayObject *ret; - PyArray_Descr *dtype; - - if (PyArray_HASFIELDS((PyArrayObject *)obj)) { - PyErr_SetString(PyExc_RuntimeError, - "field-NA is not supported yet"); - return NULL; - } - - dtype = PyArray_DescrFromType(NPY_BOOL); - if (dtype == NULL) { - return NULL; - } - - /* - * TODO: Could change this to use raw iteration to - * avoid the iterator creation overhead. - */ - if (PyArray_HASMASKNA((PyArrayObject *)obj)) { - NpyIter *iter; - PyArrayObject *op[2] = {(PyArrayObject *)obj, NULL}; - npy_uint32 flags, op_flags[2]; - PyArray_Descr *op_dtypes[2] = {NULL, dtype}; - - flags = NPY_ITER_EXTERNAL_LOOP | - NPY_ITER_ZEROSIZE_OK | - NPY_ITER_REFS_OK; - /* - * This USE_MASKNA causes there to be 3 operands, where operand - * 2 is the mask for operand 0 - */ - op_flags[0] = NPY_ITER_READONLY | NPY_ITER_USE_MASKNA; - op_flags[1] = NPY_ITER_WRITEONLY | NPY_ITER_ALLOCATE; - - iter = NpyIter_MultiNew(2, op, flags, NPY_KEEPORDER, NPY_NO_CASTING, - op_flags, op_dtypes); - if (iter == NULL) { - Py_DECREF(dtype); - return NULL; - } - - if (NpyIter_GetIterSize(iter) > 0) { - NpyIter_IterNextFunc *iternext; - npy_intp innersize, *innerstrides; - npy_intp innerstridemask, innerstride1; - char **dataptrs, *dataptrmask, *dataptr1; - - iternext = NpyIter_GetIterNext(iter, NULL); - if (iternext == NULL) { - Py_DECREF(dtype); - return NULL; - } - innerstrides = NpyIter_GetInnerStrideArray(iter); - innerstridemask = innerstrides[2]; - innerstride1 = innerstrides[1]; - /* Because buffering is disabled, the innersize is fixed */ - innersize = *NpyIter_GetInnerLoopSizePtr(iter); - dataptrs = NpyIter_GetDataPtrArray(iter); - - do { - npy_intp i; - dataptrmask = dataptrs[2]; - dataptr1 = dataptrs[1]; - - for (i = 0; i < innersize; ++i) { - /* - * Bit 0 of the mask is 0 -> NA, 1 -> available, - * so invert it and clear the rest of the bits. - */ - *dataptr1 = ~(*dataptrmask) & 0x01; - dataptrmask += innerstridemask; - dataptr1 += innerstride1; - } - } while (iternext(iter)); - } - - ret = NpyIter_GetOperandArray(iter)[1]; - Py_INCREF(ret); - Py_DECREF(dtype); - NpyIter_Deallocate(iter); - } - /* Create an array of all zeros */ - else { - npy_intp size; - ret = (PyArrayObject *)PyArray_NewLikeArray( - (PyArrayObject *)obj, NPY_KEEPORDER, dtype, 0); - if (ret == NULL) { - return NULL; - } - /* - * Can use memset because the newly allocated array is - * packed tightly in memory - */ - size = PyArray_SIZE(ret); - if (size > 0) { - memset(PyArray_DATA(ret), 0, dtype->elsize * size); - } - } - - return (PyObject *)ret; - } -} - -/* - * This function performs a reduction on the masks for an array. - * The masks are provided in raw form, with their strides conformed - * for the reduction. - * - * This is for use with a reduction where 'skipna=False'. - * - * ndim, shape: The geometry of the arrays - * src_dtype, dst_dtype: The NA mask dtypes. - * src_data, dst_data: The NA mask data pointers. - * src_strides, dst_strides: The NA mask strides, matching the geometry. - * - * Returns 0 on success, -1 on failure. - */ -static int -raw_reduce_maskna_array(int ndim, npy_intp *shape, - PyArray_Descr *src_dtype, char *src_data, npy_intp *src_strides, - PyArray_Descr *dst_dtype, char *dst_data, npy_intp *dst_strides) -{ - int idim; - npy_intp i, coord[NPY_MAXDIMS]; - npy_intp shape_it[NPY_MAXDIMS]; - npy_intp src_strides_it[NPY_MAXDIMS]; - npy_intp dst_strides_it[NPY_MAXDIMS]; - - /* Confirm that dst is not larger than src */ - for (idim = 0; idim < ndim; ++idim) { - if (src_strides[idim] == 0 && dst_strides[idim] != 0) { - PyErr_SetString(PyExc_RuntimeError, - "ReduceMaskNAArray cannot reduce into a larger array"); - return -1; - } - } - - if (src_dtype->type_num != NPY_BOOL || dst_dtype->type_num != NPY_BOOL) { - PyErr_SetString(PyExc_ValueError, - "multi-NA and field-NA are not yet supported"); - return -1; - } - - /* Initialize the destination mask to all ones, exposed data */ - if (fill_raw_byte_array(ndim, shape, dst_data, dst_strides, 1) < 0) { - return -1; - } - - /* - * Sort axes based on 'src', which has more non-zero strides, - * by making it the first operand here - */ - if (PyArray_PrepareTwoRawArrayIter(ndim, shape, - src_data, src_strides, - dst_data, dst_strides, - &ndim, shape_it, - &src_data, src_strides_it, - &dst_data, dst_strides_it) < 0) { - return NPY_FAIL; - } - - /* Special case a reduction in the inner loop */ - if (dst_strides_it[0] == 0) { - /* Special case a contiguous reduction in the inner loop */ - if (src_strides_it[0] == 1) { - NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { - /* If there's a zero in src, set dst to zero */ - if (memchr(src_data, 0, shape_it[0]) != NULL) { - *dst_data = 0; - } - } NPY_RAW_ITER_TWO_NEXT(idim, ndim, coord, shape_it, - src_data, src_strides_it, - dst_data, dst_strides_it); - } - else { - NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { - char *src_d = src_data; - /* If there's a zero in src, set dst to zero */ - for (i = 0; i < shape_it[0]; ++i) { - if (*src_d == 0) { - *dst_data = 0; - break; - } - src_d += src_strides_it[0]; - } - } NPY_RAW_ITER_TWO_NEXT(idim, ndim, coord, shape_it, - src_data, src_strides_it, - dst_data, dst_strides_it); - } - } - else { - NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { - char *src_d = src_data, *dst_d = dst_data; - for (i = 0; i < shape_it[0]; ++i) { - *dst_d &= *src_d; - src_d += src_strides_it[0]; - dst_d += dst_strides_it[0]; - } - } NPY_RAW_ITER_TWO_NEXT(idim, ndim, coord, shape_it, - src_data, src_strides_it, - dst_data, dst_strides_it); - } - - return 0; -} - -/* - * This function performs a reduction on the masks for an array. - * - * This is for use with a reduction where 'skipna=False'. - * - * operand: The operand for which the reduction is being done. This array - * must have an NA mask. - * result: The result array, which should have the same 'ndim' as - * 'operand' but with dimensions of size one for every reduction - * axis. This array must have an NA mask. - * wheremask: NOT SUPPORTED YET, but is for a boolean mask which can - * broadcast to 'result', indicating where to do the reduction. - * Should pass in NULL. - * skipwhichna: NOT SUPPORTED YET, but for future expansion to multi-NA, - * where reductions can be done on NAs with a subset of - * the possible payloads. Should pass in NULL. - * - * Returns 0 on success, -1 on failure. - */ -NPY_NO_EXPORT int -PyArray_ReduceMaskNAArray(PyArrayObject *operand, PyArrayObject *result, - PyArrayObject *wheremask, npy_bool *skipwhichna) -{ - int idim, ndim; - npy_intp result_strides[NPY_MAXDIMS]; - npy_intp *result_shape, *operand_shape; - npy_intp *result_maskna_strides; - - ndim = PyArray_NDIM(operand); - if (ndim != PyArray_NDIM(result)) { - PyErr_SetString(PyExc_ValueError, - "result and operand must have the same 'ndim' in " - "ReduceMaskNAArray"); - return -1; - } - if (!PyArray_HASMASKNA(result) || !PyArray_HASMASKNA(operand)) { - PyErr_SetString(PyExc_ValueError, - "both result and operand must have NA masks in " - "ReduceMaskNAArray"); - return -1; - } - - /* Validate that the parameters for future expansion are NULL */ - if (wheremask != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "the NA mask reduction operation in NumPy does not " - "yet support a where mask"); - return -1; - } - if (skipwhichna != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "multi-NA support is not yet implemented in " - "the NA mask reduction operation"); - return -1; - } - - /* Need to make sure the appropriate strides are 0 in 'result' */ - result_shape = PyArray_SHAPE(result); - operand_shape = PyArray_SHAPE(operand); - result_maskna_strides = PyArray_MASKNA_STRIDES(result); - for (idim = 0; idim < ndim; ++idim) { - if (result_shape[idim] == 1) { - result_strides[idim] = 0; - } - else if (result_shape[idim] != operand_shape[idim]) { - PyErr_SetString(PyExc_ValueError, - "the result shape must match the operand shape wherever " - "it is not 1 in ReduceMaskNAArray"); - return -1; - } - else { - result_strides[idim] = result_maskna_strides[idim]; - } - } - - return raw_reduce_maskna_array(ndim, PyArray_DIMS(operand), - PyArray_MASKNA_DTYPE(operand), - PyArray_MASKNA_DATA(operand), - PyArray_MASKNA_STRIDES(operand), - PyArray_MASKNA_DTYPE(result), - PyArray_MASKNA_DATA(result), - result_strides); -} - -static void -_strided_bool_mask_inversion(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *NPY_UNUSED(opdata)) -{ - while (N > 0) { - *dst = ((*src) ^ 0x01) & 0x01; - dst += dst_stride; - src += src_stride; - --N; - } -} - -NPY_NO_EXPORT int -PyArray_GetMaskInversionFunction( - npy_intp dst_mask_stride, npy_intp src_mask_stride, - PyArray_Descr *mask_dtype, - PyArray_StridedUnaryOp **out_unop, NpyAuxData **out_opdata) -{ - /* Will use the opdata with the field version */ - if (PyDataType_HASFIELDS(mask_dtype)) { - PyErr_SetString(PyExc_RuntimeError, - "field-based masks are not supported yet"); - return -1; - } - - if (mask_dtype->type_num != NPY_BOOL && mask_dtype->type_num != NPY_MASK) { - PyErr_SetString(PyExc_RuntimeError, - "unsupported data type for mask"); - return -1; - } - - /* TODO: Specialize for contiguous data */ - - *out_unop = &_strided_bool_mask_inversion; - *out_opdata = NULL; - return 0; -} - -static void -_strided_bool_mask_noinv0_noinv1_and(char *dst, npy_intp dst_stride, - char *src0, npy_intp src0_stride, - char *src1, npy_intp src1_stride, - npy_intp N, NpyAuxData *NPY_UNUSED(opdata)) -{ - while (N > 0) { - *dst = (*src0) & (*src1); - dst += dst_stride; - src0 += src0_stride; - src1 += src1_stride; - --N; - } -} - -static void -_strided_bool_mask_inv0_noinv1_and(char *dst, npy_intp dst_stride, - char *src0, npy_intp src0_stride, - char *src1, npy_intp src1_stride, - npy_intp N, NpyAuxData *NPY_UNUSED(opdata)) -{ - while (N > 0) { - *dst = ((*src0) ^ 0x01) & (*src1); - dst += dst_stride; - src0 += src0_stride; - src1 += src1_stride; - --N; - } -} - -static void -_strided_bool_mask_noinv0_inv1_and(char *dst, npy_intp dst_stride, - char *src0, npy_intp src0_stride, - char *src1, npy_intp src1_stride, - npy_intp N, NpyAuxData *NPY_UNUSED(opdata)) -{ - while (N > 0) { - *dst = (*src0) & ((*src1) ^ 0x01); - dst += dst_stride; - src0 += src0_stride; - src1 += src1_stride; - --N; - } -} - -static void -_strided_bool_mask_inv0_inv1_and(char *dst, npy_intp dst_stride, - char *src0, npy_intp src0_stride, - char *src1, npy_intp src1_stride, - npy_intp N, NpyAuxData *NPY_UNUSED(opdata)) -{ - while (N > 0) { - *dst = ((*src0) | (*src1)) ^ 0x01; - dst += dst_stride; - src0 += src0_stride; - src1 += src1_stride; - --N; - } -} - -/* - * Gets a function which ANDs together two masks, possibly inverting - * one or both of the masks as well. - * - * The dtype of the output must match 'mask0_dtype'. - */ -NPY_NO_EXPORT int -PyArray_GetMaskAndFunction( - npy_intp mask0_stride, PyArray_Descr *mask0_dtype, int invert_mask0, - npy_intp mask1_stride, PyArray_Descr *mask1_dtype, int invert_mask1, - PyArray_StridedBinaryOp **out_binop, NpyAuxData **out_opdata) -{ - /* Will use the opdata with the field version */ - if (PyDataType_HASFIELDS(mask0_dtype) || - PyDataType_HASFIELDS(mask1_dtype)) { - PyErr_SetString(PyExc_RuntimeError, - "field-based masks are not supported yet"); - return -1; - } - - if (mask0_dtype->type_num == NPY_MASK || - mask1_dtype->type_num == NPY_MASK) { - PyErr_SetString(PyExc_RuntimeError, - "multi-NA masks are not supported yet"); - return -1; - } - - if (mask0_dtype->type_num != NPY_BOOL || - mask1_dtype->type_num != NPY_BOOL) { - PyErr_SetString(PyExc_RuntimeError, - "unsupported data type for mask"); - return -1; - } - - /* TODO: Specialize for contiguous data */ - - if (invert_mask0) { - if (invert_mask1) { - *out_binop = &_strided_bool_mask_inv0_inv1_and; - } - else { - *out_binop = &_strided_bool_mask_inv0_noinv1_and; - } - } - else { - if (invert_mask1) { - *out_binop = &_strided_bool_mask_noinv0_inv1_and; - } - else { - *out_binop = &_strided_bool_mask_noinv0_noinv1_and; - } - } - *out_opdata = NULL; - return 0; -} diff --git a/numpy/core/src/multiarray/na_mask.h b/numpy/core/src/multiarray/na_mask.h deleted file mode 100644 index 160327de2..000000000 --- a/numpy/core/src/multiarray/na_mask.h +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef _NPY_PRIVATE__NA_MASK_H_ -#define _NPY_PRIVATE__NA_MASK_H_ - -#include "lowlevel_strided_loops.h" - -/* - * A ufunc-like function, which returns a boolean or an array - * of booleans indicating which values are NA. - */ -NPY_NO_EXPORT PyObject * -PyArray_IsNA(PyObject *obj); - -/* - * Gets a strided unary operation which inverts mask values. - */ -NPY_NO_EXPORT int -PyArray_GetMaskInversionFunction(npy_intp dst_mask_stride, - npy_intp src_mask_stride, - PyArray_Descr *mask_dtype, - PyArray_StridedUnaryOp **out_unop, - NpyAuxData **out_opdata); - -/* - * Gets a function which ANDs together two masks, possibly inverting - * one or both of the masks as well. - * - * The dtype of the output must match 'mask0_dtype'. - */ -NPY_NO_EXPORT int -PyArray_GetMaskAndFunction( - npy_intp mask0_stride, PyArray_Descr *mask0_dtype, int invert_mask0, - npy_intp mask1_stride, PyArray_Descr *mask1_dtype, int invert_mask1, - PyArray_StridedBinaryOp **out_binop, NpyAuxData **out_opdata); - -/* - * This function performs a reduction on the masks for an array. - * - * This is for use with a reduction where 'skipna=False'. - * - * operand: The operand for which the reduction is being done. This array - * must have an NA mask. - * result: The result array, which should have the same 'ndim' as - * 'operand' but with dimensions of size one for every reduction - * axis. This array must have an NA mask. - * wheremask: NOT SUPPORTED YET, but is for a boolean mask which can - * broadcast to 'result', indicating where to do the reduction. - * Should pass in NULL. - * skipwhichna: NOT SUPPORTED YET, but for future expansion to multi-NA, - * where reductions can be done on NAs with a subset of - * the possible payloads. Should pass in NULL. - * - * Returns 0 on success, -1 on failure. - */ -NPY_NO_EXPORT int -PyArray_ReduceMaskNAArray(PyArrayObject *operand, PyArrayObject *result, - PyArrayObject *wheremask, npy_bool *skipwhichna); - -#endif diff --git a/numpy/core/src/multiarray/na_object.c b/numpy/core/src/multiarray/na_object.c deleted file mode 100644 index 01f944df9..000000000 --- a/numpy/core/src/multiarray/na_object.c +++ /dev/null @@ -1,802 +0,0 @@ -/* - * This file implements the missing value NA singleton object for NumPy. - * - * Written by Mark Wiebe (mwwiebe@gmail.com) - * Copyright (c) 2011 by Enthought, Inc. - * - * See LICENSE.txt for the license. - */ - -#define PY_SSIZE_T_CLEAN -#include <Python.h> - -#define NPY_NO_DEPRECATED_API NPY_API_VERSION -#define _MULTIARRAYMODULE -#include <numpy/arrayobject.h> -#include <numpy/arrayscalars.h> - -#include "npy_config.h" -#include "numpy/npy_3kcompat.h" - -#include "descriptor.h" -#include "common.h" -#include "na_object.h" - -static PyObject * -na_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) -{ - NpyNA_fields *self; - - self = (NpyNA_fields *)subtype->tp_alloc(subtype, 0); - if (self != NULL) { - self->payload = NPY_NA_NOPAYLOAD; - self->dtype = NULL; - self->is_singleton = 0; - } - - return (PyObject *)self; -} - -static int -na_init(NpyNA_fields *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"payload", "dtype", NULL}; - int payload = NPY_MAX_INT; - PyArray_Descr *dtype = NULL; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|iO&:NA", kwlist, - &payload, - &PyArray_DescrConverter, &dtype)) { - Py_XDECREF(dtype); - return -1; - } - - /* Using NPY_MAX_INT as the default for 'payload' */ - if (payload == NPY_MAX_INT) { - self->payload = NPY_NA_NOPAYLOAD; - } - else if (payload < 0 || payload > 127) { - PyErr_Format(PyExc_ValueError, - "out of bounds payload for NumPy NA, " - "%d is not in the range [0,127]", payload); - Py_XDECREF(dtype); - return -1; - } - else { - self->payload = (npy_uint8)payload; - } - - Py_XDECREF(self->dtype); - self->dtype = dtype; - - return 0; -} - -/* - * The call function proxies to the na_init function to handle - * the payload and dtype parameters. - */ -static PyObject * -na_call(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) -{ - NpyNA_fields *ret; - - ret = (NpyNA_fields *)na_new(&NpyNA_Type, NULL, NULL); - if (ret != NULL) { - if (na_init(ret, args, kwds) < 0) { - Py_DECREF(ret); - return NULL; - } - } - - return (PyObject *)ret; -} - -static void -na_dealloc(NpyNA_fields *self) -{ - Py_XDECREF(self->dtype); - self->dtype = NULL; - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static PyObject * -na_repr(NpyNA_fields *self) -{ - if (self->dtype == NULL) { - if (self->payload == NPY_NA_NOPAYLOAD) { - return PyUString_FromString("NA"); - } - else { - return PyUString_FromFormat("NA(%d)", (int)self->payload); - } - } - else { - PyObject *s; - if (self->payload == NPY_NA_NOPAYLOAD) { - s = PyUString_FromString("NA(dtype="); - } - else { - s = PyUString_FromFormat("NA(%d, dtype=", (int)self->payload); - } - PyUString_ConcatAndDel(&s, - arraydescr_construction_repr(self->dtype, 1, 0)); - PyUString_ConcatAndDel(&s, - PyUString_FromString(")")); - return s; - } -} - -/* - * The str function is the same as repr, except it throws away - * the dtype. It is always either "NA" or "NA(payload)". - */ -static PyObject * -na_str(NpyNA_fields *self) -{ - if (self->payload == NPY_NA_NOPAYLOAD) { - return PyUString_FromString("NA"); - } - else { - return PyUString_FromFormat("NA(%d)", (int)self->payload); - } -} - -/* - * Any comparison with NA produces an NA. - */ -static PyObject * -na_richcompare(NpyNA_fields *self, PyObject *other, int cmp_op) -{ - /* If an ndarray is compared directly with NA, let the array handle it */ - if (PyArray_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - /* Otherwise always return the NA singleton */ - else { - PyArray_Descr *bool_dtype; - NpyNA *ret; - bool_dtype = PyArray_DescrFromType(NPY_BOOL); - if (bool_dtype == NULL) { - return NULL; - } - ret = NpyNA_FromDTypeAndPayload(bool_dtype, 0, 0); - Py_DECREF(bool_dtype); - return (PyObject *)ret; - } -} - -static PyObject * -na_payload_get(NpyNA_fields *self) -{ - if (self->payload == NPY_NA_NOPAYLOAD) { - Py_INCREF(Py_None); - return Py_None; - } - else { - return PyInt_FromLong(self->payload); - } -} - -static int -na_payload_set(NpyNA_fields *self, PyObject *value) -{ - long payload; - - if (value == NULL) { - PyErr_SetString(PyExc_AttributeError, - "Cannot delete NA payload attribute"); - return -1; - } - - /* Don't allow changing the static singleton instance */ - if (self->is_singleton) { - PyErr_SetString(PyExc_RuntimeError, - "cannot change the payload of the NumPy NA singleton, " - "make a new copy like 'numpy.NA(payload)'"); - return -1; - } - else if (value == Py_None) { - self->payload = NPY_NA_NOPAYLOAD; - } - else { - /* Use PyNumber_Index to ensure an integer in Python >= 2.5*/ -#if PY_VERSION_HEX >= 0x02050000 - value = PyNumber_Index(value); - if (value == NULL) { - return -1; - } -#else - Py_INCREF(value); -#endif - payload = PyInt_AsLong(value); - Py_DECREF(value); - if (payload == -1 && PyErr_Occurred()) { - return -1; - } - else if (payload < 0 || payload > 127) { - PyErr_Format(PyExc_ValueError, - "out of bounds payload for NumPy NA, " - "%ld is not in the range [0,127]", payload); - return -1; - } - else { - self->payload = (npy_uint8)payload; - } - } - - return 0; -} - -static PyObject * -na_dtype_get(NpyNA_fields *self) -{ - if (self->dtype == NULL) { - Py_INCREF(Py_None); - return Py_None; - } - else { - Py_INCREF(self->dtype); - return (PyObject *)self->dtype; - } -} - -static int -na_dtype_set(NpyNA_fields *self, PyObject *value) -{ - PyArray_Descr *dtype = NULL; - - if (value == NULL) { - PyErr_SetString(PyExc_AttributeError, - "Cannot delete NA dtype attribute"); - return -1; - } - /* Don't allow changing the static singleton instance */ - if (self->is_singleton) { - PyErr_SetString(PyExc_RuntimeError, - "cannot change the dtype of the NumPy NA singleton, " - "make a new copy like 'numpy.NA(dtype=val)'"); - return -1; - } - /* Convert the input into a dtype object */ - else if (!PyArray_DescrConverter(value, &dtype)) { - return -1; - } - - /* Replace the existing dtype in self */ - Py_XDECREF(self->dtype); - self->dtype = dtype; - - return 0; -} - -static PyGetSetDef na_getsets[] = { - {"payload", - (getter)na_payload_get, - (setter)na_payload_set, - NULL, NULL}, - {"dtype", - (getter)na_dtype_get, - (setter)na_dtype_set, - NULL, NULL}, - - {NULL, NULL, NULL, NULL, NULL} -}; - -/* Combines two NA values together, merging their payloads and dtypes. */ -NPY_NO_EXPORT NpyNA * -NpyNA_CombineNA(NpyNA *na1, NpyNA *na2) -{ - NpyNA_fields *ret, *fna1, *fna2; - - fna1 = (NpyNA_fields *)na1; - fna2 = (NpyNA_fields *)na2; - - ret = (NpyNA_fields *)na_new(&NpyNA_Type, NULL, NULL); - if (ret == NULL) { - return NULL; - } - - /* Combine the payloads */ - ret->payload = NpyNA_CombinePayloads(fna1->payload, fna2->payload); - - /* Combine the dtypes */ - Py_XDECREF(ret->dtype); - ret->dtype = NULL; - if (fna1->dtype != NULL && fna2->dtype != NULL) { - ret->dtype = PyArray_PromoteTypes(fna1->dtype, fna2->dtype); - if (ret->dtype == NULL) { - Py_DECREF(ret); - return NULL; - } - } - else if (fna1->dtype != NULL) { - ret->dtype = fna1->dtype; - Py_INCREF(ret->dtype); - } - else if (fna2->dtype != NULL) { - ret->dtype = fna2->dtype; - Py_INCREF(ret->dtype); - } - - return (NpyNA *)ret; -} - -/* - * Combines an NA with an object, raising an error if the object has - * no extractable NumPy dtype. - */ -NPY_NO_EXPORT NpyNA * -NpyNA_CombineNAWithObject(NpyNA *na, PyObject *obj) -{ - NpyNA_fields *ret, *fna; - PyArray_Descr *dtype = NULL; - - fna = (NpyNA_fields *)na; - - /* If 'obj' is NA, handle it specially */ - if (NpyNA_Check(obj)) { - return NpyNA_CombineNA(na, (NpyNA *)obj); - } - - /* Extract a dtype from 'obj' */ - if (PyArray_IsScalar(obj, Generic)) { - dtype = PyArray_DescrFromScalar(obj); - if (dtype == NULL) { - return NULL; - } - } - else if (PyArray_Check(obj)) { - /* TODO: This needs to be more complicated... */ - dtype = PyArray_DESCR((PyArrayObject *)obj); - Py_INCREF(dtype); - } - else { - dtype = _array_find_python_scalar_type(obj); - if (dtype == NULL) { - PyErr_SetString(PyExc_TypeError, - "numpy.NA only supports operations with scalars " - "and NumPy arrays"); - return NULL; - } - } - - ret = (NpyNA_fields *)na_new(&NpyNA_Type, NULL, NULL); - if (ret == NULL) { - return NULL; - } - - /* Copy the payload */ - ret->payload = fna->payload; - - /* Combine the dtypes */ - Py_XDECREF(ret->dtype); - if (fna->dtype == NULL) { - ret->dtype = dtype; - } - else { - ret->dtype = PyArray_PromoteTypes(fna->dtype, dtype); - Py_DECREF(dtype); - if (ret->dtype == NULL) { - Py_DECREF(ret); - return NULL; - } - } - - return (NpyNA *)ret; -} - -/*NUMPY_API - * - * Returns the *dtype* field of the NA object, which is NULL when - * the NA has no dtype. Does not raise an error. - */ -NPY_NO_EXPORT PyArray_Descr * -NpyNA_GetDType(NpyNA* na) -{ - NpyNA_fields *fna = (NpyNA_fields *)na; - return fna->dtype; -} - -/*NUMPY_API - * - * Returns true if the NA has a multi-NA payload, false otherwise. - */ -NPY_NO_EXPORT npy_bool -NpyNA_IsMultiNA(NpyNA* na) -{ - NpyNA_fields *fna = (NpyNA_fields *)na; - return fna->payload != NPY_NA_NOPAYLOAD; -} - -/*NUMPY_API - * - * Gets the multi-NA payload of the NA, or 0 if *na* doesn't have - * a multi-NA payload. - */ -NPY_NO_EXPORT int -NpyNA_GetPayload(NpyNA* na) -{ - NpyNA_fields *fna = (NpyNA_fields *)na; - return fna->payload == NPY_NA_NOPAYLOAD ? 0 : fna->payload; -} - - -/*NUMPY_API - * - * Converts an object into an NA if possible. - * - * If 'suppress_error' is enabled, doesn't raise an error when something - * isn't NA. - */ -NPY_NO_EXPORT NpyNA * -NpyNA_FromObject(PyObject *obj, int suppress_error) -{ - /* Pass through existing NAs */ - if (NpyNA_Check(obj)) { - Py_INCREF(obj); - return (NpyNA *)obj; - } - /* Convert zero-dimensional masked elements into NAs */ - else if (PyArray_Check(obj)) { - if (PyArray_NDIM((PyArrayObject *)obj) == 0 && - !PyArray_HASFIELDS((PyArrayObject *)obj)) { - if (PyArray_HASMASKNA((PyArrayObject *)obj)) { - npy_mask maskvalue; - NpyNA_fields *fna; - - maskvalue = (npy_mask)*PyArray_MASKNA_DATA( - (PyArrayObject *)obj); - if (NpyMaskValue_IsExposed(maskvalue)) { - if (!suppress_error) { - PyErr_SetString(PyExc_ValueError, - "Cannot convert zero-dimensional array with " - "valid value into NA"); - } - return NULL; - } - - fna = (NpyNA_fields *)na_new(&NpyNA_Type, NULL, NULL); - if (fna == NULL) { - return NULL; - } - - fna->dtype = PyArray_DESCR((PyArrayObject *)obj); - Py_INCREF(fna->dtype); - - if (PyArray_MASKNA_DTYPE((PyArrayObject *)obj)->type_num == - NPY_MASK) { - fna->payload = NpyMaskValue_GetPayload(maskvalue); - } - - return (NpyNA *)fna; - } - } - else { - if (!suppress_error) { - PyErr_SetString(PyExc_ValueError, - "Cannot convert array with one or more dimensions " - "into an NA"); - } - return NULL; - } - } - - if (!suppress_error) { - PyErr_SetString(PyExc_ValueError, "Cannot convert object into an NA"); - } - return NULL; -} - -/*NUMPY_API - * - * Converts a dtype reference and payload value into an NA. - * Doesn't steal the 'dtype' reference. Raises an error - * if the payload is invalid - */ -NPY_NO_EXPORT NpyNA * -NpyNA_FromDTypeAndPayload(PyArray_Descr *dtype, int multina, int payload) -{ - NpyNA_fields *fna; - - if (dtype == NULL && multina == 0 && payload == 0) { - Py_INCREF(Npy_NA); - return (NpyNA *)Npy_NA; - } - - fna = (NpyNA_fields *)na_new(&NpyNA_Type, NULL, NULL); - if (fna == NULL) { - return NULL; - } - - fna->dtype = dtype; - Py_XINCREF(fna->dtype); - - if (multina) { - if (payload < 0 || payload > 0x7f) { - PyErr_Format(PyExc_ValueError, - "Given NA payload, %d, is out of bounds [0, 128)", - payload); - Py_DECREF(fna); - } - fna->payload = (npy_uint8)payload; - } - else if (payload != 0) { - PyErr_SetString(PyExc_ValueError, - "NA payload must be zero when multi-NA is disabled"); - Py_DECREF(fna); - return NULL; - } - - return (NpyNA *)fna; -} - -/* - * Returns a mask value corresponding to the NA. - */ -NPY_NO_EXPORT npy_mask -NpyNA_AsMaskValue(NpyNA *na) -{ - return NpyMaskValue_Create(0, NpyNA_GetPayload(na)); -} - -/* An NA unary op simply passes along the same NA */ -static PyObject * -na_unaryop(PyObject *self) -{ - Py_INCREF(self); - return self; -} - -static PyObject * -na_binaryop(PyObject *op1, PyObject *op2) -{ - /* If an ndarray is operated on with NA, let the array handle it */ - if (PyArray_Check(op1) || PyArray_Check(op2)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - /* Combine NAs according to standard rules */ - else { - if (NpyNA_Check(op1)) { - return (PyObject *)NpyNA_CombineNAWithObject((NpyNA *)op1, op2); - } - else if (NpyNA_Check(op2)) { - return (PyObject *)NpyNA_CombineNAWithObject((NpyNA *)op2, op1); - } - else { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - } -} - -static PyObject * -na_power(PyObject *op1, PyObject *op2, PyObject *NPY_UNUSED(op3)) -{ - return na_binaryop(op1, op2); -} - -/* Special case bitwise <and> with a boolean 'other' */ -static PyObject * -na_and(PyObject *op1, PyObject *op2) -{ - NpyNA *na; - PyObject *other; - - if (NpyNA_Check(op1)) { - na = (NpyNA *)op1; - other = op2; - } - else if (NpyNA_Check(op2)) { - na = (NpyNA *)op2; - other = op1; - } - else { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - - /* If an ndarray is operated on with NA, let the array handle it */ - if (PyArray_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - /* NA & False is False */ - else if (other == Py_False || - ((Py_TYPE(other) == &PyBoolArrType_Type) && - ((PyBoolScalarObject *)other)->obval == 0)) { - Py_INCREF(Py_False); - return Py_False; - } - /* Combine NAs according to standard rules */ - else { - return (PyObject *)NpyNA_CombineNAWithObject(na, other); - } -} - -/* Special case bitwise <or> with a boolean 'other' */ -static PyObject * -na_or(PyObject *op1, PyObject *op2) -{ - NpyNA *na; - PyObject *other; - - if (NpyNA_Check(op1)) { - na = (NpyNA *)op1; - other = op2; - } - else if (NpyNA_Check(op2)) { - na = (NpyNA *)op2; - other = op1; - } - else { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - - /* If an ndarray is operated on with NA, let the array handle it */ - if (PyArray_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - /* NA & True is True */ - else if (other == Py_True || - ((Py_TYPE(other) == &PyBoolArrType_Type) && - ((PyBoolScalarObject *)other)->obval != 0)) { - Py_INCREF(Py_True); - return Py_True; - } - /* Combine NAs according to standard rules */ - else { - return (PyObject *)NpyNA_CombineNAWithObject(na, other); - } -} - -/* Using NA in an if statement is always an error */ -static int -na_nonzero(PyObject *NPY_UNUSED(self)) -{ - PyErr_SetString(PyExc_ValueError, - "numpy.NA represents an unknown missing value, " - "so its truth value cannot be determined"); - return -1; -} - -NPY_NO_EXPORT PyNumberMethods na_as_number = { - (binaryfunc)na_binaryop, /*nb_add*/ - (binaryfunc)na_binaryop, /*nb_subtract*/ - (binaryfunc)na_binaryop, /*nb_multiply*/ -#if defined(NPY_PY3K) -#else - (binaryfunc)na_binaryop, /*nb_divide*/ -#endif - (binaryfunc)na_binaryop, /*nb_remainder*/ - (binaryfunc)na_binaryop, /*nb_divmod*/ - (ternaryfunc)na_power, /*nb_power*/ - (unaryfunc)na_unaryop, /*nb_neg*/ - (unaryfunc)na_unaryop, /*nb_pos*/ - (unaryfunc)na_unaryop, /*nb_abs,*/ - (inquiry)na_nonzero, /*nb_nonzero*/ - (unaryfunc)na_unaryop, /*nb_invert*/ - (binaryfunc)na_binaryop, /*nb_lshift*/ - (binaryfunc)na_binaryop, /*nb_rshift*/ - (binaryfunc)na_and, /*nb_and*/ - (binaryfunc)na_binaryop, /*nb_xor*/ - (binaryfunc)na_or, /*nb_or*/ -#if defined(NPY_PY3K) -#else - 0, /*nb_coerce*/ -#endif - 0, /*nb_int*/ -#if defined(NPY_PY3K) - 0, /*nb_reserved*/ -#else - 0, /*nb_long*/ -#endif - 0, /*nb_float*/ -#if defined(NPY_PY3K) -#else - 0, /*nb_oct*/ - 0, /*nb_hex*/ -#endif - 0, /*inplace_add*/ - 0, /*inplace_subtract*/ - 0, /*inplace_multiply*/ -#if defined(NPY_PY3K) -#else - 0, /*inplace_divide*/ -#endif - 0, /*inplace_remainder*/ - 0, /*inplace_power*/ - 0, /*inplace_lshift*/ - 0, /*inplace_rshift*/ - 0, /*inplace_and*/ - 0, /*inplace_xor*/ - 0, /*inplace_or*/ - (binaryfunc)na_binaryop, /*nb_floor_divide*/ - (binaryfunc)na_binaryop, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ -#if PY_VERSION_HEX >= 0x02050000 - 0, /*nb_index*/ -#endif -}; - -NPY_NO_EXPORT PyTypeObject NpyNA_Type = { -#if defined(NPY_PY3K) - PyVarObject_HEAD_INIT(NULL, 0) -#else - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ -#endif - "numpy.NAType", /* tp_name */ - sizeof(NpyNA_fields), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)na_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ -#if defined(NPY_PY3K) - 0, /* tp_reserved */ -#else - 0, /* tp_compare */ -#endif - (reprfunc)na_repr, /* tp_repr */ - &na_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc)na_call, /* tp_call */ - (reprfunc)na_str, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT /* tp_flags */ -#if !defined(NPY_PY3K) - | Py_TPFLAGS_CHECKTYPES, -#else - , -#endif - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - (richcmpfunc)na_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - na_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)na_init, /* tp_init */ - 0, /* tp_alloc */ - na_new, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ -#if PY_VERSION_HEX >= 0x02060000 - 0, /* tp_version_tag */ -#endif -}; - -NPY_NO_EXPORT NpyNA_fields _Npy_NASingleton = { - PyObject_HEAD_INIT(&NpyNA_Type) - NPY_NA_NOPAYLOAD, /* payload */ - NULL, /* dtype */ - 1 /* is_singleton */ -}; - -/* This symbol is exported in the NumPy C API */ -NPY_NO_EXPORT PyObject *Npy_NA = (PyObject *)&_Npy_NASingleton; diff --git a/numpy/core/src/multiarray/na_object.h b/numpy/core/src/multiarray/na_object.h deleted file mode 100644 index 3e602ccdf..000000000 --- a/numpy/core/src/multiarray/na_object.h +++ /dev/null @@ -1,60 +0,0 @@ -#ifndef _NPY_PRIVATE__NA_SINGLETON_H_ -#define _NPY_PRIVATE__NA_SINGLETON_H_ - -/* Direct access to the fields of the NA object is just internal to NumPy. */ -typedef struct { - PyObject_HEAD - /* NA payload, 0 by default */ - npy_uint8 payload; - /* NA dtype, NULL by default */ - PyArray_Descr *dtype; - /* Internal flag, whether this is the singleton numpy.NA or not */ - int is_singleton; -} NpyNA_fields; - -#define NPY_NA_NOPAYLOAD (255) - -static NPY_INLINE npy_uint8 -NpyNA_CombinePayloads(npy_uint p1, npy_uint p2) -{ - if (p1 == NPY_NA_NOPAYLOAD || p2 == NPY_NA_NOPAYLOAD) { - return NPY_NA_NOPAYLOAD; - } - else { - return (p1 < p2) ? p1 : p2; - } -} - -/* Combines two NA values together, merging their payloads and dtypes. */ -NPY_NO_EXPORT NpyNA * -NpyNA_CombineNA(NpyNA *na1, NpyNA *na2); - -/* - * Combines an NA with an object, raising an error if the object has - * no extractable NumPy dtype. - */ -NPY_NO_EXPORT NpyNA * -NpyNA_CombineNAWithObject(NpyNA *na, PyObject *obj); - -/* - * Returns a mask value corresponding to the NA. - */ -NPY_NO_EXPORT npy_mask -NpyNA_AsMaskValue(NpyNA *na); - -/* - * Returns True if the object is an NA in the form of a 0-dimensional - * array. - */ -static NPY_INLINE npy_bool -NpyNA_IsZeroDimArrayNA(PyObject *obj) -{ - return PyArray_Check(obj) && - PyArray_NDIM((PyArrayObject *)obj) == 0 && - PyArray_HASMASKNA((PyArrayObject *)obj) && - !PyArray_HASFIELDS((PyArrayObject *)obj) && - !NpyMaskValue_IsExposed((npy_mask)*PyArray_MASKNA_DATA( - (PyArrayObject *)obj)); -} - -#endif diff --git a/numpy/core/src/multiarray/nditer_api.c b/numpy/core/src/multiarray/nditer_api.c index cceec518b..09e572f10 100644 --- a/numpy/core/src/multiarray/nditer_api.c +++ b/numpy/core/src/multiarray/nditer_api.c @@ -274,8 +274,7 @@ NpyIter_Reset(NpyIter *iter, char **errmsg) /*NUMPY_API * Resets the iterator to its initial state, with new base data pointers. - * This function requires great caution, even more so if any - * NPY_ITER_USE_MASKNA operands were specified. + * This function requires great caution. * * If errmsg is non-NULL, it should point to a variable which will * receive the error message, and no Python exception will be set. @@ -819,35 +818,6 @@ NpyIter_GetNOp(NpyIter *iter) } /*NUMPY_API - * Gets the index of the first operand which is the - * mask for an NPY_ITER_USE_MASKNA operand. - */ -NPY_NO_EXPORT int -NpyIter_GetFirstMaskNAOp(NpyIter *iter) -{ - return NIT_FIRST_MASKNA_OP(iter); -} - -/*NUMPY_API - * Gets the correspondences between the operands with - * NPY_ITER_USEMASKNA set and their corresponding masks. - * - * If i < NpyIter_GetFirstMaskNAOp(iter), then - * NpyIter_GetMaskNAIndices(iter)[i] is either -1 or - * an index >= NpyIter_GetFirstMaskNAOp(iter) of the corresponding - * mask. - * - * If i >= NpyIter_GetFirstMaskNAOp(iter), then - * NpyIter_GetMaskNAIndices(iter)[i] is the index - * of the corresponding maskna operand for the mask. - */ -NPY_NO_EXPORT npy_int8 * -NpyIter_GetMaskNAIndexArray(NpyIter *iter) -{ - return NIT_MASKNA_INDICES(iter); -} - -/*NUMPY_API * Gets the number of elements being iterated */ NPY_NO_EXPORT npy_intp @@ -1104,7 +1074,6 @@ NpyIter_GetIterView(NpyIter *iter, npy_intp i) npy_uint32 itflags = NIT_ITFLAGS(iter); int idim, ndim = NIT_NDIM(iter); int nop = NIT_NOP(iter); - int first_maskna_op = NIT_FIRST_MASKNA_OP(iter); npy_intp shape[NPY_MAXDIMS], strides[NPY_MAXDIMS]; PyArrayObject *obj, *view; @@ -1113,9 +1082,8 @@ NpyIter_GetIterView(NpyIter *iter, npy_intp i) NpyIter_AxisData *axisdata; npy_intp sizeof_axisdata; int writeable; - npy_int8 *maskna_indices = NIT_MASKNA_INDICES(iter); - if (i < 0 || i >= first_maskna_op) { + if (i < 0) { PyErr_SetString(PyExc_IndexError, "index provided for an iterator view was out of bounds"); return NULL; @@ -1159,29 +1127,6 @@ NpyIter_GetIterView(NpyIter *iter, npy_intp i) } /* Make sure all the flags are good */ PyArray_UpdateFlags(view, NPY_ARRAY_UPDATE_ALL); - /* - * Add the mask to the view if the operand was NPY_ITER_USE_MASKNA. - */ - if (maskna_indices[i] >= 0) { - PyArrayObject_fields *fview = (PyArrayObject_fields *)view; - int i_maskna = maskna_indices[i]; - npy_intp *maskna_strides = fview->maskna_strides; - - fview->maskna_dtype = PyArray_MASKNA_DTYPE(obj); - Py_INCREF(fview->maskna_dtype); - fview->maskna_data = NIT_RESETDATAPTR(iter)[i_maskna]; - - axisdata = NIT_AXISDATA(iter); - for (idim = 0; idim < ndim; ++idim) { - maskna_strides[ndim-idim-1] = NAD_STRIDES(axisdata)[i_maskna]; - - NIT_ADVANCE_AXISDATA(axisdata, 1); - } - - /* This view doesn't own the mask */ - fview->flags |= NPY_ARRAY_MASKNA; - fview->flags &= ~NPY_ARRAY_OWNMASKNA; - } return view; } @@ -1464,20 +1409,10 @@ NpyIter_DebugPrint(NpyIter *iter) printf("REDUCE "); if (itflags&NPY_ITFLAG_REUSE_REDUCE_LOOPS) printf("REUSE_REDUCE_LOOPS "); - if (itflags&NPY_ITFLAG_HAS_MASKNA_OP) - printf("HAS_MASKNA_OP "); printf("\n"); printf("| NDim: %d\n", (int)ndim); printf("| NOp: %d\n", (int)nop); - if (itflags&NPY_ITFLAG_HAS_MASKNA_OP) { - printf("| First MaskNA Op: %d\n", (int)NIT_FIRST_MASKNA_OP(iter)); - printf("| MaskNA Indices: "); - for (iop = 0; iop < nop; ++iop) { - printf("%d ", (int)NIT_MASKNA_INDICES(iter)[iop]); - } - printf("\n"); - } if (NIT_MASKOP(iter) >= 0) { printf("| MaskOp: %d\n", (int)NIT_MASKOP(iter)); } @@ -1887,7 +1822,6 @@ npyiter_copy_from_buffers(NpyIter *iter) int ndim = NIT_NDIM(iter); int iop, nop = NIT_NOP(iter); int maskop = NIT_MASKOP(iter); - int first_maskna_op = NIT_FIRST_MASKNA_OP(iter); npyiter_opitflags *op_itflags = NIT_OPITFLAGS(iter); NpyIter_BufferData *bufferdata = NIT_BUFFERDATA(iter); @@ -1903,7 +1837,6 @@ npyiter_copy_from_buffers(NpyIter *iter) char **ptrs = NBF_PTRS(bufferdata), **ad_ptrs = NAD_PTRS(axisdata); char **buffers = NBF_BUFFERS(bufferdata); char *buffer; - npy_int8 *maskna_indices = NIT_MASKNA_INDICES(iter); npy_intp reduce_outerdim = 0; npy_intp *reduce_outerstrides = NULL; @@ -2008,46 +1941,19 @@ npyiter_copy_from_buffers(NpyIter *iter) "operand %d (%d items)\n", (int)iop, (int)op_transfersize); - /* USE_MASKNA operand */ - if (iop < first_maskna_op && maskna_indices[iop] >= 0) { - int iop_maskna = maskna_indices[iop]; - npy_mask *maskptr; - /* TODO: support WRITEMASKED + USE_MASKNA together */ - - /* - * The mask pointer may be in the buffer or in - * the array, detect which one. - */ - if ((op_itflags[iop_maskna]&NPY_OP_ITFLAG_USINGBUFFER) != 0) { - maskptr = (npy_mask *)buffers[iop_maskna]; - } - else { - maskptr = (npy_mask *)ad_ptrs[iop_maskna]; - } - - PyArray_TransferMaskedStridedToNDim(ndim_transfer, - ad_ptrs[iop], dst_strides, axisdata_incr, - buffer, src_stride, - maskptr, strides[iop_maskna], - dst_coords, axisdata_incr, - dst_shape, axisdata_incr, - op_transfersize, dtypes[iop]->elsize, - (PyArray_MaskedStridedUnaryOp *)stransfer, - transferdata); - } /* WRITEMASKED operand */ - else if (op_itflags[iop] & NPY_OP_ITFLAG_WRITEMASKED) { - npy_mask *maskptr; + if (op_itflags[iop] & NPY_OP_ITFLAG_WRITEMASKED) { + npy_bool *maskptr; /* * The mask pointer may be in the buffer or in * the array, detect which one. */ if ((op_itflags[maskop]&NPY_OP_ITFLAG_USINGBUFFER) != 0) { - maskptr = (npy_mask *)buffers[maskop]; + maskptr = (npy_bool *)buffers[maskop]; } else { - maskptr = (npy_mask *)ad_ptrs[maskop]; + maskptr = (npy_bool *)ad_ptrs[maskop]; } PyArray_TransferMaskedStridedToNDim(ndim_transfer, @@ -2111,7 +2017,6 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) npy_uint32 itflags = NIT_ITFLAGS(iter); int ndim = NIT_NDIM(iter); int iop, nop = NIT_NOP(iter); - int first_maskna_op = NIT_FIRST_MASKNA_OP(iter); npyiter_opitflags *op_itflags = NIT_OPITFLAGS(iter); NpyIter_BufferData *bufferdata = NIT_BUFFERDATA(iter); @@ -2472,13 +2377,7 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) npy_bool skip_transfer = 0; - /* Need to pick the right item size for the data vs mask */ - if (iop < first_maskna_op) { - src_itemsize = PyArray_DTYPE(operands[iop])->elsize; - } - else { - src_itemsize = PyArray_MASKNA_DTYPE(operands[iop])->elsize; - } + src_itemsize = PyArray_DTYPE(operands[iop])->elsize; /* If stransfer wasn't set to NULL, buffering is required */ any_buffered = 1; diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index 180c55063..91768fa86 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -36,7 +36,7 @@ npyiter_prepare_one_operand(PyArrayObject **op, npy_uint32 flags, npy_uint32 op_flags, npyiter_opitflags *op_itflags); static int -npyiter_prepare_operands(int nop, int first_maskna_op, +npyiter_prepare_operands(int nop, PyArrayObject **op_in, PyArrayObject **op, char **op_dataptr, @@ -44,8 +44,7 @@ npyiter_prepare_operands(int nop, int first_maskna_op, PyArray_Descr **op_dtype, npy_uint32 flags, npy_uint32 *op_flags, npyiter_opitflags *op_itflags, - npy_int8 *out_maskop, - npy_int8 *out_maskna_indices); + npy_int8 *out_maskop); static int npyiter_check_casting(int nop, PyArrayObject **op, PyArray_Descr **op_dtype, @@ -73,7 +72,7 @@ npyiter_reverse_axis_ordering(NpyIter *iter); static void npyiter_find_best_axis_ordering(NpyIter *iter); static PyArray_Descr * -npyiter_get_common_dtype(int first_maskna_op, PyArrayObject **op, +npyiter_get_common_dtype(int nop, PyArrayObject **op, npyiter_opitflags *op_itflags, PyArray_Descr **op_dtype, PyArray_Descr **op_request_dtypes, int only_inputs, int output_scalars); @@ -88,10 +87,8 @@ npyiter_allocate_arrays(NpyIter *iter, PyArray_Descr **op_dtype, PyTypeObject *subtype, npy_uint32 *op_flags, npyiter_opitflags *op_itflags, int **op_axes, int output_scalars); -static int -npyiter_fill_maskna_axisdata(NpyIter *iter, int **op_axes); static void -npyiter_get_priority_subtype(int first_maskna_op, PyArrayObject **op, +npyiter_get_priority_subtype(int nop, PyArrayObject **op, npyiter_opitflags *op_itflags, double *subtype_priority, PyTypeObject **subtype); static int @@ -112,7 +109,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, { npy_uint32 itflags = NPY_ITFLAG_IDENTPERM; int idim, ndim; - int iop, maskna_nop, first_maskna_op; + int iop; /* The iterator being constructed */ NpyIter *iter; @@ -186,18 +183,6 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, NPY_IT_TIME_POINT(c_calculate_ndim); - /* Calculate maskna_nop */ - maskna_nop = 0; - for (iop = 0; iop < nop; ++iop) { - if (op_flags[iop]&NPY_ITER_USE_MASKNA) { - itflags |= NPY_ITFLAG_HAS_MASKNA_OP; - ++maskna_nop; - } - } - /* Adjust nop to include the masks at the end */ - first_maskna_op = nop; - nop += maskna_nop; - /* Allocate memory for the iterator */ iter = (NpyIter*) PyArray_malloc(NIT_SIZEOF_ITERATOR(itflags, ndim, nop)); @@ -208,7 +193,6 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, NIT_ITFLAGS(iter) = itflags; NIT_NDIM(iter) = ndim; NIT_NOP(iter) = nop; - NIT_FIRST_MASKNA_OP(iter) = first_maskna_op; NIT_MASKOP(iter) = -1; NIT_ITERINDEX(iter) = 0; memset(NIT_BASEOFFSETS(iter), 0, (nop+1)*NPY_SIZEOF_INTP); @@ -219,12 +203,11 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, op_dataptr = NIT_RESETDATAPTR(iter); /* Prepare all the operands */ - if (!npyiter_prepare_operands(nop, first_maskna_op, op_in, op, op_dataptr, + if (!npyiter_prepare_operands(nop, op_in, op, op_dataptr, op_request_dtypes, op_dtype, flags, op_flags, op_itflags, - &NIT_MASKOP(iter), - NIT_MASKNA_INDICES(iter))) { + &NIT_MASKOP(iter))) { PyArray_free(iter); return NULL; } @@ -296,7 +279,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, NPY_IT_TIME_POINT(c_apply_forced_iteration_order); /* Set some flags for allocated outputs */ - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { if (op[iop] == NULL) { /* Flag this so later we can avoid flipping axes */ any_allocate = 1; @@ -335,7 +318,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, NPY_IT_TIME_POINT(c_find_best_axis_ordering); if (need_subtype) { - npyiter_get_priority_subtype(first_maskna_op, op, op_itflags, + npyiter_get_priority_subtype(nop, op, op_itflags, &subtype_priority, &subtype); } @@ -352,7 +335,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, op = NIT_OPERANDS(iter); op_dtype = NIT_DTYPES(iter); - dtype = npyiter_get_common_dtype(first_maskna_op, op, + dtype = npyiter_get_common_dtype(nop, op, op_itflags, op_dtype, op_request_dtypes, only_inputs, @@ -364,7 +347,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, if (flags & NPY_ITER_COMMON_DTYPE) { NPY_IT_DBG_PRINT("Iterator: Replacing all data types\n"); /* Replace all the data types */ - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { if (op_dtype[iop] != dtype) { Py_XDECREF(op_dtype[iop]); Py_INCREF(dtype); @@ -375,7 +358,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, else { NPY_IT_DBG_PRINT("Iterator: Setting unset output data types\n"); /* Replace the NULL data types */ - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { if (op_dtype[iop] == NULL) { Py_INCREF(dtype); op_dtype[iop] = dtype; @@ -392,8 +375,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, * to check that data type conversions are following the * casting rules. */ - if (!npyiter_check_casting(first_maskna_op, op, - op_dtype, casting, op_itflags)) { + if (!npyiter_check_casting(nop, op, op_dtype, casting, op_itflags)) { NpyIter_Deallocate(iter); return NULL; } @@ -412,58 +394,6 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, return NULL; } - /* - * If there were any NA masks added to the iteration, fill in - * the strides and other data they need. This is being done - * after all the 'allocate' arrays are finished. - */ - if (maskna_nop > 0) { - npy_int8 *maskna_indices = NIT_MASKNA_INDICES(iter); - - /* Copy op references to the maskna op's masks */ - for (iop = first_maskna_op; iop < nop; ++iop) { - int iop_maskna = maskna_indices[iop]; - op[iop] = op[iop_maskna]; - Py_INCREF(op[iop]); - /* If the operand has a mask, use its dtype */ - if (PyArray_HASMASKNA(op[iop])) { - op_dtype[iop] = PyArray_MASKNA_DTYPE(op[iop]); - Py_INCREF(op_dtype[iop]); - } - /* Otherwise a virtual all-ones operand will be used */ - else { - if (PyArray_HASFIELDS(op[iop])) { - PyErr_SetString(PyExc_ValueError, - "struct-NA is not supported yet"); - NpyIter_Deallocate(iter); - return NULL; - } - else { - op_dtype[iop] = PyArray_DescrFromType(NPY_BOOL); - if (op_dtype[iop] == NULL) { - NpyIter_Deallocate(iter); - return NULL; - } - } - } - /* Propagate select flags from the main operand */ - op_itflags[iop] = op_itflags[iop_maskna] & - (NPY_OP_ITFLAG_WRITE | - NPY_OP_ITFLAG_READ | - NPY_OP_ITFLAG_BUFNEVER | - NPY_OP_ITFLAG_REDUCE | - NPY_OP_ITFLAG_WRITEMASKED); - /* The mask is always aligned (alignment 1) */ - op_itflags[iop] |= NPY_OP_ITFLAG_ALIGNED; - } - - /* Fill in the strides for the masks */ - if (!npyiter_fill_maskna_axisdata(iter, op_axes)) { - NpyIter_Deallocate(iter); - return NULL; - } - } - NPY_IT_TIME_POINT(c_allocate_arrays); /* @@ -507,7 +437,7 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, * reference arrays and flag it if so. */ if (flags & NPY_ITER_REFS_OK) { - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { PyArray_Descr *rdt = op_dtype[iop]; if ((rdt->flags & (NPY_ITEM_REFCOUNT | NPY_ITEM_IS_POINTER | @@ -986,18 +916,6 @@ npyiter_check_per_op_flags(npy_uint32 op_flags, npyiter_opitflags *op_itflags) "be used together with ARRAYMASK"); return 0; } - /* - * When WRITEMASKED and USE_MASKNA are supported together, - * it will probably require that buffering is enabled as well, - * because that will need yet another temporary mask buffer to combine - * the two masks before doing the masked copies. - */ - if ((op_flags & NPY_ITER_USE_MASKNA) != 0) { - PyErr_SetString(PyExc_ValueError, - "The combination of iterator flags WRITEMASKED " - "and USE_MASKNA is not yet supported"); - return 0; - } *op_itflags |= NPY_OP_ITFLAG_WRITEMASKED; } @@ -1108,27 +1026,6 @@ npyiter_prepare_one_operand(PyArrayObject **op, "Iteration of zero-sized operands is not enabled"); return 0; } - /* - * Writeable USE_MASKNA operands must have a mask - * (or NA dtype, later) - */ - if ((op_flags & NPY_ITER_USE_MASKNA) != 0 && - ((*op_itflags) & NPY_OP_ITFLAG_WRITE) != 0 && - !PyArray_HASMASKNA(*op)) { - PyErr_SetString(PyExc_ValueError, - "Operand is writeable and flagged USE_MASKNA, " - "but the operand does not have an NA mask"); - return 0; - } - /* Arrays with NA masks must have USE_MASKNA specified */ - tmp = op_flags & (NPY_ITER_USE_MASKNA | NPY_ITER_IGNORE_MASKNA); - if (tmp != NPY_ITER_USE_MASKNA && tmp != NPY_ITER_IGNORE_MASKNA && - PyArray_HASMASKNA(*op)) { - PyErr_SetString(PyExc_ValueError, - "Operand has an NA mask but the operation does " - "not support NA via the flag USE_MASKNA"); - return 0; - } *op_dataptr = PyArray_BYTES(*op); /* PyArray_DESCR does not give us a reference */ *op_dtype = PyArray_DESCR(*op); @@ -1220,26 +1117,23 @@ npyiter_prepare_one_operand(PyArrayObject **op, * can replace the arrays if copying is necessary. */ static int -npyiter_prepare_operands(int nop, int first_maskna_op, PyArrayObject **op_in, +npyiter_prepare_operands(int nop, PyArrayObject **op_in, PyArrayObject **op, char **op_dataptr, PyArray_Descr **op_request_dtypes, PyArray_Descr **op_dtype, npy_uint32 flags, npy_uint32 *op_flags, npyiter_opitflags *op_itflags, - npy_int8 *out_maskop, - npy_int8 *out_maskna_indices) + npy_int8 *out_maskop) { - int iop, iop_maskna = first_maskna_op, i; + int iop, i; npy_int8 maskop = -1; int any_writemasked_ops = 0; /* - * Here we just prepare the provided operands. The masks for - * the maskna operands get prepared later, after any 'allocate' - * operands are allocated. + * Here we just prepare the provided operands. */ - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { op[iop] = op_in[iop]; Py_XINCREF(op[iop]); op_dtype[iop] = NULL; @@ -1274,16 +1168,6 @@ npyiter_prepare_operands(int nop, int first_maskna_op, PyArrayObject **op_in, any_writemasked_ops = 1; } - /* Link the operands to their maskna operands */ - if (op_flags[iop] & NPY_ITER_USE_MASKNA) { - out_maskna_indices[iop] = iop_maskna; - out_maskna_indices[iop_maskna] = iop; - ++iop_maskna; - } - else { - out_maskna_indices[iop] = -1; - } - /* * Prepare the operand. This produces an op_dtype[iop] reference * on success. @@ -1302,17 +1186,10 @@ npyiter_prepare_operands(int nop, int first_maskna_op, PyArrayObject **op_in, } } - /* Initialize the mask virtual operands to NULL for now */ - for (iop = first_maskna_op; iop < nop; ++iop) { - op[iop] = NULL; - op_dataptr[iop] = NULL; - op_dtype[iop] = NULL; - } - /* If all the operands were NULL, it's an error */ if (op[0] == NULL) { int all_null = 1; - for (iop = 1; iop < first_maskna_op; ++iop) { + for (iop = 1; iop < nop; ++iop) { if (op[iop] != NULL) { all_null = 0; break; @@ -1416,14 +1293,14 @@ npyiter_shape_string(npy_intp n, npy_intp *vals, char *ending) } static int -npyiter_check_casting(int first_maskna_op, PyArrayObject **op, +npyiter_check_casting(int nop, PyArrayObject **op, PyArray_Descr **op_dtype, NPY_CASTING casting, npyiter_opitflags *op_itflags) { int iop; - for(iop = 0; iop < first_maskna_op; ++iop) { + for(iop = 0; iop < nop; ++iop) { NPY_IT_DBG_PRINT1("Iterator: Checking casting for operand %d\n", (int)iop); #if NPY_IT_DBG_TRACING @@ -1566,7 +1443,6 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf npy_uint32 itflags = NIT_ITFLAGS(iter); int idim, ndim = NIT_NDIM(iter); int iop, nop = NIT_NOP(iter); - int first_maskna_op = NIT_FIRST_MASKNA_OP(iter); int maskop = NIT_MASKOP(iter); int ondim; @@ -1590,7 +1466,7 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf } } } - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { op_cur = op[iop]; if (op_cur != NULL) { npy_intp *shape = PyArray_DIMS(op_cur); @@ -1671,8 +1547,7 @@ npyiter_fill_axisdata(NpyIter *iter, npy_uint32 flags, npyiter_opitflags *op_itf NAD_INDEX(axisdata) = 0; memcpy(NAD_PTRS(axisdata), op_dataptr, NPY_SIZEOF_INTP*nop); - /* Not processing the maskna masks until after allocation */ - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { op_cur = op[iop]; if (op_axes == NULL || op_axes[iop] == NULL) { @@ -1825,7 +1700,7 @@ broadcast_error: { if (errmsg == NULL) { return 0; } - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { if (op[iop] != NULL) { tmp = npyiter_shape_string(PyArray_NDIM(op[iop]), PyArray_DIMS(op[iop]), @@ -1868,7 +1743,7 @@ broadcast_error: { errmsg = PyUString_FromString("operands could not be broadcast " "together with remapped shapes " "[original->remapped]: "); - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { if (op[iop] != NULL) { int *axes = op_axes[iop]; @@ -2206,7 +2081,6 @@ npyiter_apply_forced_iteration_order(NpyIter *iter, NPY_ORDER order) /*npy_uint32 itflags = NIT_ITFLAGS(iter);*/ int ndim = NIT_NDIM(iter); int iop, nop = NIT_NOP(iter); - int first_maskna_op = NIT_FIRST_MASKNA_OP(iter); switch (order) { case NPY_CORDER: @@ -2227,7 +2101,7 @@ npyiter_apply_forced_iteration_order(NpyIter *iter, NPY_ORDER order) int forder = 1; /* Check that all the array inputs are fortran order */ - for (iop = 0; iop < first_maskna_op; ++iop, ++op) { + for (iop = 0; iop < nop; ++iop, ++op) { if (*op && !PyArray_CHKFLAGS(*op, NPY_ARRAY_F_CONTIGUOUS)) { forder = 0; break; @@ -2257,7 +2131,6 @@ npyiter_flip_negative_strides(NpyIter *iter) npy_uint32 itflags = NIT_ITFLAGS(iter); int idim, ndim = NIT_NDIM(iter); int iop, nop = NIT_NOP(iter); - int first_maskna_op = NIT_FIRST_MASKNA_OP(iter); npy_intp istrides, nstrides = NAD_NSTRIDES(); NpyIter_AxisData *axisdata, *axisdata0; @@ -2274,7 +2147,7 @@ npyiter_flip_negative_strides(NpyIter *iter) /* * Check the signs of all the operand strides. */ - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { if (strides[iop] < 0) { any_negative = 1; } @@ -2286,7 +2159,7 @@ npyiter_flip_negative_strides(NpyIter *iter) * If at least one stride is negative and none are positive, * flip all the strides for this dimension. */ - if (any_negative && iop == first_maskna_op) { + if (any_negative && iop == nop) { npy_intp shapem1 = NAD_SHAPE(axisdata) - 1; for (istrides = 0; istrides < nstrides; ++istrides) { @@ -2380,7 +2253,6 @@ npyiter_find_best_axis_ordering(NpyIter *iter) npy_uint32 itflags = NIT_ITFLAGS(iter); int idim, ndim = NIT_NDIM(iter); int iop, nop = NIT_NOP(iter); - int first_maskna_op = NIT_FIRST_MASKNA_OP(iter); npy_intp ax_i0, ax_i1, ax_ipos; npy_int8 ax_j0, ax_j1; @@ -2412,7 +2284,7 @@ npyiter_find_best_axis_ordering(NpyIter *iter) strides1 = NAD_STRIDES(NIT_INDEX_AXISDATA(axisdata, ax_j1)); - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { if (strides0[iop] != 0 && strides1[iop] != 0) { if (intp_abs(strides1[iop]) <= intp_abs(strides0[iop])) { @@ -2519,7 +2391,7 @@ npyiter_find_best_axis_ordering(NpyIter *iter) * are not read from out of the calculation. */ static PyArray_Descr * -npyiter_get_common_dtype(int first_maskna_op, PyArrayObject **op, +npyiter_get_common_dtype(int nop, PyArrayObject **op, npyiter_opitflags *op_itflags, PyArray_Descr **op_dtype, PyArray_Descr **op_request_dtypes, int only_inputs, int output_scalars) @@ -2532,7 +2404,7 @@ npyiter_get_common_dtype(int first_maskna_op, PyArrayObject **op, NPY_IT_DBG_PRINT("Iterator: Getting a common data type from operands\n"); - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { if (op_dtype[iop] != NULL && (!only_inputs || (op_itflags[iop] & NPY_OP_ITFLAG_READ))) { /* If no dtype was requested and the op is a scalar, pass the op */ @@ -2826,7 +2698,6 @@ npyiter_allocate_arrays(NpyIter *iter, npy_uint32 itflags = NIT_ITFLAGS(iter); int idim, ndim = NIT_NDIM(iter); int iop, nop = NIT_NOP(iter); - int first_maskna_op = NIT_FIRST_MASKNA_OP(iter); int check_writemasked_reductions = 0; @@ -2837,7 +2708,7 @@ npyiter_allocate_arrays(NpyIter *iter, bufferdata = NIT_BUFFERDATA(iter); } - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { /* * Check whether there are any WRITEMASKED REDUCE operands * which should be validated after all the strides are filled @@ -2873,17 +2744,6 @@ npyiter_allocate_arrays(NpyIter *iter, op[iop] = out; /* - * Add an NA mask to the array if needed. When NA dtypes - * are supported, this should skip allocating the mask - * if the allocated array has an NA dtype. - */ - if (op_flags[iop] & NPY_ITER_USE_MASKNA) { - if (PyArray_AllocateMaskNA(out, 1, 0, 1) < 0) { - return 0; - } - } - - /* * Now we need to replace the pointers and strides with values * from the new array. */ @@ -2912,15 +2772,6 @@ npyiter_allocate_arrays(NpyIter *iter, if (temp == NULL) { return 0; } - /* - * Add an NA mask if needed, defaulting to all NAs because - * the data is uninitialized - */ - if (PyArray_HASMASKNA(op[iop])) { - if (PyArray_AllocateMaskNA(temp, 1, 0, 0) < 0) { - return 0; - } - } if (PyArray_CopyInto(temp, op[iop]) != 0) { Py_DECREF(temp); return 0; @@ -2962,12 +2813,6 @@ npyiter_allocate_arrays(NpyIter *iter, if (temp == NULL) { return 0; } - /* Add an NA mask if needed */ - if (PyArray_HASMASKNA(op[iop])) { - if (PyArray_AllocateMaskNA(temp, 1, 0, 1) < 0) { - return 0; - } - } /* * If the data will be read, copy it into temp. @@ -3103,7 +2948,7 @@ npyiter_allocate_arrays(NpyIter *iter, } if (check_writemasked_reductions) { - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { /* * Check whether there are any WRITEMASKED REDUCE operands * which should be validated now that all the strides are filled @@ -3131,167 +2976,19 @@ npyiter_allocate_arrays(NpyIter *iter, } /* - * Prepares the maskna virtual operands for the constructor - * operands, and fills in the axisdata. Fills in 'op_maskna_dataptr', - * 'op_maskna_dtype', and may modify 'op_maskna_itflags'. - * - * This needs to be called after any 'allocate' operands have - * been allocated. There is no validation of the shape/strides done, - * because the shape of a mask exactly matches the shape of the - * operand to which it attached. - * - * Returns 1 on success, 0 on failure. - */ -static int -npyiter_fill_maskna_axisdata(NpyIter *iter, int **op_axes) -{ - npy_uint32 itflags = NIT_ITFLAGS(iter); - int idim, ndim = NIT_NDIM(iter); - int iop, iop_maskna, nop = NIT_NOP(iter); - int first_maskna_op = NIT_FIRST_MASKNA_OP(iter); - npy_int8 *perm; - - npyiter_opitflags *op_itflags = NIT_OPITFLAGS(iter); - npy_int8 *maskna_indices = NIT_MASKNA_INDICES(iter); - NpyIter_AxisData *axisdata; - npy_intp sizeof_axisdata; - PyArrayObject **op = NIT_OPERANDS(iter), *op_cur; - char **op_dataptr = NIT_RESETDATAPTR(iter); - NpyIter_BufferData *bufferdata = NULL; - - axisdata = NIT_AXISDATA(iter); - sizeof_axisdata = NIT_AXISDATA_SIZEOF(itflags, ndim, nop); - perm = NIT_PERM(iter); - - if (itflags & NPY_ITFLAG_BUFFER) { - bufferdata = NIT_BUFFERDATA(iter); - } - - /* Fill in the reset dataptr array with the mask pointers */ - for (iop = first_maskna_op; iop < nop; ++iop) { - /* If there's a mask, process that */ - if (PyArray_HASMASKNA(op[iop])) { - op_dataptr[iop] = PyArray_MASKNA_DATA(op[iop]); - } - /* - * Otherwise we create a virtual operand a single one value - * broadcast everywhere - */ - else { - static char ones_virtual_mask_data = 1; - - op_itflags[iop] |= (NPY_OP_ITFLAG_VIRTUAL | - NPY_OP_ITFLAG_BUFNEVER); - op_dataptr[iop] = &ones_virtual_mask_data; - if (itflags & NPY_ITFLAG_BUFFER) { - NBF_PTRS(bufferdata)[iop] = op_dataptr[iop]; - } - } - } - - /* Process the maskna operands, filling in the axisdata */ - for (idim = 0; idim < ndim; ++idim) { - npy_intp *strides = NAD_STRIDES(axisdata); - npy_int8 p; - int idim_permuted; - - p = perm[idim]; - if (p < 0) { - idim_permuted = -1-p; - } - else { - idim_permuted = p; - } - for (iop = first_maskna_op; iop < nop; ++iop) { - /* - * iop_maskna is the index of the USE_MASKNA input, - * iop is the index of the corresponding mask. - */ - iop_maskna = maskna_indices[iop]; - op_cur = op[iop_maskna]; - - /* - * The strides of the mask will be zero exactly - * where they're zero for the main data, or will - * be zero always if the operand has no NA support and - * a virtual mask of all ones is being used. - */ - if (strides[iop_maskna] == 0 || !PyArray_HASMASKNA(op_cur)) { - strides[iop] = 0; - } - else { - int i; - - if (op_axes == NULL || op_axes[iop_maskna] == NULL) { - i = PyArray_NDIM(op_cur) - idim_permuted - 1; - } - else { - i = op_axes[iop_maskna][ndim-idim_permuted-1]; - } - - strides[iop] = PyArray_MASKNA_STRIDES(op_cur)[i]; - /* Reverse the axis if necessary */ - if (p < 0) { - op_dataptr[iop] += (NAD_SHAPE(axisdata)-1) * strides[iop]; - strides[iop] = -strides[iop]; - } - } - } - - NIT_ADVANCE_AXISDATA(axisdata, 1); - } - - /* Initialize the mask data pointers */ - axisdata = NIT_AXISDATA(iter); - for (idim = 0; idim < ndim; ++idim) { - memcpy(NAD_PTRS(axisdata) + first_maskna_op, - op_dataptr + first_maskna_op, - NPY_SIZEOF_INTP*(nop - first_maskna_op)); - - NIT_ADVANCE_AXISDATA(axisdata, 1); - } - - /* Initialize the strides of any BUFNEVER mask operands */ - if (itflags & NPY_ITFLAG_BUFFER) { - npy_intp *strides = NBF_STRIDES(bufferdata); - - for (iop = first_maskna_op; iop < nop; ++iop) { - if (op_itflags[iop] & NPY_OP_ITFLAG_BUFNEVER) { - if (PyArray_HASMASKNA(op[iop])) { - axisdata = NIT_AXISDATA(iter); - /* Find stride of the first non-empty shape */ - for (idim = 0; idim < ndim; ++idim) { - if (NAD_SHAPE(axisdata) != 1) { - strides[iop] = NAD_STRIDES(axisdata)[iop]; - break; - } - NIT_ADVANCE_AXISDATA(axisdata, 1); - } - } - else { - strides[iop] = 0; - } - } - } - } - - return 1; -} - -/* * The __array_priority__ attribute of the inputs determines * the subtype of any output arrays. This function finds the * subtype of the input array with highest priority. */ static void -npyiter_get_priority_subtype(int first_maskna_op, PyArrayObject **op, +npyiter_get_priority_subtype(int nop, PyArrayObject **op, npyiter_opitflags *op_itflags, double *subtype_priority, PyTypeObject **subtype) { int iop; - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { if (op[iop] != NULL && op_itflags[iop] & NPY_OP_ITFLAG_READ) { double priority = PyArray_GetPriority((PyObject *)op[iop], 0.0); if (priority > *subtype_priority) { @@ -3308,7 +3005,6 @@ npyiter_allocate_transfer_functions(NpyIter *iter) npy_uint32 itflags = NIT_ITFLAGS(iter); /*int ndim = NIT_NDIM(iter);*/ int iop = 0, nop = NIT_NOP(iter); - int first_maskna_op = NIT_FIRST_MASKNA_OP(iter); npy_intp i; npyiter_opitflags *op_itflags = NIT_OPITFLAGS(iter); @@ -3321,7 +3017,6 @@ npyiter_allocate_transfer_functions(NpyIter *iter) **writetransferfn = NBF_WRITETRANSFERFN(bufferdata); NpyAuxData **readtransferdata = NBF_READTRANSFERDATA(bufferdata), **writetransferdata = NBF_WRITETRANSFERDATA(bufferdata); - npy_int8 *maskna_indices = NIT_MASKNA_INDICES(iter); PyArray_StridedUnaryOp *stransfer = NULL; NpyAuxData *transferdata = NULL; @@ -3341,18 +3036,13 @@ npyiter_allocate_transfer_functions(NpyIter *iter) * allocate the appropriate transfer functions */ if (!(flags & NPY_OP_ITFLAG_BUFNEVER)) { - PyArray_Descr *op_orig_dtype; - - /* Get either the array's or the array namask's dtype */ - op_orig_dtype = (iop < first_maskna_op) ? PyArray_DESCR(op[iop]) - : PyArray_MASKNA_DTYPE(op[iop]); if (flags & NPY_OP_ITFLAG_READ) { int move_references = 0; if (PyArray_GetDTypeTransferFunction( (flags & NPY_OP_ITFLAG_ALIGNED) != 0, op_stride, op_dtype[iop]->elsize, - op_orig_dtype, + PyArray_DESCR(op[iop]), op_dtype[iop], move_references, &stransfer, @@ -3369,40 +3059,8 @@ npyiter_allocate_transfer_functions(NpyIter *iter) if (flags & NPY_OP_ITFLAG_WRITE) { int move_references = 1; - /* - * If the operand has USE_MASKNA, use a masked transfer fn. - * The masks for the maskna operands can be copied straight - * unless the operand is also WRITEMASKED. - */ - if (iop < first_maskna_op && maskna_indices[iop] >= 0) { - /* TODO: support USE_MASKNA + WRITEMASKED together */ - PyArray_Descr *mask_dtype = - PyArray_MASKNA_DTYPE(op[iop]); - int iop_maskna = maskna_indices[iop]; - - /* - * If the mask's stride is contiguous, use it, otherwise - * the mask may or may not be buffered, so the stride - * could be inconsistent. - */ - if (PyArray_GetMaskedDTypeTransferFunction( - (flags & NPY_OP_ITFLAG_ALIGNED) != 0, - op_dtype[iop]->elsize, - op_stride, - (flags & NPY_OP_ITFLAG_REDUCE) ? NPY_MAX_INTP : - strides[iop_maskna], - op_dtype[iop], - op_orig_dtype, - mask_dtype, - move_references, - (PyArray_MaskedStridedUnaryOp **)&stransfer, - &transferdata, - &needs_api) != NPY_SUCCEED) { - goto fail; - } - } /* If the operand is WRITEMASKED, use a masked transfer fn */ - else if (flags & NPY_OP_ITFLAG_WRITEMASKED) { + if (flags & NPY_OP_ITFLAG_WRITEMASKED) { int maskop = NIT_MASKOP(iter); PyArray_Descr *mask_dtype = PyArray_DESCR(op[maskop]); @@ -3419,7 +3077,7 @@ npyiter_allocate_transfer_functions(NpyIter *iter) mask_dtype->elsize : NPY_MAX_INTP, op_dtype[iop], - op_orig_dtype, + PyArray_DESCR(op[iop]), mask_dtype, move_references, (PyArray_MaskedStridedUnaryOp **)&stransfer, @@ -3434,7 +3092,7 @@ npyiter_allocate_transfer_functions(NpyIter *iter) op_dtype[iop]->elsize, op_stride, op_dtype[iop], - op_orig_dtype, + PyArray_DESCR(op[iop]), move_references, &stransfer, &transferdata, diff --git a/numpy/core/src/multiarray/nditer_impl.h b/numpy/core/src/multiarray/nditer_impl.h index ef10308fb..bc4d01ff9 100644 --- a/numpy/core/src/multiarray/nditer_impl.h +++ b/numpy/core/src/multiarray/nditer_impl.h @@ -101,8 +101,6 @@ #define NPY_ITFLAG_REDUCE 0x1000 /* Reduce iteration doesn't need to recalculate reduce loops next time */ #define NPY_ITFLAG_REUSE_REDUCE_LOOPS 0x2000 -/* The iterator has one or more operands with NPY_ITER_USE_MASKNA set */ -#define NPY_ITFLAG_HAS_MASKNA_OP 0x4000 /* Internal iterator per-operand iterator flags */ @@ -137,7 +135,7 @@ struct NpyIter_InternalOnly { /* Initial fixed position data */ npy_uint32 itflags; - npy_uint8 ndim, nop, first_maskna_op; + npy_uint8 ndim, nop; npy_int8 maskop; npy_intp itersize, iterstart, iterend; /* iterindex is only used if RANGED or BUFFERED is set */ @@ -154,8 +152,6 @@ typedef npy_int16 npyiter_opitflags; /* Byte sizes of the iterator members */ #define NIT_PERM_SIZEOF(itflags, ndim, nop) \ NPY_INTP_ALIGNED(NPY_MAXDIMS) -#define NIT_MASKNA_INDICES_SIZEOF(itflags, ndim, nop) \ - NPY_INTP_ALIGNED(NPY_MAXDIMS) #define NIT_DTYPES_SIZEOF(itflags, ndim, nop) \ ((NPY_SIZEOF_INTP)*(nop)) #define NIT_RESETDATAPTR_SIZEOF(itflags, ndim, nop) \ @@ -172,12 +168,9 @@ typedef npy_int16 npyiter_opitflags; /* Byte offsets of the iterator members starting from iter->iter_flexdata */ #define NIT_PERM_OFFSET() \ (0) -#define NIT_MASKNA_INDICES_OFFSET(itflags, ndim, nop) \ +#define NIT_DTYPES_OFFSET(itflags, ndim, nop) \ (NIT_PERM_OFFSET() + \ NIT_PERM_SIZEOF(itflags, ndim, nop)) -#define NIT_DTYPES_OFFSET(itflags, ndim, nop) \ - (NIT_MASKNA_INDICES_OFFSET(itflags, ndim, nop) + \ - NIT_MASKNA_INDICES_SIZEOF(itflags, ndim, nop)) #define NIT_RESETDATAPTR_OFFSET(itflags, ndim, nop) \ (NIT_DTYPES_OFFSET(itflags, ndim, nop) + \ NIT_DTYPES_SIZEOF(itflags, ndim, nop)) @@ -204,8 +197,6 @@ typedef npy_int16 npyiter_opitflags; ((iter)->ndim) #define NIT_NOP(iter) \ ((iter)->nop) -#define NIT_FIRST_MASKNA_OP(iter) \ - ((iter)->first_maskna_op) #define NIT_MASKOP(iter) \ ((iter)->maskop) #define NIT_ITERSIZE(iter) \ @@ -218,8 +209,6 @@ typedef npy_int16 npyiter_opitflags; (iter->iterindex) #define NIT_PERM(iter) ((npy_int8 *)( \ &(iter)->iter_flexdata + NIT_PERM_OFFSET())) -#define NIT_MASKNA_INDICES(iter) ((npy_int8 *)( \ - &(iter)->iter_flexdata + NIT_MASKNA_INDICES_OFFSET(itflags, ndim, nop))) #define NIT_DTYPES(iter) ((PyArray_Descr **)( \ &(iter)->iter_flexdata + NIT_DTYPES_OFFSET(itflags, ndim, nop))) #define NIT_RESETDATAPTR(iter) ((char **)( \ diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 3a6c632e0..d6029e814 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -374,11 +374,6 @@ NpyIter_OpFlagsConverter(PyObject *op_flags_in, flag = NPY_ITER_UPDATEIFCOPY; } break; - case 's': - if (strcmp(str, "use_maskna") == 0) { - flag = NPY_ITER_USE_MASKNA; - } - break; } break; case 'v': @@ -618,7 +613,6 @@ npyiter_convert_ops(PyObject *op_in, PyObject *op_flags_in, int *nop_out) { int iop, nop; - int any_maskna; /* nop and op */ if (PyTuple_Check(op_in) || PyList_Check(op_in)) { @@ -688,7 +682,7 @@ npyiter_convert_ops(PyObject *op_in, PyObject *op_flags_in, for (iop = 0; iop < nop; ++iop) { if (op[iop] != NULL) { PyArrayObject *ao; - int fromanyflags = NPY_ARRAY_ALLOWNA; + int fromanyflags = 0; if (op_flags[iop]&(NPY_ITER_READWRITE|NPY_ITER_WRITEONLY)) { fromanyflags |= NPY_ARRAY_UPDATEIFCOPY; @@ -714,33 +708,6 @@ npyiter_convert_ops(PyObject *op_in, PyObject *op_flags_in, } } - /* - * Because the Python exposure of nditer knows how to deal with - * NA-masked arrays, we automatically add NPY_ITER_USE_MASKNA - * flags for convenience. - */ - any_maskna = 0; - for (iop = 0; iop < nop; ++iop) { - /* Enable MASKNA iteration if the op needs it */ - if (op[iop] != NULL && PyArray_HASMASKNA(op[iop])) { - op_flags[iop] |= NPY_ITER_USE_MASKNA; - any_maskna = 1; - } - } - /* - * If any operands had an NA-mask, add it to the 'allocate' ones too. - * This causes the Python exposure nditer to have slightly less control - * than the C NpyIter usage, but is generally going to be what people - * want. - */ - if (any_maskna) { - for (iop = 0; iop < nop; ++iop) { - if (op[iop] == NULL) { - op_flags[iop] |= NPY_ITER_USE_MASKNA; - } - } - } - return 1; } @@ -1420,7 +1387,7 @@ static PyObject *npyiter_value_get(NewNpyArrayIterObject *self) { PyObject *ret; - npy_intp iop, first_maskna_op; + npy_intp iop, nop; if (self->iter == NULL || self->finished) { PyErr_SetString(PyExc_ValueError, @@ -1428,18 +1395,18 @@ static PyObject *npyiter_value_get(NewNpyArrayIterObject *self) return NULL; } - first_maskna_op = NpyIter_GetFirstMaskNAOp(self->iter); + nop = NpyIter_GetNOp(self->iter); /* Return an array or tuple of arrays with the values */ - if (first_maskna_op == 1) { + if (nop == 1) { ret = npyiter_seq_item(self, 0); } else { - ret = PyTuple_New(first_maskna_op); + ret = PyTuple_New(nop); if (ret == NULL) { return NULL; } - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { PyObject *a = npyiter_seq_item(self, iop); if (a == NULL) { Py_DECREF(ret); @@ -1456,7 +1423,7 @@ static PyObject *npyiter_operands_get(NewNpyArrayIterObject *self) { PyObject *ret; - npy_intp iop, first_maskna_op; + npy_intp iop, nop; PyArrayObject **operands; if (self->iter == NULL) { @@ -1465,14 +1432,14 @@ static PyObject *npyiter_operands_get(NewNpyArrayIterObject *self) return NULL; } - first_maskna_op = NpyIter_GetFirstMaskNAOp(self->iter); + nop = NpyIter_GetNOp(self->iter); operands = self->operands; - ret = PyTuple_New(first_maskna_op); + ret = PyTuple_New(nop); if (ret == NULL) { return NULL; } - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { PyObject *operand = (PyObject *)operands[iop]; Py_INCREF(operand); @@ -1486,7 +1453,7 @@ static PyObject *npyiter_itviews_get(NewNpyArrayIterObject *self) { PyObject *ret; - npy_intp iop, first_maskna_op; + npy_intp iop, nop; if (self->iter == NULL) { PyErr_SetString(PyExc_ValueError, @@ -1494,13 +1461,13 @@ static PyObject *npyiter_itviews_get(NewNpyArrayIterObject *self) return NULL; } - first_maskna_op = NpyIter_GetFirstMaskNAOp(self->iter); + nop = NpyIter_GetNOp(self->iter); - ret = PyTuple_New(first_maskna_op); + ret = PyTuple_New(nop); if (ret == NULL) { return NULL; } - for (iop = 0; iop < first_maskna_op; ++iop) { + for (iop = 0; iop < nop; ++iop) { PyArrayObject *view = NpyIter_GetIterView(self->iter, iop); if (view == NULL) { @@ -1951,11 +1918,7 @@ static PyObject *npyiter_nop_get(NewNpyArrayIterObject *self) return NULL; } - /* - * We only expose the provided operands, which is everything - * before the first MASKNA operand. - */ - return PyInt_FromLong(NpyIter_GetFirstMaskNAOp(self->iter)); + return PyInt_FromLong(NpyIter_GetNOp(self->iter)); } static PyObject *npyiter_itersize_get(NewNpyArrayIterObject *self) @@ -1986,11 +1949,7 @@ npyiter_seq_length(NewNpyArrayIterObject *self) return 0; } else { - /* - * We only expose the provided operands, which is everything - * before the first MASKNA operand. - */ - return NpyIter_GetFirstMaskNAOp(self->iter); + return NpyIter_GetNOp(self->iter); } } @@ -1999,7 +1958,6 @@ npyiter_seq_item(NewNpyArrayIterObject *self, Py_ssize_t i) { PyArrayObject *ret; - npy_int8 *maskna_indices; npy_intp ret_ndim; npy_intp nop, innerloopsize, innerstride; char *dataptr; @@ -2020,11 +1978,7 @@ npyiter_seq_item(NewNpyArrayIterObject *self, Py_ssize_t i) return NULL; } - /* - * We only expose the provided operands, which is everything - * before the first MASKNA operand. - */ - nop = NpyIter_GetFirstMaskNAOp(self->iter); + nop = NpyIter_GetNOp(self->iter); /* Negative indexing */ if (i < 0) { @@ -2055,7 +2009,6 @@ npyiter_seq_item(NewNpyArrayIterObject *self, Py_ssize_t i) dataptr = self->dataptrs[i]; dtype = self->dtypes[i]; has_external_loop = NpyIter_HasExternalLoop(self->iter); - maskna_indices = NpyIter_GetMaskNAIndexArray(self->iter); if (has_external_loop) { innerloopsize = *self->innerloopsizeptr; @@ -2080,22 +2033,6 @@ npyiter_seq_item(NewNpyArrayIterObject *self, Py_ssize_t i) return NULL; } - /* If this is a USE_MASKNA operand, include the mask */ - if (maskna_indices[i] >= 0) { - PyArrayObject_fields *fret = (PyArrayObject_fields *)ret; - int i_maskna = maskna_indices[i]; - - fret->maskna_dtype = NpyIter_GetDescrArray(self->iter)[i_maskna]; - Py_INCREF(fret->maskna_dtype); - fret->maskna_data = self->dataptrs[i_maskna]; - if (has_external_loop) { - fret->maskna_strides[0] = self->innerstrides[i_maskna]; - } - - fret->flags |= NPY_ARRAY_MASKNA; - fret->flags &= ~NPY_ARRAY_OWNMASKNA; - } - PyArray_UpdateFlags(ret, NPY_ARRAY_UPDATE_ALL); return (PyObject *)ret; @@ -2122,11 +2059,7 @@ npyiter_seq_slice(NewNpyArrayIterObject *self, return NULL; } - /* - * We only expose the provided operands, which is everything - * before the first MASKNA operand. - */ - nop = NpyIter_GetFirstMaskNAOp(self->iter); + nop = NpyIter_GetNOp(self->iter); if (ilow < 0) { ilow = 0; } @@ -2160,7 +2093,6 @@ npyiter_seq_ass_item(NewNpyArrayIterObject *self, Py_ssize_t i, PyObject *v) { npy_intp nop, innerloopsize, innerstride; - npy_int8 *maskna_indices; char *dataptr; PyArray_Descr *dtype; PyArrayObject *tmp; @@ -2187,11 +2119,7 @@ npyiter_seq_ass_item(NewNpyArrayIterObject *self, Py_ssize_t i, PyObject *v) return -1; } - /* - * We only expose the provided operands, which is everything - * before the first MASKNA operand. - */ - nop = NpyIter_GetFirstMaskNAOp(self->iter); + nop = NpyIter_GetNOp(self->iter); /* Negative indexing */ if (i < 0) { @@ -2222,8 +2150,6 @@ npyiter_seq_ass_item(NewNpyArrayIterObject *self, Py_ssize_t i, PyObject *v) innerstride = 0; } - maskna_indices = NpyIter_GetMaskNAIndexArray(self->iter); - /* TODO - there should be a better way than this... */ Py_INCREF(dtype); tmp = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, @@ -2233,24 +2159,6 @@ npyiter_seq_ass_item(NewNpyArrayIterObject *self, Py_ssize_t i, PyObject *v) if (tmp == NULL) { return -1; } - /* If this is a USE_MASKNA operand, include the mask */ - if (maskna_indices[i] >= 0) { - PyArrayObject_fields *ftmp = (PyArrayObject_fields *)tmp; - int i_maskna = maskna_indices[i]; - - ftmp->maskna_dtype = NpyIter_GetDescrArray(self->iter)[i_maskna]; - Py_INCREF(ftmp->maskna_dtype); - ftmp->maskna_data = self->dataptrs[i_maskna]; - if (has_external_loop) { - ftmp->maskna_strides[0] = self->innerstrides[i_maskna]; - } - else { - ftmp->maskna_strides[0] = 0; - } - - ftmp->flags |= NPY_ARRAY_MASKNA; - ftmp->flags &= ~NPY_ARRAY_OWNMASKNA; - } PyArray_UpdateFlags(tmp, NPY_ARRAY_UPDATE_ALL); @@ -2285,11 +2193,7 @@ npyiter_seq_ass_slice(NewNpyArrayIterObject *self, Py_ssize_t ilow, return -1; } - /* - * We only expose the provided operands, which is everything - * before the first MASKNA operand. - */ - nop = NpyIter_GetFirstMaskNAOp(self->iter); + nop = NpyIter_GetNOp(self->iter); if (ilow < 0) { ilow = 0; } diff --git a/numpy/core/src/multiarray/reduction.c b/numpy/core/src/multiarray/reduction.c index fc62c9e1f..f466c6d0d 100644 --- a/numpy/core/src/multiarray/reduction.c +++ b/numpy/core/src/multiarray/reduction.c @@ -18,7 +18,6 @@ #include "numpy/npy_3kcompat.h" #include "lowlevel_strided_loops.h" -#include "na_mask.h" #include "reduction.h" /* @@ -28,9 +27,7 @@ * from the result must be done later by the caller once the * computation is complete. * - * This function never adds an NA mask to the allocated - * result, that is the responsibility of the caller. It also - * always allocates a base class ndarray. + * This function always allocates a base class ndarray. * * If 'dtype' isn't NULL, this function steals its reference. */ @@ -155,7 +152,7 @@ conform_reduce_result(int ndim, npy_bool *axis_flags, ndim, shape, strides, PyArray_DATA(out), - PyArray_FLAGS(out) & ~(NPY_ARRAY_MASKNA|NPY_ARRAY_OWNMASKNA), + PyArray_FLAGS(out), NULL); if (ret == NULL) { return NULL; @@ -166,82 +163,25 @@ conform_reduce_result(int ndim, npy_bool *axis_flags, return NULL; } - /* Take a view of the mask if it exists */ - if (PyArray_HASMASKNA(out)) { - npy_intp *strides_ret = ret->maskna_strides; - strides_out = PyArray_MASKNA_STRIDES(out); - idim_out = 0; - for (idim = 0; idim < ndim; ++idim) { - if (axis_flags[idim]) { - strides_ret[idim] = 0; - } - else { - strides_ret[idim] = strides_out[idim_out]; - ++idim_out; - } - } - - ret->maskna_dtype = PyArray_MASKNA_DTYPE(out); - Py_INCREF(ret->maskna_dtype); - ret->maskna_data = PyArray_MASKNA_DATA(out); - ret->flags |= NPY_ARRAY_MASKNA; - } - return (PyArrayObject *)ret; } -/* - * Creates a result for reducing 'operand' along the axes specified - * in 'axis_flags'. If 'dtype' isn't NULL, this function steals a - * reference to 'dtype'. - * - * If 'out' isn't NULL, this function creates a view conforming - * to the number of dimensions of 'operand', adding a singleton dimension - * for each reduction axis specified. In this case, 'dtype' is ignored - * (but its reference is still stolen), and the caller must handle any - * type conversion/validity check for 'out'. When 'need_namask' is true, - * raises an exception if 'out' doesn't have an NA mask. - * - * If 'subok' is true, creates a result with the subtype of 'operand', - * otherwise creates on with the base ndarray class. - * - * If 'out' is NULL, it allocates a new array whose shape matches - * that of 'operand', except for at the reduction axes. An NA mask - * is added if 'need_namask' is true. If 'dtype' is NULL, the dtype - * of 'operand' is used for the result. - */ NPY_NO_EXPORT PyArrayObject * PyArray_CreateReduceResult(PyArrayObject *operand, PyArrayObject *out, - PyArray_Descr *dtype, npy_bool *axis_flags, - int need_namask, int keepdims, int subok, - const char *funcname) + PyArray_Descr *dtype, npy_bool *axis_flags, + int keepdims, int subok, + const char *funcname) { PyArrayObject *result; if (out == NULL) { /* This function steals the reference to 'dtype' */ result = allocate_reduce_result(operand, axis_flags, dtype, subok); - - /* Allocate an NA mask if necessary */ - if (need_namask && result != NULL) { - if (PyArray_AllocateMaskNA(result, 1, 0, 1) < 0) { - Py_DECREF(result); - return NULL; - } - } } else { /* Steal the dtype reference */ Py_XDECREF(dtype); - if (need_namask && !PyArray_HASMASKNA(out)) { - PyErr_Format(PyExc_ValueError, - "output parameter for reduction operation %s " - "needs an NA mask, but the array provided does " - "not have one", funcname); - return NULL; - } - result = conform_reduce_result(PyArray_NDIM(operand), axis_flags, out, keepdims, funcname); } @@ -275,306 +215,13 @@ check_nonreorderable_axes(int ndim, npy_bool *axis_flags, const char *funcname) return 0; } -/* - * Initializes the reduce result for skipna reductions where operand - * has more than one dimension. - * - * 'operand must have an NA mask, 'result' may or may not have an - * NA mask, and 'skipna' must be True to call this function. - */ -static PyArrayObject * -initialize_reduce_result_noidentity_skipna( - PyArrayObject *operand, PyArrayObject *result, - npy_bool *axis_flags, const char *funcname) -{ - PyArrayObject *initialized = NULL; - npy_intp initialized_countdown; - npy_intp op_itemsize; - PyArray_Descr *bool_dtype; - - /* Iterator parameters */ - NpyIter *iter = NULL; - PyArrayObject *op[3]; - npy_uint32 flags, op_flags[3]; - npy_intp fixed_strides[4]; - - /* Data transfer function */ - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; - int needs_api; - - op_itemsize = PyArray_DTYPE(operand)->elsize; - - /* - * Create a view of operand which owns its its own mask, so that - * we can change it. - */ - operand = (PyArrayObject *)PyArray_View(operand, NULL, &PyArray_Type); - if (operand == NULL) { - goto fail; - } - if (PyArray_AllocateMaskNA(operand, 1, 0, 1) < 0) { - goto fail; - } - - /* - * Allocate a flag array to keep track of which elements in the result - * have already been initialized. - * - * This reference to bool_dtype gets stolen by NewLikeArray. - */ - bool_dtype = PyArray_DescrFromType(NPY_BOOL); - if (bool_dtype == NULL) { - goto fail; - } - initialized = (PyArrayObject *)PyArray_NewLikeArray(result, - NPY_KEEPORDER, bool_dtype, 0); - if (initialized == NULL) { - goto fail; - } - if (PyArray_AssignZero(initialized, NULL, 0, NULL) < 0) { - Py_DECREF(initialized); - goto fail; - } - - /* Set up the iterator for copying the elements */ - op[0] = operand; - op[1] = result; - op[2] = initialized; - op_flags[0] = NPY_ITER_READWRITE | NPY_ITER_USE_MASKNA; - op_flags[1] = NPY_ITER_READWRITE | NPY_ITER_IGNORE_MASKNA; - op_flags[2] = NPY_ITER_READWRITE; - flags = NPY_ITER_EXTERNAL_LOOP | - NPY_ITER_REFS_OK | - NPY_ITER_REDUCE_OK | - NPY_ITER_ZEROSIZE_OK | - NPY_ITER_DONT_NEGATE_STRIDES; - - iter = NpyIter_MultiNew(3, op, flags, - NPY_KEEPORDER, NPY_UNSAFE_CASTING, - op_flags, - NULL); - if (iter == NULL) { - goto fail; - } - NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); - needs_api = NpyIter_IterationNeedsAPI(iter); - - /* Get a function for copying the elements */ - if (PyArray_GetDTypeTransferFunction( - PyArray_ISALIGNED(operand) && PyArray_ISALIGNED(result), - fixed_strides[0], fixed_strides[1], - PyArray_DTYPE(operand), PyArray_DTYPE(result), - 0, - &stransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { - goto fail; - } - - /* - * Track how many initializations we've done, both to - * short circuit completion and to raise an error if - * any remained uninitialized. - */ - initialized_countdown = PyArray_SIZE(result); - - if (NpyIter_GetIterSize(iter) != 0) { - NpyIter_IterNextFunc *iternext; - char **dataptr; - npy_intp *strideptr; - npy_intp *countptr, count, subcount; - NPY_BEGIN_THREADS_DEF; - - iternext = NpyIter_GetIterNext(iter, NULL); - if (iternext == NULL) { - goto fail; - } - dataptr = NpyIter_GetDataPtrArray(iter); - strideptr = NpyIter_GetInnerStrideArray(iter); - countptr = NpyIter_GetInnerLoopSizePtr(iter); - - if (!needs_api) { - NPY_BEGIN_THREADS; - } - - do { - char *op_d = dataptr[0], *res_d = dataptr[1]; - char *init_d = dataptr[2], *op_namask_d = dataptr[3]; - npy_intp op_s = strideptr[0], res_s = strideptr[1]; - npy_intp init_s = strideptr[2], op_namask_s = strideptr[3]; - - count = *countptr; - - /* If the result stride is 0, copy at most one value */ - if (res_s == 0) { - npy_intp i; - for (i = 0; i < count; ++i) { - if (*init_d == 0 && NpyMaskValue_IsExposed( - *(npy_mask *)op_namask_d)) { - - /* Mark it as initialized */ - *init_d = 1; - stransfer(res_d, 0, op_d + i * op_s, op_s, - 1, op_itemsize, transferdata); - - --initialized_countdown; - if (initialized_countdown == 0) { - goto finish_loop; - } - break; - } - - init_d += init_s; - op_namask_d += op_namask_s; - } - } - /* Otherwise process the data in runs as large as possible */ - else { - do { - /* Skip values that are initialized or masked */ - subcount = 0; - while (subcount < count && (*init_d == 1 || - !NpyMaskValue_IsExposed( - *(npy_mask *)op_namask_d))) { - ++subcount; - init_d += init_s; - op_namask_d += op_namask_s; - } - op_d += subcount * op_s; - res_d += subcount * res_s; - count -= subcount; - - /* Transfer values that are uninitialized and exposed */ - subcount = 0; - while (subcount < count && (*init_d == 0 && - NpyMaskValue_IsExposed( - *(npy_mask *)op_namask_d))) { - ++subcount; - /* Mark it as initialized */ - *init_d = 1; - init_d += init_s; - op_namask_d += op_namask_s; - } - stransfer(res_d, res_s, op_d, op_s, - subcount, op_itemsize, transferdata); - op_d += subcount * op_s; - res_d += subcount * res_s; - count -= subcount; - - initialized_countdown -= subcount; - if (initialized_countdown == 0) { - goto finish_loop; - } - } while (count > 0); - } - } while (iternext(iter)); - - finish_loop: - if (!needs_api) { - NPY_END_THREADS; - } - } - - if (needs_api && PyErr_Occurred()) { - goto fail; - } - - /* Since this ufunc has no identity, all elements must be initialized */ - if (initialized_countdown != 0) { - PyErr_Format(PyExc_ValueError, - "reduction operation %s with skipna=True " - "had an output element with all its inputs NA", funcname); - goto fail; - } - - /* If 'result' has an NA mask, set it to all exposed */ - if (PyArray_HASMASKNA(result)) { - if (PyArray_AssignMaskNA(result, 1, NULL, 0, NULL) < 0) { - goto fail; - } - } - - Py_DECREF(initialized); - NpyIter_Deallocate(iter); - NPY_AUXDATA_FREE(transferdata); - return operand; - -fail: - Py_XDECREF(operand); - Py_XDECREF(initialized); - if (iter != NULL) { - NpyIter_Deallocate(iter); - } - NPY_AUXDATA_FREE(transferdata); - - return NULL; -} - -/* - * This function initializes a result array for a reduction operation - * which has no identity. This means it needs to copy the first element - * it sees along the reduction axes to result, then return a view of - * the operand which excludes that element. - * - * If a reduction has an identity, such as 0 or 1, the result should - * be initialized by calling PyArray_AssignZero(result, NULL, !skipna, NULL) - * or PyArray_AssignOne(result, NULL, !skipna, NULL), because this - * function raises an exception when there are no elements to reduce. - * - * For regular reduction, this means it copies the subarray indexed - * at zero along each reduction axis into 'result', then returns a view - * into 'operand' excluding those copied elements. If 'operand' has - * an NA mask in this case, the caller should have already done - * the reduction on the mask. This function copies the subarray with - * 'replacena' set to True, so that the already accumulated NA mask - * in result doesn't get overwritten. - * - * For 'skipna' reduction, this is more complicated. In the one dimensional - * case, it searches for the first non-NA element, copies that element - * to 'result', then returns a view into the rest of 'operand'. For - * multi-dimensional reductions, the initial elements may be scattered - * throughout the array. - * - * To deal with this, a view of 'operand' is taken, and given its own - * copy of the NA mask. Additionally, an array of flags is created, - * matching the shape of 'result', and initialized to all False. - * Then, the elements of the 'operand' view are gone through, and any time - * an exposed element is encounted which isn't already flagged in the - * auxiliary array, it is copied into 'result' and flagged as copied. - * The element is masked as an NA in the view of 'operand', so that the - * later reduction step will skip it during processing. - * - * result : The array into which the result is computed. This must have - * the same number of dimensions as 'operand', but for each - * axis i where 'axis_flags[i]' is True, it has a single element. - * operand : The array being reduced. - * axis_flags : An array of boolean flags, one for each axis of 'operand'. - * When a flag is True, it indicates to reduce along that axis. - * reorderable : If True, the reduction being done is reorderable, which - * means specifying multiple axes of reduction at once is ok, - * and the reduction code may calculate the reduction in an - * arbitrary order. The calculation may be reordered because - * of cache behavior or multithreading requirements. - * skipna : If True, indicates that the reduction is being calculated - * as if the NA values are being dropped from the computation - * instead of accumulating into an NA result. - * out_skip_first_count : This gets populated with the number of first-visit - * elements that should be skipped during the - * iteration loop. - * funcname : The name of the reduction operation, for the purpose of - * better quality error messages. For example, "numpy.max" - * would be a good name for NumPy's max function. - * - * Returns a view which contains the remaining elements on which to do - * the reduction. - */ NPY_NO_EXPORT PyArrayObject * PyArray_InitializeReduceResult( PyArrayObject *result, PyArrayObject *operand, - npy_bool *axis_flags, int reorderable, int skipna, + npy_bool *axis_flags, int reorderable, npy_intp *out_skip_first_count, const char *funcname) { - npy_intp *strides, *shape, shape_orig[NPY_MAXDIMS], shape0; + npy_intp *strides, *shape, shape_orig[NPY_MAXDIMS]; PyArrayObject *op_view = NULL; int idim, ndim, nreduce_axes; @@ -592,80 +239,18 @@ PyArray_InitializeReduceResult( return NULL; } - /* - * If 'skipna' is False, or 'operand' has no NA mask in which - * case the 'skipna' flag does nothing. - */ - if (!skipna || !PyArray_HASMASKNA(operand)) { - if (PyArray_SIZE(operand) == 0) { - PyErr_Format(PyExc_ValueError, - "zero-size array to reduction operation %s " - "which has no identity", - funcname); - return NULL; - } - - /* Take a view into 'operand' which we can modify. */ - op_view = (PyArrayObject *)PyArray_View(operand, NULL, &PyArray_Type); - if (op_view == NULL) { - return NULL; - } + if (PyArray_SIZE(operand) == 0) { + PyErr_Format(PyExc_ValueError, + "zero-size array to reduction operation %s " + "which has no identity", + funcname); + return NULL; } - /* - * Otherwise 'skipna' is True and 'operand' has an NA mask. Deal - * with the simple one-dimensional case first - */ - else if (ndim == 1) { - char *data, *maskna_data; - npy_intp *maskna_strides; - - ndim = PyArray_NDIM(operand); - - op_view = (PyArrayObject *)PyArray_View(operand, NULL, &PyArray_Type); - if (op_view == NULL) { - return NULL; - } - - shape = PyArray_DIMS(op_view); - shape0 = shape[0]; - data = PyArray_DATA(op_view); - strides = PyArray_STRIDES(op_view); - maskna_data = PyArray_MASKNA_DATA(op_view); - maskna_strides = PyArray_MASKNA_STRIDES(op_view); - - /* Shrink the array from the start until we find an exposed element */ - while (shape0 > 0 && - !NpyMaskValue_IsExposed((npy_mask)*maskna_data)) { - --shape0; - data += strides[0]; - maskna_data += maskna_strides[0]; - } - if (shape0 == 0) { - Py_DECREF(op_view); - PyErr_Format(PyExc_ValueError, - "fully NA array with skipna=True to reduction operation " - "%s which has no identity", funcname); - return NULL; - } - - /* - * With the first element exposed, fall through to the code - * which copies the element and adjusts the view just as in the - * non-skipna case. - */ - shape[0] = shape0; - ((PyArrayObject_fields *)op_view)->data = data; - ((PyArrayObject_fields *)op_view)->maskna_data = maskna_data; - } - /* - * Here 'skipna' is True and 'operand' has an NA mask, but - * 'operand' has more than one dimension, so it's the complicated - * case - */ - else { - return initialize_reduce_result_noidentity_skipna( - operand, result, axis_flags, funcname); + /* Take a view into 'operand' which we can modify. */ + op_view = (PyArrayObject *)PyArray_View(operand, NULL, &PyArray_Type); + if (op_view == NULL) { + return NULL; } /* @@ -687,12 +272,9 @@ PyArray_InitializeReduceResult( } /* - * Copy the elements into the result to start, with - * 'preservena' set to True so that we don't overwrite - * what we already calculated in ReduceNAMask. + * Copy the elements into the result to start. */ - if (PyArray_AssignArray(result, op_view, NULL, NPY_UNSAFE_CASTING, - 1, NULL) < 0) { + if (PyArray_AssignArray(result, op_view, NULL, NPY_UNSAFE_CASTING) < 0) { Py_DECREF(op_view); return NULL; } @@ -709,15 +291,6 @@ PyArray_InitializeReduceResult( ((PyArrayObject_fields *)op_view)->data += strides[idim]; } } - if (PyArray_HASMASKNA(op_view)) { - strides = PyArray_MASKNA_STRIDES(op_view); - for (idim = 0; idim < ndim; ++idim) { - if (axis_flags[idim]) { - ((PyArrayObject_fields *)op_view)->maskna_data += - strides[idim]; - } - } - } } /* If there are zero reduction axes, make the view empty */ else if (nreduce_axes == 0) { @@ -741,7 +314,7 @@ PyArray_InitializeReduceResult( } /*NUMPY_API - * + * * This function executes all the standard NumPy reduction function * boilerplate code, just calling assign_identity and the appropriate * inner loop function where necessary. @@ -760,10 +333,6 @@ PyArray_InitializeReduceResult( * and the reduction code may calculate the reduction in an * arbitrary order. The calculation may be reordered because * of cache behavior or multithreading requirements. - * skipna : If true, NAs are skipped instead of propagating. - * whichskipna : NOT YET SUPPORTED, but this parameter is placed here - * so that support can be added for multi-NA without - * breaking API compatibility. Pass in NULL. * keepdims : If true, leaves the reduction dimensions in the result * with size one. * subok : If true, the result uses the subclass of operand, otherwise @@ -772,14 +341,6 @@ PyArray_InitializeReduceResult( * this function is called to initialize the result to * the reduction's unit. * loop : The loop which does the reduction. - * masked_loop : The loop which does the reduction with a mask. - * advanced_masked_loop: If non-NULL, this is a loop which uses a mask from - * both the operand and the result. The 'result' is - * initialized to a usual reduction of the operand's mask, - * but both the operand's mask and the result's mask - * are provided so that the loop may decide to expose - * elements, which normally would not be exposed by the - * normal NA propagation rules, based on the input data. * data : Data which is passed to assign_identity and the inner loop. * buffersize : Buffer size for the iterator. For the default, pass in 0. * funcname : The name of the reduction function, for error messages. @@ -791,15 +352,12 @@ PyArray_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, PyArray_Descr *result_dtype, NPY_CASTING casting, npy_bool *axis_flags, int reorderable, - int skipna, npy_bool *skipwhichna, int keepdims, + int keepdims, int subok, PyArray_AssignReduceIdentityFunc *assign_identity, PyArray_ReduceLoopFunc *loop, - PyArray_ReduceLoopFunc *masked_loop, - PyArray_ReduceLoopFunc *advanced_masked_loop, void *data, npy_intp buffersize, const char *funcname) { - int use_maskna; PyArrayObject *result = NULL, *op_view = NULL; npy_intp skip_first_count = 0; @@ -816,34 +374,6 @@ PyArray_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, "a where mask"); return NULL; } - if (skipwhichna != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "multi-NA support is not yet implemented in " - "reduce operations"); - return NULL; - } - - use_maskna = PyArray_HASMASKNA(operand); - - /* - * If 'operand' has an NA mask, but 'out' doesn't, validate that 'operand' - * contains no NA values so we can ignore the mask entirely. - */ - if (use_maskna && !skipna && out != NULL && !PyArray_HASMASKNA(out)) { - int containsna = PyArray_ContainsNA(operand, wheremask, NULL); - if (containsna == -1) { - goto fail; - } - else if (containsna) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA to an array which " - "does not support NAs"); - goto fail; - } - else { - use_maskna = 0; - } - } /* * This either conforms 'out' to the ndim of 'operand', or allocates @@ -851,35 +381,13 @@ PyArray_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, */ Py_INCREF(result_dtype); result = PyArray_CreateReduceResult(operand, out, - result_dtype, axis_flags, !skipna && use_maskna, + result_dtype, axis_flags, keepdims, subok, funcname); if (result == NULL) { goto fail; } /* - * Do the reduction on the NA mask before the data. This way - * we can avoid modifying the outputs which end up masked, obeying - * the required NA masking semantics. - */ - if (use_maskna && !skipna) { - if (PyArray_ReduceMaskNAArray(operand, result, NULL, NULL) < 0) { - goto fail; - } - - /* - * Short circuit any calculation if the result is a 0-dim NA - * and the advanced masked loop which could expose it isn't - * provided. - */ - if (advanced_masked_loop == NULL && PyArray_SIZE(result) == 1 && - !NpyMaskValue_IsExposed( - (npy_mask)*PyArray_MASKNA_DATA(result))) { - goto finish; - } - } - - /* * Initialize the result to the reduction unit if possible, * otherwise copy the initial values and get a view to the rest. */ @@ -893,7 +401,7 @@ PyArray_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, goto fail; } - if (assign_identity(result, !skipna, data) < 0) { + if (assign_identity(result, data) < 0) { goto fail; } op_view = operand; @@ -901,7 +409,7 @@ PyArray_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, } else { op_view = PyArray_InitializeReduceResult(result, operand, - axis_flags, reorderable, skipna, + axis_flags, reorderable, &skip_first_count, funcname); if (op_view == NULL) { goto fail; @@ -932,36 +440,6 @@ PyArray_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, op_flags[1] = NPY_ITER_READONLY | NPY_ITER_ALIGNED; - /* Add mask-related flags */ - if (use_maskna) { - if (skipna) { - /* The output's mask has been set to all exposed already */ - op_flags[0] |= NPY_ITER_IGNORE_MASKNA; - /* Need the input's mask to determine what to skip */ - op_flags[1] |= NPY_ITER_USE_MASKNA; - } - else { - /* Iterate over the output's mask */ - op_flags[0] |= NPY_ITER_USE_MASKNA; - if (advanced_masked_loop == NULL) { - /* Input's mask is already incorporated in the output's mask */ - op_flags[1] |= NPY_ITER_IGNORE_MASKNA; - } - else { - /* The reduction wants to use the operand's mask as well */ - op_flags[1] |= NPY_ITER_USE_MASKNA; - } - } - } - else { - /* - * If 'out' had no mask, and 'operand' did, we checked that 'operand' - * contains no NA values and can ignore the masks. - */ - op_flags[0] |= NPY_ITER_IGNORE_MASKNA; - op_flags[1] |= NPY_ITER_IGNORE_MASKNA; - } - iter = NpyIter_AdvancedNew(2, op, flags, NPY_KEEPORDER, casting, op_flags, @@ -989,40 +467,17 @@ PyArray_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, needs_api = NpyIter_IterationNeedsAPI(iter); /* Straightforward reduction */ - if (!use_maskna) { - if (loop == NULL) { - PyErr_Format(PyExc_RuntimeError, - "reduction operation %s did not supply an " - "unmasked inner loop function", funcname); - goto fail; - } - - if (loop(iter, dataptr, strideptr, countptr, - iternext, needs_api, skip_first_count, data) < 0) { - - goto fail; - } - } - /* Masked reduction with both masks */ - else if (!skipna && advanced_masked_loop != NULL) { - if (advanced_masked_loop(iter, dataptr, strideptr, countptr, - iternext, needs_api, skip_first_count, data) < 0) { - goto fail; - } + if (loop == NULL) { + PyErr_Format(PyExc_RuntimeError, + "reduction operation %s did not supply an " + "inner loop function", funcname); + goto fail; } - /* Regular masked reduction with just one mask */ - else { - if (masked_loop == NULL) { - PyErr_Format(PyExc_RuntimeError, - "reduction operation %s did not supply a " - "masked inner loop function", funcname); - goto fail; - } - if (masked_loop(iter, dataptr, strideptr, countptr, - iternext, needs_api, skip_first_count, data) < 0) { - goto fail; - } + if (loop(iter, dataptr, strideptr, countptr, + iternext, needs_api, skip_first_count, data) < 0) { + + goto fail; } } @@ -1053,140 +508,3 @@ fail: return NULL; } - -/* - * This function counts the number of elements that a reduction - * will see along the reduction directions, given the provided options. - * - * If the reduction operand has no NA mask or 'skipna' is false, this - * is simply the prod`uct of all the reduction axis sizes. A NumPy - * scalar is returned in this case. - * - * If the reduction operand has an NA mask and 'skipna' is true, this - * counts the number of elements which are not NA along the reduction - * dimensions, and returns an array with the counts. - */ -NPY_NO_EXPORT PyObject * -PyArray_CountReduceItems(PyArrayObject *operand, - npy_bool *axis_flags, int skipna, int keepdims) -{ - int idim, ndim = PyArray_NDIM(operand); - - /* The product of the reduction dimensions in this case */ - if (!skipna || !PyArray_HASMASKNA(operand)) { - npy_intp count = 1, *shape = PyArray_SHAPE(operand); - PyArray_Descr *dtype; - PyObject *ret; - - for (idim = 0; idim < ndim; ++idim) { - if (axis_flags[idim]) { - count *= shape[idim]; - } - } - - dtype = PyArray_DescrFromType(NPY_INTP); - if (dtype == NULL) { - return NULL; - } - ret = PyArray_Scalar(&count, dtype, NULL); - Py_DECREF(dtype); - return ret; - } - /* Otherwise we need to do a count based on the NA mask */ - else { - npy_intp *strides; - PyArrayObject *result; - PyArray_Descr *result_dtype; - - npy_intp i, coord[NPY_MAXDIMS]; - npy_intp shape_it[NPY_MAXDIMS]; - npy_intp operand_strides_it[NPY_MAXDIMS]; - npy_intp result_strides_it[NPY_MAXDIMS]; - char *operand_data = NULL, *result_data = NULL; - - /* - * To support field-NA, we would create a result type - * with an INTP matching each field, then separately count - * the available elements per-field. - */ - if (PyArray_HASFIELDS(operand)) { - PyErr_SetString(PyExc_RuntimeError, - "field-NA isn't implemented yet"); - return NULL; - } - - /* - * TODO: The loop below is specialized for NPY_BOOL masks, - * will need another version for NPY_MASK masks. - */ - if (PyArray_MASKNA_DTYPE(operand)->type_num != NPY_BOOL) { - PyErr_SetString(PyExc_RuntimeError, - "multi-NA isn't implemented yet"); - return NULL; - } - - /* Allocate an array for the reduction counting */ - result_dtype = PyArray_DescrFromType(NPY_INTP); - if (result_dtype == NULL) { - return NULL; - } - result = PyArray_CreateReduceResult(operand, NULL, - result_dtype, axis_flags, 0, - keepdims, 0, "count_reduce_items"); - if (result == NULL) { - return NULL; - } - - /* Initialize result to all zeros */ - if (PyArray_AssignZero(result, NULL, 0, NULL) < 0) { - Py_DECREF(result); - return NULL; - } - - /* - * Set all the reduction strides to 0 in result so - * we can use them for raw iteration - */ - strides = PyArray_STRIDES(result); - for (idim = 0; idim < ndim; ++idim) { - if (axis_flags[idim]) { - strides[idim] = 0; - } - } - - /* - * Sort axes based on 'operand', which has more non-zero strides, - * by making it the first operand here - */ - if (PyArray_PrepareTwoRawArrayIter(ndim, PyArray_SHAPE(operand), - PyArray_MASKNA_DATA(operand), PyArray_MASKNA_STRIDES(operand), - PyArray_DATA(result), PyArray_STRIDES(result), - &ndim, shape_it, - &operand_data, operand_strides_it, - &result_data, result_strides_it) < 0) { - Py_DECREF(result); - return NULL; - } - - /* - * NOTE: The following only works for NPY_BOOL masks. - */ - NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { - char *operand_d = operand_data, *result_d = result_data; - for (i = 0; i < shape_it[0]; ++i) { - *(npy_intp *)result_d += *operand_d; - - operand_d += operand_strides_it[0]; - result_d += result_strides_it[0]; - } - } NPY_RAW_ITER_TWO_NEXT(idim, ndim, coord, shape_it, - operand_data, operand_strides_it, - result_data, result_strides_it); - - /* Remove the reduction axes and return the result */ - if (!keepdims) { - PyArray_RemoveAxesInPlace(result, axis_flags); - } - return PyArray_Return(result); - } -} diff --git a/numpy/core/src/multiarray/reduction.h b/numpy/core/src/multiarray/reduction.h index 268fdaf29..76ab9e18a 100644 --- a/numpy/core/src/multiarray/reduction.h +++ b/numpy/core/src/multiarray/reduction.h @@ -2,54 +2,19 @@ #define _NPY_PRIVATE__REDUCTION_H_ /* - * This function counts the number of elements that a reduction - * will see along the reduction directions, given the provided options. - * - * If the reduction operand has no NA mask or 'skipna' is false, this - * is simply the prod`uct of all the reduction axis sizes. A NumPy - * scalar is returned in this case. - * - * If the reduction operand has an NA mask and 'skipna' is true, this - * counts the number of elements which are not NA along the reduction - * dimensions, and returns an array with the counts. - */ -NPY_NO_EXPORT PyObject * -PyArray_CountReduceItems(PyArrayObject *operand, - npy_bool *axis_flags, int skipna, int keepdims); - -/* * This function initializes a result array for a reduction operation * which has no identity. This means it needs to copy the first element * it sees along the reduction axes to result, then return a view of * the operand which excludes that element. * * If a reduction has an identity, such as 0 or 1, the result should - * be initialized by calling PyArray_AssignZero(result, NULL, !skipna, NULL) - * or PyArray_AssignOne(result, NULL, !skipna, NULL), because this + * be initialized by calling PyArray_AssignZero(result, NULL, NULL) + * or PyArray_AssignOne(result, NULL, NULL), because this * function raises an exception when there are no elements to reduce. * - * For regular reduction, this means it copies the subarray indexed - * at zero along each reduction axis into 'result', then returns a view - * into 'operand' excluding those copied elements. If 'operand' has - * an NA mask in this case, the caller should have already done - * the reduction on the mask. This function copies the subarray with - * 'replacena' set to True, so that the already accumulated NA mask - * in result doesn't get overwritten. - * - * For 'skipna' reduction, this is more complicated. In the one dimensional - * case, it searches for the first non-NA element, copies that element - * to 'result', then returns a view into the rest of 'operand'. For - * multi-dimensional reductions, the initial elements may be scattered - * throughout the array. - * - * To deal with this, a view of 'operand' is taken, and given its own - * copy of the NA mask. Additionally, an array of flags is created, - * matching the shape of 'result', and initialized to all False. - * Then, the elements of the 'operand' view are gone through, and any time - * an exposed element is encounted which isn't already flagged in the - * auxiliary array, it is copied into 'result' and flagged as copied. - * The element is masked as an NA in the view of 'operand', so that the - * later reduction step will skip it during processing. + * This means it copies the subarray indexed at zero along each reduction axis + * into 'result', then returns a view into 'operand' excluding those copied + * elements. * * result : The array into which the result is computed. This must have * the same number of dimensions as 'operand', but for each @@ -62,9 +27,6 @@ PyArray_CountReduceItems(PyArrayObject *operand, * and the reduction code may calculate the reduction in an * arbitrary order. The calculation may be reordered because * of cache behavior or multithreading requirements. - * skipna : If True, indicates that the reduction is being calculated - * as if the NA values are being dropped from the computation - * instead of accumulating into an NA result. * out_skip_first_count : This gets populated with the number of first-visit * elements that should be skipped during the * iteration loop. @@ -78,7 +40,7 @@ PyArray_CountReduceItems(PyArrayObject *operand, NPY_NO_EXPORT PyArrayObject * PyArray_InitializeReduceResult( PyArrayObject *result, PyArrayObject *operand, - npy_bool *axis_flags, int reorderable, int skipna, + npy_bool *axis_flags, int reorderable, npy_intp *out_skip_first_count, const char *funcname); /* @@ -90,21 +52,19 @@ PyArray_InitializeReduceResult( * to the number of dimensions of 'operand', adding a singleton dimension * for each reduction axis specified. In this case, 'dtype' is ignored * (but its reference is still stolen), and the caller must handle any - * type conversion/validity check for 'out'. When 'need_namask' is true, - * raises an exception if 'out' doesn't have an NA mask. + * type conversion/validity check for 'out' * * If 'subok' is true, creates a result with the subtype of 'operand', * otherwise creates on with the base ndarray class. * - * If 'out' is NULL, it allocates a new array whose shape matches - * that of 'operand', except for at the reduction axes. An NA mask - * is added if 'need_namask' is true. If 'dtype' is NULL, the dtype + * If 'out' is NULL, it allocates a new array whose shape matches that of + * 'operand', except for at the reduction axes. If 'dtype' is NULL, the dtype * of 'operand' is used for the result. */ NPY_NO_EXPORT PyArrayObject * PyArray_CreateReduceResult(PyArrayObject *operand, PyArrayObject *out, PyArray_Descr *dtype, npy_bool *axis_flags, - int need_namask, int keepdims, int subok, + int keepdims, int subok, const char *funcname); #endif diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c index 8b61c797d..dd0e0158b 100644 --- a/numpy/core/src/multiarray/scalarapi.c +++ b/numpy/core/src/multiarray/scalarapi.c @@ -18,7 +18,6 @@ #include "scalartypes.h" #include "common.h" -#include "na_object.h" static PyArray_Descr * _descr_from_subtype(PyObject *type) @@ -802,18 +801,7 @@ PyArray_Return(PyArrayObject *mp) } if (PyArray_NDIM(mp) == 0) { PyObject *ret; - if (PyArray_HASMASKNA(mp)) { - npy_mask maskvalue = (npy_mask)(*PyArray_MASKNA_DATA(mp)); - if (NpyMaskValue_IsExposed(maskvalue)) { - ret = PyArray_ToScalar(PyArray_DATA(mp), mp); - } - else { - ret = (PyObject *)NpyNA_FromObject((PyObject *)mp, 0); - } - } - else { - ret = PyArray_ToScalar(PyArray_DATA(mp), mp); - } + ret = PyArray_ToScalar(PyArray_DATA(mp), mp); Py_DECREF(mp); return ret; } diff --git a/numpy/core/src/multiarray/sequence.c b/numpy/core/src/multiarray/sequence.c index cb3b30b3a..5a43f0565 100644 --- a/numpy/core/src/multiarray/sequence.c +++ b/numpy/core/src/multiarray/sequence.c @@ -71,7 +71,7 @@ array_slice(PyArrayObject *self, Py_ssize_t ilow, Py_ssize_t ihigh) ret = (PyArrayObject *)PyArray_NewFromDescr(Py_TYPE(self), dtype, PyArray_NDIM(self), shape, PyArray_STRIDES(self), data, - PyArray_FLAGS(self) & ~(NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA), + PyArray_FLAGS(self), (PyObject *)self); if (ret == NULL) { return NULL; @@ -83,27 +83,6 @@ array_slice(PyArrayObject *self, Py_ssize_t ilow, Py_ssize_t ihigh) } PyArray_UpdateFlags(ret, NPY_ARRAY_UPDATE_ALL); - /* Also take a view of the NA mask if it exists */ - if (PyArray_HASMASKNA(self)) { - PyArrayObject_fields *fret = (PyArrayObject_fields *)ret; - - fret->maskna_dtype = PyArray_MASKNA_DTYPE(self); - Py_INCREF(fret->maskna_dtype); - - data = PyArray_MASKNA_DATA(self); - if (ilow < ihigh) { - data += ilow * PyArray_MASKNA_STRIDES(self)[0]; - } - fret->maskna_data = data; - - memcpy(fret->maskna_strides, PyArray_MASKNA_STRIDES(self), - PyArray_NDIM(self) * sizeof(npy_intp)); - - /* This view doesn't own the mask */ - fret->flags |= NPY_ARRAY_MASKNA; - fret->flags &= ~NPY_ARRAY_OWNMASKNA; - } - return (PyObject *)ret; } diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c index d8cb5b784..3ed46eac4 100644 --- a/numpy/core/src/multiarray/shape.c +++ b/numpy/core/src/multiarray/shape.c @@ -19,15 +19,14 @@ static int _check_ones(PyArrayObject *self, int newnd, - npy_intp* newdims, npy_intp *strides, npy_intp *masknastrides); + npy_intp* newdims, npy_intp *strides); static int _fix_unknown_dimension(PyArray_Dims *newshape, npy_intp s_original); static int _attempt_nocopy_reshape(PyArrayObject *self, int newnd, npy_intp* newdims, - npy_intp *newstrides, npy_intp *newmasknastrides, - int is_f_order); + npy_intp *newstrides, int is_f_order); static void _putzero(char *optr, PyObject *zero, PyArray_Descr *dtype); @@ -150,7 +149,6 @@ PyArray_Resize(PyArrayObject *self, PyArray_Dims *newshape, int refcheck, } ((PyArrayObject_fields *)self)->dimensions = dimptr; ((PyArrayObject_fields *)self)->strides = dimptr + new_nd; - ((PyArrayObject_fields *)self)->maskna_strides = dimptr + 2*new_nd; } /* make new_strides variable */ @@ -184,8 +182,7 @@ PyArray_Newshape(PyArrayObject *self, PyArray_Dims *newdims, npy_bool same, incref = NPY_TRUE; npy_intp *strides = NULL; npy_intp newstrides[NPY_MAXDIMS]; - npy_intp newmasknastrides[NPY_MAXDIMS]; - int flags, build_maskna_strides = 0; + int flags; if (order == NPY_ANYORDER) { order = PyArray_ISFORTRAN(self); @@ -213,12 +210,11 @@ PyArray_Newshape(PyArrayObject *self, PyArray_Dims *newdims, * In this case we don't need to do anything but update strides and * dimensions. So, we can handle non single-segment cases. */ - i = _check_ones(self, ndim, dimensions, newstrides, newmasknastrides); + i = _check_ones(self, ndim, dimensions, newstrides); if (i == 0) { strides = newstrides; } - flags = PyArray_FLAGS(self) & ~(NPY_ARRAY_OWNMASKNA | - NPY_ARRAY_MASKNA); + flags = PyArray_FLAGS(self); if (strides == NULL) { /* @@ -241,7 +237,7 @@ PyArray_Newshape(PyArrayObject *self, PyArray_Dims *newdims, order == NPY_CORDER)) && (PyArray_NDIM(self) > 1))) { int success = 0; success = _attempt_nocopy_reshape(self, ndim, dimensions, - newstrides, newmasknastrides, order); + newstrides, order); if (success) { /* no need to copy the array after all */ strides = newstrides; @@ -254,13 +250,7 @@ PyArray_Newshape(PyArrayObject *self, PyArray_Dims *newdims, } incref = NPY_FALSE; self = (PyArrayObject *)newcopy; - build_maskna_strides = 1; } - flags = PyArray_FLAGS(self) & ~(NPY_ARRAY_OWNMASKNA | - NPY_ARRAY_MASKNA); - } - else { - build_maskna_strides = 1; } /* We always have to interpret the contiguous buffer correctly */ @@ -324,34 +314,6 @@ PyArray_Newshape(PyArrayObject *self, PyArray_Dims *newdims, return NULL; } - /* If there's an NA mask, make sure to view it too */ - if (PyArray_HASMASKNA(self)) { - PyArrayObject_fields *fa = (PyArrayObject_fields *)ret; - fa->maskna_dtype = PyArray_MASKNA_DTYPE(self); - Py_INCREF(fa->maskna_dtype); - fa->maskna_data = PyArray_MASKNA_DATA(self); - if (build_maskna_strides) { - npy_intp stride = 1; - if (order == NPY_FORTRANORDER) { - for (i = 0; i < ndim; ++i) { - fa->maskna_strides[i] = stride; - stride *= fa->dimensions[i]; - } - } - else { - for (i = ndim-1; i >= 0; --i) { - fa->maskna_strides[i] = stride; - stride *= fa->dimensions[i]; - } - } - } - else { - memcpy(fa->maskna_strides, newmasknastrides, - fa->nd * sizeof(npy_intp)); - } - fa->flags |= NPY_ARRAY_MASKNA; - } - PyArray_UpdateFlags(ret, NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS); return (PyObject *)ret; @@ -386,13 +348,12 @@ PyArray_Reshape(PyArrayObject *self, PyObject *shape) /* inserts 0 for strides where dimension will be 1 */ static int _check_ones(PyArrayObject *self, int newnd, - npy_intp* newdims, npy_intp *strides, npy_intp *masknastrides) + npy_intp* newdims, npy_intp *strides) { int nd; npy_intp *dims; npy_bool done=NPY_FALSE; int j, k; - int has_maskna = PyArray_HASMASKNA(self); nd = PyArray_NDIM(self); dims = PyArray_DIMS(self); @@ -400,17 +361,11 @@ _check_ones(PyArrayObject *self, int newnd, for (k = 0, j = 0; !done && (j < nd || k < newnd);) { if ((j<nd) && (k<newnd) && (newdims[k] == dims[j])) { strides[k] = PyArray_STRIDES(self)[j]; - if (has_maskna) { - masknastrides[k] = PyArray_MASKNA_STRIDES(self)[j]; - } j++; k++; } else if ((k < newnd) && (newdims[k] == 1)) { strides[k] = 0; - if (has_maskna) { - masknastrides[k] = 0; - } k++; } else if ((j<nd) && (dims[j] == 1)) { @@ -475,24 +430,19 @@ _putzero(char *optr, PyObject *zero, PyArray_Descr *dtype) */ static int _attempt_nocopy_reshape(PyArrayObject *self, int newnd, npy_intp* newdims, - npy_intp *newstrides, npy_intp *newmasknastrides, - int is_f_order) + npy_intp *newstrides, int is_f_order) { int oldnd; npy_intp olddims[NPY_MAXDIMS]; - npy_intp oldstrides[NPY_MAXDIMS], oldmasknastrides[NPY_MAXDIMS]; + npy_intp oldstrides[NPY_MAXDIMS]; int oi, oj, ok, ni, nj, nk; int np, op; - int has_maskna = PyArray_HASMASKNA(self); oldnd = 0; for (oi = 0; oi < PyArray_NDIM(self); oi++) { if (PyArray_DIMS(self)[oi]!= 1) { olddims[oldnd] = PyArray_DIMS(self)[oi]; oldstrides[oldnd] = PyArray_STRIDES(self)[oi]; - if (has_maskna) { - oldmasknastrides[oldnd] = PyArray_MASKNA_STRIDES(self)[oi]; - } oldnd++; } } @@ -543,18 +493,14 @@ _attempt_nocopy_reshape(PyArrayObject *self, int newnd, npy_intp* newdims, for (ok = oi; ok < oj - 1; ok++) { if (is_f_order) { - if (oldstrides[ok+1] != olddims[ok]*oldstrides[ok] || - (has_maskna && oldmasknastrides[ok+1] != - olddims[ok]*oldmasknastrides[ok])) { + if (oldstrides[ok+1] != olddims[ok]*oldstrides[ok]) { /* not contiguous enough */ return 0; } } else { /* C order */ - if (oldstrides[ok] != olddims[ok+1]*oldstrides[ok+1] || - (has_maskna && oldmasknastrides[ok] != - olddims[ok+1]*oldmasknastrides[ok+1])) { + if (oldstrides[ok] != olddims[ok+1]*oldstrides[ok+1]) { /* not contiguous enough */ return 0; } @@ -566,13 +512,6 @@ _attempt_nocopy_reshape(PyArrayObject *self, int newnd, npy_intp* newdims, for (nk = ni + 1; nk < nj; nk++) { newstrides[nk] = newstrides[nk - 1]*newdims[nk - 1]; } - if (has_maskna) { - newmasknastrides[ni] = oldmasknastrides[oi]; - for (nk = ni + 1; nk < nj; nk++) { - newmasknastrides[nk] = - newmasknastrides[nk - 1]*newdims[nk - 1]; - } - } } else { /* C order */ @@ -580,13 +519,6 @@ _attempt_nocopy_reshape(PyArrayObject *self, int newnd, npy_intp* newdims, for (nk = nj - 1; nk > ni; nk--) { newstrides[nk - 1] = newstrides[nk]*newdims[nk]; } - if (has_maskna) { - newmasknastrides[nj - 1] = oldmasknastrides[oj - 1]; - for (nk = nj - 1; nk > ni; nk--) { - newmasknastrides[nk - 1] = - newmasknastrides[nk]*newdims[nk]; - } - } } ni = nj++; oi = oj++; @@ -879,7 +811,7 @@ PyArray_Transpose(PyArrayObject *ap, PyArray_Dims *permute) PyArray_DESCR(ap), n, PyArray_DIMS(ap), NULL, PyArray_DATA(ap), - flags & ~(NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA), + flags, (PyObject *)ap); if (ret == NULL) { return NULL; @@ -891,21 +823,6 @@ PyArray_Transpose(PyArrayObject *ap, PyArray_Dims *permute) return NULL; } - /* Take a view of the NA mask as well if necessary */ - if (flags & NPY_ARRAY_MASKNA) { - PyArrayObject_fields *fa = (PyArrayObject_fields *)ret; - - fa->maskna_dtype = PyArray_MASKNA_DTYPE(ap); - Py_INCREF(fa->maskna_dtype); - fa->maskna_data = PyArray_MASKNA_DATA(ap); - - for (i = 0; i < n; i++) { - fa->maskna_strides[i] = - PyArray_MASKNA_STRIDES(ap)[permutation[i]]; - } - fa->flags |= NPY_ARRAY_MASKNA; - } - /* fix the dimensions and strides of the return-array */ for (i = 0; i < n; i++) { PyArray_DIMS(ret)[i] = PyArray_DIMS(ap)[permutation[i]]; @@ -1134,33 +1051,12 @@ PyArray_Ravel(PyArrayObject *arr, NPY_ORDER order) 1, val, &stride, PyArray_BYTES(arr), - PyArray_FLAGS(arr) & ~(NPY_ARRAY_OWNMASKNA | NPY_ARRAY_MASKNA), + PyArray_FLAGS(arr), (PyObject *)arr); if (ret == NULL) { return NULL; } - /* Take a view of the NA mask as well if necessary */ - if (PyArray_HASMASKNA(arr)) { - PyArrayObject_fields *fa = - (PyArrayObject_fields *)ret; - - fa->maskna_dtype = PyArray_MASKNA_DTYPE(arr); - Py_INCREF(fa->maskna_dtype); - fa->maskna_data = PyArray_MASKNA_DATA(arr); - - /* - * Because the strides of the NA mask always match up - * layout-wise with the strides of the data, we don't - * have to also check them the same way. This is due - * to the fact that PyArray_AllocateMaskNA is the only - * mechanism ever used to create an NA mask. - */ - fa->maskna_strides[0] = - PyArray_MASKNA_STRIDES(arr)[strideperm[ndim-1].perm]; - fa->flags |= NPY_ARRAY_MASKNA; - } - PyArray_UpdateFlags(ret, NPY_ARRAY_C_CONTIGUOUS|NPY_ARRAY_F_CONTIGUOUS); Py_INCREF(arr); @@ -1201,13 +1097,6 @@ PyArray_Flatten(PyArrayObject *a, NPY_ORDER order) return NULL; } - if (PyArray_HASMASKNA(a)) { - if (PyArray_AllocateMaskNA(ret, 1, 0, 1) < 0) { - Py_DECREF(ret); - return NULL; - } - } - if (PyArray_CopyAsFlat(ret, a, order) < 0) { Py_DECREF(ret); return NULL; @@ -1295,18 +1184,6 @@ PyArray_RemoveAxesInPlace(PyArrayObject *arr, npy_bool *flags) } } - /* Compress the mask strides if the result has an NA mask */ - if (PyArray_HASMASKNA(arr)) { - strides = fa->maskna_strides; - idim_out = 0; - for (idim = 0; idim < ndim; ++idim) { - if (!flags[idim]) { - strides[idim_out] = strides[idim]; - ++idim_out; - } - } - } - /* The final number of dimensions */ fa->nd = idim_out; } diff --git a/numpy/core/src/private/lowlevel_strided_loops.h b/numpy/core/src/private/lowlevel_strided_loops.h index 6ffa36ee9..a73dd12cf 100644 --- a/numpy/core/src/private/lowlevel_strided_loops.h +++ b/numpy/core/src/private/lowlevel_strided_loops.h @@ -33,11 +33,11 @@ typedef void (PyArray_StridedUnaryOp)(char *dst, npy_intp dst_stride, * which values are transformed. * * In particular, the 'i'-th element is operated on if and only if - * NpyMaskValue_IsExposed(mask[i*mask_stride]). + * mask[i*mask_stride] is true. */ typedef void (PyArray_MaskedStridedUnaryOp)(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, - npy_mask *mask, npy_intp mask_stride, + npy_bool *mask, npy_intp mask_stride, npy_intp N, npy_intp src_itemsize, NpyAuxData *transferdata); @@ -209,10 +209,10 @@ PyArray_GetDTypeTransferFunction(int aligned, int *out_needs_api); /* - * This is identical to PyArray_GetDTypeTransferFunction, but - * returns a transfer function which also takes a mask as a parameter. - * Bit zero of the mask is used to determine which values to copy, - * and data is transfered exactly when NpyMask_IsExposed(mask[i*mask_stride]). + * This is identical to PyArray_GetDTypeTransferFunction, but returns a + * transfer function which also takes a mask as a parameter. The mask is used + * to determine which values to copy, and data is transfered exactly when + * mask[i*mask_stride] is true. * * If move_references is true, values which are not copied to the * destination will still have their source reference decremented. @@ -251,20 +251,6 @@ PyArray_CastRawArrays(npy_intp count, int move_references); /* - * Casts the elements from one n-dimensional array to another n-dimensional - * array with identical shape but possibly different strides and dtypes. - * Does not account for overlap. - * - * Returns NPY_SUCCEED or NPY_FAIL. - */ -NPY_NO_EXPORT int -PyArray_CastRawNDimArrays(int ndim, npy_intp *shape, - char *src, char *dst, - npy_intp *src_strides, npy_intp *dst_strides, - PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - int move_references); - -/* * These two functions copy or convert the data of an n-dimensional array * to/from a 1-dimensional strided buffer. These functions will only call * 'stransfer' with the provided dst_stride/src_stride and @@ -330,7 +316,7 @@ NPY_NO_EXPORT npy_intp PyArray_TransferMaskedStridedToNDim(npy_intp ndim, char *dst, npy_intp *dst_strides, npy_intp dst_strides_inc, char *src, npy_intp src_stride, - npy_mask *mask, npy_intp mask_stride, + npy_bool *mask, npy_intp mask_stride, npy_intp *coords, npy_intp coords_inc, npy_intp *shape, npy_intp shape_inc, npy_intp count, npy_intp src_itemsize, @@ -345,8 +331,7 @@ PyArray_TransferMaskedStridedToNDim(npy_intp ndim, * * This is intended for simple, lightweight iteration over arrays * where no buffering of any kind is needed, and the array may - * not be stored as a PyArrayObject. For example, to iterate over - * the NA mask of an array. + * not be stored as a PyArrayObject. * * You can use this together with NPY_RAW_ITER_START and * NPY_RAW_ITER_ONE_NEXT to handle the looping boilerplate of everything @@ -410,34 +395,6 @@ PyArray_PrepareThreeRawArrayIter(int ndim, npy_intp *shape, char **out_dataB, npy_intp *out_stridesB, char **out_dataC, npy_intp *out_stridesC); -/* - * The same as PyArray_PrepareOneRawArrayIter, but for four - * operands instead of one. Any broadcasting of the four operands - * should have already been done before calling this function, - * as the ndim and shape is only specified once for all operands. - * - * Only the strides of the first operand are used to reorder - * the dimensions, no attempt to consider all the strides together - * is made, as is done in the NpyIter object. - * - * You can use this together with NPY_RAW_ITER_START and - * NPY_RAW_ITER_FOUR_NEXT to handle the looping boilerplate of everything - * but the innermost loop (which is for idim == 0). - * - * Returns 0 on success, -1 on failure. - */ -NPY_NO_EXPORT int -PyArray_PrepareFourRawArrayIter(int ndim, npy_intp *shape, - char *dataA, npy_intp *stridesA, - char *dataB, npy_intp *stridesB, - char *dataC, npy_intp *stridesC, - char *dataD, npy_intp *stridesD, - int *out_ndim, npy_intp *out_shape, - char **out_dataA, npy_intp *out_stridesA, - char **out_dataB, npy_intp *out_stridesB, - char **out_dataC, npy_intp *out_stridesC, - char **out_dataD, npy_intp *out_stridesD); - /* Start raw iteration */ #define NPY_RAW_ITER_START(idim, ndim, coord, shape) \ memset((coord), 0, (ndim) * sizeof(coord[0])); \ diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 3b62e150f..8de2dad39 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -38,6 +38,7 @@ #include "numpy/arrayobject.h" #include "numpy/ufuncobject.h" +#include "numpy/arrayscalars.h" #include "lowlevel_strided_loops.h" #include "ufunc_type_resolution.h" @@ -710,8 +711,7 @@ static int get_ufunc_arguments(PyUFuncObject *ufunc, PyObject **out_extobj, PyObject **out_typetup, int *out_subok, - PyArrayObject **out_wheremask, - int *out_use_maskna) + PyArrayObject **out_wheremask) { int i, nargs, nin = ufunc->nin, nout = ufunc->nout; PyObject *obj, *context; @@ -719,7 +719,6 @@ static int get_ufunc_arguments(PyUFuncObject *ufunc, char *ufunc_name; int any_flexible = 0, any_object = 0; - int any_non_maskna_out = 0, any_maskna_out = 0; ufunc_name = ufunc->name ? ufunc->name : "<unnamed ufunc>"; @@ -736,9 +735,6 @@ static int get_ufunc_arguments(PyUFuncObject *ufunc, return -1; } - /* Need USE_MASKNA mode if any input has an NA mask */ - *out_use_maskna = 0; - /* Get input arguments */ for (i = 0; i < nin; ++i) { obj = PyTuple_GET_ITEM(args, i); @@ -756,15 +752,11 @@ static int get_ufunc_arguments(PyUFuncObject *ufunc, context = NULL; } out_op[i] = (PyArrayObject *)PyArray_FromAny(obj, - NULL, 0, 0, NPY_ARRAY_ALLOWNA, context); + NULL, 0, 0, 0, context); Py_XDECREF(context); if (out_op[i] == NULL) { return -1; } - /* If the array has an NA mask, enable USE_MASKNA mode */ - if (PyArray_HASMASKNA(out_op[i])) { - *out_use_maskna = 1; - } if (!any_flexible && PyTypeNum_ISFLEXIBLE(PyArray_DESCR(out_op[i])->type_num)) { any_flexible = 1; @@ -801,13 +793,6 @@ static int get_ufunc_arguments(PyUFuncObject *ufunc, } Py_INCREF(obj); out_op[i] = (PyArrayObject *)obj; - - if (PyArray_HASMASKNA((PyArrayObject *)obj)) { - any_maskna_out = 1; - } - else { - any_non_maskna_out = 1; - } } else { PyErr_SetString(PyExc_TypeError, @@ -900,13 +885,6 @@ static int get_ufunc_arguments(PyUFuncObject *ufunc, } Py_INCREF(value); out_op[nin] = (PyArrayObject *)value; - - if (PyArray_HASMASKNA((PyArrayObject *)value)) { - any_maskna_out = 1; - } - else { - any_non_maskna_out = 1; - } } else { PyErr_SetString(PyExc_TypeError, @@ -978,44 +956,6 @@ static int get_ufunc_arguments(PyUFuncObject *ufunc, } Py_XDECREF(str_key_obj); - /* - * If NA mask support is enabled and there are non-maskNA outputs, - * only proceed if all the inputs contain no NA values. - */ - if (*out_use_maskna && any_non_maskna_out) { - /* Check all the inputs for NA */ - for (i = 0; i < nin; ++i) { - if (PyArray_HASMASKNA(out_op[i])) { - int containsna = PyArray_ContainsNA(out_op[i], NULL, NULL); - if (containsna == -1) { - return -1; - } - else if (containsna) { - PyErr_SetString(PyExc_ValueError, - "Cannot assign NA value to an array which " - "does not support NAs"); - return -1; - } - } - } - - /* Disable MASKNA - the inner loop uses NPY_ITER_IGNORE_MASKNA */ - *out_use_maskna = 0; - } - /* - * If we're not using a masked loop, but an output has an NA mask, - * set it to all exposed. - */ - else if (!(*out_use_maskna) && any_maskna_out) { - for (i = nin; i < nin+nout; ++i) { - if (PyArray_HASMASKNA(out_op[i])) { - if (PyArray_AssignMaskNA(out_op[i], 1, NULL, 0, NULL) < 0) { - return -1; - } - } - } - } - return 0; fail: @@ -1482,43 +1422,9 @@ execute_legacy_ufunc_loop(PyUFuncObject *ufunc, } /* - * This function combines the 'nin' input masks together, copying the - * result into each of the 'nout' output masks. - */ -static void -combine_ufunc_maskna(char **masks, npy_intp *strides, npy_intp count, - int nin, int nout) -{ - char *masks_copies[NPY_MAXARGS]; - npy_intp i; - int iop; - - /* Make copies of the mask pointers to modify */ - memcpy(masks_copies, masks, (nin + nout) * sizeof(char *)); - - /* - * TODO: This code only works for NPY_BOOL masks, will need to - * generalize this for multi-NA. - */ - for (i = 0; i < count; ++i) { - char maskvalue = *masks_copies[0]; - masks_copies[0] += strides[0]; - for (iop = 1; iop < nin; ++iop) { - maskvalue &= *masks_copies[iop]; - masks_copies[iop] += strides[iop]; - } - for (iop = nin; iop < nin + nout; ++iop) { - *masks_copies[iop] = maskvalue; - masks_copies[iop] += strides[iop]; - } - } -} - -/* * nin - number of inputs * nout - number of outputs * wheremask - if not NULL, the 'where=' parameter to the ufunc. - * use_maskna - if non-zero, flag USE_MASKNA for all the operands * op - the operands (nin + nout of them) * order - the loop execution order/output memory order * buffersize - how big of a buffer to use @@ -1527,9 +1433,8 @@ combine_ufunc_maskna(char **masks, npy_intp *strides, npy_intp count, * innerloopdata - data to pass to the inner loop */ static int -execute_ufunc_masked_loop(PyUFuncObject *ufunc, +execute_fancy_ufunc_loop(PyUFuncObject *ufunc, PyArrayObject *wheremask, - int use_maskna, PyArrayObject **op, PyArray_Descr **dtypes, NPY_ORDER order, @@ -1564,20 +1469,6 @@ execute_ufunc_masked_loop(PyUFuncObject *ufunc, default_op_out_flags |= NPY_ITER_WRITEMASKED; } - if (use_maskna) { - default_op_in_flags |= NPY_ITER_USE_MASKNA; - default_op_out_flags |= NPY_ITER_USE_MASKNA; - } - /* - * Some operands may still have NA masks, but they will - * have been checked to ensure they have no NAs using - * PyArray_ContainsNA. Thus we flag to ignore MASKNA here. - */ - else { - default_op_in_flags |= NPY_ITER_IGNORE_MASKNA; - default_op_out_flags |= NPY_ITER_IGNORE_MASKNA; - } - /* Set up the flags */ for (i = 0; i < nin; ++i) { op_flags[i] = default_op_in_flags | @@ -1689,26 +1580,12 @@ execute_ufunc_masked_loop(PyUFuncObject *ufunc, NPY_UF_DBG_PRINT("Actual inner loop:\n"); /* Execute the loop */ - if (wheremask != NULL) { - do { - NPY_UF_DBG_PRINT1("iterator loop count %d\n", (int)*countptr); - innerloop(dataptr, strides, - dataptr[nop], strides[nop], - *countptr, innerloopdata); - } while (iternext(iter)); - } - else { - do { - NPY_UF_DBG_PRINT1("iterator loop count %d\n", (int)*countptr); - /* Combine the input NA masks for the output */ - combine_ufunc_maskna(&dataptr[nop], &strides[nop], *countptr, - nin, nout); - /* Evaluate the ufunc wherever the NA mask says */ - innerloop(dataptr, strides, - dataptr[nop + nin], strides[nop + nin], - *countptr, innerloopdata); - } while (iternext(iter)); - } + do { + NPY_UF_DBG_PRINT1("iterator loop count %d\n", (int)*countptr); + innerloop(dataptr, strides, + dataptr[nop], strides[nop], + *countptr, innerloopdata); + } while (iternext(iter)); if (!needs_api) { NPY_END_THREADS; @@ -1796,7 +1673,6 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, npy_intp *inner_strides_tmp, *ax_strides_tmp[NPY_MAXDIMS]; int core_dim_ixs_size, *core_dim_ixs; - int use_maskna = 0; /* The __array_prepare__ function to call for each output */ PyObject *arr_prep[NPY_MAXARGS]; @@ -1837,17 +1713,11 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, /* Get all the arguments */ retval = get_ufunc_arguments(ufunc, args, kwds, op, &order, &casting, &extobj, - &type_tup, &subok, NULL, &use_maskna); + &type_tup, &subok, NULL); if (retval < 0) { goto fail; } - if (use_maskna) { - PyErr_SetString(PyExc_ValueError, - "Generalized ufuncs do not support ndarrays with NA masks"); - goto fail; - } - /* Figure out the number of dimensions needed by the iterator */ broadcast_ndim = 0; for (i = 0; i < nin; ++i) { @@ -2181,7 +2051,7 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, int i, nop; char *ufunc_name; int retval = -1, subok = 1; - int usemaskedloop = 0; + int need_fancy = 0; PyArray_Descr *dtypes[NPY_MAXARGS]; @@ -2201,7 +2071,7 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, */ PyObject *arr_prep_args = NULL; - int trivial_loop_ok = 0, use_maskna = 0; + int trivial_loop_ok = 0; NPY_ORDER order = NPY_KEEPORDER; /* Use the default assignment casting rule */ @@ -2238,28 +2108,16 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, /* Get all the arguments */ retval = get_ufunc_arguments(ufunc, args, kwds, op, &order, &casting, &extobj, - &type_tup, &subok, &wheremask, &use_maskna); + &type_tup, &subok, &wheremask); if (retval < 0) { goto fail; } /* - * Use the masked loop if either an input had an NA mask or a wheremask - * was specified. + * Use the masked loop if a wheremask was specified. */ - if (wheremask != NULL || use_maskna) { - usemaskedloop = 1; - - /* - * TODO: Implement support for this (requires more work in the - * iterator first) - */ - if (wheremask && use_maskna) { - PyErr_SetString(PyExc_RuntimeError, - "Ufuncs do not work with NA masked arrays and " - "the where= parameter at the same time yet"); - goto fail; - } + if (wheremask != NULL) { + need_fancy = 1; } /* Get the buffersize, errormask, and error object globals */ @@ -2287,7 +2145,7 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, } /* Only do the trivial loop check for the unmasked version. */ - if (!usemaskedloop) { + if (!need_fancy) { /* * This checks whether a trivial loop is ok, making copies of * scalar and one dimensional operands if that will help. @@ -2353,15 +2211,15 @@ PyUFunc_GenericFunction(PyUFuncObject *ufunc, PyUFunc_clearfperr(); /* Do the ufunc loop */ - if (usemaskedloop) { - NPY_UF_DBG_PRINT("Executing masked inner loop\n"); + if (need_fancy) { + NPY_UF_DBG_PRINT("Executing fancy inner loop\n"); - retval = execute_ufunc_masked_loop(ufunc, wheremask, use_maskna, + retval = execute_fancy_ufunc_loop(ufunc, wheremask, op, dtypes, order, buffersize, arr_prep, arr_prep_args); } else { - NPY_UF_DBG_PRINT("Executing unmasked inner loop\n"); + NPY_UF_DBG_PRINT("Executing legacy inner loop\n"); if (ufunc->legacy_inner_loop_selector != NULL) { retval = execute_legacy_ufunc_loop(ufunc, trivial_loop_ok, @@ -2574,15 +2432,15 @@ reduce_type_resolver(PyUFuncObject *ufunc, PyArrayObject *arr, } static int -assign_reduce_identity_zero(PyArrayObject *result, int preservena, void *data) +assign_reduce_identity_zero(PyArrayObject *result, void *data) { - return PyArray_AssignZero(result, NULL, preservena, NULL); + return PyArray_FillWithScalar(result, PyArrayScalar_False); } static int -assign_reduce_identity_one(PyArrayObject *result, int preservena, void *data) +assign_reduce_identity_one(PyArrayObject *result, void *data) { - return PyArray_AssignOne(result, NULL, preservena, NULL); + return PyArray_FillWithScalar(result, PyArrayScalar_True); } static int @@ -2673,112 +2531,6 @@ finish_loop: return (needs_api && PyErr_Occurred()) ? -1 : 0; } -static int -masked_reduce_loop(NpyIter *iter, char **dataptrs, npy_intp *strides, - npy_intp *countptr, NpyIter_IterNextFunc *iternext, - int needs_api, npy_intp skip_first_count, void *data) -{ - PyArray_Descr *dtypes[3], **iter_dtypes; - npy_intp fixed_strides[3], fixed_mask_stride; - PyUFuncObject *ufunc = (PyUFuncObject *)data; - char *dataptrs_copy[3]; - npy_intp strides_copy[3]; - - /* The masked selected inner loop */ - PyUFunc_MaskedStridedInnerLoopFunc *innerloop = NULL; - NpyAuxData *innerloopdata = NULL; - - NPY_BEGIN_THREADS_DEF; - - /* Get the inner loop */ - NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); - fixed_mask_stride = fixed_strides[2]; - fixed_strides[2] = fixed_strides[0]; - iter_dtypes = NpyIter_GetDescrArray(iter); - dtypes[0] = iter_dtypes[0]; - dtypes[1] = iter_dtypes[1]; - dtypes[2] = iter_dtypes[0]; - if (ufunc->masked_inner_loop_selector(ufunc, dtypes, iter_dtypes[2], - fixed_strides, fixed_mask_stride, - &innerloop, &innerloopdata, &needs_api) < 0) { - return -1; - } - - if (!needs_api) { - NPY_BEGIN_THREADS; - } - - if (skip_first_count > 0) { - do { - npy_intp count = *countptr; - - /* Skip any first-visit elements */ - if (NpyIter_IsFirstVisit(iter, 0)) { - if (strides[0] == 0) { - --count; - --skip_first_count; - dataptrs[1] += strides[1]; - dataptrs[2] += strides[2]; - } - else { - skip_first_count -= count; - count = 0; - } - } - - /* Turn the two items into three for the inner loop */ - dataptrs_copy[0] = dataptrs[0]; - dataptrs_copy[1] = dataptrs[1]; - dataptrs_copy[2] = dataptrs[0]; - strides_copy[0] = strides[0]; - strides_copy[1] = strides[1]; - strides_copy[2] = strides[0]; - /* - * If skipna=True, this masks based on the mask in 'arr', - * otherwise it masks based on the mask in 'result' - */ - innerloop(dataptrs_copy, strides_copy, - dataptrs[2], strides[2], - count, innerloopdata); - - /* Jump to the faster loop when skipping is done */ - if (skip_first_count == 0) { - if (iternext(iter)) { - break; - } - else { - goto finish_loop; - } - } - } while (iternext(iter)); - } - do { - /* Turn the two items into three for the inner loop */ - dataptrs_copy[0] = dataptrs[0]; - dataptrs_copy[1] = dataptrs[1]; - dataptrs_copy[2] = dataptrs[0]; - strides_copy[0] = strides[0]; - strides_copy[1] = strides[1]; - strides_copy[2] = strides[0]; - /* - * If skipna=True, this masks based on the mask in 'arr', - * otherwise it masks based on the mask in 'result' - */ - innerloop(dataptrs_copy, strides_copy, - dataptrs[2], strides[2], - *countptr, innerloopdata); - } while (iternext(iter)); - -finish_loop: - if (!needs_api) { - NPY_END_THREADS; - } - - NPY_AUXDATA_FREE(innerloopdata); - - return (needs_api && PyErr_Occurred()) ? -1 : 0; -} - /* * The implementation of the reduction operators with the new iterator * turned into a bit of a long function here, but I think the design @@ -2798,7 +2550,7 @@ finish_loop: */ static PyArrayObject * PyUFunc_Reduce(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, - int naxes, int *axes, PyArray_Descr *odtype, int skipna, int keepdims) + int naxes, int *axes, PyArray_Descr *odtype, int keepdims) { int iaxes, reorderable, ndim; npy_bool axis_flags[NPY_MAXDIMS]; @@ -2875,11 +2627,9 @@ PyUFunc_Reduce(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, result = PyArray_ReduceWrapper(arr, out, NULL, dtype, dtype, NPY_UNSAFE_CASTING, axis_flags, reorderable, - skipna, NULL, keepdims, 0, + keepdims, 0, assign_identity, reduce_loop, - masked_reduce_loop, - NULL, ufunc, buffersize, ufunc_name); Py_DECREF(dtype); @@ -2890,7 +2640,7 @@ PyUFunc_Reduce(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, static PyObject * PyUFunc_Accumulate(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, - int axis, int otype, int skipna) + int axis, int otype) { PyArrayObject *op[2]; PyArray_Descr *op_dtypes[2] = {NULL, NULL}; @@ -2898,7 +2648,7 @@ PyUFunc_Accumulate(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, int *op_axes[2] = {op_axes_arrays[0], op_axes_arrays[1]}; npy_uint32 op_flags[2]; int idim, ndim, otype_final; - int needs_api, need_outer_iterator, use_maskna = 0; + int needs_api, need_outer_iterator; NpyIter *iter = NULL, *iter_inner = NULL; @@ -2922,17 +2672,6 @@ PyUFunc_Accumulate(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, printf("\n"); #endif - use_maskna = PyArray_HASMASKNA(arr); - if (use_maskna) { - PyErr_SetString(PyExc_RuntimeError, - "ufunc accumulate doesn't support NA masked arrays yet"); - return NULL; - } - /* If there's no NA mask, there are no NAs to skip */ - else { - skipna = 0; - } - if (PyUFunc_GetPyValues("accumulate", &buffersize, &errormask, &errobj) < 0) { return NULL; } @@ -2995,11 +2734,6 @@ PyUFunc_Accumulate(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, NPY_ITER_NO_SUBTYPE; op_flags[1] = NPY_ITER_READONLY; - if (use_maskna) { - op_flags[0] |= NPY_ITER_USE_MASKNA; - op_flags[1] |= NPY_ITER_USE_MASKNA; - } - op[0] = out; op[1] = arr; @@ -3076,11 +2810,6 @@ PyUFunc_Accumulate(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, goto fail; } - if (use_maskna) { - if (PyArray_AllocateMaskNA(out, 1, 0, 1) < 0) { - goto fail; - } - } } } @@ -3279,7 +3008,7 @@ fail: */ static PyObject * PyUFunc_Reduceat(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *ind, - PyArrayObject *out, int axis, int otype, int skipna) + PyArrayObject *out, int axis, int otype) { PyArrayObject *op[3]; PyArray_Descr *op_dtypes[3] = {NULL, NULL, NULL}; @@ -3308,12 +3037,6 @@ PyUFunc_Reduceat(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *ind, NPY_BEGIN_THREADS_DEF; - if (PyArray_HASMASKNA(arr)) { - PyErr_SetString(PyExc_RuntimeError, - "ufunc reduceat doesn't support NA masked arrays yet"); - return NULL; - } - reduceat_ind = (npy_intp *)PyArray_DATA(ind); ind_size = PyArray_DIM(ind, 0); red_axis_size = PyArray_DIM(arr, axis); @@ -3658,11 +3381,11 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, PyArrayObject *indices = NULL; PyArray_Descr *otype = NULL; PyArrayObject *out = NULL; - int skipna = 0, keepdims = 0; + int keepdims = 0; static char *kwlist1[] = {"array", "axis", "dtype", - "out", "skipna", "keepdims", NULL}; + "out", "keepdims", NULL}; static char *kwlist2[] = {"array", "indices", "axis", - "dtype", "out", "skipna", NULL}; + "dtype", "out", NULL}; static char *_reduce_type[] = {"reduce", "accumulate", "reduceat", NULL}; if (ufunc == NULL) { @@ -3691,13 +3414,12 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, if (operation == UFUNC_REDUCEAT) { PyArray_Descr *indtype; indtype = PyArray_DescrFromType(NPY_INTP); - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO&O&i", kwlist2, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO&O&", kwlist2, &op, &obj_ind, &axes_in, PyArray_DescrConverter2, &otype, - PyArray_OutputConverter, &out, - &skipna)) { + PyArray_OutputConverter, &out)) { Py_XDECREF(otype); return NULL; } @@ -3709,24 +3431,22 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, } } else if (operation == UFUNC_ACCUMULATE) { - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO&O&ii", kwlist1, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO&O&i", kwlist1, &op, &axes_in, PyArray_DescrConverter2, &otype, PyArray_OutputConverter, &out, - &skipna, &keepdims)) { Py_XDECREF(otype); return NULL; } } else { - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO&O&ii", kwlist1, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO&O&i", kwlist1, &op, &axes_in, PyArray_DescrConverter2, &otype, - PyArray_OutputAllowNAConverter, &out, - &skipna, + PyArray_OutputConverter, &out, &keepdims)) { Py_XDECREF(otype); return NULL; @@ -3739,8 +3459,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, else { context = NULL; } - mp = (PyArrayObject *)PyArray_FromAny(op, NULL, 0, 0, - NPY_ARRAY_ALLOWNA, context); + mp = (PyArrayObject *)PyArray_FromAny(op, NULL, 0, 0, 0, context); Py_XDECREF(context); if (mp == NULL) { return NULL; @@ -3899,7 +3618,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, switch(operation) { case UFUNC_REDUCE: ret = PyUFunc_Reduce(ufunc, mp, out, naxes, axes, - otype, skipna, keepdims); + otype, keepdims); break; case UFUNC_ACCUMULATE: if (naxes != 1) { @@ -3910,7 +3629,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, return NULL; } ret = (PyArrayObject *)PyUFunc_Accumulate(ufunc, mp, out, axes[0], - otype->type_num, skipna); + otype->type_num); break; case UFUNC_REDUCEAT: if (naxes != 1) { @@ -3921,7 +3640,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, return NULL; } ret = (PyArrayObject *)PyUFunc_Reduceat(ufunc, mp, indices, out, - axes[0], otype->type_num, skipna); + axes[0], otype->type_num); Py_DECREF(indices); break; } diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index 3c697f1ee..da1b2e811 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -1329,8 +1329,7 @@ unmasked_ufunc_loop_as_masked( do { /* Skip masked values */ subloopsize = 0; - while (subloopsize < loopsize && - !NpyMaskValue_IsExposed(*(npy_mask *)mask)) { + while (subloopsize < loopsize && !*mask) { ++subloopsize; mask += mask_stride; } @@ -1343,8 +1342,7 @@ unmasked_ufunc_loop_as_masked( * mess with the 'args' pointer values) */ subloopsize = 0; - while (subloopsize < loopsize && - NpyMaskValue_IsExposed(*(npy_mask *)mask)) { + while (subloopsize < loopsize && *mask) { ++subloopsize; mask += mask_stride; } diff --git a/numpy/core/tests/test_api.py b/numpy/core/tests/test_api.py index 55d3c4ea8..3c365992b 100644 --- a/numpy/core/tests/test_api.py +++ b/numpy/core/tests/test_api.py @@ -101,36 +101,6 @@ def test_copyto_fromscalar(): np.copyto(a.T, 4.5, where=mask) assert_equal(a, [[2.5,4.5,4.5],[4.5,4.5,3.5]]) - # Simple copy to NA-masked - a_orig = a - a = a_orig.view(maskna=True) - a[...] = np.NA - np.copyto(a, 0.5) - assert_equal(a, 0.5) - a[...] = np.NA - np.copyto(a.T, 1.5) - assert_equal(a, 1.5) - - # Where-masked copy to NA-masked - a[0,0] = np.NA - a[1,1] = np.NA - mask = np.array([[1,0,1],[0,0,1]], dtype='?') - np.copyto(a, 2.5, where=mask) - assert_equal(np.isna(a), [[0,0,0],[0,1,0]]) - assert_equal(a_orig, [[2.5,1.5,2.5],[1.5,1.5,2.5]]) - - # Simple preservena=True copy - a[0,0] = np.NA - np.copyto(a, 3.5, preservena=True) - assert_equal(np.isna(a), [[1,0,0],[0,1,0]]) - assert_equal(a_orig, [[2.5,3.5,3.5],[3.5,1.5,3.5]]) - - # Where-masked preservena=True copy - mask = np.array([[1,0,1],[0,0,1]], dtype='?') - np.copyto(a, 4.5, where=mask, preservena=True) - assert_equal(np.isna(a), [[1,0,0],[0,1,0]]) - assert_equal(a_orig, [[2.5,3.5,4.5],[3.5,1.5,4.5]]) - def test_copyto(): a = np.arange(6, dtype='i4').reshape(2,3) @@ -167,79 +137,6 @@ def test_copyto(): # 'dst' must be an array assert_raises(TypeError, np.copyto, [1,2,3], [2,3,4]) -def test_copyto_maskna(): - a_orig = np.zeros((2,3), dtype='f8') - a = a_orig.view(maskna=True) - - # Simple copy from non-masked to NA-masked - a[...] = np.NA - np.copyto(a, np.arange(6).reshape(2,3)) - assert_equal(a, [[0,1,2],[3,4,5]]) - a[...] = np.NA - np.copyto(a.T, np.arange(6).reshape(3,2) + 1) - assert_equal(a, [[1,3,5],[2,4,6]]) - - # Simple copy from NA-masked to NA-masked - a[...] = np.NA - a[1,2] = 12 - tmp = np.arange(6, maskna=True).reshape(2,3) - tmp[0,1] = np.NA - tmp[1,2] = np.NA - np.copyto(a, tmp) - assert_equal(a_orig, [[0,3,2],[3,4,12]]) - assert_equal(np.isna(a), [[0,1,0],[0,0,1]]) - - # Where-masked copy from non-masked to NA-masked - a[...] = np.NA - a[0,2] = 6 - mask = np.array([[0,0,1],[0,1,0]], dtype='?') - tmp = np.arange(6).reshape(2,3) + 1 - np.copyto(a, tmp, where=mask) - assert_equal(a_orig, [[0,3,3],[3,5,12]]) - assert_equal(np.isna(a), ~mask) - - # Where-masked copy from NA-masked to NA-masked - a[1,2] = 12 - mask = np.array([[0,1,1],[0,0,1]], dtype='?') - tmp = np.arange(6, maskna=True).reshape(2,3) + 3 - tmp[0,0] = np.NA - tmp[0,1] = np.NA - tmp[1,2] = np.NA - np.copyto(a, tmp, where=mask) - assert_equal(a_orig, [[0,3,5],[3,5,12]]) - assert_equal(np.isna(a), [[1,1,0],[1,0,1]]) - - # Preserve-NA copy from non-masked to NA-masked - np.copyto(a, np.arange(6).reshape(2,3), preservena=True) - assert_equal(a_orig, [[0,3,2],[3,4,12]]) - assert_equal(np.isna(a), [[1,1,0],[1,0,1]]) - - # Preserve-NA copy from NA-masked to NA-masked - tmp = np.arange(6, maskna=True).reshape(2,3) + 1 - tmp[0,0] = np.NA - tmp[1,1] = np.NA - np.copyto(a, tmp, preservena=True) - assert_equal(a_orig, [[0,3,3],[3,4,12]]) - assert_equal(np.isna(a), [[1,1,0],[1,1,1]]) - - # Where-masked preserve-NA copy from non-masked to NA-masked - tmp = np.arange(6).reshape(2,3) + 3 - a[1,2] = 12 - mask = np.array([[0,1,1],[0,1,0]], dtype='?') - np.copyto(a, tmp, where=mask, preservena=True) - assert_equal(a_orig, [[0,3,5],[3,4,12]]) - assert_equal(np.isna(a), [[1,1,0],[1,1,0]]) - - # Where-masked preserve-NA copy from NA-masked to NA-masked - a[0,0] = 0 - mask = np.array([[0,1,1],[1,0,1]], dtype='?') - tmp = np.arange(6, maskna=True).reshape(2,3) + 1 - tmp[1,0] = np.NA - tmp[1,2] = np.NA - np.copyto(a, tmp, where=mask, preservena=True) - assert_equal(a_orig, [[0,3,3],[3,4,12]]) - assert_equal(np.isna(a), [[0,1,0],[1,1,1]]) - def test_copy_order(): a = np.arange(24).reshape(2,3,4) b = a.copy(order='F') diff --git a/numpy/core/tests/test_indexerrors.py b/numpy/core/tests/test_indexerrors.py index 230ca4955..af10df9b3 100644 --- a/numpy/core/tests/test_indexerrors.py +++ b/numpy/core/tests/test_indexerrors.py @@ -11,19 +11,6 @@ class TestIndexErrors(TestCase): assert_raises(IndexError, x.take, [0], axis=2) assert_raises(IndexError, x.take, [1], axis=2) - def test_maskna_take_1D(self): - # Check exception taking from masked array - a = np.array([1, np.NA, 2, np.NA], maskna=True) - assert_raises(IndexError, a.take, [6]) - a = np.array(np.NA, maskna=True) - assert_raises(IndexError, a.take, [6]) - - # Check exception taking from masked 0-d array - d = np.empty((5, 0), maskna=True) - assert_raises(IndexError, d.take, [1], axis=1) - assert_raises(IndexError, d.take, [0], axis=1) - assert_raises(IndexError, d.take, [0]) - def test_take_from_object(self): # Check exception taking from object array d = np.zeros(5, dtype=object) @@ -40,19 +27,11 @@ class TestIndexErrors(TestCase): assert_raises(IndexError, a.item, 20) a = np.empty((5, 0), dtype=object) assert_raises(IndexError, a.item, (0, 0)) - a = np.empty(5, dtype=object, maskna=True) - assert_raises(IndexError, a.item, 20) - a = np.empty((5, 0), dtype=object, maskna=True) - assert_raises(IndexError, a.item, (0, 0)) a = np.empty(5, dtype=object) assert_raises(IndexError, a.itemset, 20, 0) a = np.empty((5, 0), dtype=object) assert_raises(IndexError, a.itemset, (0, 0), 0) - a = np.empty(5, dtype=object, maskna=True) - assert_raises(IndexError, a.itemset, 20, 0) - a = np.empty((5, 0), dtype=object, maskna=True) - assert_raises(IndexError, a.itemset, (0, 0), 0) def test_put_exceptions(self): a = np.zeros((5, 5)) @@ -116,13 +95,6 @@ class TestIndexErrors(TestCase): assert_raises(IndexError, lambda: a[(1, 0)]) assert_raises(IndexError, lambda: assign(a, (1, 0), 1)) - a = np.zeros((3,5), maskna=True) - assert_raises(IndexError, lambda: a[(10, 20)]) - assert_raises(IndexError, lambda: assign(a, (10, 20), 1)) - a = np.zeros((3,0), maskna=True) - assert_raises(IndexError, lambda: a[(1, 0)]) - assert_raises(IndexError, lambda: assign(a, (1, 0), 1)) - a = np.zeros((10,)) assert_raises(IndexError, lambda: assign(a, 10, 1)) a = np.zeros((0,)) diff --git a/numpy/core/tests/test_maskna.py b/numpy/core/tests/test_maskna.py deleted file mode 100644 index c19cd70c2..000000000 --- a/numpy/core/tests/test_maskna.py +++ /dev/null @@ -1,1621 +0,0 @@ -import numpy as np -from numpy.compat import asbytes -from numpy.testing import * -import sys, warnings -from numpy.testing.utils import WarningManager - - -def combinations(iterable, r): - # copied from 2.7 documentation in order to support - # Python < 2.6 - # combinations('ABCD', 2) --> AB AC AD BC BD CD - # combinations(range(4), 3) --> 012 013 023 123 - pool = tuple(iterable) - n = len(pool) - if r > n: - return - indices = range(r) - yield tuple(pool[i] for i in indices) - while True: - for i in reversed(range(r)): - if indices[i] != i + n - r: - break - else: - return - indices[i] += 1 - for j in range(i+1, r): - indices[j] = indices[j-1] + 1 - yield tuple(pool[i] for i in indices) - - -def test_array_maskna_flags(): - a = np.arange(3) - assert_(not a.flags.maskna) - assert_(not a.flags.ownmaskna) - assert_(not a.flags['MASKNA']) - assert_(not a.flags['OWNMASKNA']) - # Add a mask by setting the flag - a.flags.maskna = True - assert_(a.flags.maskna) - assert_(a.flags.ownmaskna) - assert_(a.flags['MASKNA']) - assert_(a.flags['OWNMASKNA']) - # Can't remove the mask once it's created - def setmaskna(x, v): - x.maskna = v - assert_raises(ValueError, setmaskna, a.flags, False) - def setownmaskna(x, v): - x.ownmaskna = v - assert_raises(ValueError, setownmaskna, a.flags, False) - - -def test_array_maskna_construction(): - # Construction with NA inputs - a = np.array([1.0, 2.0, np.NA, 7.0], maskna=True) - assert_equal(a.dtype, np.dtype('f8')) - assert_(a.flags.maskna) - assert_equal(type(a[2]), np.NAType) - # Without the 'maskna=True', still produces an NA mask if NA is there - a = np.array([1.0, 2.0, np.NA, 7.0]) - assert_equal(a.dtype, np.dtype('f8')) - assert_(a.flags.maskna) - assert_equal(type(a[2]), np.NAType) - # Without any NAs, does not produce an NA mask - a = np.array([1.0, 2.0, 4.0, 7.0]) - assert_equal(a.dtype, np.dtype('f8')) - assert_(not a.flags.maskna) - - # From np.NA as a straight scalar - a = np.array(np.NA, maskna=True) - assert_equal(type(a), np.ndarray) - assert_(np.isna(a)) - - # As a special case, converting np.NA to an array produces - # a zero-dimensional masked array - a = np.array(np.NA) - assert_equal(type(a), np.ndarray) - assert_(np.isna(a)) - - # The data type defaults to the same as an empty array if all is NA - a = np.array([np.NA], maskna=True) - b = np.array([]) - assert_equal(a.dtype, b.dtype) - assert_(np.isna(a)) - - a = np.zeros((3,)) - assert_(not a.flags.maskna) - a = np.zeros((3,), maskna=True) - assert_(a.flags.maskna) - assert_equal(np.isna(a), False) - - a = np.ones((3,)) - assert_(not a.flags.maskna) - a = np.ones((3,), maskna=True) - assert_(a.flags.maskna) - assert_equal(np.isna(a), False) - - # np.empty returns all NAs if maskna is set to True - a = np.empty((3,)) - assert_(not a.flags.maskna) - a = np.empty((3,), maskna=True) - assert_(a.flags.maskna) - assert_equal(np.isna(a), True) - - # np.empty_like returns all NAs if maskna is set to True - tmp = np.arange(3) - a = np.empty_like(tmp) - assert_(not a.flags.maskna) - a = np.empty_like(tmp, maskna=True) - assert_(a.flags.maskna) - assert_equal(np.isna(a), True) - -@dec.skipif(sys.version_info < (2, 6)) -def test_array_maskna_pep3118(): - global memoryview - if sys.version_info[:2] == (2, 6): - from numpy.core.multiarray import memorysimpleview as memoryview - - a = np.array([0, 1, np.NA], maskna=True) - # The buffer protocol doesn't support NA masks, should raise an error - assert_raises(TypeError, memoryview, a) - -def test_array_maskna_asarray(): - a = np.arange(6).reshape(2,3) - - # Should not add an NA mask by default - res = np.asarray(a) - assert_(res is a) - assert_(not res.flags.maskna) - - # Should add an NA mask if requested - res = np.asarray(a, maskna=True) - assert_(res.flags.maskna) - assert_(res.flags.ownmaskna) - res = np.asarray(a, ownmaskna=True) - assert_(res.flags.maskna) - assert_(res.flags.ownmaskna) - - a.flags.maskna = True - - # Should view or create a copy of the NA mask - res = np.asarray(a) - assert_(res is a) - res = np.asarray(a, maskna=True) - assert_(res is a) - res = np.asarray(a, ownmaskna=True) - assert_(res is a) - - b = a.view() - assert_(not b.flags.ownmaskna) - - res = np.asarray(b) - assert_(res is b) - res = np.asarray(b, maskna=True) - assert_(res is b) - res = np.asarray(b, ownmaskna=True) - assert_(not (res is b)) - assert_(res.flags.ownmaskna) - - -def test_array_maskna_copy(): - a = np.array([1,2,3]) - b = np.array([2,3,4], maskna=True) - c = np.array([3,4,np.NA], maskna=True) - - # Make a copy, adding a mask - res = a.copy(maskna=True) - assert_equal(res, a) - assert_(res.flags.maskna) - assert_(res.flags.ownmaskna) - - res = np.copy(a, maskna=True) - assert_equal(res, a) - assert_(res.flags.maskna) - assert_(res.flags.ownmaskna) - - # Make a copy, removing a mask - res = b.copy(maskna=False) - assert_equal(res, b) - assert_(not res.flags.maskna) - assert_(not res.flags.ownmaskna) - - res = np.copy(b, maskna=False) - assert_equal(res, b) - assert_(not res.flags.maskna) - assert_(not res.flags.ownmaskna) - - # Copy with removing a mask doesn't work if there are NAs - assert_raises(ValueError, c.copy, maskna=False) - assert_raises(ValueError, np.copy, c, maskna=False) - - # Make a copy, preserving non-masked - res = a.copy() - assert_equal(res, a) - assert_(not res.flags.maskna) - assert_(not res.flags.ownmaskna) - - res = np.copy(a) - assert_equal(res, a) - assert_(not res.flags.maskna) - assert_(not res.flags.ownmaskna) - - # Make a copy, preserving masked - res = b.copy() - assert_equal(res, b) - assert_(res.flags.maskna) - assert_(res.flags.ownmaskna) - - res = np.copy(b) - assert_equal(res, b) - assert_(res.flags.maskna) - assert_(res.flags.ownmaskna) - - # Make a copy, preserving masked with an NA - res = c.copy() - assert_array_equal(res, c) - assert_(res.flags.maskna) - assert_(res.flags.ownmaskna) - - res = np.copy(c) - assert_array_equal(res, c) - assert_(res.flags.maskna) - assert_(res.flags.ownmaskna) - - -def test_array_maskna_astype(): - dtsrc = [np.dtype(d) for d in '?bhilqpBHILQPefdgFDGSUO'] - #dtsrc.append(np.dtype([('b', np.int, (1,))])) - dtsrc.append(np.dtype('datetime64[D]')) - dtsrc.append(np.dtype('timedelta64[s]')) - - dtdst = [np.dtype(d) for d in '?bhilqpBHILQPefdgFDGSUO'] - #dtdst.append(np.dtype([('b', np.int, (1,))])) - dtdst.append(np.dtype('datetime64[D]')) - dtdst.append(np.dtype('timedelta64[s]')) - - warn_ctx = WarningManager() - warn_ctx.__enter__() - try: - warnings.simplefilter("ignore", np.ComplexWarning) - for dt1 in dtsrc: - a = np.ones(2, dt1, maskna=1) - a[1] = np.NA - for dt2 in dtdst: - msg = 'type %s to %s conversion' % (dt1, dt2) - b = a.astype(dt2) - assert_(b.flags.maskna, msg) - assert_(b.flags.ownmaskna, msg) - assert_(np.isna(b[1]), msg) - finally: - warn_ctx.__exit__() - - -def test_array_maskna_repr(): - # Test some simple reprs with NA in them - a = np.array(np.NA, maskna=True) - assert_equal(repr(a), 'array(NA, dtype=float64)') - a = np.array(3, maskna=True) - assert_equal(repr(a), 'array(3, maskna=True)') - a = np.array([np.NA, 3], maskna=True) - assert_equal(repr(a), 'array([NA, 3])') - a = np.array([np.NA, np.NA]) - assert_equal(repr(a), 'array([ NA, NA], dtype=float64)') - a = np.array([3.5, np.NA], maskna=True) - assert_equal(repr(a), 'array([ 3.5, NA])') - a = np.array([3.75, 6.25], maskna=True) - assert_equal(repr(a), 'array([ 3.75, 6.25], maskna=True)') - a = np.array([3.75, 6.25], maskna=True, dtype='f4') - assert_equal(repr(a), 'array([ 3.75, 6.25], maskna=True, dtype=float32)') - - -def test_isna(): - # Objects which are not np.NA or ndarray all return False - assert_equal(np.isna(True), False) - assert_equal(np.isna("abc"), False) - assert_equal(np.isna([1,2,3]), False) - assert_equal(np.isna({3:5}), False) - # Various NA values return True - assert_equal(np.isna(np.NA), True) - assert_equal(np.isna(np.NA()), True) - assert_equal(np.isna(np.NA(5)), True) - assert_equal(np.isna(np.NA(dtype='f4')), True) - assert_equal(np.isna(np.NA(12,dtype='f4')), True) - - -def test_array_maskna_item(): - # With a zero-dimensional array - a = np.array(np.NA, maskna=True) - - # Should return NA as the item - assert_equal(type(a.item()), np.NAType) - - # Should be able to set the item - a.itemset(1.5) - assert_(not np.isna(a)) - assert_equal(a, 1.5) - a.itemset(np.NA) - assert_(np.isna(a)) - - # With a one-dimensional array - a = np.array([1, np.NA, 2, np.NA], maskna=True) - - # Should return the scalar or NA as the item - assert_(not np.isna(a.item(0))) - assert_equal(type(a.item(1)), np.NAType) - - # Should be able to set the items - a.itemset(0, np.NA) - assert_(np.isna(a[0])) - a.itemset(1, 12) - assert_(not np.isna(a[1])) - assert_equal(a[1], 12) - - # With a two-dimensional array - a = np.arange(6, maskna=True).reshape(2,3) - a[0,1] = np.NA - # Should return the scalar or NA as the item - assert_(not np.isna(a.item((0,0)))) - assert_equal(type(a.item((0,1))), np.NAType) - - # Should be able to set the items - a.itemset((0,1), 8) - assert_(not np.isna(a[0,1])) - assert_equal(a[0,1], 8) - a.itemset((1,1), np.NA) - assert_(np.isna(a[1,1])) - - -def test_array_maskna_payload(): - # Single numbered index - a = np.zeros((2,), maskna=True) - a[0] = np.NA - assert_equal(a[0].payload, None) - - # Tuple index - a = np.zeros((2,3), maskna=True) - a[1,1] = np.NA - assert_equal(a[1,1].payload, None) - - -def test_array_maskna_isna_1D(): - a = np.arange(10) - - # With no mask, it returns all False - assert_equal(np.isna(a), False) - assert_equal(np.isna(a).shape, (10,)) - - # With a mask but no NAs, it still returns all False - a.flags.maskna = True - assert_equal(np.isna(a), False) - assert_equal(np.isna(a).shape, (10,)) - - # Checking isna of a single value - assert_equal(np.isna(a[4]), False) - # Assigning NA to a single value - a[3] = np.NA - assert_equal(np.isna(a), [0,0,0,1,0,0,0,0,0,0]) - # Checking isna of a single value - assert_equal(np.isna(a[3]), True) - - # Checking isna of a slice - assert_equal(np.isna(a[1:6]), [0,0,1,0,0]) - # Assigning NA to a slice - a[5:7] = np.NA - assert_equal(np.isna(a), [0,0,0,1,0,1,1,0,0,0]) - - # Checking isna of a strided slice - assert_equal(np.isna(a[1:8:2]), [0,1,1,0]) - # Assigning NA to a strided slice - a[2:10:3] = np.NA - assert_equal(np.isna(a), [0,0,1,1,0,1,1,0,1,0]) - - # Checking isna of a boolean mask index - mask = np.array([1,1,0,0,0,1,0,1,1,0], dtype='?') - assert_equal(np.isna(a[mask]), [0,0,1,0,1]) - # Assigning NA to a boolean masked index - a[mask] = np.NA - assert_equal(np.isna(a), [1,1,1,1,0,1,1,1,1,0]) - - # TODO: fancy indexing is next... - - -def test_array_maskna_isna_2D(): - a = np.zeros((3,4)) - - # With no mask, it returns all False - assert_equal(np.isna(a), False) - assert_equal(np.isna(a).shape, (3,4)) - - # With a mask but no NAs, it still returns all False - a.flags.maskna = True - assert_equal(np.isna(a), False) - assert_equal(np.isna(a).shape, (3,4)) - - # Checking isna of a single value - assert_equal(np.isna(a[1,2]), False) - # Assigning NA to a single value - a[1,2] = np.NA - assert_equal(np.isna(a), [[0,0,0,0],[0,0,1,0],[0,0,0,0]]) - # Checking isna of a single value - assert_equal(np.isna(a[1,2]), True) - - # Checking isna of a slice - assert_equal(np.isna(a[1:4,1:3]), [[0,1],[0,0]]) - # Assigning NA to a slice - a[1:3,0:2] = np.NA - assert_equal(np.isna(a), [[0,0,0,0],[1,1,1,0],[1,1,0,0]]) - - # Checking isna of a strided slice - assert_equal(np.isna(a[1:,1:5:2]), [[1,0],[1,0]]) - # Assigning NA to a strided slice - a[::2,::2] = np.NA - assert_equal(np.isna(a), [[1,0,1,0],[1,1,1,0],[1,1,1,0]]) - - # Checking isna of a boolean mask index - mask = np.array([[1,1,0,0],[0,1,0,1],[0,0,1,0]], dtype='?') - assert_equal(np.isna(a[mask]), [1,0,1,0,1]) - # Assigning NA to a boolean masked index - a[mask] = np.NA - assert_equal(np.isna(a), [[1,1,1,0],[1,1,1,1],[1,1,1,0]]) - - # TODO: fancy indexing is next... - - -def test_array_maskna_to_nomask(): - # Assignment from an array with NAs to a non-masked array, - # excluding the NAs with a mask - a = np.array([[2,np.NA,5],[1,6,np.NA]], maskna=True) - mask = np.array([[1,0,0],[1,1,0]], dtype='?') - badmask = np.array([[1,0,0],[0,1,1]], dtype='?') - expected = np.array([[2,1,2],[1,6,5]]) - - # With masked indexing - b = np.arange(6).reshape(2,3) - b[mask] = a[mask] - assert_array_equal(b, expected) - - # With copyto - b = np.arange(6).reshape(2,3) - np.copyto(b, a, where=mask) - assert_array_equal(b, expected) - - # With masked indexing - b = np.arange(6).reshape(2,3) - def asn(): - b[badmask] = a[badmask] - assert_raises(ValueError, asn) - - # With copyto - b = np.arange(6).reshape(2,3) - assert_raises(ValueError, np.copyto, b, a, where=badmask) - - -def test_array_maskna_view_function(): - a = np.arange(10) - - # Taking a view of a non-masked array, making sure there's a mask - b = a.view(maskna=True) - assert_(not a.flags.maskna) - assert_(b.flags.maskna) - assert_(b.flags.ownmaskna) - - # Taking a view of a non-masked array, making sure there's an owned mask - b = a.view(ownmaskna=True) - assert_(not a.flags.maskna) - assert_(b.flags.maskna) - assert_(b.flags.ownmaskna) - - # Taking a view of a masked array - c = b.view() - assert_(b.flags.maskna) - assert_(b.flags.ownmaskna) - assert_(c.flags.maskna) - assert_(not c.flags.ownmaskna) - - # Taking a view of a masked array with maskna=False is invalid - assert_raises(ValueError, b.view, maskna=False) - - # Taking a view of a masked array, making sure there's a mask - c = b.view(maskna = True) - assert_(b.flags.maskna) - assert_(b.flags.ownmaskna) - assert_(c.flags.maskna) - assert_(not c.flags.ownmaskna) - - # Taking a view of a masked array, making sure there's an owned mask - c = b.view(ownmaskna = True) - assert_(b.flags.maskna) - assert_(b.flags.ownmaskna) - assert_(c.flags.maskna) - assert_(c.flags.ownmaskna) - -def test_array_maskna_view_dtype(): - tcs = np.typecodes['AllFloat'] + \ - np.typecodes['AllInteger'] + \ - np.typecodes['Complex'] - - same_size = [] - diff_size = [] - for x in combinations(tcs, 2): - if np.dtype(x[0]).itemsize == np.dtype(x[1]).itemsize: - same_size.append(x) - else: diff_size.append(x) - - for (from_type, to_type) in diff_size: - a = np.arange(10, dtype=from_type, maskna=True) - - # Ensure that a view of a masked array cannot change to - # different sized dtype - assert_raises(TypeError, a.view, to_type) - - for (from_type, to_type) in same_size: - a = np.arange(10, dtype=from_type, maskna=True) - - # Ensure that a view of a masked array can change to - # same sized dtype - b = a.view(dtype=to_type) - -def test_array_maskna_array_function_1D(): - a = np.arange(10) - a_ref = a.copy() - b = a.view(maskna=True) - b[3:10:2] = np.NA - b_view = b.view() - - # Ensure the setup is correct - assert_(not a.flags.maskna) - assert_(b.flags.maskna) - assert_(b.flags.ownmaskna) - assert_(b_view.flags.maskna) - assert_(not b_view.flags.ownmaskna) - - # Should be able to add a mask with 'maskna=' - c = np.array(a, maskna=True) - assert_(c.flags.maskna) - assert_(c.flags.ownmaskna) - assert_(not (c is b)) - - # Should be able to add a mask with 'ownmaskna=' - c = np.array(a, ownmaskna=True) - assert_(c.flags.maskna) - assert_(c.flags.ownmaskna) - assert_(not (c is b)) - - # Should propagate mask - c = np.array(b) - assert_(c.flags.maskna) - assert_(c.flags.ownmaskna) - assert_equal(np.isna(b), np.isna(c)) - assert_(not (c is b)) - - # Should propagate mask with 'maskna=True' - c = np.array(b, maskna=True) - assert_(c.flags.maskna) - assert_(c.flags.ownmaskna) - assert_equal(np.isna(b), np.isna(c)) - assert_(not (c is b)) - - # Should propagate mask with 'ownmaskna=True' - c = np.array(b, ownmaskna=True) - assert_(c.flags.maskna) - assert_(c.flags.ownmaskna) - assert_equal(np.isna(b), np.isna(c)) - assert_(not (c is b)) - - # Should be able to pass it through - c = np.array(b, copy=False) - assert_(c is b) - - # Should be able to pass it through with 'maskna=True' - c = np.array(b, copy=False, maskna=True) - assert_(c is b) - - # Should be able to pass it through with 'maskna=True' - c = np.array(b_view, copy=False, maskna=True) - assert_(c is b_view) - - # Should be able to pass an owned mask through with 'ownmaskna=True' - c = np.array(b, copy=False, ownmaskna=True) - assert_(c is b) - - # Should produce a view with an owned mask with 'ownmaskna=True' - c = np.array(b_view, copy=False, ownmaskna=True) - assert_(c.base is b_view.base) - assert_(c.flags.ownmaskna) - assert_(not (c is b_view)) - - # Should produce a view whose base is 'c', because 'c' owns - # the data for its mask - d = c.view() - assert_(d.base is c) - assert_(d.flags.maskna) - assert_(not d.flags.ownmaskna) - - -def test_array_maskna_setasflat(): - # Copy from a C to a F array with some NAs - a_orig = np.empty((2,3), order='C') - b_orig = np.empty((3,2), order='F') - a = a_orig.view(maskna=True) - b = b_orig.view(maskna=True) - a[...] = 1 - a[0,1] = np.NA - a[1,2] = np.NA - b[...] = 2 - b.setasflat(a) - assert_equal(np.isna(a), [[0,1,0],[0,0,1]]) - assert_equal(b_orig, [[1,2],[1,1],[1,2]]) - assert_equal(np.isna(b), [[0,1],[0,0],[0,1]]) - - -def test_array_maskna_ravel(): - # From a C array - a = np.zeros((2,3), maskna=True, order='C') - a[0,1] = np.NA - a[1,2] = np.NA - - # Ravel in C order returns a view - b = np.ravel(a) - assert_(b.base is a) - assert_equal(b.shape, (6,)) - assert_(b.flags.maskna) - assert_(not b.flags.ownmaskna) - assert_equal(np.isna(b), [0,1,0,0,0,1]) - - # Ravel in F order returns a copy - b = np.ravel(a, order='F') - assert_(b.base is None) - assert_equal(b.shape, (6,)) - assert_(b.flags.maskna) - assert_(b.flags.ownmaskna) - assert_equal(np.isna(b), [0,0,1,0,0,1]) - - a = np.arange(12, maskna=True).reshape(2,3,2).swapaxes(1,2) - assert_equal(a.ravel(order='K'), np.arange(12)) - - -def test_array_maskna_reshape(): - # Simple reshape 1D -> 2D - a = np.arange(6, maskna=True) - a[1] = np.NA - a[5] = np.NA - - # Reshape from 1D to C order - b = a.reshape(2,3) - assert_(b.base is a) - assert_equal(b.shape, (2,3)) - assert_(b.flags.maskna) - assert_(not b.flags.ownmaskna) - assert_equal(np.isna(b), [[0,1,0],[0,0,1]]) - - # Reshape from 1D to F order - b = a.reshape(2,3,order='F') - assert_(b.base is a) - assert_equal(b.shape, (2,3)) - assert_(b.flags.maskna) - assert_(not b.flags.ownmaskna) - assert_equal(np.isna(b), [[0,0,0],[1,0,1]]) - - # Add a new axis using 'newaxis' - a = np.array(np.NA, maskna=True) - assert_equal(np.isna(a[np.newaxis]), [True]) - - -def test_array_maskna_view_NA_assignment_1D(): - a = np.arange(10) - a_ref = a.copy() - - # Make sure that assigning NA doesn't affect the original data - b = a.view(maskna=True) - b[...] = np.NA - assert_equal(np.isna(b), True) - assert_equal(a, a_ref) - - b = a.view(maskna=True) - b[:] = np.NA - assert_equal(np.isna(b), True) - assert_equal(a, a_ref) - - b = a.view(maskna=True) - b[3:5] = np.NA - assert_equal(np.isna(b), [0,0,0,1,1,0,0,0,0,0]) - assert_equal(a, a_ref) - - b = a.view(maskna=True) - b[3:10:3] = np.NA - assert_equal(np.isna(b), [0,0,0,1,0,0,1,0,0,1]) - assert_equal(a, a_ref) - - b = a.view(maskna=True) - b[3] = np.NA - assert_equal(np.isna(b), [0,0,0,1,0,0,0,0,0,0]) - assert_equal(a, a_ref) - - b = a.view(maskna=True) - mask = np.array([0,1,0,1,1,0,0,0,1,1], dtype='?') - b[mask] = np.NA - assert_equal(np.isna(b), mask) - assert_equal(a, a_ref) - - # TODO: fancy indexing is next... - - -def test_array_maskna_view_NA_assignment_2D(): - a = np.arange(6).reshape(2,3) - a_ref = a.copy() - - # Make sure that assigning NA doesn't affect the original data - b = a.view(maskna=True) - b[...] = np.NA - assert_equal(np.isna(b), True) - assert_equal(a, a_ref) - - b = a.view(maskna=True) - b[:] = np.NA - assert_equal(np.isna(b), True) - assert_equal(a, a_ref) - - b = a.view(maskna=True) - b[0,:] = np.NA - assert_equal(np.isna(b[0]), True) - assert_equal(np.isna(b[1]), False) - assert_equal(a, a_ref) - - b = a.view(maskna=True) - b[1:,1:3] = np.NA - assert_equal(np.isna(b), [[0,0,0],[0,1,1]]) - assert_equal(a, a_ref) - - b = a.view(maskna=True) - b[1,::2] = np.NA - assert_equal(np.isna(b), [[0,0,0],[1,0,1]]) - assert_equal(a, a_ref) - - b = a.view(maskna=True) - b[0,2] = np.NA - assert_equal(np.isna(b), [[0,0,1],[0,0,0]]) - assert_equal(a, a_ref) - - b = a.view(maskna=True) - mask = np.array([[1,0,1],[1,1,0]], dtype='?') - b[mask] = np.NA - assert_equal(np.isna(b), mask) - assert_equal(a, a_ref) - - # TODO: fancy indexing is next... - - -def test_array_maskna_view_array_assignment_1D(): - a = np.arange(5) - b = a.view(maskna=True) - - # Assigning a constant scalar should unmask the values - b[...] = np.NA - b[...] = 3 - assert_equal(a, 3) - assert_equal(np.isna(b), False) - - # Assigning from a list should unmask the values - b[...] = np.NA - b[...] = [2] - assert_equal(a, [2,2,2,2,2]) - assert_equal(np.isna(b), False) - - # Assigning from a list should unmask the values - b[...] = np.NA - b[...] = [2,3,4,5,6] - assert_equal(a, [2,3,4,5,6]) - assert_equal(np.isna(b), False) - - # Assigning from a list with NAs should unmask the non-NA values - b[...] = np.NA - b[...] = [7,np.NA,2,0,np.NA] - assert_equal(a, [7,3,2,0,6]) - assert_equal(np.isna(b), [0,1,0,0,1]) - - # Assigning from an unmasked array should unmask the values - b[...] = np.NA - b[...] = np.arange(5) - assert_equal(a, np.arange(5)) - assert_equal(np.isna(b), False) - - # Assigning from a masked array with no NAs should unmask the values - b[...] = np.NA - tmp = np.arange(5) + 1 - tmp.flags.maskna = True - b[...] = tmp - assert_equal(a, np.arange(5) + 1) - assert_equal(np.isna(b), False) - - # Assigning from a masked array with some NAs should unmask most - # of the values, and leave the value behind the NAs untouched - b[...] = np.NA - tmp = np.arange(5) + 5 - tmp.flags.maskna = True - tmp[2] = np.NA - b[...] = tmp - assert_equal(a, [5,6,3,8,9]) - assert_equal(np.isna(b), [0,0,1,0,0]) - - # Assigning to a single element should unmask the value - b[...] = np.NA - b[2] = 10 - assert_equal(a, [5,6,10,8,9]) - assert_equal(np.isna(b), [1,1,0,1,1]) - - # Assigning to a simple slice should unmask the values - b[...] = np.NA - b[2:] = 4 - assert_equal(a, [5,6,4,4,4]) - assert_equal(np.isna(b), [1,1,0,0,0]) - - # Assigning to a strided slice should unmask the values - b[...] = np.NA - b[3::-2] = 12 - assert_equal(a, [5,12,4,12,4]) - assert_equal(np.isna(b), [1,0,1,0,1]) - - # Assigning to a boolean index should unmask the values - b[...] = np.NA - mask = np.array([0,1,1,0,1], dtype='?') - b[mask] = 7 - assert_equal(a, [5,7,7,12,7]) - assert_equal(np.isna(b), [1,0,0,1,0]) - - # Assigning a list to a boolean index should unmask the values - b[...] = np.NA - mask = np.array([1,0,0,0,1], dtype='?') - b[mask] = [8,1] - assert_equal(a, [8,7,7,12,1]) - assert_equal(np.isna(b), [0,1,1,1,0]) - - # Assigning a list with NA to a boolean index should unmask non-NA values - b[...] = np.NA - mask = np.array([0,1,1,0,0], dtype='?') - b[mask] = np.array([8,np.NA], maskna=True) - assert_equal(a, [8,8,7,12,1]) - assert_equal(np.isna(b), [1,0,1,1,1]) - - # TODO: fancy indexing is next... - - -def test_maskna_nonzero_1D(): - a = np.zeros((5,), maskna=True) - - # The nonzeros without any NAs - assert_equal(np.count_nonzero(a), 0) - assert_equal(np.nonzero(a)[0], []) - a[2] = 3 - assert_equal(np.count_nonzero(a), 1) - assert_equal(np.nonzero(a)[0], [2]) - a[3:] = 2 - assert_equal(np.count_nonzero(a), 3) - assert_equal(np.nonzero(a)[0], [2,3,4]) - - # The nonzeros with an NA - a[2] = np.NA - assert_(np.isna(np.count_nonzero(a))) - assert_raises(ValueError, np.nonzero, a) - - -def test_maskna_take_1D(): - a = np.arange(5, maskna=True) - b = np.arange(3) - c = b.view(maskna=True) - - # Take without any NAs - assert_equal(a.take([0,2,4]), [0,2,4]) - - # Take without any NAs, into non-NA output parameter - a.take([0,2,4], out=b) - assert_equal(b, [0,2,4]) - - # Take without any NAs, into NA output parameter - b[...] = 1 - c[...] = np.NA - a.take([0,2,4], out=c) - assert_equal(c, [0,2,4]) - - # Take with some NAs - a[2] = np.NA - a[3] = np.NA - ret = a.take([0,2,4]) - assert_equal([ret[0], ret[2]], [0,4]) - assert_equal(np.isna(ret), [0,1,0]) - - # Take with some NAs, into NA output parameter - b[...] = 1 - c[...] = np.NA - a.take([0,2,4], out=c) - assert_equal(b, [0,1,4]) - assert_equal([c[0], c[2]], [0,4]) - assert_equal(np.isna(c), [0,1,0]) - - c[...] = 1 - a.take([0,2,4], out=c) - assert_equal(b, [0,1,4]) - assert_equal([c[0], c[2]], [0,4]) - assert_equal(np.isna(c), [0,1,0]) - - # Take with an NA just at the start - a = np.arange(5, maskna=True) - a[0] = np.NA - res = a.take([1,2,3,4]) - assert_equal(res, [1,2,3,4]) - - -def test_maskna_ufunc_1D(): - a_orig = np.arange(3) - a = a_orig.view(maskna=True) - b_orig = np.array([5,4,3]) - b = b_orig.view(maskna=True) - c_orig = np.array([0,0,0]) - c = c_orig.view(maskna=True) - - # An NA mask is produced if an operand has one - res = a + b_orig - assert_(res.flags.maskna) - assert_equal(res, [5,5,5]) - - res = b_orig + a - assert_(res.flags.maskna) - assert_equal(res, [5,5,5]) - - # Can still output to a non-NA array if there are no NAs - np.add(a, b, out=c_orig) - assert_equal(c_orig, [5,5,5]) - - # Should unmask everything if the output has NA support but - # the inputs don't - c_orig[...] = 0 - c[...] = np.NA - np.add(a_orig, b_orig, out=c) - assert_equal(c, [5,5,5]) - - # If the input has NA support but an output parameter doesn't, - # should work as long as the inputs contain no NAs - c_orig[...] = 0 - np.add(a, b, out=c_orig) - assert_equal(c_orig, [5,5,5]) - - # An NA is produced if either operand has one - a[0] = np.NA - b[1] = np.NA - res = a + b - assert_equal(np.isna(res), [1,1,0]) - assert_equal(res[2], 5) - - # If the output contains NA, can't have out= parameter without - # NA support - assert_raises(ValueError, np.add, a, b, out=c_orig) - - # Divide in-place with NA - a_orig = np.array([[3], [12.]]) - a = a_orig.view(maskna=True) - a[0,0] = np.NA - a /= 3 - # Shouldn't have touched the masked element - assert_array_equal(a_orig, [[3], [4.]]) - assert_array_equal(a, [[np.NA], [4.]]) - # double-check assertions - assert_equal(np.isna(a), [[1], [0]]) - assert_equal(a[~np.isna(a)], [4.]) - - -def test_maskna_ufunc_sum_1D(): - check_maskna_ufunc_sum_1D(np.sum) - - -def test_maskna_ufunc_add_reduce_1D(): - check_maskna_ufunc_sum_1D(np.add.reduce) - - -def check_maskna_ufunc_sum_1D(sum_func): - a = np.arange(3.0, maskna=True) - b = np.array(0.5) - c_orig = np.array(0.5) - c = c_orig.view(maskna=True) - - # Since 'a' has no NA values, this should work - sum_func(a, out=b) - assert_equal(b, 3.0) - b[...] = 7 - sum_func(a, skipna=True, out=b) - assert_equal(b, 3.0) - - ret = sum_func(a) - assert_equal(ret, 3.0) - ret = sum_func(a, skipna=True) - assert_equal(ret, 3.0) - - # With an NA value, the reduce should throw with the non-NA output param - a[1] = np.NA - assert_raises(ValueError, sum_func, a, out=b) - - # With an NA value, the output parameter can still be an NA-array - c_orig[...] = 0.5 - sum_func(a, out=c) - assert_equal(c_orig, 0.5) - assert_(np.isna(c)) - - # Should not touch the out= element when assigning NA - b[...] = 1.0 - d = b.view(maskna=True) - sum_func(a, out=d) - assert_(np.isna(d)) - assert_equal(b, 1.0) - - # Without an output parameter, return NA - ret = sum_func(a) - assert_(np.isna(ret)) - - # With 'skipna=True' - ret = sum_func(a, skipna=True) - assert_equal(ret, 2.0) - - # With 'skipna=True', and out= parameter - b[...] = 0.5 - sum_func(a, skipna=True, out=b) - assert_equal(b, 2.0) - - # With 'skipna=True', and out= parameter with a mask - c[...] = 0.5 - c[...] = np.NA - sum_func(a, skipna=True, out=c) - assert_(not np.isna(c)) - assert_equal(c, 2.0) - - -def test_ufunc_max_1D(): - check_ufunc_max_1D(np.max) - - -def test_ufunc_maximum_reduce_1D(): - check_ufunc_max_1D(np.maximum.reduce) - - -def check_ufunc_max_1D(max_func): - a_orig = np.array([0, 3, 2, 10, -1, 5, 7, -2]) - a = a_orig.view(maskna=True) - - # Straightforward reduce with no NAs - b = max_func(a) - assert_equal(b, 10) - - # Set the biggest value to NA - a[3] = np.NA - b = max_func(a) - assert_(np.isna(b)) - - # Skip the NA - b = max_func(a, skipna=True) - assert_(not b.flags.maskna) - assert_(not np.isna(b)) - assert_equal(b, 7) - - # Set the first value to NA - a[0] = np.NA - b = max_func(a, skipna=True) - assert_(not b.flags.maskna) - assert_(not np.isna(b)) - assert_equal(b, 7) - - # Set all the values to NA - should raise the same error as - # for an empty array - a[...] = np.NA - assert_raises(ValueError, max_func, a, skipna=True) - - -def test_ufunc_skipna_max_3D(): - check_ufunc_skipna_max_3D(np.max) - - -def test_ufunc_skipna_maximum_reduce_3D(): - check_ufunc_skipna_max_3D(np.maximum.reduce) - - -def check_ufunc_skipna_max_3D(max_func): - a_orig = np.array([[[29, 6, 24, 11, 24], - [17, 26, 10, 29, 21], - [ 4, 4, 7, 9, 30], - [ 9, 20, 5, 12, 23]], - [[ 8, 9, 10, 31, 22], - [ 5, 20, 2, 29, 27], - [21, 22, 13, 30, 20], - [24, 27, 9, 20, 31]], - [[14, 0, 13, 11, 22], - [ 0, 16, 16, 14, 2], - [ 0, 2, 1, 29, 12], - [24, 25, 12, 11, 9]]]) - a = a_orig.view(maskna=True) - b = a_orig.copy() - - def check_all_axis_combos(x, y, badaxes=()): - if 0 not in badaxes: - res = max_func(x, axis=0, skipna=True) - assert_array_equal(res, max_func(y, axis=0, skipna=True)) - if 1 not in badaxes: - res = max_func(x, axis=1, skipna=True) - assert_array_equal(res, max_func(y, axis=1, skipna=True)) - if 2 not in badaxes: - res = max_func(x, axis=2, skipna=True) - assert_array_equal(res, max_func(y, axis=2, skipna=True)) - res = max_func(x, axis=(0,1), skipna=True) - assert_array_equal(res, max_func(y, axis=(0,1), skipna=True)) - res = max_func(x, axis=(0,2), skipna=True) - assert_array_equal(res, max_func(y, axis=(0,2), skipna=True)) - res = max_func(x, axis=(1,2), skipna=True) - assert_array_equal(res, max_func(y, axis=(1,2), skipna=True)) - res = max_func(x, axis=(0,1,2), skipna=True) - assert_array_equal(res, max_func(y, axis=(0,1,2), skipna=True)) - - # Straightforward reduce with no NAs - check_all_axis_combos(a, b) - - # Set a few values in 'a' to NA, and set the corresponding - # values in 'b' to -1 to definitely eliminate them from the maximum - for coord in [(0,1,2), (1,2,2), (0,2,4), (2,1,0)]: - a[coord] = np.NA - b[coord] = -1 - check_all_axis_combos(a, b) - - # Set a few more values in 'a' to NA - for coord in [(2,1,1), (2,1,2), (2,1,3), (0,0,4), (0,3,4)]: - a[coord] = np.NA - b[coord] = -1 - check_all_axis_combos(a, b) - - # Set it so that there's a full set of NAs along the third dimension - for coord in [(2,1,4)]: - a[coord] = np.NA - b[coord] = -1 - check_all_axis_combos(a, b, badaxes=(2,)) - assert_raises(ValueError, max_func, a, axis=2, skipna=True) - - # Set it so that there's a full set of NAs along the second dimension - for coord in [(0,1,4)]: - a[coord] = np.NA - b[coord] = -1 - check_all_axis_combos(a, b, badaxes=(1,2)) - assert_raises(ValueError, max_func, a, axis=1, skipna=True) - assert_raises(ValueError, max_func, a, axis=2, skipna=True) - - -def test_ufunc_ndarray_any(): - a = np.array([0,0,0,0,0], dtype='?', maskna=True) - assert_array_equal(a.any(), False) - a[0] = np.NA - assert_array_equal(a.any(), np.NA) - assert_array_equal(a.any(skipna=True), False) - a[0] = 0 - a[-1] = np.NA - assert_array_equal(a.any(), np.NA) - assert_array_equal(a.any(skipna=True), False) - a[0] = 1 - assert_array_equal(a.any(), True) - assert_array_equal(a.any(skipna=True), True) - a[-1] = 1 - a[-2] = np.NA - assert_array_equal(a.any(), True) - assert_array_equal(a.any(skipna=True), True) - - a = np.array([[0,0,0],[0,np.NA,0]], dtype='?') - assert_array_equal(a.any(axis=0), [False, np.NA, False]) - assert_array_equal(a.any(axis=1), [False, np.NA]) - assert_array_equal(a.any(axis=0, skipna=True), [False, False, False]) - assert_array_equal(a.any(axis=1, skipna=True), [False, False]) - - a[0,1] = 1 - assert_array_equal(a.any(axis=0), [False, True, False]) - assert_array_equal(a.any(axis=1), [True, np.NA]) - assert_array_equal(a.any(axis=0, skipna=True), [False, True, False]) - assert_array_equal(a.any(axis=1, skipna=True), [True, False]) - - a[0,1] = np.NA - a[1,1] = 0 - a[0,2] = 1 - assert_array_equal(a.any(axis=0), [False, np.NA, True]) - assert_array_equal(a.any(axis=1), [True, False]) - assert_array_equal(a.any(axis=0, skipna=True), [False, False, True]) - assert_array_equal(a.any(axis=1, skipna=True), [True, False]) - - -def test_ufunc_ndarray_all(): - a = np.array([1,1,1,1,1], dtype='?', maskna=True) - assert_array_equal(a.all(), True) - a[0] = np.NA - assert_array_equal(a.all(), np.NA) - assert_array_equal(a.all(skipna=True), True) - a[0] = 1 - a[-1] = np.NA - assert_array_equal(a.all(), np.NA) - assert_array_equal(a.all(skipna=True), True) - a[0] = 0 - assert_array_equal(a.all(), False) - assert_array_equal(a.all(skipna=True), False) - a[-1] = 0 - a[-2] = np.NA - assert_array_equal(a.all(), False) - assert_array_equal(a.all(skipna=True), False) - - a = np.array([[1,1,1],[1,np.NA,1]], dtype='?') - assert_array_equal(a.all(axis=0), [True, np.NA, True]) - assert_array_equal(a.all(axis=1), [True, np.NA]) - assert_array_equal(a.all(axis=0, skipna=True), [True, True, True]) - assert_array_equal(a.all(axis=1, skipna=True), [True, True]) - - a[0,1] = 0 - assert_array_equal(a.all(axis=0), [True, False, True]) - assert_array_equal(a.all(axis=1), [False, np.NA]) - assert_array_equal(a.all(axis=0, skipna=True), [True, False, True]) - assert_array_equal(a.all(axis=1, skipna=True), [False, True]) - - a[0,1] = np.NA - a[1,1] = 1 - a[0,2] = 0 - assert_array_equal(a.all(axis=0), [True, np.NA, False]) - assert_array_equal(a.all(axis=1), [False, True]) - assert_array_equal(a.all(axis=0, skipna=True), [True, True, False]) - assert_array_equal(a.all(axis=1, skipna=True), [False, True]) - - -def test_count_reduce_items(): - # np.count_reduce_items - - # When skipna is False, it should always return the - # product of the reduction axes as a NumPy intp scalar - a = np.zeros((2,3,4)) - - res = np.count_reduce_items(a) - assert_equal(res, 24) - assert_equal(type(res), np.intp) - - res = np.count_reduce_items(a, axis=0) - assert_equal(res, 2) - assert_equal(type(res), np.intp) - - res = np.count_reduce_items(a, axis=(1,2)) - assert_equal(res, 12) - assert_equal(type(res), np.intp) - - # This still holds if 'a' has an NA mask and some NA values - a = np.zeros((2,3,4), maskna=True) - a[1,2,2] = np.NA - a[0,1,2] = np.NA - a[1,0,3] = np.NA - - res = np.count_reduce_items(a) - assert_equal(res, 24) - assert_equal(type(res), np.intp) - - res = np.count_reduce_items(a, axis=0) - assert_equal(res, 2) - assert_equal(type(res), np.intp) - - res = np.count_reduce_items(a, axis=(1,2)) - assert_equal(res, 12) - assert_equal(type(res), np.intp) - - # If skipna is True, but the array has no NA mask, the result - # should still be the product of the reduction axes - a = np.zeros((2,3,4)) - - res = np.count_reduce_items(a, skipna=True) - assert_equal(res, 24) - assert_equal(type(res), np.intp) - - res = np.count_reduce_items(a, axis=0, skipna=True) - assert_equal(res, 2) - assert_equal(type(res), np.intp) - - res = np.count_reduce_items(a, axis=(1,2), skipna=True) - assert_equal(res, 12) - assert_equal(type(res), np.intp) - - # Finally, when skipna is True AND the array has an NA mask, - # we get an array of counts - a = np.zeros((2,3,4), maskna=True) - a[1,2,2] = np.NA - a[0,1,2] = np.NA - a[1,0,3] = np.NA - - # When doing a full reduction, should still get the scalar - res = np.count_reduce_items(a, skipna=True) - assert_equal(res, 21) - assert_equal(res.dtype, np.dtype(np.intp)) - - res = np.count_reduce_items(a, axis=0, skipna=True) - assert_equal(res, [[2,2,2,1], [2,2,1,2], [2,2,1,2]]) - assert_equal(res.dtype, np.dtype(np.intp)) - - res = np.count_reduce_items(a, axis=(1,2), skipna=True) - assert_equal(res, [11,10]) - assert_equal(res.dtype, np.dtype(np.intp)) - - -def test_array_maskna_clip_method(): - # ndarray.clip - a = np.array([2, np.NA, 10, 4, np.NA, 7], maskna=True) - - b = np.clip(a, 3, None) - assert_equal(np.isna(b), [0,1,0,0,1,0]) - assert_equal(b[~np.isna(b)], [3, 10, 4, 7]) - - res = np.clip(a, None, 6) - assert_equal(np.isna(res), [0,1,0,0,1,0]) - assert_equal(res[~np.isna(res)], [2, 6, 4, 6]) - - res = np.clip(a, 4, 7) - assert_equal(np.isna(res), [0,1,0,0,1,0]) - assert_equal(res[~np.isna(res)], [4, 7, 4, 7]) - - -def test_array_maskna_max_min_ptp_methods(): - # ndarray.max, ndarray.min, ndarray.ptp - a = np.array([[2, np.NA, 10], - [4, 8, 7], - [12, 4, np.NA]], maskna=True) - - res = a.max(axis=0) - assert_equal(np.isna(res), [0,1,1]) - assert_equal(res[~np.isna(res)], [12]) - - res = a.max(axis=-1) - assert_equal(np.isna(res), [1,0,1]) - assert_equal(res[~np.isna(res)], [8]) - - res = a.min(axis=0) - assert_equal(np.isna(res), [0,1,1]) - assert_equal(res[~np.isna(res)], [2]) - - res = a.min(axis=-1) - assert_equal(np.isna(res), [1,0,1]) - assert_equal(res[~np.isna(res)], [4]) - - res = a.ptp(axis=0) - assert_equal(np.isna(res), [0,1,1]) - assert_equal(res[~np.isna(res)], [10]) - - res = a.ptp(axis=-1) - assert_equal(np.isna(res), [1,0,1]) - assert_equal(res[~np.isna(res)], [4]) - - -def test_array_maskna_sum_prod_methods(): - # ndarray.sum, ndarray.prod - a = np.array([[2, np.NA, 10], - [4, 8, 7], - [12, 4, np.NA], - [3, 2, 5]], maskna=True) - - res = a.sum(axis=0) - assert_equal(np.isna(res), [0,1,1]) - assert_equal(res[~np.isna(res)], [21]) - - res = a.sum(axis=-1) - assert_equal(np.isna(res), [1,0,1,0]) - assert_equal(res[~np.isna(res)], [19,10]) - - res = a.prod(axis=0) - assert_equal(np.isna(res), [0,1,1]) - assert_equal(res[~np.isna(res)], [2*4*12*3]) - - res = a.prod(axis=-1) - assert_equal(np.isna(res), [1,0,1,0]) - assert_equal(res[~np.isna(res)], [4*8*7,3*2*5]) - - # Check also with Fortran-order - a = np.array([[2, np.NA, 10], - [4, 8, 7], - [12, 4, np.NA], - [3, 2, 5]], maskna=True, order='F') - - res = a.sum(axis=0) - assert_equal(np.isna(res), [0,1,1]) - assert_equal(res[~np.isna(res)], [21]) - - res = a.sum(axis=-1) - assert_equal(np.isna(res), [1,0,1,0]) - assert_equal(res[~np.isna(res)], [19,10]) - - -def test_array_maskna_std_mean_methods(): - # ndarray.std, ndarray.mean - a = np.array([[2, np.NA, 10], - [4, 8, 7], - [12, 4, np.NA]], maskna=True) - - res = a.mean(axis=0) - assert_equal(np.isna(res), [0,1,1]) - assert_equal(res[~np.isna(res)], [np.array([2,4,12]).mean()]) - - res = a.mean(axis=-1) - assert_equal(np.isna(res), [1,0,1]) - assert_equal(res[~np.isna(res)], [np.array([4,8,7]).mean()]) - - res = a.std(axis=0) - assert_equal(np.isna(res), [0,1,1]) - assert_equal(res[~np.isna(res)], [np.array([2,4,12]).std()]) - - res = a.std(axis=-1) - assert_equal(np.isna(res), [1,0,1]) - assert_equal(res[~np.isna(res)], [np.array([4,8,7]).std()]) - - -def test_array_maskna_conjugate_method(): - # ndarray.conjugate - a = np.array([1j, 2+4j, np.NA, 2-1.5j, np.NA], maskna=True) - - b = a.conjugate() - assert_equal(np.isna(b), [0,0,1,0,1]) - assert_equal(b[~np.isna(b)], [-1j, 2-4j, 2+1.5j]) - - -def test_array_maskna_diagonal(): - # ndarray.diagonal - a = np.arange(6, maskna=True) - a.shape = (2,3) - a[0,1] = np.NA - - res = a.diagonal() - assert_(res.flags.maskna) - assert_(res.flags.ownmaskna) - assert_equal(res, [0, 4]) - - res = a.diagonal(-1) - assert_equal(res, [3]) - - res = a.diagonal(-2) - assert_equal(res, []) - - # This diagonal has the NA - res = a.diagonal(1) - assert_equal(np.isna(res), [1,0]) - assert_equal(res[~np.isna(res)], [5]) - - res = a.diagonal(2) - assert_equal(res, [2]) - - res = a.diagonal(3) - assert_equal(res, []) - - -def test_array_maskna_concatenate(): - # np.concatenate - a = np.arange(6, maskna=True, dtype='i4').reshape(2,3) - a[1,0] = np.NA - - b = np.array([[12],[13]], dtype='i4') - res = np.concatenate([a, b], axis=1) - assert_equal(np.isna(res), [[0,0,0,0], [1,0,0,0]]) - assert_equal(res[~np.isna(res)], [0,1,2,12,4,5,13]) - assert_equal(res.strides, (16, 4)) - - b = np.array([[10, np.NA, 11]], maskna=True, dtype='i4') - res = np.concatenate([a,b], axis=0) - assert_equal(np.isna(res), [[0,0,0], [1,0,0], [0,1,0]]) - assert_equal(res[~np.isna(res)], [0,1,2,4,5,10,11]) - assert_equal(res.strides, (12, 4)) - - b = np.array([[np.NA, 10]], order='F', maskna=True, dtype='i4') - res = np.concatenate([a.T, b], axis=0) - assert_equal(np.isna(res), [[0,1], [0,0], [0,0], [1,0]]) - assert_equal(res[~np.isna(res)], [0,1,4,2,5,10]) - assert_equal(res.strides, (4, 16)) - - -def test_array_maskna_column_stack(): - # np.column_stack - a = np.array((1,2,3), maskna=True) - b = np.array((2,3,4), maskna=True) - b[2] = np.NA - res = np.column_stack((a,b)) - assert_equal(np.isna(res), [[0,0], [0,0], [0,1]]) - assert_equal(res[~np.isna(res)], [1,2,2,3,3]) - - -def test_array_maskna_compress(): - # ndarray.compress - a = np.arange(5., maskna=True) - a[0] = np.NA - - mask = np.array([0,1,1,1,1], dtype='?') - res = a.compress(mask) - assert_equal(res, [1,2,3,4]) - - -def test_array_maskna_squeeze(): - # np.squeeze - a = np.zeros((1,3,1,1,4,2,1), maskna=True) - a[0,1,0,0,3,0,0] = np.NA - - res = np.squeeze(a) - assert_equal(res.shape, (3,4,2)) - assert_(np.isna(res[1,3,0])) - - res = np.squeeze(a, axis=(0,2,6)) - assert_equal(res.shape, (3,1,4,2)) - assert_(np.isna(res[1,0,3,0])) - - -def test_array_maskna_mean(): - # np.mean - - # With an NA mask, but no NA - a = np.arange(6, maskna=True).reshape(2,3) - - res = np.mean(a) - assert_equal(res, 2.5) - res = np.mean(a, axis=0) - assert_equal(res, [1.5, 2.5, 3.5]) - - # With an NA and skipna=False - a = np.arange(6, maskna=True).reshape(2,3) - a[0,1] = np.NA - - res = np.mean(a) - assert_(type(res) is np.NAType) - - res = np.mean(a, axis=0) - assert_array_equal(res, [1.5, np.NA, 3.5]) - - res = np.mean(a, axis=1) - assert_array_equal(res, [np.NA, 4.0]) - - # With an NA and skipna=True - res = np.mean(a, skipna=True) - assert_almost_equal(res, 2.8) - - res = np.mean(a, axis=0, skipna=True) - assert_array_equal(res, [1.5, 4.0, 3.5]) - - res = np.mean(a, axis=1, skipna=True) - assert_array_equal(res, [1.0, 4.0]) - - -def test_array_maskna_var_std(): - # np.var, np.std - - # With an NA and skipna=False - a = np.arange(6, maskna=True).reshape(2,3) - a[0,1] = np.NA - - res = np.var(a) - assert_(type(res) is np.NAType) - res = np.std(a) - assert_(type(res) is np.NAType) - - res = np.var(a, axis=0) - assert_array_equal(res, [2.25, np.NA, 2.25]) - res = np.std(a, axis=0) - assert_array_equal(res, [1.5, np.NA, 1.5]) - - res = np.var(a, axis=1) - assert_array_almost_equal(res, [np.NA, 0.66666666666666663]) - res = np.std(a, axis=1) - assert_array_almost_equal(res, [np.NA, 0.81649658092772603]) - - # With an NA and skipna=True - a = np.arange(6, maskna=True).reshape(2,3) - a[0,1] = np.NA - - res = np.var(a, skipna=True) - assert_almost_equal(res, 2.96) - res = np.std(a, skipna=True) - assert_almost_equal(res, 1.7204650534085253) - - res = np.var(a, axis=0, skipna=True) - assert_array_equal(res, [2.25, 0, 2.25]) - res = np.std(a, axis=0, skipna=True) - assert_array_equal(res, [1.5, 0, 1.5]) - - res = np.var(a, axis=1, skipna=True) - assert_array_almost_equal(res, [1.0, 0.66666666666666663]) - res = np.std(a, axis=1, skipna=True) - assert_array_almost_equal(res, [1.0, 0.81649658092772603]) - - -def test_array_maskna_linspace_logspace(): - # np.linspace, np.logspace - - a = np.linspace(2.0, 3.0, num=5) - b = np.linspace(2.0, 3.0, num=5, maskna=True) - assert_equal(a, b) - assert_(not a.flags.maskna) - assert_(b.flags.maskna) - - a = np.logspace(2.0, 3.0, num=4) - b = np.logspace(2.0, 3.0, num=4, maskna=True) - assert_equal(a, b) - assert_(not a.flags.maskna) - assert_(b.flags.maskna) - - -from numpy.testing import dec -@dec.knownfailureif(True, "eye is not implemented for maskna") -def test_array_maskna_eye_identity(): - # np.eye - - # By default there should be no NA mask - a = np.eye(3) - assert_(not a.flags.maskna) - a = np.identity(3) - assert_(not a.flags.maskna) - - a = np.eye(3, maskna=True) - assert_(a.flags.maskna) - assert_(a.flags.ownmaskna) - assert_equal(a, np.eye(3)) - - a = np.eye(3, k=2, maskna=True) - assert_(a.flags.maskna) - assert_(a.flags.ownmaskna) - assert_equal(a, np.eye(3, k=2)) - - a = np.identity(3, maskna=True) - assert_(a.flags.maskna) - assert_(a.flags.ownmaskna) - assert_equal(a, np.identity(3)) - -if __name__ == "__main__": - run_module_suite() diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index e9c7a9c4f..bab83031b 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -965,24 +965,6 @@ class TestMethods(TestCase): assert_equal(a.ravel(order='K'), [2,3,0,1]) assert_(a.ravel(order='K').flags.owndata) - def test_setasflat(self): - # In this case, setasflat can treat a as a flat array, - # and must treat b in chunks of 3 - a = np.arange(3*3*4).reshape(3,3,4) - b = np.arange(3*4*3, dtype='f4').reshape(3,4,3).T - - assert_(not np.all(a.ravel() == b.ravel())) - a.setasflat(b) - assert_equal(a.ravel(), b.ravel()) - - # A case where the strides of neither a nor b can be collapsed - a = np.arange(3*2*4).reshape(3,2,4)[:,:,:-1] - b = np.arange(3*3*3, dtype='f4').reshape(3,3,3).T[:,:,:-1] - - assert_(not np.all(a.ravel() == b.ravel())) - a.setasflat(b) - assert_equal(a.ravel(), b.ravel()) - class TestSubscripting(TestCase): def test_test_zero_rank(self): x = array([1,2,3]) @@ -2608,8 +2590,7 @@ if sys.version_info >= (2, 6): def test_multiarray_flags_writable_attribute_deletion(self): a = np.ones(2).flags - attr = ['updateifcopy', 'aligned', 'writeable', 'maskna', - 'ownmaskna'] + attr = ['updateifcopy', 'aligned', 'writeable'] for s in attr: assert_raises(AttributeError, delattr, a, s) diff --git a/numpy/core/tests/test_na.py b/numpy/core/tests/test_na.py deleted file mode 100644 index eb92a0035..000000000 --- a/numpy/core/tests/test_na.py +++ /dev/null @@ -1,198 +0,0 @@ -import numpy as np -from numpy.compat import asbytes -from numpy.testing import * -import sys, warnings - -def test_na_construction(): - # construct a new NA object - v = np.NA() - assert_(not v is np.NA) - assert_equal(v.payload, None) - assert_equal(v.dtype, None) - - # Construct with a payload - v = np.NA(3) - assert_equal(v.payload, 3) - assert_equal(v.dtype, None) - - # Construct with a dtype - v = np.NA(dtype='f4') - assert_equal(v.payload, None) - assert_equal(v.dtype, np.dtype('f4')) - - # Construct with both a payload and a dtype - v = np.NA(5, dtype='f4,i2') - assert_equal(v.payload, 5) - assert_equal(v.dtype, np.dtype('f4,i2')) - - # min and max payload values - v = np.NA(0) - assert_equal(v.payload, 0) - v = np.NA(127) - assert_equal(v.payload, 127) - - # Out of bounds payload values - assert_raises(ValueError, np.NA, -1) - assert_raises(ValueError, np.NA, 128) - -def test_na_str(): - # With no payload or dtype - assert_equal(str(np.NA), 'NA') - assert_equal(str(np.NA()), 'NA') - - # With a payload - assert_equal(str(np.NA(10)), 'NA(10)') - - # With just a dtype - assert_equal(str(np.NA(dtype='c16')), 'NA') - - # With a payload and a dtype - assert_equal(str(np.NA(10, dtype='f4')), 'NA(10)') - -def test_na_repr(): - # With no payload or dtype - assert_equal(repr(np.NA), 'NA') - assert_equal(repr(np.NA()), 'NA') - - # With a payload - assert_equal(repr(np.NA(10)), 'NA(10)') - - # With just a dtype - assert_equal(repr(np.NA(dtype='?')), "NA(dtype='bool')") - if sys.byteorder == 'little': - assert_equal(repr(np.NA(dtype='<c16')), "NA(dtype='complex128')") - assert_equal(repr(np.NA(dtype='>c16')), "NA(dtype='>c16')") - else: - assert_equal(repr(np.NA(dtype='>c16')), "NA(dtype='complex128')") - assert_equal(repr(np.NA(dtype='<c16')), "NA(dtype='<c16')") - - # With a payload and a dtype - if sys.byteorder == 'little': - assert_equal(repr(np.NA(10, dtype='<f4')), "NA(10, dtype='float32')") - assert_equal(repr(np.NA(10, dtype='>f4')), "NA(10, dtype='>f4')") - else: - assert_equal(repr(np.NA(10, dtype='>f4')), "NA(10, dtype='float32')") - assert_equal(repr(np.NA(10, dtype='<f4')), "NA(10, dtype='<f4')") - -def test_na_comparison(): - # NA cannot be converted to a boolean - assert_raises(ValueError, bool, np.NA) - - # Comparison results should be np.NA(dtype='bool') - def check_comparison_result(res): - assert_(np.isna(res)) - assert_(res.dtype == np.dtype('bool')) - - # Comparison with different objects produces an NA with boolean type - check_comparison_result(np.NA < 3) - check_comparison_result(np.NA <= 3) - check_comparison_result(np.NA == 3) - check_comparison_result(np.NA != 3) - check_comparison_result(np.NA >= 3) - check_comparison_result(np.NA > 3) - - # Should work with NA on the other side too - check_comparison_result(3 < np.NA) - check_comparison_result(3 <= np.NA) - check_comparison_result(3 == np.NA) - check_comparison_result(3 != np.NA) - check_comparison_result(3 >= np.NA) - check_comparison_result(3 > np.NA) - - # Comparison with an array should produce an array - a = np.array([0,1,2]) < np.NA - assert_equal(np.isna(a), [1,1,1]) - assert_equal(a.dtype, np.dtype('bool')) - a = np.array([0,1,2]) == np.NA - assert_equal(np.isna(a), [1,1,1]) - assert_equal(a.dtype, np.dtype('bool')) - a = np.array([0,1,2]) != np.NA - assert_equal(np.isna(a), [1,1,1]) - assert_equal(a.dtype, np.dtype('bool')) - - # Comparison with an array should work on the other side too - a = np.NA > np.array([0,1,2]) - assert_equal(np.isna(a), [1,1,1]) - assert_equal(a.dtype, np.dtype('bool')) - a = np.NA == np.array([0,1,2]) - assert_equal(np.isna(a), [1,1,1]) - assert_equal(a.dtype, np.dtype('bool')) - a = np.NA != np.array([0,1,2]) - assert_equal(np.isna(a), [1,1,1]) - assert_equal(a.dtype, np.dtype('bool')) - -def test_na_operations(): - # The minimum of the payload is taken - assert_equal((np.NA + np.NA(3)).payload, None) - assert_equal((np.NA(12) + np.NA()).payload, None) - assert_equal((np.NA(2) - np.NA(6)).payload, 2) - assert_equal((np.NA(5) - np.NA(1)).payload, 1) - - # The dtypes are promoted like np.promote_types - assert_equal((np.NA(dtype='f4') * np.NA(dtype='f8')).dtype, - np.dtype('f8')) - assert_equal((np.NA(dtype='c8') * np.NA(dtype='f8')).dtype, - np.dtype('c16')) - assert_equal((np.NA * np.NA(dtype='i8')).dtype, - np.dtype('i8')) - assert_equal((np.NA(dtype='i2') / np.NA).dtype, - np.dtype('i2')) - -def test_na_other_operations(): - # Make sure we get NAs for all these operations - assert_equal(type(np.NA + 3), np.NAType) - assert_equal(type(3 + np.NA), np.NAType) - assert_equal(type(np.NA - 3.0), np.NAType) - assert_equal(type(3.0 - np.NA), np.NAType) - assert_equal(type(np.NA * 2j), np.NAType) - assert_equal(type(2j * np.NA), np.NAType) - assert_equal(type(np.NA / 2j), np.NAType) - assert_equal(type(2j / np.NA), np.NAType) - assert_equal(type(np.NA // 2j), np.NAType) - assert_equal(type(np.NA % 6), np.NAType) - assert_equal(type(6 % np.NA), np.NAType) - assert_equal(type(np.NA ** 2), np.NAType) - assert_equal(type(2 ** np.NA), np.NAType) - assert_equal(type(np.NA & 2), np.NAType) - assert_equal(type(2 & np.NA), np.NAType) - assert_equal(type(np.NA | 2), np.NAType) - assert_equal(type(2 | np.NA), np.NAType) - assert_equal(type(np.NA << 2), np.NAType) - assert_equal(type(2 << np.NA), np.NAType) - assert_equal(type(np.NA >> 2), np.NAType) - assert_equal(type(2 >> np.NA), np.NAType) - assert_(abs(np.NA) is np.NA) - assert_((-np.NA) is np.NA) - assert_((+np.NA) is np.NA) - assert_((~np.NA) is np.NA) - - # The NA should get the dtype from the other operand - assert_equal((np.NA + 3).dtype, np.array(3).dtype) - assert_equal((np.NA - 3.0).dtype, np.array(3.0).dtype) - assert_equal((np.NA * 2j).dtype, np.array(2j).dtype) - - # Should have type promotion if the NA already has a dtype - assert_equal((np.NA(dtype='f4') ** 3.0).dtype, np.dtype('f8')) - - # Bitwise and/or are specialized slightly - # NOTE: The keywords 'and' and 'or' coerce to boolean, so we cannot - # properly support them. - assert_equal(np.NA & False, False) - assert_equal(False & np.NA, False) - assert_equal(np.NA | True, True) - assert_equal(True | np.NA, True) - assert_equal(type(np.NA | False), np.NAType) - assert_equal(type(np.NA & True), np.NAType) - assert_equal((np.NA | False).dtype, np.array(False).dtype) - assert_equal((np.NA & True).dtype, np.array(True).dtype) - - -def test_na_writable_attributes_deletion(): - a = np.NA(2) - attr = ['payload', 'dtype'] - for s in attr: - assert_raises(AttributeError, delattr, a, s) - - -if __name__ == "__main__": - run_module_suite() diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index 7a316ac3a..4abe5e2ff 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -2418,145 +2418,6 @@ def test_iter_writemasked(): # were copied back assert_equal(a, [3,3,2.5]) -def test_iter_maskna(): - a_orig = np.zeros((3,), dtype='f8') - b_orig = np.zeros((3,), dtype='f4') - a = a_orig.view(maskna=True) - b = b_orig.view(maskna=True) - - # Default iteration with NA mask - a[...] = np.NA - it = np.nditer(a) - for x in it: - assert_equal(np.isna(x), True) - - # readonly USE_MASKNA iteration of an array without an NA mask - # creates a virtual mask - it = np.nditer([a_orig,b_orig], [], [['readonly','use_maskna']]*2) - for x, y in it: - assert_(x.flags.maskna) - assert_(not x.flags.ownmaskna) - assert_(y.flags.maskna) - assert_(not y.flags.ownmaskna) - assert_equal(np.isna(x), False) - assert_equal(np.isna(y), False) - - # buffered readonly USE_MASKNA iteration of an array without an NA mask - # creates a virtual mask - it = np.nditer([a_orig,b_orig], ['buffered'], [['readonly','use_maskna']]*2, - op_dtypes=['i4','i8'], casting='unsafe') - for x, y in it: - assert_(x.flags.maskna) - assert_(not x.flags.ownmaskna) - assert_(y.flags.maskna) - assert_(not y.flags.ownmaskna) - assert_equal(np.isna(x), False) - assert_equal(np.isna(y), False) - - # writeable USE_MASKNA iteration of an array without an NA mask - # is disallowed - assert_raises(ValueError, np.nditer, a_orig, [], - [['readwrite','use_maskna']]) - assert_raises(ValueError, np.nditer, a_orig, [], - [['writeonly','use_maskna']]) - assert_raises(ValueError, np.nditer, a_orig, ['buffered'], - [['readwrite','use_maskna']]) - assert_raises(ValueError, np.nditer, a_orig, ['buffered'], - [['writeonly','use_maskna']]) - - # Assigning NAs and values in an iteration - a[...] = [0,1,2] - b_orig[...] = [1,2,2] - it = np.nditer([a,b_orig], [], [['writeonly','use_maskna'], ['readonly']]) - for x, y in it: - if y == 2: - x[...] = np.NA - else: - x[...] = 5 - assert_equal(a[0], 5) - assert_equal(np.isna(a), [0,1,1]) - - # Copying NA values in an iteration - b.flags.maskna = True - a[...] = [np.NA, np.NA, 1] - b[...] = [np.NA, 0, np.NA] - it = np.nditer([a,b], [], [['writeonly','use_maskna'], - ['readonly','use_maskna']]) - for x, y in it: - x[...] = y - assert_equal(a[1], 0) - assert_equal(np.isna(a), [1,0,1]) - - # Copying NA values with buffering - a_orig[...] = [1.5,2.5,3.5] - b_orig[...] = [4.5,5.5,6.5] - a[...] = [np.NA, np.NA, 5.5] - b[...] = [np.NA, 3.5, np.NA] - it = np.nditer([a,b], ['buffered'], [['writeonly','use_maskna'], - ['readonly','use_maskna']], - op_dtypes=['i4','i4'], - casting='unsafe') - for x, y in it: - x[...] = y - # The 3.5 in b gets truncated to 3, because the iterator is processing - # elements as int32 values. - assert_equal(a[1], 3) - assert_equal(np.isna(a), [1,0,1]) - assert_equal(a_orig, [1.5,3,5.5]) - - # Copying NA values with buffering and external_loop - a_orig[...] = [1.5,2.5,3.5] - b_orig[...] = [4.5,5.5,6.5] - a[...] = [np.NA, np.NA, 5.5] - b[...] = [np.NA, 3.5, np.NA] - it = np.nditer([a,b], ['buffered','external_loop'], - [['writeonly','use_maskna'], - ['readonly','use_maskna']], - op_dtypes=['i4','i4'], - casting='unsafe') - for x, y in it: - assert_equal(x.size, 3) - x[...] = y - # The 3.5 in b gets truncated to 3, because the iterator is processing - # elements as int32 values. - assert_equal(a[1], 3) - assert_equal(np.isna(a), [1,0,1]) - assert_equal(a_orig, [1.5,3,5.5]) - - # WRITEMASKED and MASKNA aren't supported together yet - mask = np.array([1,1,0], dtype='?') - assert_raises(ValueError, np.nditer, [a,b,mask], [], - [['writeonly','use_maskna','writemasked'], - ['readonly','use_maskna'], - ['readonly','arraymask']]) - # when they are supported together, will probably require buffering - assert_raises(ValueError, np.nditer, [a,b,mask], ['buffered'], - [['writeonly','use_maskna','writemasked'], - ['readonly','use_maskna'], - ['readonly','arraymask']]) - -def test_iter_maskna_default_use_maskna(): - # The Python exposure of nditer adds the USE_MASKNA flag automatically - a = np.array([3, 5, np.NA, 2, 1]) - b = np.array([1, 1.0, 4.5, 2, 0]) - - # The output should automatically get an NA mask - it = np.nditer([a,b,None]) - for x,y,z in it: - z[...] = x+y - assert_(it.operands[2].flags.maskna) - assert_array_equal(it.operands[2], a+b) - - # This holds even when we specify the op_flags - it = np.nditer([a,b.copy(),None], op_flags=[['readonly'], - ['readwrite'], ['writeonly', 'allocate']]) - for x,y,z in it: - y[...] = y[...] + 1 - z[...] = x+y - assert_(it.operands[2].flags.maskna) - assert_array_equal(it.operands[2], a+b+1) - - def test_iter_non_writable_attribute_deletion(): it = np.nditer(np.ones(2)) attr = ["value", "shape", "operands", "itviews", "has_delayed_bufalloc", diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index 31d818778..2cedb87e9 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -606,39 +606,6 @@ class TestNonzero(TestCase): np.count_nonzero(a, axis=0, out=res) assert_equal(res, [1,2,0]) - # A 3-dimensional array with an NA - a = array([[[0,1,0],[2,np.NA,0]], [[0,1,0],[2,3,0]]], maskna=True) - - # Test that the NA reduces correctly - assert_array_equal(np.count_nonzero(a, axis=()), - [[[0,1,0],[1,np.NA,0]], [[0,1,0],[1,1,0]]]) - assert_array_equal(np.count_nonzero(a, axis=0), [[0,2,0], [2,np.NA,0]]) - assert_array_equal(np.count_nonzero(a, axis=1), [[1,np.NA,0], [1,2,0]]) - assert_array_equal(np.count_nonzero(a, axis=2), [[1,np.NA], [1,2]]) - assert_array_equal(np.count_nonzero(a, axis=(0,1)), [2,np.NA,0]) - assert_array_equal(np.count_nonzero(a, axis=(0,2)), [2,np.NA]) - assert_array_equal(np.count_nonzero(a, axis=(1,2)), [np.NA,3]) - assert_array_equal(np.count_nonzero(a, axis=(0,1,2)), - np.NA(dtype=np.intp)) - assert_array_equal(np.count_nonzero(a, axis=None), - np.NA(dtype=np.intp)) - - # Test that the NA gets skipped correctly - assert_array_equal(np.count_nonzero(a, axis=(), skipna=True), - [[[0,1,0],[1,0,0]], [[0,1,0],[1,1,0]]]) - assert_array_equal(np.count_nonzero(a, axis=0, skipna=True), - [[0,2,0], [2,1,0]]) - assert_array_equal(np.count_nonzero(a, axis=1, skipna=True), - [[1,1,0], [1,2,0]]) - assert_array_equal(np.count_nonzero(a, axis=2, skipna=True), - [[1,1], [1,2]]) - assert_array_equal(np.count_nonzero(a, axis=(0,1), skipna=True), - [2,3,0]) - assert_array_equal(np.count_nonzero(a, axis=(0,2), skipna=True), [2,3]) - assert_array_equal(np.count_nonzero(a, axis=(1,2), skipna=True), [2,3]) - assert_array_equal(np.count_nonzero(a, axis=(0,1,2), skipna=True), 5) - assert_array_equal(np.count_nonzero(a, axis=None, skipna=True), 5) - class TestIndex(TestCase): def test_boolean(self): a = rand(3,5,8) @@ -1323,17 +1290,6 @@ class TestIsclose(object): # Ensure that the mask isn't modified... assert_array_equal([True, True, False], y.mask) - def test_maskna_arrays(self): - x = array([NA, 1, 2, 3]) - y = array([0, 1, 2, NA]) - assert_array_equal(isclose(x, y), array([NA, True, True, NA])) - - assert_array_equal(isclose(NA, arange(3)), array([NA, NA, NA])) - - x = array([NA, nan, 2, 3]) - y = array([nan, 1, 2, NA]) - assert_array_equal(isclose(x, y), array([NA, False, True, NA])) - def test_scalar_return(self): assert_(isscalar(isclose(1, 1))) @@ -1464,20 +1420,6 @@ class TestLikeFuncs(TestCase): b = like_function(a, subok=False) assert_(not (type(b) is np.matrix)) - # Test that 'maskna=True' works - a = np.arange(6).reshape(2,3) - res = like_function(a, maskna=True) - assert_(res.flags.maskna) - assert_(res.flags.ownmaskna) - assert_equal(res.shape, a.shape) - assert_equal(res.dtype, a.dtype) - - # Test that no NA mask is created when the prototype is NA-masked - a = np.arange(6, maskna=True).reshape(2,3) - assert_(a.flags.maskna) - res = like_function(a) - assert_(not res.flags.maskna) - def test_ones_like(self): self.check_like_function(np.ones_like, 1) diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 7905d38cd..ac36aa479 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -1692,10 +1692,5 @@ class TestRegression(TestCase): a = np.array(['abc'], dtype=np.unicode)[0] del a - def test_maskna_deallocation(self): - # This caused a segfault when running under python-debug - a = np.array([1]).view(maskna=True) - del a - if __name__ == "__main__": run_module_suite() |