diff options
Diffstat (limited to 'numpy/ma')
| -rw-r--r-- | numpy/ma/LICENSE | 24 | ||||
| -rw-r--r-- | numpy/ma/__init__.py | 22 | ||||
| -rw-r--r-- | numpy/ma/bench.py | 198 | ||||
| -rw-r--r-- | numpy/ma/core.py | 2979 | ||||
| -rw-r--r-- | numpy/ma/extras.py | 722 | ||||
| -rw-r--r-- | numpy/ma/morestats.py | 406 | ||||
| -rw-r--r-- | numpy/ma/mrecords.py | 717 | ||||
| -rw-r--r-- | numpy/ma/mstats.py | 433 | ||||
| -rw-r--r-- | numpy/ma/setup.py | 19 | ||||
| -rw-r--r-- | numpy/ma/tests/test_core.py | 1305 | ||||
| -rw-r--r-- | numpy/ma/tests/test_extras.py | 331 | ||||
| -rw-r--r-- | numpy/ma/tests/test_morestats.py | 114 | ||||
| -rw-r--r-- | numpy/ma/tests/test_mrecords.py | 181 | ||||
| -rw-r--r-- | numpy/ma/tests/test_mstats.py | 174 | ||||
| -rw-r--r-- | numpy/ma/tests/test_subclassing.py | 183 | ||||
| -rw-r--r-- | numpy/ma/testutils.py | 220 | ||||
| -rw-r--r-- | numpy/ma/timer_comparison.py | 461 | ||||
| -rw-r--r-- | numpy/ma/version.py | 11 |
18 files changed, 8500 insertions, 0 deletions
diff --git a/numpy/ma/LICENSE b/numpy/ma/LICENSE new file mode 100644 index 000000000..b41aae0c8 --- /dev/null +++ b/numpy/ma/LICENSE @@ -0,0 +1,24 @@ +* Copyright (c) 2006, University of Georgia and Pierre G.F. Gerard-Marchant +* All rights reserved. +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the distribution. +* * Neither the name of the University of Georgia nor the +* names of its contributors may be used to endorse or promote products +* derived from this software without specific prior written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY +* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file diff --git a/numpy/ma/__init__.py b/numpy/ma/__init__.py new file mode 100644 index 000000000..ccbb6d3c6 --- /dev/null +++ b/numpy/ma/__init__.py @@ -0,0 +1,22 @@ +"""Masked arrays add-ons. + +A collection of utilities for maskedarray + +:author: Pierre GF Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: __init__.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import core +from core import * + +import extras +from extras import * + +__all__ = ['core', 'extras'] +__all__ += core.__all__ +__all__ += extras.__all__ diff --git a/numpy/ma/bench.py b/numpy/ma/bench.py new file mode 100644 index 000000000..cdb010df1 --- /dev/null +++ b/numpy/ma/bench.py @@ -0,0 +1,198 @@ +#! python + +import timeit +#import IPython.ipapi +#ip = IPython.ipapi.get() +#from IPython import ipmagic +import numpy +import maskedarray +from maskedarray import filled +from maskedarray.testutils import assert_equal + + +#####--------------------------------------------------------------------------- +#---- --- Global variables --- +#####--------------------------------------------------------------------------- + +# Small arrays .................................. +xs = numpy.random.uniform(-1,1,6).reshape(2,3) +ys = numpy.random.uniform(-1,1,6).reshape(2,3) +zs = xs + 1j * ys +m1 = [[True, False, False], [False, False, True]] +m2 = [[True, False, True], [False, False, True]] +nmxs = numpy.ma.array(xs, mask=m1) +nmys = numpy.ma.array(ys, mask=m2) +nmzs = numpy.ma.array(zs, mask=m1) +mmxs = maskedarray.array(xs, mask=m1) +mmys = maskedarray.array(ys, mask=m2) +mmzs = maskedarray.array(zs, mask=m1) +# Big arrays .................................... +xl = numpy.random.uniform(-1,1,100*100).reshape(100,100) +yl = numpy.random.uniform(-1,1,100*100).reshape(100,100) +zl = xl + 1j * yl +maskx = xl > 0.8 +masky = yl < -0.8 +nmxl = numpy.ma.array(xl, mask=maskx) +nmyl = numpy.ma.array(yl, mask=masky) +nmzl = numpy.ma.array(zl, mask=maskx) +mmxl = maskedarray.array(xl, mask=maskx, shrink=True) +mmyl = maskedarray.array(yl, mask=masky, shrink=True) +mmzl = maskedarray.array(zl, mask=maskx, shrink=True) + +#####--------------------------------------------------------------------------- +#---- --- Functions --- +#####--------------------------------------------------------------------------- + +def timer(s, v='', nloop=500, nrep=3): + units = ["s", "ms", "\xb5s", "ns"] + scaling = [1, 1e3, 1e6, 1e9] + print "%s : %-50s : " % (v,s), + varnames = ["%ss,nm%ss,mm%ss,%sl,nm%sl,mm%sl" % tuple(x*6) for x in 'xyz'] + setup = 'from __main__ import numpy, maskedarray, %s' % ','.join(varnames) + Timer = timeit.Timer(stmt=s, setup=setup) + best = min(Timer.repeat(nrep, nloop)) / nloop + if best > 0.0: + order = min(-int(numpy.floor(numpy.log10(best)) // 3), 3) + else: + order = 3 + print "%d loops, best of %d: %.*g %s per loop" % (nloop, nrep, + 3, + best * scaling[order], + units[order]) +# ip.magic('timeit -n%i %s' % (nloop,s)) + + + +def compare_functions_1v(func, nloop=500, test=True, + xs=xs, nmxs=nmxs, mmxs=mmxs, + xl=xl, nmxl=nmxl, mmxl=mmxl): + funcname = func.__name__ + print "-"*50 + print "%s on small arrays" % funcname + if test: + assert_equal(filled(eval("numpy.ma.%s(nmxs)" % funcname),0), + filled(eval("maskedarray.%s(mmxs)" % funcname),0)) + for (module, data) in zip(("numpy", "numpy.ma","maskedarray"), + ("xs","nmxs","mmxs")): + timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) + # + print "%s on large arrays" % funcname + if test: + assert_equal(filled(eval("numpy.ma.%s(nmxl)" % funcname),0), + filled(eval("maskedarray.%s(mmxl)" % funcname),0)) + for (module, data) in zip(("numpy", "numpy.ma","maskedarray"), + ("xl","nmxl","mmxl")): + timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) + return + +def compare_methods(methodname, args, vars='x', nloop=500, test=True, + xs=xs, nmxs=nmxs, mmxs=mmxs, + xl=xl, nmxl=nmxl, mmxl=mmxl): + print "-"*50 + print "%s on small arrays" % methodname + if test: + assert_equal(filled(eval("nm%ss.%s(%s)" % (vars,methodname,args)),0), + filled(eval("mm%ss.%s(%s)" % (vars,methodname,args)),0)) + for (data, ver) in zip(["nm%ss" % vars, "mm%ss" % vars], ('numpy.ma ','maskedarray')): + timer("%(data)s.%(methodname)s(%(args)s)" % locals(), v=ver, nloop=nloop) + # + print "%s on large arrays" % methodname + if test: + assert_equal(filled(eval("nm%sl.%s(%s)" % (vars,methodname,args)),0), + filled(eval("mm%sl.%s(%s)" % (vars,methodname,args)),0)) + for (data, ver) in zip(["nm%sl" % vars, "mm%sl" % vars], ('numpy.ma ','maskedarray')): + timer("%(data)s.%(methodname)s(%(args)s)" % locals(), v=ver, nloop=nloop) + return + +def compare_functions_2v(func, nloop=500, test=True, + xs=xs, nmxs=nmxs, mmxs=mmxs, + ys=ys, nmys=nmys, mmys=mmys, + xl=xl, nmxl=nmxl, mmxl=mmxl, + yl=yl, nmyl=nmyl, mmyl=mmyl): + funcname = func.__name__ + print "-"*50 + print "%s on small arrays" % funcname + if test: + assert_equal(filled(eval("numpy.ma.%s(nmxs,nmys)" % funcname),0), + filled(eval("maskedarray.%s(mmxs,mmys)" % funcname),0)) + for (module, data) in zip(("numpy", "numpy.ma","maskedarray"), + ("xs,ys","nmxs,nmys","mmxs,mmys")): + timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) + # + print "%s on large arrays" % funcname + if test: + assert_equal(filled(eval("numpy.ma.%s(nmxl, nmyl)" % funcname),0), + filled(eval("maskedarray.%s(mmxl, mmyl)" % funcname),0)) + for (module, data) in zip(("numpy", "numpy.ma","maskedarray"), + ("xl,yl","nmxl,nmyl","mmxl,mmyl")): + timer("%(module)s.%(funcname)s(%(data)s)" % locals(), v="%11s" % module, nloop=nloop) + return + + +############################################################################### + + +################################################################################ +if __name__ == '__main__': +# # Small arrays .................................. +# xs = numpy.random.uniform(-1,1,6).reshape(2,3) +# ys = numpy.random.uniform(-1,1,6).reshape(2,3) +# zs = xs + 1j * ys +# m1 = [[True, False, False], [False, False, True]] +# m2 = [[True, False, True], [False, False, True]] +# nmxs = numpy.ma.array(xs, mask=m1) +# nmys = numpy.ma.array(ys, mask=m2) +# nmzs = numpy.ma.array(zs, mask=m1) +# mmxs = maskedarray.array(xs, mask=m1) +# mmys = maskedarray.array(ys, mask=m2) +# mmzs = maskedarray.array(zs, mask=m1) +# # Big arrays .................................... +# xl = numpy.random.uniform(-1,1,100*100).reshape(100,100) +# yl = numpy.random.uniform(-1,1,100*100).reshape(100,100) +# zl = xl + 1j * yl +# maskx = xl > 0.8 +# masky = yl < -0.8 +# nmxl = numpy.ma.array(xl, mask=maskx) +# nmyl = numpy.ma.array(yl, mask=masky) +# nmzl = numpy.ma.array(zl, mask=maskx) +# mmxl = maskedarray.array(xl, mask=maskx, shrink=True) +# mmyl = maskedarray.array(yl, mask=masky, shrink=True) +# mmzl = maskedarray.array(zl, mask=maskx, shrink=True) +# + compare_functions_1v(numpy.sin) + compare_functions_1v(numpy.log) + compare_functions_1v(numpy.sqrt) + #.................................................................... + compare_functions_2v(numpy.multiply) + compare_functions_2v(numpy.divide) + compare_functions_2v(numpy.power) + #.................................................................... + compare_methods('ravel','', nloop=1000) + compare_methods('conjugate','','z', nloop=1000) + compare_methods('transpose','', nloop=1000) + compare_methods('compressed','', nloop=1000) + compare_methods('__getitem__','0', nloop=1000) + compare_methods('__getitem__','(0,0)', nloop=1000) + compare_methods('__getitem__','[0,-1]', nloop=1000) + compare_methods('__setitem__','0, 17', nloop=1000, test=False) + compare_methods('__setitem__','(0,0), 17', nloop=1000, test=False) + #.................................................................... + print "-"*50 + print "__setitem__ on small arrays" + timer('nmxs.__setitem__((-1,0),numpy.ma.masked)', 'numpy.ma ',nloop=10000) + timer('mmxs.__setitem__((-1,0),maskedarray.masked)', 'maskedarray',nloop=10000) + print "-"*50 + print "__setitem__ on large arrays" + timer('nmxl.__setitem__((-1,0),numpy.ma.masked)', 'numpy.ma ',nloop=10000) + timer('mmxl.__setitem__((-1,0),maskedarray.masked)', 'maskedarray',nloop=10000) + #.................................................................... + print "-"*50 + print "where on small arrays" + assert_equal(eval("numpy.ma.where(nmxs>2,nmxs,nmys)"), + eval("maskedarray.where(mmxs>2, mmxs,mmys)")) + timer('numpy.ma.where(nmxs>2,nmxs,nmys)', 'numpy.ma ',nloop=1000) + timer('maskedarray.where(mmxs>2, mmxs,mmys)', 'maskedarray',nloop=1000) + print "-"*50 + print "where on large arrays" + timer('numpy.ma.where(nmxl>2,nmxl,nmyl)', 'numpy.ma ',nloop=100) + timer('maskedarray.where(mmxl>2, mmxl,mmyl)', 'maskedarray',nloop=100) diff --git a/numpy/ma/core.py b/numpy/ma/core.py new file mode 100644 index 000000000..0f2ab64ff --- /dev/null +++ b/numpy/ma/core.py @@ -0,0 +1,2979 @@ +# pylint: disable-msg=E1002 +"""MA: a facility for dealing with missing observations +MA is generally used as a numpy.array look-alike. +by Paul F. Dubois. + +Copyright 1999, 2000, 2001 Regents of the University of California. +Released for unlimited redistribution. +Adapted for numpy_core 2005 by Travis Oliphant and +(mainly) Paul Dubois. + +Subclassing of the base ndarray 2006 by Pierre Gerard-Marchant. +pgmdevlist_AT_gmail_DOT_com +Improvements suggested by Reggie Dugard (reggie_AT_merfinllc_DOT_com) + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: core.py 3639 2007-12-13 03:39:17Z pierregm $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: pierregm $)" +__version__ = '1.0' +__revision__ = "$Revision: 3639 $" +__date__ = '$Date: 2007-12-13 05:39:17 +0200 (Thu, 13 Dec 2007) $' + +__docformat__ = "restructuredtext en" + +__all__ = ['MAError', 'MaskType', 'MaskedArray', + 'bool_', 'complex_', 'float_', 'int_', 'object_', + 'abs', 'absolute', 'add', 'all', 'allclose', 'allequal', 'alltrue', + 'amax', 'amin', 'anom', 'anomalies', 'any', 'arange', + 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctan2', + 'arctanh', 'argmax', 'argmin', 'argsort', 'around', + 'array', 'asarray','asanyarray', + 'bitwise_and', 'bitwise_or', 'bitwise_xor', + 'ceil', 'choose', 'compressed', 'concatenate', 'conjugate', + 'cos', 'cosh', 'count', + 'default_fill_value', 'diagonal', 'divide', 'dump', 'dumps', + 'empty', 'empty_like', 'equal', 'exp', + 'fabs', 'fmod', 'filled', 'floor', 'floor_divide','fix_invalid', + 'getmask', 'getmaskarray', 'greater', 'greater_equal', 'hypot', + 'ids', 'inner', 'innerproduct', + 'isMA', 'isMaskedArray', 'is_mask', 'is_masked', 'isarray', + 'left_shift', 'less', 'less_equal', 'load', 'loads', 'log', 'log10', + 'logical_and', 'logical_not', 'logical_or', 'logical_xor', + 'make_mask', 'make_mask_none', 'mask_or', 'masked', + 'masked_array', 'masked_equal', 'masked_greater', + 'masked_greater_equal', 'masked_inside', 'masked_less', + 'masked_less_equal', 'masked_not_equal', 'masked_object', + 'masked_outside', 'masked_print_option', 'masked_singleton', + 'masked_values', 'masked_where', 'max', 'maximum', 'mean', 'min', + 'minimum', 'multiply', + 'negative', 'nomask', 'nonzero', 'not_equal', + 'ones', 'outer', 'outerproduct', + 'power', 'product', 'ptp', 'put', 'putmask', + 'rank', 'ravel', 'remainder', 'repeat', 'reshape', 'resize', + 'right_shift', 'round_', + 'shape', 'sin', 'sinh', 'size', 'sometrue', 'sort', 'sqrt', 'std', + 'subtract', 'sum', 'swapaxes', + 'take', 'tan', 'tanh', 'transpose', 'true_divide', + 'var', 'where', + 'zeros'] + +import sys +import types +import cPickle +import operator +# +import numpy +from numpy.core import bool_, complex_, float_, int_, object_, str_ + +import numpy.core.umath as umath +import numpy.core.fromnumeric as fromnumeric +import numpy.core.numeric as numeric +import numpy.core.numerictypes as ntypes +from numpy import bool_, dtype, typecodes, amax, amin, ndarray +from numpy import expand_dims as n_expand_dims +from numpy import array as narray +import warnings + + +MaskType = bool_ +nomask = MaskType(0) + +divide_tolerance = 1.e-35 +numpy.seterr(all='ignore') + + + +#####-------------------------------------------------------------------------- +#---- --- Exceptions --- +#####-------------------------------------------------------------------------- +class MAError(Exception): + "Class for MA related errors." + def __init__ (self, args=None): + "Creates an exception." + Exception.__init__(self,args) + self.args = args + def __str__(self): + "Calculates the string representation." + return str(self.args) + __repr__ = __str__ + +#####-------------------------------------------------------------------------- +#---- --- Filling options --- +#####-------------------------------------------------------------------------- +# b: boolean - c: complex - f: floats - i: integer - O: object - S: string +default_filler = {'b': True, + 'c' : 1.e20 + 0.0j, + 'f' : 1.e20, + 'i' : 999999, + 'O' : '?', + 'S' : 'N/A', + 'u' : 999999, + 'V' : '???', + } +max_filler = ntypes._minvals +max_filler.update([(k,-numpy.inf) for k in [numpy.float32, numpy.float64]]) +min_filler = ntypes._maxvals +min_filler.update([(k,numpy.inf) for k in [numpy.float32, numpy.float64]]) +if 'float128' in ntypes.typeDict: + max_filler.update([(numpy.float128,-numpy.inf)]) + min_filler.update([(numpy.float128, numpy.inf)]) + +def default_fill_value(obj): + """Calculates the default fill value for the argument object. + """ + if hasattr(obj,'dtype'): + defval = default_filler[obj.dtype.kind] + elif isinstance(obj, numeric.dtype): + defval = default_filler[obj.kind] + elif isinstance(obj, float): + defval = default_filler['f'] + elif isinstance(obj, int) or isinstance(obj, long): + defval = default_filler['i'] + elif isinstance(obj, str): + defval = default_filler['S'] + elif isinstance(obj, complex): + defval = default_filler['c'] + else: + defval = default_filler['O'] + return defval + +def minimum_fill_value(obj): + "Calculates the default fill value suitable for taking the minimum of ``obj``." + if hasattr(obj, 'dtype'): + objtype = obj.dtype + filler = min_filler[objtype] + if filler is None: + raise TypeError, 'Unsuitable type for calculating minimum.' + return filler + elif isinstance(obj, float): + return min_filler[ntypes.typeDict['float_']] + elif isinstance(obj, int): + return min_filler[ntypes.typeDict['int_']] + elif isinstance(obj, long): + return min_filler[ntypes.typeDict['uint']] + elif isinstance(obj, numeric.dtype): + return min_filler[obj] + else: + raise TypeError, 'Unsuitable type for calculating minimum.' + +def maximum_fill_value(obj): + "Calculates the default fill value suitable for taking the maximum of ``obj``." + if hasattr(obj, 'dtype'): + objtype = obj.dtype + filler = max_filler[objtype] + if filler is None: + raise TypeError, 'Unsuitable type for calculating minimum.' + return filler + elif isinstance(obj, float): + return max_filler[ntypes.typeDict['float_']] + elif isinstance(obj, int): + return max_filler[ntypes.typeDict['int_']] + elif isinstance(obj, long): + return max_filler[ntypes.typeDict['uint']] + elif isinstance(obj, numeric.dtype): + return max_filler[obj] + else: + raise TypeError, 'Unsuitable type for calculating minimum.' + +def set_fill_value(a, fill_value): + """Sets the filling value of a, if a is a masked array. +Otherwise, does nothing. + +*Returns*: + None + """ + if isinstance(a, MaskedArray): + a.set_fill_value(fill_value) + return + +def get_fill_value(a): + """Returns the filling value of a, if any. +Otherwise, returns the default filling value for that type. + """ + if isinstance(a, MaskedArray): + result = a.fill_value + else: + result = default_fill_value(a) + return result + +def common_fill_value(a, b): + """Returns the common filling value of a and b, if any. + If a and b have different filling values, returns None.""" + t1 = get_fill_value(a) + t2 = get_fill_value(b) + if t1 == t2: + return t1 + return None + +#####-------------------------------------------------------------------------- +def filled(a, value = None): + """Returns a as an array with masked data replaced by value. +If value is None, get_fill_value(a) is used instead. +If a is already a ndarray, a itself is returned. + +*Parameters*: + a : {var} + An input object. + value : {var}, optional + Filling value. If not given, the output of get_fill_value(a) is used instead. + +*Returns*: + A ndarray. + + """ + if hasattr(a, 'filled'): + return a.filled(value) + elif isinstance(a, ndarray): # and a.flags['CONTIGUOUS']: + return a + elif isinstance(a, dict): + return narray(a, 'O') + else: + return narray(a) + +#####-------------------------------------------------------------------------- +def get_masked_subclass(*arrays): + """Returns the youngest subclass of MaskedArray from a list of (masked) arrays. +In case of siblings, the first takes over.""" + if len(arrays) == 1: + arr = arrays[0] + if isinstance(arr, MaskedArray): + rcls = type(arr) + else: + rcls = MaskedArray + else: + arrcls = [type(a) for a in arrays] + rcls = arrcls[0] + if not issubclass(rcls, MaskedArray): + rcls = MaskedArray + for cls in arrcls[1:]: + if issubclass(cls, rcls): + rcls = cls + return rcls + +#####-------------------------------------------------------------------------- +def get_data(a, subok=True): + """Returns the _data part of a (if any), or a as a ndarray. + +*Parameters* : + a : {ndarray} + A ndarray or a subclass of. + subok : {boolean} + Whether to force the output to a 'pure' ndarray (False) or to return a subclass + of ndarray if approriate (True). + """ + data = getattr(a, '_data', numpy.array(a, subok=subok)) + if not subok: + return data.view(ndarray) + return data +getdata = get_data + +def fix_invalid(a, copy=True, fill_value=None): + """Returns (a copy of) a where invalid data (nan/inf) are masked and replaced +by fill_value. +Note that a copy is performed by default (just in case...). + +*Parameters*: + a : {ndarray} + A (subclass of) ndarray. + copy : {boolean} + Whether to use a copy of a (True) or to fix a in place (False). + fill_value : {var}, optional + Value used for fixing invalid data. + If not given, the output of get_fill_value(a) is used instead. + +*Returns* : + MaskedArray object + """ + a = masked_array(a, copy=copy, subok=True) + invalid = (numpy.isnan(a._data) | numpy.isinf(a._data)) + a._mask |= invalid + if fill_value is None: + fill_value = a.fill_value + a._data[invalid] = fill_value + return a + + + +#####-------------------------------------------------------------------------- +#---- --- Ufuncs --- +#####-------------------------------------------------------------------------- +ufunc_domain = {} +ufunc_fills = {} + +class _DomainCheckInterval: + """Defines a valid interval, so that : +``domain_check_interval(a,b)(x) = true`` where ``x < a`` or ``x > b``.""" + def __init__(self, a, b): + "domain_check_interval(a,b)(x) = true where x < a or y > b" + if (a > b): + (a, b) = (b, a) + self.a = a + self.b = b + + def __call__ (self, x): + "Executes the call behavior." + return umath.logical_or(umath.greater (x, self.b), + umath.less(x, self.a)) +#............................ +class _DomainTan: + """Defines a valid interval for the `tan` function, so that: +``domain_tan(eps) = True`` where ``abs(cos(x)) < eps``""" + def __init__(self, eps): + "domain_tan(eps) = true where abs(cos(x)) < eps)" + self.eps = eps + def __call__ (self, x): + "Executes the call behavior." + return umath.less(umath.absolute(umath.cos(x)), self.eps) +#............................ +class _DomainSafeDivide: + """Defines a domain for safe division.""" + def __init__ (self, tolerance=divide_tolerance): + self.tolerance = tolerance + def __call__ (self, a, b): + return umath.absolute(a) * self.tolerance >= umath.absolute(b) +#............................ +class _DomainGreater: + "DomainGreater(v)(x) = true where x <= v" + def __init__(self, critical_value): + "DomainGreater(v)(x) = true where x <= v" + self.critical_value = critical_value + + def __call__ (self, x): + "Executes the call behavior." + return umath.less_equal(x, self.critical_value) +#............................ +class _DomainGreaterEqual: + "DomainGreaterEqual(v)(x) = true where x < v" + def __init__(self, critical_value): + "DomainGreaterEqual(v)(x) = true where x < v" + self.critical_value = critical_value + + def __call__ (self, x): + "Executes the call behavior." + return umath.less(x, self.critical_value) + +#.............................................................................. +class _MaskedUnaryOperation: + """Defines masked version of unary operations, where invalid values are pre-masked. + +:IVariables: + f : function. + fill : Default filling value *[0]*. + domain : Default domain *[None]*. + """ + def __init__ (self, mufunc, fill=0, domain=None): + """ _MaskedUnaryOperation(aufunc, fill=0, domain=None) + aufunc(fill) must be defined + self(x) returns aufunc(x) + with masked values where domain(x) is true or getmask(x) is true. + """ + self.f = mufunc + self.fill = fill + self.domain = domain + self.__doc__ = getattr(mufunc, "__doc__", str(mufunc)) + self.__name__ = getattr(mufunc, "__name__", str(mufunc)) + ufunc_domain[mufunc] = domain + ufunc_fills[mufunc] = fill + # + def __call__ (self, a, *args, **kwargs): + "Executes the call behavior." + m = getmask(a) + d1 = get_data(a) + if self.domain is not None: + dm = narray(self.domain(d1), copy=False) + m = numpy.logical_or(m, dm) + # The following two lines control the domain filling methods. + d1 = d1.copy() +# d1[dm] = self.fill + numpy.putmask(d1, dm, self.fill) + # Take care of the masked singletong first ... + if not m.ndim and m: + return masked + # Get the result.............................. + if isinstance(a, MaskedArray): + result = self.f(d1, *args, **kwargs).view(type(a)) + else: + result = self.f(d1, *args, **kwargs).view(MaskedArray) + # Fix the mask if we don't have a scalar + if result.ndim > 0: + result._mask = m + return result + # + def __str__ (self): + return "Masked version of %s. [Invalid values are masked]" % str(self.f) + +#.............................................................................. +class _MaskedBinaryOperation: + """Defines masked version of binary operations, +where invalid values are pre-masked. + +:IVariables: + f : function. + fillx : Default filling value for the first argument *[0]*. + filly : Default filling value for the second argument *[0]*. + domain : Default domain *[None]*. + """ + def __init__ (self, mbfunc, fillx=0, filly=0): + """abfunc(fillx, filly) must be defined. + abfunc(x, filly) = x for all x to enable reduce. + """ + self.f = mbfunc + self.fillx = fillx + self.filly = filly + self.__doc__ = getattr(mbfunc, "__doc__", str(mbfunc)) + self.__name__ = getattr(mbfunc, "__name__", str(mbfunc)) + ufunc_domain[mbfunc] = None + ufunc_fills[mbfunc] = (fillx, filly) + # + def __call__ (self, a, b, *args, **kwargs): + "Execute the call behavior." + m = mask_or(getmask(a), getmask(b)) + (d1, d2) = (get_data(a), get_data(b)) + result = self.f(d1, d2, *args, **kwargs).view(get_masked_subclass(a,b)) + if result.size > 1: + if m is not nomask: + result._mask = make_mask_none(result.shape) + result._mask.flat = m + elif m: + return masked + return result + # + def reduce (self, target, axis=0, dtype=None): + """Reduces `target` along the given `axis`.""" + if isinstance(target, MaskedArray): + tclass = type(target) + else: + tclass = MaskedArray + m = getmask(target) + t = filled(target, self.filly) + if t.shape == (): + t = t.reshape(1) + if m is not nomask: + m = make_mask(m, copy=1) + m.shape = (1,) + if m is nomask: + return self.f.reduce(t, axis).view(tclass) + t = t.view(tclass) + t._mask = m + tr = self.f.reduce(getdata(t), axis, dtype=dtype or t.dtype) + mr = umath.logical_and.reduce(m, axis) + tr = tr.view(tclass) + if mr.ndim > 0: + tr._mask = mr + return tr + elif mr: + return masked + return tr + + def outer (self, a, b): + "Returns the function applied to the outer product of a and b." + ma = getmask(a) + mb = getmask(b) + if ma is nomask and mb is nomask: + m = nomask + else: + ma = getmaskarray(a) + mb = getmaskarray(b) + m = umath.logical_or.outer(ma, mb) + if (not m.ndim) and m: + return masked + rcls = get_masked_subclass(a,b) +# d = self.f.outer(filled(a, self.fillx), filled(b, self.filly)).view(rcls) + d = self.f.outer(getdata(a), getdata(b)).view(rcls) + if d.ndim > 0: + d._mask = m + return d + + def accumulate (self, target, axis=0): + """Accumulates `target` along `axis` after filling with y fill value.""" + if isinstance(target, MaskedArray): + tclass = type(target) + else: + tclass = masked_array + t = filled(target, self.filly) + return self.f.accumulate(t, axis).view(tclass) + + def __str__ (self): + return "Masked version of " + str(self.f) + +#.............................................................................. +class _DomainedBinaryOperation: + """Defines binary operations that have a domain, like divide. + +They have no reduce, outer or accumulate. + +:IVariables: + f : function. + domain : Default domain. + fillx : Default filling value for the first argument *[0]*. + filly : Default filling value for the second argument *[0]*. + """ + def __init__ (self, dbfunc, domain, fillx=0, filly=0): + """abfunc(fillx, filly) must be defined. + abfunc(x, filly) = x for all x to enable reduce. + """ + self.f = dbfunc + self.domain = domain + self.fillx = fillx + self.filly = filly + self.__doc__ = getattr(dbfunc, "__doc__", str(dbfunc)) + self.__name__ = getattr(dbfunc, "__name__", str(dbfunc)) + ufunc_domain[dbfunc] = domain + ufunc_fills[dbfunc] = (fillx, filly) + + def __call__(self, a, b): + "Execute the call behavior." + ma = getmask(a) + mb = getmask(b) + d1 = getdata(a) + d2 = get_data(b) + t = narray(self.domain(d1, d2), copy=False) + if t.any(None): + mb = mask_or(mb, t) + # The following two lines control the domain filling + d2 = d2.copy() + numpy.putmask(d2, t, self.filly) + m = mask_or(ma, mb) + if (not m.ndim) and m: + return masked + result = self.f(d1, d2).view(get_masked_subclass(a,b)) + if result.ndim > 0: + result._mask = m + return result + + def __str__ (self): + return "Masked version of " + str(self.f) + +#.............................................................................. +# Unary ufuncs +exp = _MaskedUnaryOperation(umath.exp) +conjugate = _MaskedUnaryOperation(umath.conjugate) +sin = _MaskedUnaryOperation(umath.sin) +cos = _MaskedUnaryOperation(umath.cos) +tan = _MaskedUnaryOperation(umath.tan) +arctan = _MaskedUnaryOperation(umath.arctan) +arcsinh = _MaskedUnaryOperation(umath.arcsinh) +sinh = _MaskedUnaryOperation(umath.sinh) +cosh = _MaskedUnaryOperation(umath.cosh) +tanh = _MaskedUnaryOperation(umath.tanh) +abs = absolute = _MaskedUnaryOperation(umath.absolute) +fabs = _MaskedUnaryOperation(umath.fabs) +negative = _MaskedUnaryOperation(umath.negative) +floor = _MaskedUnaryOperation(umath.floor) +ceil = _MaskedUnaryOperation(umath.ceil) +around = _MaskedUnaryOperation(fromnumeric.round_) +logical_not = _MaskedUnaryOperation(umath.logical_not) +# Domained unary ufuncs ....................................................... +sqrt = _MaskedUnaryOperation(umath.sqrt, 0.0, + _DomainGreaterEqual(0.0)) +log = _MaskedUnaryOperation(umath.log, 1.0, + _DomainGreater(0.0)) +log10 = _MaskedUnaryOperation(umath.log10, 1.0, + _DomainGreater(0.0)) +tan = _MaskedUnaryOperation(umath.tan, 0.0, + _DomainTan(1.e-35)) +arcsin = _MaskedUnaryOperation(umath.arcsin, 0.0, + _DomainCheckInterval(-1.0, 1.0)) +arccos = _MaskedUnaryOperation(umath.arccos, 0.0, + _DomainCheckInterval(-1.0, 1.0)) +arccosh = _MaskedUnaryOperation(umath.arccosh, 1.0, + _DomainGreaterEqual(1.0)) +arctanh = _MaskedUnaryOperation(umath.arctanh, 0.0, + _DomainCheckInterval(-1.0+1e-15, 1.0-1e-15)) +# Binary ufuncs ............................................................... +add = _MaskedBinaryOperation(umath.add) +subtract = _MaskedBinaryOperation(umath.subtract) +multiply = _MaskedBinaryOperation(umath.multiply, 1, 1) +arctan2 = _MaskedBinaryOperation(umath.arctan2, 0.0, 1.0) +equal = _MaskedBinaryOperation(umath.equal) +equal.reduce = None +not_equal = _MaskedBinaryOperation(umath.not_equal) +not_equal.reduce = None +less_equal = _MaskedBinaryOperation(umath.less_equal) +less_equal.reduce = None +greater_equal = _MaskedBinaryOperation(umath.greater_equal) +greater_equal.reduce = None +less = _MaskedBinaryOperation(umath.less) +less.reduce = None +greater = _MaskedBinaryOperation(umath.greater) +greater.reduce = None +logical_and = _MaskedBinaryOperation(umath.logical_and) +alltrue = _MaskedBinaryOperation(umath.logical_and, 1, 1).reduce +logical_or = _MaskedBinaryOperation(umath.logical_or) +sometrue = logical_or.reduce +logical_xor = _MaskedBinaryOperation(umath.logical_xor) +bitwise_and = _MaskedBinaryOperation(umath.bitwise_and) +bitwise_or = _MaskedBinaryOperation(umath.bitwise_or) +bitwise_xor = _MaskedBinaryOperation(umath.bitwise_xor) +hypot = _MaskedBinaryOperation(umath.hypot) +# Domained binary ufuncs ...................................................... +divide = _DomainedBinaryOperation(umath.divide, _DomainSafeDivide(), 0, 1) +true_divide = _DomainedBinaryOperation(umath.true_divide, + _DomainSafeDivide(), 0, 1) +floor_divide = _DomainedBinaryOperation(umath.floor_divide, + _DomainSafeDivide(), 0, 1) +remainder = _DomainedBinaryOperation(umath.remainder, + _DomainSafeDivide(), 0, 1) +fmod = _DomainedBinaryOperation(umath.fmod, _DomainSafeDivide(), 0, 1) + + +#####-------------------------------------------------------------------------- +#---- --- Mask creation functions --- +#####-------------------------------------------------------------------------- +def get_mask(a): + """Returns the mask of a, if any, or nomask. +To get a full array of booleans of the same shape as a, use getmaskarray. + """ + return getattr(a, '_mask', nomask) +getmask = get_mask + +def getmaskarray(a): + """Returns the mask of a, if any, or a boolean array of the shape of a, full of False. + """ + m = getmask(a) + if m is nomask: + m = make_mask_none(fromnumeric.shape(a)) + return m + +def is_mask(m): + """Returns True if m is a legal mask. +Does not check contents, only type. + """ + try: + return m.dtype.type is MaskType + except AttributeError: + return False +# +def make_mask(m, copy=False, shrink=True, flag=None): + """Returns m as a mask, creating a copy if necessary or requested. +The function can accept any sequence of integers or nomask. +Does not check that contents must be 0s and 1s. + +*Parameters*: + m : {ndarray} + Potential mask. + copy : {boolean} + Whether to return a copy of m (True) or m itself (False). + shrink : {boolean} + Whether to shrink m to nomask if all its values are False. + """ + if flag is not None: + warnings.warn("The flag 'flag' is now called 'shrink'!", + DeprecationWarning) + shrink = flag + if m is nomask: + return nomask + elif isinstance(m, ndarray): + m = filled(m, True) + if m.dtype.type is MaskType: + if copy: + result = narray(m, dtype=MaskType, copy=copy) + else: + result = m + else: + result = narray(m, dtype=MaskType) + else: + result = narray(filled(m, True), dtype=MaskType) + # Bas les masques ! + if shrink and not result.any(): + return nomask + else: + return result + +def make_mask_none(s): + """Returns a mask of shape s, filled with False. + +*Parameters*: + s : {tuple} + A tuple indicating the shape of the final mask. + """ + result = numeric.zeros(s, dtype=MaskType) + return result + +def mask_or (m1, m2, copy=False, shrink=True): + """Returns the combination of two masks m1 and m2. +The masks are combined with the *logical_or* operator, treating nomask as False. +The result may equal m1 or m2 if the other is nomask. + +*Parameters*: + m1 : {ndarray} + First mask. + m2 : {ndarray} + Second mask + copy : {boolean} + Whether to return a copy. + shrink : {boolean} + Whether to shrink m to nomask if all its values are False. + """ + if m1 is nomask: + return make_mask(m2, copy=copy, shrink=shrink) + if m2 is nomask: + return make_mask(m1, copy=copy, shrink=shrink) + if m1 is m2 and is_mask(m1): + return m1 + return make_mask(umath.logical_or(m1, m2), copy=copy, shrink=shrink) + +#####-------------------------------------------------------------------------- +#--- --- Masking functions --- +#####-------------------------------------------------------------------------- +def masked_where(condition, a, copy=True): + """Returns a as an array masked where condition is true. +Masked values of a or condition are kept. + +*Parameters*: + condition : {ndarray} + Masking condition. + a : {ndarray} + Array to mask. + copy : {boolean} + Whether to return a copy of a (True) or modify a in place. + """ + cond = filled(condition,1) + a = narray(a, copy=copy, subok=True) + if hasattr(a, '_mask'): + cond = mask_or(cond, a._mask) + cls = type(a) + else: + cls = MaskedArray + result = a.view(cls) + result._mask = cond + return result + +def masked_greater(x, value, copy=True): + "Shortcut to masked_where, with condition = (x > value)." + return masked_where(greater(x, value), x, copy=copy) + +def masked_greater_equal(x, value, copy=True): + "Shortcut to masked_where, with condition = (x >= value)." + return masked_where(greater_equal(x, value), x, copy=copy) + +def masked_less(x, value, copy=True): + "Shortcut to masked_where, with condition = (x < value)." + return masked_where(less(x, value), x, copy=copy) + +def masked_less_equal(x, value, copy=True): + "Shortcut to masked_where, with condition = (x <= value)." + return masked_where(less_equal(x, value), x, copy=copy) + +def masked_not_equal(x, value, copy=True): + "Shortcut to masked_where, with condition = (x != value)." + return masked_where((x != value), x, copy=copy) + +# +def masked_equal(x, value, copy=True): + """Shortcut to masked_where, with condition = (x == value). +For floating point, consider `masked_values(x, value)` instead. + """ + return masked_where((x == value), x, copy=copy) +# d = filled(x, 0) +# c = umath.equal(d, value) +# m = mask_or(c, getmask(x)) +# return array(d, mask=m, copy=copy) + +def masked_inside(x, v1, v2, copy=True): + """Shortcut to masked_where, where condition is True for x inside the interval +[v1,v2] (v1 <= x <= v2). +The boundaries v1 and v2 can be given in either order. + +*Note*: + The array x is prefilled with its filling value. + """ + if v2 < v1: + (v1, v2) = (v2, v1) + xf = filled(x) + condition = (xf >= v1) & (xf <= v2) + return masked_where(condition, x, copy=copy) + +def masked_outside(x, v1, v2, copy=True): + """Shortcut to masked_where, where condition is True for x outside the interval +[v1,v2] (x < v1)|(x > v2). +The boundaries v1 and v2 can be given in either order. + +*Note*: + The array x is prefilled with its filling value. + """ + if v2 < v1: + (v1, v2) = (v2, v1) + xf = filled(x) + condition = (xf < v1) | (xf > v2) + return masked_where(condition, x, copy=copy) + +# +def masked_object(x, value, copy=True): + """Masks the array x where the data are exactly equal to value. + +This function is suitable only for object arrays: for floating point, please use +``masked_values`` instead. + +*Notes*: + The mask is set to `nomask` if posible. + """ + if isMaskedArray(x): + condition = umath.equal(x._data, value) + mask = x._mask + else: + condition = umath.equal(fromnumeric.asarray(x), value) + mask = nomask + mask = mask_or(mask, make_mask(condition, shrink=True)) + return masked_array(x, mask=mask, copy=copy, fill_value=value) + +def masked_values(x, value, rtol=1.e-5, atol=1.e-8, copy=True): + """Masks the array x where the data are approximately equal to value +(abs(x - value) <= atol+rtol*abs(value)). +Suitable only for floating points. For integers, please use ``masked_equal``. +The mask is set to nomask if posible. + +*Parameters*: + x : {ndarray} + Array to fill. + value : {float} + Masking value. + rtol : {float} + Tolerance parameter. + atol : {float}, *[1e-8]* + Tolerance parameter. + copy : {boolean} + Whether to return a copy of x. + """ + abs = umath.absolute + xnew = filled(x, value) + if issubclass(xnew.dtype.type, numeric.floating): + condition = umath.less_equal(abs(xnew-value), atol+rtol*abs(value)) + mask = getattr(x, '_mask', nomask) + else: + condition = umath.equal(xnew, value) + mask = nomask + mask = mask_or(mask, make_mask(condition, shrink=True)) + return masked_array(xnew, mask=mask, copy=copy, fill_value=value) + +def masked_invalid(a, copy=True): + """Masks the array for invalid values (nans or infs). + Any preexisting mask is conserved.""" + a = narray(a, copy=copy, subok=True) + condition = (numpy.isnan(a) | numpy.isinf(a)) + if hasattr(a, '_mask'): + condition = mask_or(condition, a._mask) + cls = type(a) + else: + cls = MaskedArray + result = a.view(cls) + result._mask = cond + return result + + +#####-------------------------------------------------------------------------- +#---- --- Printing options --- +#####-------------------------------------------------------------------------- +class _MaskedPrintOption: + """Handles the string used to represent missing data in a masked array.""" + def __init__ (self, display): + "Creates the masked_print_option object." + self._display = display + self._enabled = True + + def display(self): + "Displays the string to print for masked values." + return self._display + + def set_display (self, s): + "Sets the string to print for masked values." + self._display = s + + def enabled(self): + "Is the use of the display value enabled?" + return self._enabled + + def enable(self, shrink=1): + "Set the enabling shrink to `shrink`." + self._enabled = shrink + + def __str__ (self): + return str(self._display) + + __repr__ = __str__ + +#if you single index into a masked location you get this object. +masked_print_option = _MaskedPrintOption('--') + +#####-------------------------------------------------------------------------- +#---- --- MaskedArray class --- +#####-------------------------------------------------------------------------- + +#............................................................................... +class _arraymethod(object): + """Defines a wrapper for basic array methods. +Upon call, returns a masked array, where the new _data array is the output of +the corresponding method called on the original _data. + +If onmask is True, the new mask is the output of the method called on the initial +mask. Otherwise, the new mask is just a reference to the initial mask. + +:IVariables: + _name : String + Name of the function to apply on data. + _onmask : {boolean} *[True]* + Whether the mask must be processed also (True) or left alone (False). + obj : Object + The object calling the arraymethod + """ + def __init__(self, funcname, onmask=True): + self._name = funcname + self._onmask = onmask + self.obj = None + self.__doc__ = self.getdoc() + # + def getdoc(self): + "Returns the doc of the function (from the doc of the method)." + methdoc = getattr(ndarray, self._name, None) + methdoc = getattr(numpy, self._name, methdoc) + if methdoc is not None: + return methdoc.__doc__ + # + def __get__(self, obj, objtype=None): + self.obj = obj + return self + # + def __call__(self, *args, **params): + methodname = self._name + data = self.obj._data + mask = self.obj._mask + cls = type(self.obj) + result = getattr(data, methodname)(*args, **params).view(cls) + result._update_from(self.obj) + #result._shrinkmask = self.obj._shrinkmask + if result.ndim: + if not self._onmask: + result.__setmask__(mask) + elif mask is not nomask: + result.__setmask__(getattr(mask, methodname)(*args, **params)) + else: + if mask.ndim and mask.all(): + return masked + return result +#.......................................................... + +class FlatIter(object): + "Defines an interator." + def __init__(self, ma): + self.ma = ma + self.ma_iter = numpy.asarray(ma).flat + + if ma._mask is nomask: + self.maskiter = None + else: + self.maskiter = ma._mask.flat + + def __iter__(self): + return self + + ### This won't work is ravel makes a copy + def __setitem__(self, index, value): + a = self.ma.ravel() + a[index] = value + + def next(self): + d = self.ma_iter.next() + if self.maskiter is not None and self.maskiter.next(): + d = masked + return d + + +class MaskedArray(numeric.ndarray): + """Arrays with possibly masked values. +Masked values of True exclude the corresponding element from any computation. + +Construction: + x = MaskedArray(data, mask=nomask, dtype=None, copy=True, fill_value=None, + mask = nomask, fill_value=None, shrink=True) + +*Parameters*: + data : {var} + Input data. + mask : {nomask, sequence} + Mask. + Must be convertible to an array of booleans with the same shape as data: + True indicates a masked (eg., invalid) data. + dtype : {dtype} + Data type of the output. If None, the type of the data argument is used. + If dtype is not None and different from data.dtype, a copy is performed. + copy : {boolean} + Whether to copy the input data (True), or to use a reference instead. + Note: data are NOT copied by default. + fill_value : {var} + Value used to fill in the masked values when necessary. If None, a default + based on the datatype is used. + keep_mask : {True, boolean} + Whether to combine mask with the mask of the input data, if any (True), + or to use only mask for the output (False). + hard_mask : {False, boolean} + Whether to use a hard mask or not. With a hard mask, masked values cannot + be unmasked. + subok : {True, boolean} + Whether to return a subclass of MaskedArray (if possible) or a plain + MaskedArray. + """ + + __array_priority__ = 15 + _defaultmask = nomask + _defaulthardmask = False + _baseclass = numeric.ndarray + + def __new__(cls, data=None, mask=nomask, dtype=None, copy=False, fill_value=None, + keep_mask=True, hard_mask=False, flag=None, + subok=True, **options): + """Creates a new masked array from scratch. + Note: you can also create an array with the .view(MaskedArray) method... + """ + if flag is not None: + warnings.warn("The flag 'flag' is now called 'shrink'!", + DeprecationWarning) + shrink = flag + # Process data............ + _data = narray(data, dtype=dtype, copy=copy, subok=True) + _baseclass = getattr(data, '_baseclass', type(_data)) + _basedict = getattr(data, '_basedict', getattr(data, '__dict__', None)) + if not isinstance(data, MaskedArray) or not subok: + _data = _data.view(cls) + else: + _data = _data.view(type(data)) + # Backwards compatibility w/ numpy.core.ma ....... + if hasattr(data,'_mask') and not isinstance(data, ndarray): + _data._mask = data._mask + _sharedmask = True + # Process mask ........... + if mask is nomask: + if not keep_mask: + _data._mask = nomask + if copy: + _data._mask = _data._mask.copy() + _data._sharedmask = False + else: + _data._sharedmask = True + else: + mask = narray(mask, dtype=MaskType, copy=copy) + if mask.shape != _data.shape: + (nd, nm) = (_data.size, mask.size) + if nm == 1: + mask = numeric.resize(mask, _data.shape) + elif nm == nd: + mask = fromnumeric.reshape(mask, _data.shape) + else: + msg = "Mask and data not compatible: data size is %i, "+\ + "mask size is %i." + raise MAError, msg % (nd, nm) + copy = True + if _data._mask is nomask: + _data._mask = mask + _data._sharedmask = not copy + else: + if not keep_mask: + _data._mask = mask + _data._sharedmask = not copy + else: + _data._mask = umath.logical_or(mask, _data._mask) + _data._sharedmask = False + + # Update fill_value....... + if fill_value is None: + _data._fill_value = getattr(data, '_fill_value', + default_fill_value(_data)) + else: + _data._fill_value = fill_value + # Process extra options .. + _data._hardmask = hard_mask + _data._baseclass = _baseclass + _data._basedict = _basedict + return _data + # + def _update_from(self, obj): + """Copies some attributes of obj to self. + """ + self._hardmask = getattr(obj, '_hardmask', self._defaulthardmask) + self._sharedmask = getattr(obj, '_sharedmask', False) + if obj is not None: + self._baseclass = getattr(obj, '_baseclass', type(obj)) + else: + self._baseclass = ndarray + self._fill_value = getattr(obj, '_fill_value', None) + return + #........................ + def __array_finalize__(self,obj): + """Finalizes the masked array. + """ + # Get main attributes ......... + self._mask = getattr(obj, '_mask', nomask) + self._update_from(obj) + # Update special attributes ... + self._basedict = getattr(obj, '_basedict', getattr(obj, '__dict__', None)) + if self._basedict is not None: + self.__dict__.update(self._basedict) + # Finalize the mask ........... + if self._mask is not nomask: + self._mask.shape = self.shape + return + #.................................. + def __array_wrap__(self, obj, context=None): + """Special hook for ufuncs. +Wraps the numpy array and sets the mask according to context. + """ + result = obj.view(type(self)) + result._update_from(self) + #.......... + if context is not None: + result._mask = result._mask.copy() + (func, args, _) = context + m = reduce(mask_or, [getmask(arg) for arg in args]) + # Get the domain mask................ + domain = ufunc_domain.get(func, None) + if domain is not None: + if len(args) > 2: + d = reduce(domain, args) + else: + d = domain(*args) + # Fill the result where the domain is wrong + try: + # Binary domain: take the last value + fill_value = ufunc_fills[func][-1] + except TypeError: + # Unary domain: just use this one + fill_value = ufunc_fills[func] + except KeyError: + # Domain not recognized, use fill_value instead + fill_value = self.fill_value + result = result.copy() + numpy.putmask(result, d, fill_value) + # Update the mask + if m is nomask: + if d is not nomask: + m = d + else: + m |= d + # Make sure the mask has the proper size + if result.shape == () and m: + return masked + else: + result._mask = m + result._sharedmask = False + #.... + return result + #............................................. + def __getitem__(self, indx): + """x.__getitem__(y) <==> x[y] +Returns the item described by i, as a masked array. + """ + # This test is useful, but we should keep things light... +# if getmask(indx) is not nomask: +# msg = "Masked arrays must be filled before they can be used as indices!" +# raise IndexError, msg + dout = ndarray.__getitem__(self.view(ndarray), indx) + m = self._mask + if not getattr(dout,'ndim', False): + # Just a scalar............ + if m is not nomask and m[indx]: + return masked + else: + # Force dout to MA ........ + dout = dout.view(type(self)) + # Inherit attributes from self + dout._update_from(self) + # Update the mask if needed + if m is not nomask: + dout._mask = ndarray.__getitem__(m, indx).reshape(dout.shape) +# Note: Don't try to check for m.any(), that'll take too long... +# mask = ndarray.__getitem__(m, indx).reshape(dout.shape) +# if self._shrinkmask and not m.any(): +# dout._mask = nomask +# else: +# dout._mask = mask + return dout + #........................ + def __setitem__(self, indx, value): + """x.__setitem__(i, y) <==> x[i]=y +Sets item described by index. If value is masked, masks those locations. + """ + if self is masked: + raise MAError, 'Cannot alter the masked element.' +# if getmask(indx) is not nomask: +# msg = "Masked arrays must be filled before they can be used as indices!" +# raise IndexError, msg + #.... + if value is masked: + m = self._mask + if m is nomask: + m = numpy.zeros(self.shape, dtype=MaskType) + m[indx] = True + self._mask = m + self._sharedmask = False + return + #.... + dval = getdata(value).astype(self.dtype) + valmask = getmask(value) + if self._mask is nomask: + if valmask is not nomask: + self._mask = numpy.zeros(self.shape, dtype=MaskType) + self._mask[indx] = valmask + elif not self._hardmask: + # Unshare the mask if necessary to avoid propagation + self.unshare_mask() + self._mask[indx] = valmask + elif hasattr(indx, 'dtype') and (indx.dtype==bool_): + indx = indx * umath.logical_not(self._mask) + else: + mindx = mask_or(self._mask[indx], valmask, copy=True) + dindx = self._data[indx] + if dindx.size > 1: + dindx[~mindx] = dval + elif mindx is nomask: + dindx = dval + dval = dindx + self._mask[indx] = mindx + # Set data .......... + ndarray.__setitem__(self._data,indx,dval) + #............................................ + def __getslice__(self, i, j): + """x.__getslice__(i, j) <==> x[i:j] +Returns the slice described by (i, j). +The use of negative indices is not supported.""" + return self.__getitem__(slice(i,j)) + #........................ + def __setslice__(self, i, j, value): + """x.__setslice__(i, j, value) <==> x[i:j]=value +Sets the slice (i,j) of a to value. If value is masked, masks those locations. + """ + self.__setitem__(slice(i,j), value) + #............................................ + def __setmask__(self, mask, copy=False): + """Sets the mask.""" + if mask is not nomask: + mask = narray(mask, copy=copy, dtype=MaskType) +# if self._shrinkmask and not mask.any(): +# mask = nomask + if self._mask is nomask: + self._mask = mask + elif self._hardmask: + if mask is not nomask: + self._mask.__ior__(mask) + else: + # This one is tricky: if we set the mask that way, we may break the + # propagation. But if we don't, we end up with a mask full of False + # and a test on nomask fails... + if mask is nomask: + self._mask = nomask + else: + self.unshare_mask() + self._mask.flat = mask + if self._mask.shape: + self._mask = numeric.reshape(self._mask, self.shape) + _set_mask = __setmask__ + #.... + def _get_mask(self): + """Returns the current mask.""" + return self._mask +# return self._mask.reshape(self.shape) + mask = property(fget=_get_mask, fset=__setmask__, doc="Mask") + #............................................ + def harden_mask(self): + "Forces the mask to hard." + self._hardmask = True + + def soften_mask(self): + "Forces the mask to soft." + self._hardmask = False + + def unshare_mask(self): + "Copies the mask and set the sharedmask flag to False." + if self._sharedmask: + self._mask = self._mask.copy() + self._sharedmask = False + + def shrink_mask(self): + "Reduces a mask to nomask when possible." + m = self._mask + if m.ndim and not m.any(): + self._mask = nomask + + #............................................ + def _get_data(self): + "Returns the current data, as a view of the original underlying data." + return self.view(self._baseclass) + _data = property(fget=_get_data) + data = property(fget=_get_data) + + def raw_data(self): + """Returns the _data part of the MaskedArray. +DEPRECATED: You should really use ``.data`` instead...""" + return self._data + #............................................ + def _get_flat(self): + "Returns a flat iterator." + return FlatIter(self) + # + def _set_flat (self, value): + "Sets a flattened version of self to value." + y = self.ravel() + y[:] = value + # + flat = property(fget=_get_flat, fset=_set_flat, + doc="Flat version of the array.") + #............................................ + def get_fill_value(self): + "Returns the filling value." + if self._fill_value is None: + self._fill_value = default_fill_value(self) + return self._fill_value + + def set_fill_value(self, value=None): + """Sets the filling value to value. +If value is None, uses a default based on the data type.""" + if value is None: + value = default_fill_value(self) + self._fill_value = value + + fill_value = property(fget=get_fill_value, fset=set_fill_value, + doc="Filling value.") + + def filled(self, fill_value=None): + """Returns a copy of self._data, where masked values are filled with +fill_value. + +If fill_value is None, self.fill_value is used instead. + +*Note*: + + Subclassing is preserved + + The result is NOT a MaskedArray ! + +*Examples*: + >>> x = array([1,2,3,4,5], mask=[0,0,1,0,1], fill_value=-999) + >>> x.filled() + array([1,2,-999,4,-999]) + >>> type(x.filled()) + <type 'numpy.ndarray'> + """ + m = self._mask + if m is nomask or not m.any(): + return self._data + # + if fill_value is None: + fill_value = self.fill_value + # + if self is masked_singleton: + result = numeric.asanyarray(fill_value) + else: + result = self._data.copy() + try: + numpy.putmask(result, m, fill_value) + except (TypeError, AttributeError): + fill_value = narray(fill_value, dtype=object) + d = result.astype(object) + result = fromnumeric.choose(m, (d, fill_value)) + except IndexError: + #ok, if scalar + if self._data.shape: + raise + elif m: + result = narray(fill_value, dtype=self.dtype) + else: + result = self._data + return result + + def compressed(self): + """Returns a 1-D array of all the non-masked data.""" + data = ndarray.ravel(self._data).view(type(self)) + data._update_from(self) + if self._mask is not nomask: + data = data[numpy.logical_not(ndarray.ravel(self._mask))] +# if not self._shrinkmask: +# data._mask = numpy.zeros(data.shape, dtype=MaskType) + return data + + #............................................ + def __str__(self): + """Calculates the string representation. + """ + if masked_print_option.enabled(): + f = masked_print_option + if self is masked: + return str(f) + m = self._mask + if m is nomask: + res = self._data + else: + if m.shape == (): + if m: + return str(f) + else: + return str(self._data) + # convert to object array to make filled work +#CHECK: the two lines below seem more robust than the self._data.astype +# res = numeric.empty(self._data.shape, object_) +# numeric.putmask(res,~m,self._data) + res = self._data.astype("|O8") + res[m] = f + else: + res = self.filled(self.fill_value) + return str(res) + + def __repr__(self): + """Calculates the repr representation. + """ + with_mask = """\ +masked_%(name)s(data = + %(data)s, + mask = + %(mask)s, + fill_value=%(fill)s) +""" + with_mask1 = """\ +masked_%(name)s(data = %(data)s, + mask = %(mask)s, + fill_value=%(fill)s) +""" + n = len(self.shape) + name = repr(self._data).split('(')[0] + if n <= 1: + return with_mask1 % { + 'name': name, + 'data': str(self), + 'mask': str(self._mask), + 'fill': str(self.fill_value), + } + return with_mask % { + 'name': name, + 'data': str(self), + 'mask': str(self._mask), + 'fill': str(self.fill_value), + } + #............................................ + def __add__(self, other): + "Adds other to self, and returns a new masked array." + return add(self, other) + # + def __sub__(self, other): + "Subtracts other to self, and returns a new masked array." + return subtract(self, other) + # + def __mul__(self, other): + "Multiplies other to self, and returns a new masked array." + return multiply(self, other) + # + def __div__(self, other): + "Divides other to self, and returns a new masked array." + return divide(self, other) + # + def __truediv__(self, other): + "Divides other to self, and returns a new masked array." + return true_divide(self, other) + # + def __floordiv__(self, other): + "Divides other to self, and returns a new masked array." + return floor_divide(self, other) + + #............................................ + def __iadd__(self, other): + "Adds other to self in place." + ndarray.__iadd__(self._data, getdata(other)) + m = getmask(other) + if self._mask is nomask: + self._mask = m + elif m is not nomask: + self._mask += m + return self + #.... + def __isub__(self, other): + "Subtracts other from self in place." + ndarray.__isub__(self._data, getdata(other)) + m = getmask(other) + if self._mask is nomask: + self._mask = m + elif m is not nomask: + self._mask += m + return self + #.... + def __imul__(self, other): + "Multiplies self by other in place." + ndarray.__imul__(self._data, getdata(other)) + m = getmask(other) + if self._mask is nomask: + self._mask = m + elif m is not nomask: + self._mask += m + return self + #.... + def __idiv__(self, other): + "Divides self by other in place." + other_data = getdata(other) + dom_mask = _DomainSafeDivide().__call__(self._data, other_data) + other_mask = getmask(other) + new_mask = mask_or(other_mask, dom_mask) + # The following 3 lines control the domain filling + if dom_mask.any(): + other_data = other_data.copy() + numpy.putmask(other_data, dom_mask, 1) + ndarray.__idiv__(self._data, other_data) + self._mask = mask_or(self._mask, new_mask) + return self + #............................................ + def __float__(self): + "Converts self to float." + if self._mask is not nomask: + warnings.warn("Warning: converting a masked element to nan.") + return numpy.nan + #raise MAError, 'Cannot convert masked element to a Python float.' + return float(self.item()) + + def __int__(self): + "Converts self to int." + if self._mask is not nomask: + raise MAError, 'Cannot convert masked element to a Python int.' + return int(self.item()) + #............................................ + def get_imag(self): + result = self._data.imag.view(type(self)) + result.__setmask__(self._mask) + return result + imag = property(fget=get_imag,doc="Imaginary part") + + def get_real(self): + result = self._data.real.view(type(self)) + result.__setmask__(self._mask) + return result + real = property(fget=get_real,doc="Real part") + + + #............................................ + def count(self, axis=None): + """Counts the non-masked elements of the array along the given axis. + +*Parameters*: + axis : {integer}, optional + Axis along which to count the non-masked elements. If not given, all the + non masked elements are counted. + +*Returns*: + A masked array where the mask is True where all data are masked. + If axis is None, returns either a scalar ot the masked singleton if all values + are masked. + """ + m = self._mask + s = self.shape + ls = len(s) + if m is nomask: + if ls == 0: + return 1 + if ls == 1: + return s[0] + if axis is None: + return self.size + else: + n = s[axis] + t = list(s) + del t[axis] + return numeric.ones(t) * n + n1 = numpy.size(m, axis) + n2 = m.astype(int_).sum(axis) + if axis is None: + return (n1-n2) + else: + return masked_array(n1 - n2) + #............................................ + flatten = _arraymethod('flatten') +# ravel = _arraymethod('ravel') + def ravel(self): + """Returns a 1D version of self, as a view.""" + r = ndarray.ravel(self._data).view(type(self)) + r._update_from(self) + if self._mask is not nomask: + r._mask = ndarray.ravel(self._mask).reshape(r.shape) + else: + r._mask = nomask + return r + repeat = _arraymethod('repeat') + # + def reshape (self, *s): + """Reshapes the array to shape s. + +*Returns*: + A new masked array. + +*Notes: + If you want to modify the shape in place, please use ``a.shape = s`` + """ + result = self._data.reshape(*s).view(type(self)) + result.__dict__.update(self.__dict__) + if result._mask is not nomask: + result._mask = self._mask.copy() + result._mask.shape = result.shape + return result + # + def resize(self, newshape, refcheck=True, order=False): + """Attempts to modify the size and the shape of the array in place. + +The array must own its own memory and not be referenced by other arrays. + +*Returns*: + None. + """ + try: + self._data.resize(newshape, refcheck, order) + if self.mask is not nomask: + self._mask.resize(newshape, refcheck, order) + except ValueError: + raise ValueError("Cannot resize an array that has been referenced " + "or is referencing another array in this way.\n" + "Use the resize function.") + return None + # + def put(self, indices, values, mode='raise'): + """Sets storage-indexed locations to corresponding values. + +a.put(values, indices, mode) sets a.flat[n] = values[n] for each n in indices. +If values is shorter than indices then it will repeat. +If values has some masked values, the initial mask is updated in consequence, +else the corresponding values are unmasked. + """ + m = self._mask + # Hard mask: Get rid of the values/indices that fall on masked data + if self._hardmask and self._mask is not nomask: + mask = self._mask[indices] + indices = narray(indices, copy=False) + values = narray(values, copy=False, subok=True) + values.resize(indices.shape) + indices = indices[~mask] + values = values[~mask] + #.... + self._data.put(indices, values, mode=mode) + #.... + if m is nomask: + m = getmask(values) + else: + m = m.copy() + if getmask(values) is nomask: + m.put(indices, False, mode=mode) + else: + m.put(indices, values._mask, mode=mode) + m = make_mask(m, copy=False, shrink=True) + self._mask = m + #............................................ + def ids (self): + """Returns the addresses of the data and mask areas.""" + return (self.ctypes.data, self._mask.ctypes.data) + #............................................ + def all(self, axis=None, out=None): + """Returns True if all entries along the given axis are True, False otherwise. +Masked values are considered as True during computation. + +*Parameters* + axis : {integer}, optional + Axis along which the operation is performed. + If None, the operation is performed on a flatten array + out : {MaskedArray}, optional + Alternate optional output. + If not None, out should be a valid MaskedArray of the same shape as the + output of self._data.all(axis). + +*Returns* + A masked array, where the mask is True if all data along the axis are masked. + +*Notes* + An exception is raised if ``out`` is not None and not of the same type as self. + """ + if out is None: + d = self.filled(True).all(axis=axis).view(type(self)) + if d.ndim > 0: + d.__setmask__(self._mask.all(axis)) + return d + elif type(out) is not type(self): + raise TypeError("The external array should have a type %s (got %s instead)" %\ + (type(self), type(out))) + self.filled(True).all(axis=axis, out=out) + if out.ndim: + out.__setmask__(self._mask.all(axis)) + return out + + + def any(self, axis=None, out=None): + """Returns True if at least one entry along the given axis is True. + +Returns False if all entries are False. +Masked values are considered as True during computation. + +*Parameters* + axis : {integer}, optional + Axis along which the operation is performed. + If None, the operation is performed on a flatten array + out : {MaskedArray}, optional + Alternate optional output. + If not None, out should be a valid MaskedArray of the same shape as the + output of self._data.all(axis). + +*Returns* + A masked array, where the mask is True if all data along the axis are masked. + +*Notes* + An exception is raised if ``out`` is not None and not of the same type as self. + """ + if out is None: + d = self.filled(False).any(axis=axis).view(type(self)) + if d.ndim > 0: + d.__setmask__(self._mask.all(axis)) + return d + elif type(out) is not type(self): + raise TypeError("The external array should have a type %s (got %s instead)" %\ + (type(self), type(out))) + self.filled(False).any(axis=axis, out=out) + if out.ndim: + out.__setmask__(self._mask.all(axis)) + return out + + + def nonzero(self): + """Returns the indices of the elements of a that are not zero nor masked, +as a tuple of arrays. + +There are as many tuples as dimensions of a, each tuple contains the indices of +the non-zero elements in that dimension. The corresponding non-zero values can +be obtained with ``a[a.nonzero()]``. + +To group the indices by element, rather than dimension, use instead: +``transpose(a.nonzero())``. + +The result of this is always a 2d array, with a row for each non-zero element. + """ + return narray(self.filled(0), copy=False).nonzero() + #............................................ + def trace(self, offset=0, axis1=0, axis2=1, dtype=None, out=None): + """a.trace(offset=0, axis1=0, axis2=1, dtype=None, out=None) +Returns the sum along the offset diagonal of the array's indicated `axis1` and `axis2`. + """ + # TODO: What are we doing with `out`? + m = self._mask + if m is nomask: + result = super(MaskedArray, self).trace(offset=offset, axis1=axis1, + axis2=axis2, out=out) + return result.astype(dtype) + else: + D = self.diagonal(offset=offset, axis1=axis1, axis2=axis2) + return D.astype(dtype).filled(0).sum(axis=None) + #............................................ + def sum(self, axis=None, dtype=None): + """Sums the array over the given axis. + +Masked elements are set to 0 internally. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. + """ + if self._mask is nomask: + mask = nomask + else: + mask = self._mask.all(axis) + if (not mask.ndim) and mask: + return masked + result = self.filled(0).sum(axis, dtype=dtype).view(type(self)) + if result.ndim > 0: + result.__setmask__(mask) + return result + + def cumsum(self, axis=None, dtype=None): + """Returns the cumulative sum of the elements of the array along the given axis. + +Masked values are set to 0 internally. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. + """ + result = self.filled(0).cumsum(axis=axis, dtype=dtype).view(type(self)) + result.__setmask__(self.mask) + return result + + def prod(self, axis=None, dtype=None): + """Returns the product of the elements of the array along the given axis. + +Masked elements are set to 1 internally. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. + """ + if self._mask is nomask: + mask = nomask + else: + mask = self._mask.all(axis) + if (not mask.ndim) and mask: + return masked + result = self.filled(1).prod(axis=axis, dtype=dtype).view(type(self)) + if result.ndim: + result.__setmask__(mask) + return result + product = prod + + def cumprod(self, axis=None, dtype=None): + """Returns the cumulative product of the elements of the array along the given axis. + +Masked values are set to 1 internally. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. + """ + result = self.filled(1).cumprod(axis=axis, dtype=dtype).view(type(self)) + result.__setmask__(self.mask) + return result + + def mean(self, axis=None, dtype=None): + """Averages the array over the given axis. Equivalent to + + a.sum(axis, dtype) / a.size(axis). + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. + """ + if self._mask is nomask: + return super(MaskedArray, self).mean(axis=axis, dtype=dtype) + else: + dsum = self.sum(axis=axis, dtype=dtype) + cnt = self.count(axis=axis) + return dsum*1./cnt + + def anom(self, axis=None, dtype=None): + """Returns the anomalies (deviations from the average) along the given axis. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. + """ + m = self.mean(axis, dtype) + if not axis: + return (self - m) + else: + return (self - expand_dims(m,axis)) + + def var(self, axis=None, dtype=None): + """Returns the variance, a measure of the spread of a distribution. + +The variance is the average of the squared deviations from the mean, +i.e. var = mean((x - x.mean())**2). + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. + +*Notes*: + The value returned is a biased estimate of the true variance. + For the (more standard) unbiased estimate, use varu. + """ + if self._mask is nomask: + # TODO: Do we keep super, or var _data and take a view ? + return super(MaskedArray, self).var(axis=axis, dtype=dtype) + else: + cnt = self.count(axis=axis) + danom = self.anom(axis=axis, dtype=dtype) + danom *= danom + dvar = narray(danom.sum(axis) / cnt).view(type(self)) + if axis is not None: + dvar._mask = mask_or(self._mask.all(axis), (cnt==1)) + dvar._update_from(self) + return dvar + + def std(self, axis=None, dtype=None): + """Returns the standard deviation, a measure of the spread of a distribution. + +The standard deviation is the square root of the average of the squared +deviations from the mean, i.e. std = sqrt(mean((x - x.mean())**2)). + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. + If not given, the current dtype is used instead. + +*Notes*: + The value returned is a biased estimate of the true standard deviation. + For the more standard unbiased estimate, use stdu. + """ + dvar = self.var(axis,dtype) + if axis is not None or dvar is not masked: + dvar = sqrt(dvar) + return dvar + + #............................................ + def argsort(self, axis=None, fill_value=None, kind='quicksort', + order=None): + """Returns a ndarray of indices that sort the array along the specified axis. + Masked values are filled beforehand to fill_value. + Returns a numpy array. + +*Parameters*: + axis : {integer}, optional + Axis to be indirectly sorted. + If not given, uses a flatten version of the array. + fill_value : {var} + Value used to fill in the masked values. + If not given, self.fill_value is used instead. + kind : {string} + Sorting algorithm (default 'quicksort') + Possible values: 'quicksort', 'mergesort', or 'heapsort' + +*Notes*: + This method executes an indirect sort along the given axis using the algorithm + specified by the kind keyword. It returns an array of indices of the same shape + as 'a' that index data along the given axis in sorted order. + + The various sorts are characterized by average speed, worst case performance + need for work space, and whether they are stable. A stable sort keeps items + with the same key in the same relative order. The three available algorithms + have the following properties: + + |------------------------------------------------------| + | kind | speed | worst case | work space | stable| + |------------------------------------------------------| + |'quicksort'| 1 | O(n^2) | 0 | no | + |'mergesort'| 2 | O(n*log(n)) | ~n/2 | yes | + |'heapsort' | 3 | O(n*log(n)) | 0 | no | + |------------------------------------------------------| + + All the sort algorithms make temporary copies of the data when the sort is not + along the last axis. Consequently, sorts along the last axis are faster and use + less space than sorts along other axis. + """ + if fill_value is None: + fill_value = default_fill_value(self) + d = self.filled(fill_value).view(ndarray) + return d.argsort(axis=axis, kind=kind, order=order) + #........................ + def argmin(self, axis=None, fill_value=None): + """Returns a ndarray of indices for the minimum values of a along the +specified axis. + +Masked values are treated as if they had the value fill_value. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + fill_value : {var}, optional + Value used to fill in the masked values. + If None, the output of minimum_fill_value(self._data) is used. + """ + if fill_value is None: + fill_value = minimum_fill_value(self) + d = self.filled(fill_value).view(ndarray) + return d.argmin(axis) + #........................ + def argmax(self, axis=None, fill_value=None): + """Returns the array of indices for the maximum values of `a` along the +specified axis. + +Masked values are treated as if they had the value fill_value. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + fill_value : {var}, optional + Value used to fill in the masked values. + If None, the output of maximum_fill_value(self._data) is used. + """ + if fill_value is None: + fill_value = maximum_fill_value(self._data) + d = self.filled(fill_value).view(ndarray) + return d.argmax(axis) + + def sort(self, axis=-1, kind='quicksort', order=None, + endwith=True, fill_value=None): + """Sort a along the given axis. + +*Parameters*: + axis : {integer} + Axis to be indirectly sorted. + kind : {string} + Sorting algorithm (default 'quicksort') + Possible values: 'quicksort', 'mergesort', or 'heapsort'. + order : {var} + If a has fields defined, then the order keyword can be the field + name to sort on or a list (or tuple) of field names to indicate + the order that fields should be used to define the sort. + fill_value : {var} + Value used to fill in the masked values. + If None, use the the output of minimum_fill_value(). + endwith : {boolean} + Whether missing values (if any) should be forced in the upper indices + (at the end of the array) (True) or lower indices (at the beginning). + +*Returns*: + When used as method, returns None. + When used as a function, returns an array. + +*Notes*: + This method sorts 'a' in place along the given axis using the algorithm + specified by the kind keyword. + + The various sorts may characterized by average speed, worst case performance + need for work space, and whether they are stable. A stable sort keeps items + with the same key in the same relative order and is most useful when used w/ + argsort where the key might differ from the items being sorted. + The three available algorithms have the following properties: + + |------------------------------------------------------| + | kind | speed | worst case | work space | stable| + |------------------------------------------------------| + |'quicksort'| 1 | O(n^2) | 0 | no | + |'mergesort'| 2 | O(n*log(n)) | ~n/2 | yes | + |'heapsort' | 3 | O(n*log(n)) | 0 | no | + |------------------------------------------------------| + + """ + if self._mask is nomask: + ndarray.sort(self,axis=axis, kind=kind, order=order) + else: + if fill_value is None: + if endwith: + filler = minimum_fill_value(self) + else: + filler = maximum_fill_value(self) + else: + filler = fill_value + idx = numpy.indices(self.shape) + idx[axis] = self.filled(filler).argsort(axis=axis,kind=kind,order=order) + idx_l = idx.tolist() + tmp_mask = self._mask[idx_l].flat + tmp_data = self._data[idx_l].flat + self.flat = tmp_data + self._mask.flat = tmp_mask + return + + #............................................ + def min(self, axis=None, fill_value=None): + """Returns the minimum of a along the given axis. + +Masked values are filled with fill_value. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + fill_value : {var}, optional + Value used to fill in the masked values. + If None, use the the output of minimum_fill_value(). + """ + mask = self._mask + # Check all/nothing case ...... + if mask is nomask: + return super(MaskedArray, self).min(axis=axis) + elif (not mask.ndim) and mask: + return masked + # Get the mask ................ + if axis is None: + mask = umath.logical_and.reduce(mask.flat) + else: + mask = umath.logical_and.reduce(mask, axis=axis) + # Get the fil value ........... + if fill_value is None: + fill_value = minimum_fill_value(self) + # Get the data ................ + result = self.filled(fill_value).min(axis=axis).view(type(self)) + if result.ndim > 0: + result._mask = mask + return result + #........................ + def max(self, axis=None, fill_value=None): + """Returns the maximum/a along the given axis. + +Masked values are filled with fill_value. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + fill_value : {var}, optional + Value used to fill in the masked values. + If None, use the the output of maximum_fill_value(). + """ + mask = self._mask + # Check all/nothing case ...... + if mask is nomask: + return super(MaskedArray, self).max(axis=axis) + elif (not mask.ndim) and mask: + return masked + # Check the mask .............. + if axis is None: + mask = umath.logical_and.reduce(mask.flat) + else: + mask = umath.logical_and.reduce(mask, axis=axis) + # Get the fill value .......... + if fill_value is None: + fill_value = maximum_fill_value(self) + # Get the data ................ + result = self.filled(fill_value).max(axis=axis).view(type(self)) + if result.ndim > 0: + result._mask = mask + return result + #........................ + def ptp(self, axis=None, fill_value=None): + """Returns the visible data range (max-min) along the given axis. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + fill_value : {var}, optional + Value used to fill in the masked values. + If None, the maximum uses the maximum default, the minimum uses + the minimum default. + """ + return self.max(axis, fill_value) - self.min(axis, fill_value) + + + # Array methods --------------------------------------- +# conj = conjugate = _arraymethod('conjugate') + copy = _arraymethod('copy') + diagonal = _arraymethod('diagonal') + take = _arraymethod('take') +# ravel = _arraymethod('ravel') + transpose = _arraymethod('transpose') + T = property(fget=lambda self:self.transpose()) + swapaxes = _arraymethod('swapaxes') + clip = _arraymethod('clip', onmask=False) + compress = _arraymethod('compress') + copy = _arraymethod('copy') + squeeze = _arraymethod('squeeze') + #-------------------------------------------- + def tolist(self, fill_value=None): + """Copies the data portion of the array to a hierarchical python list and +returns that list. + +Data items are converted to the nearest compatible Python type. +Masked values are converted to fill_value. If fill_value is None, the corresponding +entries in the output list will be ``None``. + """ + if fill_value is not None: + return self.filled(fill_value).tolist() + result = self.filled().tolist() + if self._mask is nomask: + return result + if self.ndim == 0: + return [None] + elif self.ndim == 1: + maskedidx = self._mask.nonzero()[0].tolist() + [operator.setitem(result,i,None) for i in maskedidx] + else: + for idx in zip(*[i.tolist() for i in self._mask.nonzero()]): + tmp = result + for i in idx[:-1]: + tmp = tmp[i] + tmp[idx[-1]] = None + return result + #........................ + def tostring(self, fill_value=None, order='C'): + """Returns a copy of array data as a Python string containing the raw +bytes in the array. + +*Parameters*: + fill_value : {var}, optional + Value used to fill in the masked values. + If None, uses self.fill_value instead. + order : {string} + Order of the data item in the copy {"C","F","A"}. + "C" -- C order (row major) + "Fortran" -- Fortran order (column major) + "Any" -- Current order of array. + None -- Same as "Any" + """ + return self.filled(fill_value).tostring(order=order) + #-------------------------------------------- + # Pickling + def __getstate__(self): + "Returns the internal state of the masked array, for pickling purposes." + state = (1, + self.shape, + self.dtype, + self.flags.fnc, + self._data.tostring(), + getmaskarray(self).tostring(), + self._fill_value, + ) + return state + # + def __setstate__(self, state): + """Restores the internal state of the masked array, for pickling purposes. +``state`` is typically the output of the ``__getstate__`` output, and is a 5-tuple: + + - class name + - a tuple giving the shape of the data + - a typecode for the data + - a binary string for the data + - a binary string for the mask. + """ + (ver, shp, typ, isf, raw, msk, flv) = state + ndarray.__setstate__(self, (shp, typ, isf, raw)) + self._mask.__setstate__((shp, dtype(bool), isf, msk)) + self.fill_value = flv + # + def __reduce__(self): + """Returns a 3-tuple for pickling a MaskedArray.""" + return (_mareconstruct, + (self.__class__, self._baseclass, (0,), 'b', ), + self.__getstate__()) + + +def _mareconstruct(subtype, baseclass, baseshape, basetype,): + """Internal function that builds a new MaskedArray from the information stored +in a pickle.""" + _data = ndarray.__new__(baseclass, baseshape, basetype) + _mask = ndarray.__new__(ndarray, baseshape, 'b1') + return subtype.__new__(subtype, _data, mask=_mask, dtype=basetype, shrink=False) +#MaskedArray.__dump__ = dump +#MaskedArray.__dumps__ = dumps + + + +#####-------------------------------------------------------------------------- +#---- --- Shortcuts --- +#####--------------------------------------------------------------------------- +def isMaskedArray(x): + "Is x a masked array, that is, an instance of MaskedArray?" + return isinstance(x, MaskedArray) +isarray = isMaskedArray +isMA = isMaskedArray #backward compatibility +# We define the masked singleton as a float for higher precedence... +masked_singleton = MaskedArray(0, dtype=float_, mask=True) +masked = masked_singleton + +masked_array = MaskedArray + +def array(data, dtype=None, copy=False, order=False, mask=nomask, subok=True, + keep_mask=True, hard_mask=False, fill_value=None, shrink=True): + """array(data, dtype=None, copy=True, order=False, mask=nomask, + keep_mask=True, shrink=True, fill_value=None) +Acts as shortcut to MaskedArray, with options in a different order for convenience. +And backwards compatibility... + """ + #TODO: we should try to put 'order' somwehere + return MaskedArray(data, mask=mask, dtype=dtype, copy=copy, subok=subok, + keep_mask=keep_mask, hard_mask=hard_mask, + fill_value=fill_value) +array.__doc__ = masked_array.__doc__ + +def is_masked(x): + """Returns whether x has some masked values.""" + m = getmask(x) + if m is nomask: + return False + elif m.any(): + return True + return False + + +#####--------------------------------------------------------------------------- +#---- --- Extrema functions --- +#####--------------------------------------------------------------------------- +class _extrema_operation(object): + "Generic class for maximum/minimum functions." + def __call__(self, a, b=None): + "Executes the call behavior." + if b is None: + return self.reduce(a) + return where(self.compare(a, b), a, b) + #......... + def reduce(self, target, axis=None): + "Reduces target along the given axis." + m = getmask(target) + if axis is not None: + kargs = { 'axis' : axis } + else: + kargs = {} + target = target.ravel() + if not (m is nomask): + m = m.ravel() + if m is nomask: + t = self.ufunc.reduce(target, **kargs) + else: + target = target.filled(self.fill_value_func(target)).view(type(target)) + t = self.ufunc.reduce(target, **kargs) + m = umath.logical_and.reduce(m, **kargs) + if hasattr(t, '_mask'): + t._mask = m + elif m: + t = masked + return t + #......... + def outer (self, a, b): + "Returns the function applied to the outer product of a and b." + ma = getmask(a) + mb = getmask(b) + if ma is nomask and mb is nomask: + m = nomask + else: + ma = getmaskarray(a) + mb = getmaskarray(b) + m = logical_or.outer(ma, mb) + result = self.ufunc.outer(filled(a), filled(b)) + result._mask = m + return result + +#............................ +class _minimum_operation(_extrema_operation): + "Object to calculate minima" + def __init__ (self): + """minimum(a, b) or minimum(a) +In one argument case, returns the scalar minimum. + """ + self.ufunc = umath.minimum + self.afunc = amin + self.compare = less + self.fill_value_func = minimum_fill_value + +#............................ +class _maximum_operation(_extrema_operation): + "Object to calculate maxima" + def __init__ (self): + """maximum(a, b) or maximum(a) + In one argument case returns the scalar maximum. + """ + self.ufunc = umath.maximum + self.afunc = amax + self.compare = greater + self.fill_value_func = maximum_fill_value + +#.......................................................... +def min(array, axis=None, out=None): + """Returns the minima along the given axis. +If `axis` is None, applies to the flattened array.""" + if out is not None: + raise TypeError("Output arrays Unsupported for masked arrays") + if axis is None: + return minimum(array) + else: + return minimum.reduce(array, axis) +min.__doc__ = MaskedArray.min.__doc__ +#............................ +def max(obj, axis=None, out=None): + if out is not None: + raise TypeError("Output arrays Unsupported for masked arrays") + if axis is None: + return maximum(obj) + else: + return maximum.reduce(obj, axis) +max.__doc__ = MaskedArray.max.__doc__ +#............................. +def ptp(obj, axis=None): + """a.ptp(axis=None) = a.max(axis)-a.min(axis)""" + try: + return obj.max(axis)-obj.min(axis) + except AttributeError: + return max(obj, axis=axis) - min(obj, axis=axis) +ptp.__doc__ = MaskedArray.ptp.__doc__ + + +#####--------------------------------------------------------------------------- +#---- --- Definition of functions from the corresponding methods --- +#####--------------------------------------------------------------------------- +class _frommethod: + """Defines functions from existing MaskedArray methods. +:ivar _methodname (String): Name of the method to transform. + """ + def __init__(self, methodname): + self._methodname = methodname + self.__doc__ = self.getdoc() + def getdoc(self): + "Returns the doc of the function (from the doc of the method)." + try: + return getattr(MaskedArray, self._methodname).__doc__ + except: + return getattr(numpy, self._methodname).__doc__ + def __call__(self, a, *args, **params): + if isinstance(a, MaskedArray): + return getattr(a, self._methodname).__call__(*args, **params) + #FIXME ---- + #As x is not a MaskedArray, we transform it to a ndarray with asarray + #... and call the corresponding method. + #Except that sometimes it doesn't work (try reshape([1,2,3,4],(2,2))) + #we end up with a "SystemError: NULL result without error in PyObject_Call" + #A dirty trick is then to call the initial numpy function... + method = getattr(narray(a, copy=False), self._methodname) + try: + return method(*args, **params) + except SystemError: + return getattr(numpy,self._methodname).__call__(a, *args, **params) + +all = _frommethod('all') +anomalies = anom = _frommethod('anom') +any = _frommethod('any') +conjugate = _frommethod('conjugate') +ids = _frommethod('ids') +nonzero = _frommethod('nonzero') +diagonal = _frommethod('diagonal') +maximum = _maximum_operation() +mean = _frommethod('mean') +minimum = _minimum_operation () +product = _frommethod('prod') +ptp = _frommethod('ptp') +ravel = _frommethod('ravel') +repeat = _frommethod('repeat') +std = _frommethod('std') +sum = _frommethod('sum') +swapaxes = _frommethod('swapaxes') +take = _frommethod('take') +var = _frommethod('var') + +#.............................................................................. +def power(a, b, third=None): + """Computes a**b elementwise. + Masked values are set to 1.""" + if third is not None: + raise MAError, "3-argument power not supported." + ma = getmask(a) + mb = getmask(b) + m = mask_or(ma, mb) + fa = getdata(a) + fb = getdata(b) + if fb.dtype.char in typecodes["Integer"]: + return masked_array(umath.power(fa, fb), m) + md = make_mask((fa < 0), shrink=True) + m = mask_or(m, md) + if m is nomask: + return masked_array(umath.power(fa, fb)) + else: + fa = fa.copy() + fa[(fa < 0)] = 1 + return masked_array(umath.power(fa, fb), m) + +#.............................................................................. +def argsort(a, axis=None, kind='quicksort', order=None, fill_value=None): + "Function version of the eponymous method." + if fill_value is None: + fill_value = default_fill_value(a) + d = filled(a, fill_value) + if axis is None: + return d.argsort(kind=kind, order=order) + return d.argsort(axis, kind=kind, order=order) +argsort.__doc__ = MaskedArray.argsort.__doc__ + +def argmin(a, axis=None, fill_value=None): + "Function version of the eponymous method." + if fill_value is None: + fill_value = default_fill_value(a) + d = filled(a, fill_value) + return d.argmin(axis=axis) +argmin.__doc__ = MaskedArray.argmin.__doc__ + +def argmax(a, axis=None, fill_value=None): + "Function version of the eponymous method." + if fill_value is None: + fill_value = default_fill_value(a) + try: + fill_value = - fill_value + except: + pass + d = filled(a, fill_value) + return d.argmax(axis=axis) +argmin.__doc__ = MaskedArray.argmax.__doc__ + +def sort(a, axis=-1, kind='quicksort', order=None, endwith=True, fill_value=None): + "Function version of the eponymous method." + a = narray(a, copy=False, subok=True) + if fill_value is None: + if endwith: + filler = minimum_fill_value(a) + else: + filler = maximum_fill_value(a) + else: + filler = fill_value +# return + indx = numpy.indices(a.shape).tolist() + indx[axis] = filled(a,filler).argsort(axis=axis,kind=kind,order=order) + return a[indx] +sort.__doc__ = MaskedArray.sort.__doc__ + + +def compressed(x): + """Returns a 1-D array of all the non-masked data.""" + if getmask(x) is nomask: + return x + else: + return x.compressed() + +def concatenate(arrays, axis=0): + "Concatenates the arrays along the given axis." + d = numpy.concatenate([getdata(a) for a in arrays], axis) + rcls = get_masked_subclass(*arrays) + data = d.view(rcls) + # Check whether one of the arrays has a non-empty mask... + for x in arrays: + if getmask(x) is not nomask: + break + return data + # OK, so we have to concatenate the masks + dm = numpy.concatenate([getmaskarray(a) for a in arrays], axis) + shrink = numpy.logical_or.reduce([getattr(a,'_shrinkmask',True) for a in arrays]) + if shrink and not dm.any(): + data._mask = nomask + else: + data._mask = dm.reshape(d.shape) + return data + +def count(a, axis = None): + return masked_array(a, copy=False).count(axis) +count.__doc__ = MaskedArray.count.__doc__ + + +def expand_dims(x,axis): + "Expands the shape of the array by including a new axis before the given one." + result = n_expand_dims(x,axis) + if isinstance(x, MaskedArray): + new_shape = result.shape + result = x.view() + result.shape = new_shape + if result._mask is not nomask: + result._mask.shape = new_shape + return result + +#...................................... +def left_shift (a, n): + "Left shift n bits." + m = getmask(a) + if m is nomask: + d = umath.left_shift(filled(a), n) + return masked_array(d) + else: + d = umath.left_shift(filled(a, 0), n) + return masked_array(d, mask=m) + +def right_shift (a, n): + "Right shift n bits." + m = getmask(a) + if m is nomask: + d = umath.right_shift(filled(a), n) + return masked_array(d) + else: + d = umath.right_shift(filled(a, 0), n) + return masked_array(d, mask=m) + +#...................................... +def put(a, indices, values, mode='raise'): + """Sets storage-indexed locations to corresponding values. +Values and indices are filled if necessary.""" + # We can't use 'frommethod', the order of arguments is different + try: + return a.put(indices, values, mode=mode) + except AttributeError: + return narray(a, copy=False).put(indices, values, mode=mode) + +def putmask(a, mask, values): #, mode='raise'): + """Sets a.flat[n] = values[n] for each n where mask.flat[n] is true. + +If values is not the same size of a and mask then it will repeat as necessary. +This gives different behavior than a[mask] = values.""" + # We can't use 'frommethod', the order of arguments is different + try: + return a.putmask(values, mask) + except AttributeError: + return numpy.putmask(narray(a, copy=False), mask, values) + +def transpose(a,axes=None): + """Returns a view of the array with dimensions permuted according to axes, +as a masked array. + +If ``axes`` is None (default), the output view has reversed dimensions compared +to the original. + """ + #We can't use 'frommethod', as 'transpose' doesn't take keywords + try: + return a.transpose(axes) + except AttributeError: + return narray(a, copy=False).transpose(axes).view(MaskedArray) + +def reshape(a, new_shape): + """Changes the shape of the array a to new_shape.""" + #We can't use 'frommethod', it whine about some parameters. Dmmit. + try: + return a.reshape(new_shape) + except AttributeError: + return narray(a, copy=False).reshape(new_shape).view(MaskedArray) + +def resize(x, new_shape): + """Returns a new array with the specified shape. + +The total size of the original array can be any size. +The new array is filled with repeated copies of a. If a was masked, the new array +will be masked, and the new mask will be a repetition of the old one. + """ + # We can't use _frommethods here, as N.resize is notoriously whiny. + m = getmask(x) + if m is not nomask: + m = numpy.resize(m, new_shape) + result = numpy.resize(x, new_shape).view(get_masked_subclass(x)) + if result.ndim: + result._mask = m + return result + + +#................................................ +def rank(obj): + "maskedarray version of the numpy function." + return fromnumeric.rank(getdata(obj)) +rank.__doc__ = numpy.rank.__doc__ +# +def shape(obj): + "maskedarray version of the numpy function." + return fromnumeric.shape(getdata(obj)) +shape.__doc__ = numpy.shape.__doc__ +# +def size(obj, axis=None): + "maskedarray version of the numpy function." + return fromnumeric.size(getdata(obj), axis) +size.__doc__ = numpy.size.__doc__ +#................................................ + +#####-------------------------------------------------------------------------- +#---- --- Extra functions --- +#####-------------------------------------------------------------------------- +def where (condition, x=None, y=None): + """where(condition | x, y) + +Returns a (subclass of) masked array, shaped like condition, where the elements +are x when condition is True, and y otherwise. If neither x nor y are given, +returns a tuple of indices where condition is True (a la condition.nonzero()). + +*Parameters*: + condition : {var} + The condition to meet. Must be convertible to an integer array. + x : {var}, optional + Values of the output when the condition is met + y : {var}, optional + Values of the output when the condition is not met. + """ + if x is None and y is None: + return filled(condition, 0).nonzero() + elif x is None or y is None: + raise ValueError, "Either both or neither x and y should be given." + # Get the condition ............... + fc = filled(condition, 0).astype(bool_) + notfc = numpy.logical_not(fc) + # Get the data ...................................... + xv = getdata(x) + yv = getdata(y) + if x is masked: + ndtype = yv.dtype + xm = numpy.ones(fc.shape, dtype=MaskType) + elif y is masked: + ndtype = xv.dtype + ym = numpy.ones(fc.shape, dtype=MaskType) + else: + ndtype = numpy.max([xv.dtype, yv.dtype]) + xm = getmask(x) + d = numpy.empty(fc.shape, dtype=ndtype).view(MaskedArray) + numpy.putmask(d._data, fc, xv.astype(ndtype)) + numpy.putmask(d._data, notfc, yv.astype(ndtype)) + d._mask = numpy.zeros(fc.shape, dtype=MaskType) + numpy.putmask(d._mask, fc, getmask(x)) + numpy.putmask(d._mask, notfc, getmask(y)) + d._mask |= getmaskarray(condition) + if not d._mask.any(): + d._mask = nomask + return d +# # Get the data as a (subclass of) MaskedArray +# xv = getdata(x) +# yv = getdata(y) +# d = numpy.choose(fc, (yv, xv)).view(MaskedArray) +# # Get the mask .................... +# xm = getmask(x) +# ym = getmask(y) +# d.mask = numpy.choose(fc, (ym, xm)) | getmask(condition) +# # Fix the dtype if one of the values was masked, to prevent an upload to float +# if y is masked: +# ndtype = xv.dtype +# elif x is masked: +# ndtype = yv.dtype +# else: +# ndtype = d.dtype +# return d.astype(ndtype) + +def choose (indices, t, out=None, mode='raise'): + "Returns array shaped like indices with elements chosen from t" + #TODO: implement options `out` and `mode`, if possible. + def fmask (x): + "Returns the filled array, or True if masked." + if x is masked: + return 1 + return filled(x) + def nmask (x): + "Returns the mask, True if ``masked``, False if ``nomask``." + if x is masked: + return 1 + m = getmask(x) + if m is nomask: + return 0 + return m + c = filled(indices, 0) + masks = [nmask(x) for x in t] + a = [fmask(x) for x in t] + d = numpy.choose(c, a) + m = numpy.choose(c, masks) + m = make_mask(mask_or(m, getmask(indices)), copy=0, shrink=True) + return masked_array(d, mask=m) + +def round_(a, decimals=0, out=None): + """Returns a copy of a, rounded to 'decimals' places. + +When 'decimals' is negative, it specifies the number of positions to the left of +the decimal point. The real and imaginary parts of complex numbers are rounded +separately. Nothing is done if the array is not of float type and 'decimals' is +greater than or equal to 0. + +*Parameters*: + decimals : {integer} + Number of decimals to round to. May be negative. + out : {ndarray} + Existing array to use for output. + If not given, returns a default copy of a. + +*Notes*: + If out is given and does not have a mask attribute, the mask of a is lost! + """ + if out is None: + result = fromnumeric.round_(getdata(a), decimals, out) + if isinstance(a,MaskedArray): + result = result.view(type(a)) + result._mask = a._mask + else: + result = result.view(MaskedArray) + return result + else: + fromnumeric.round_(getdata(a), decimals, out) + if hasattr(out, '_mask'): + out._mask = getmask(a) + return out + +def arange(stop, start=None, step=1, dtype=None): + "maskedarray version of the numpy function." + return numpy.arange(stop, start, step, dtype).view(MaskedArray) +arange.__doc__ = numpy.arange.__doc__ + +def inner(a, b): + "maskedarray version of the numpy function." + fa = filled(a, 0) + fb = filled(b, 0) + if len(fa.shape) == 0: + fa.shape = (1,) + if len(fb.shape) == 0: + fb.shape = (1,) + return numpy.inner(fa, fb).view(MaskedArray) +inner.__doc__ = numpy.inner.__doc__ +inner.__doc__ += "\n*Notes*:\n Masked values are replaced by 0." +innerproduct = inner + +def outer(a, b): + "maskedarray version of the numpy function." + fa = filled(a, 0).ravel() + fb = filled(b, 0).ravel() + d = numeric.outer(fa, fb) + ma = getmask(a) + mb = getmask(b) + if ma is nomask and mb is nomask: + return masked_array(d) + ma = getmaskarray(a) + mb = getmaskarray(b) + m = make_mask(1-numeric.outer(1-ma, 1-mb), copy=0) + return masked_array(d, mask=m) +outer.__doc__ = numpy.outer.__doc__ +outer.__doc__ += "\n*Notes*:\n Masked values are replaced by 0." +outerproduct = outer + +def allequal (a, b, fill_value=True): + """Returns True if all entries of a and b are equal, using fill_value +as a truth value where either or both are masked. + """ + m = mask_or(getmask(a), getmask(b)) + if m is nomask: + x = getdata(a) + y = getdata(b) + d = umath.equal(x, y) + return d.all() + elif fill_value: + x = getdata(a) + y = getdata(b) + d = umath.equal(x, y) + dm = array(d, mask=m, copy=False) + return dm.filled(True).all(None) + else: + return False + +def allclose (a, b, fill_value=True, rtol=1.e-5, atol=1.e-8): + """ Returns True if all elements of a and b are equal subject to given tolerances. +If fill_value is True, masked values are considered equal. +If fill_value is False, masked values considered unequal. +The relative error rtol should be positive and << 1.0 +The absolute error atol comes into play for those elements of b that are very small +or zero; it says how small `a` must be also. + """ + m = mask_or(getmask(a), getmask(b)) + d1 = getdata(a) + d2 = getdata(b) + x = filled(array(d1, copy=0, mask=m), fill_value).astype(float) + y = filled(array(d2, copy=0, mask=m), 1).astype(float) + d = umath.less_equal(umath.absolute(x-y), atol + rtol * umath.absolute(y)) + return fromnumeric.alltrue(fromnumeric.ravel(d)) + +#.............................................................................. +def asarray(a, dtype=None): + """asarray(data, dtype) = array(data, dtype, copy=0, subok=0) +Returns a as a MaskedArray object of the given dtype. +If dtype is not given or None, is is set to the dtype of a. +No copy is performed if a is already an array. +Subclasses are converted to the base class MaskedArray. + """ + return masked_array(a, dtype=dtype, copy=False, keep_mask=True, subok=False) + +def asanyarray(a, dtype=None): + """asanyarray(data, dtype) = array(data, dtype, copy=0, subok=1) +Returns a as an masked array. +If dtype is not given or None, is is set to the dtype of a. +No copy is performed if a is already an array. +Subclasses are conserved. + """ + return masked_array(a, dtype=dtype, copy=False, keep_mask=True, subok=True) + + +def empty(new_shape, dtype=float): + "maskedarray version of the numpy function." + return numpy.empty(new_shape, dtype).view(MaskedArray) +empty.__doc__ = numpy.empty.__doc__ + +def empty_like(a): + "maskedarray version of the numpy function." + return numpy.empty_like(a).view(MaskedArray) +empty_like.__doc__ = numpy.empty_like.__doc__ + +def ones(new_shape, dtype=float): + "maskedarray version of the numpy function." + return numpy.ones(new_shape, dtype).view(MaskedArray) +ones.__doc__ = numpy.ones.__doc__ + +def zeros(new_shape, dtype=float): + "maskedarray version of the numpy function." + return numpy.zeros(new_shape, dtype).view(MaskedArray) +zeros.__doc__ = numpy.zeros.__doc__ + +#####-------------------------------------------------------------------------- +#---- --- Pickling --- +#####-------------------------------------------------------------------------- +def dump(a,F): + """Pickles the MaskedArray `a` to the file `F`. +`F` can either be the handle of an exiting file, or a string representing a file name. + """ + if not hasattr(F,'readline'): + F = open(F,'w') + return cPickle.dump(a,F) + +def dumps(a): + """Returns a string corresponding to the pickling of the MaskedArray.""" + return cPickle.dumps(a) + +def load(F): + """Wrapper around ``cPickle.load`` which accepts either a file-like object or + a filename.""" + if not hasattr(F, 'readline'): + F = open(F,'r') + return cPickle.load(F) + +def loads(strg): + "Loads a pickle from the current string.""" + return cPickle.loads(strg) + + +############################################################################### + +if __name__ == '__main__': + from maskedarray.testutils import assert_equal, assert_almost_equal + + # Small arrays .................................. + xs = numpy.random.uniform(-1,1,6).reshape(2,3) + ys = numpy.random.uniform(-1,1,6).reshape(2,3) + zs = xs + 1j * ys + m1 = [[True, False, False], [False, False, True]] + m2 = [[True, False, True], [False, False, True]] + nmxs = numpy.ma.array(xs, mask=m1) + nmys = numpy.ma.array(ys, mask=m2) + nmzs = numpy.ma.array(zs, mask=m1) + mmxs = array(xs, mask=m1) + mmys = array(ys, mask=m2) + mmzs = array(zs, mask=m1) + # Big arrays .................................... + xl = numpy.random.uniform(-1,1,100*100).reshape(100,100) + yl = numpy.random.uniform(-1,1,100*100).reshape(100,100) + zl = xl + 1j * yl + maskx = xl > 0.8 + masky = yl < -0.8 + nmxl = numpy.ma.array(xl, mask=maskx) + nmyl = numpy.ma.array(yl, mask=masky) + nmzl = numpy.ma.array(zl, mask=maskx) + mmxl = array(xl, mask=maskx, shrink=True) + mmyl = array(yl, mask=masky, shrink=True) + mmzl = array(zl, mask=maskx, shrink=True) + # + z = empty(3,) + mmys.all(0, out=z) + + if 1: + x = numpy.array([[ 0.13, 0.26, 0.90], + [ 0.28, 0.33, 0.63], + [ 0.31, 0.87, 0.70]]) + m = numpy.array([[ True, False, False], + [False, False, False], + [True, True, False]], dtype=numpy.bool_) + mx = masked_array(x, mask=m) + xbig = numpy.array([[False, False, True], + [False, False, True], + [False, True, True]], dtype=numpy.bool_) + mxbig = (mx > 0.5) + mxsmall = (mx < 0.5) + # + assert (mxbig.all()==False) + assert (mxbig.any()==True) + assert_equal(mxbig.all(0),[False, False, True]) + assert_equal(mxbig.all(1), [False, False, True]) + assert_equal(mxbig.any(0),[False, False, True]) + assert_equal(mxbig.any(1), [True, True, True]) + + if 1: + xx = array([1+10j,20+2j], mask=[1,0]) + assert_equal(xx.imag,[10,2]) + assert_equal(xx.imag.filled(), [1e+20,2]) + assert_equal(xx.real,[1,20]) + assert_equal(xx.real.filled(), [1e+20,20]) diff --git a/numpy/ma/extras.py b/numpy/ma/extras.py new file mode 100644 index 000000000..f122d6d47 --- /dev/null +++ b/numpy/ma/extras.py @@ -0,0 +1,722 @@ +"""Masked arrays add-ons. + +A collection of utilities for maskedarray + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: extras.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +__all__ = [ +'apply_along_axis', 'atleast_1d', 'atleast_2d', 'atleast_3d', 'average', +'vstack', 'hstack', 'dstack', 'row_stack', 'column_stack', +'compress_rowcols', 'compress_rows', 'compress_cols', 'count_masked', +'dot', 'hsplit', +'mask_rowcols','mask_rows','mask_cols','masked_all','masked_all_like', +'mediff1d', 'mr_', +'notmasked_edges','notmasked_contiguous', +'stdu', 'varu', + ] + +from itertools import groupby + +import core +from core import * + +import numpy +from numpy import float_ +import numpy.core.umath as umath +import numpy.core.numeric as numeric +from numpy.core.numeric import ndarray +from numpy.core.numeric import array as nxarray +from numpy.core.fromnumeric import asarray as nxasarray + +from numpy.lib.index_tricks import concatenator +import numpy.lib.function_base as function_base + +#............................................................................... +def issequence(seq): + """Returns True if the argument is a sequence (ndarray, list or tuple).""" + if isinstance(seq, ndarray): + return True + elif isinstance(seq, tuple): + return True + elif isinstance(seq, list): + return True + return False + +def count_masked(arr, axis=None): + """Counts the number of masked elements along the given axis. + +*Parameters*: + axis : {integer}, optional + Axis along which to count. + If None (default), a flattened version of the array is used. + """ + m = getmaskarray(arr) + return m.sum(axis) + +def masked_all(shape, dtype=float_): + """Returns an empty masked array of the given shape and dtype, + where all the data are masked. + +*Parameters*: + dtype : {dtype}, optional + Data type of the output. + """ + a = masked_array(numeric.empty(shape, dtype), + mask=numeric.ones(shape, bool_)) + return a + +def masked_all_like(arr): + """Returns an empty masked array of the same shape and dtype as the array `a`, + where all the data are masked.""" + a = masked_array(numeric.empty_like(arr), + mask=numeric.ones(arr.shape, bool_)) + return a + +#####-------------------------------------------------------------------------- +#---- --- New methods --- +#####-------------------------------------------------------------------------- +def varu(a, axis=None, dtype=None): + """Returns an unbiased estimate of the variance. + i.e. var = sum((x - x.mean())**2)/(size(x,axis)-1) + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. If not given, the current dtype + is used instead. + +*Notes*: + The value returned is an unbiased estimate of the true variance. + For the (less standard) biased estimate, use var. + """ + a = asarray(a) + cnt = a.count(axis=axis) + anom = a.anom(axis=axis, dtype=dtype) + anom *= anom + dvar = anom.sum(axis) / (cnt-1) + if axis is None: + return dvar + dvar.__setmask__(mask_or(a._mask.all(axis), (cnt==1))) + return dvar +# return a.__class__(dvar, +# mask=mask_or(a._mask.all(axis), (cnt==1)), +# fill_value=a._fill_value) + +def stdu(a, axis=None, dtype=None): + """Returns an unbiased estimate of the standard deviation. + The standard deviation is the square root of the average of the squared + deviations from the mean, i.e. stdu = sqrt(varu(x)). + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + dtype : {dtype}, optional + Datatype for the intermediary computation. + If not given, the current dtype is used instead. + +*Notes*: + The value returned is an unbiased estimate of the true standard deviation. + For the (less standard) biased estimate, use std. + """ + a = asarray(a) + dvar = a.varu(axis,dtype) + if axis is None: + if dvar is masked: + return masked + else: + # Should we use umath.sqrt instead ? + return sqrt(dvar) + return sqrt(dvar) +# return a.__class__(sqrt(dvar._data), mask=dvar._mask, +# fill_value=a._fill_value) + +MaskedArray.stdu = stdu +MaskedArray.varu = varu + +#####-------------------------------------------------------------------------- +#---- --- Standard functions --- +#####-------------------------------------------------------------------------- +class _fromnxfunction: + """Defines a wrapper to adapt numpy functions to masked arrays.""" + def __init__(self, funcname): + self._function = funcname + self.__doc__ = self.getdoc() + def getdoc(self): + "Retrieves the __doc__ string from the function." + return getattr(numpy, self._function).__doc__ +\ + "*Notes*:\n (The function is applied to both the _data and the _mask, if any.)" + def __call__(self, *args, **params): + func = getattr(numpy, self._function) + if len(args)==1: + x = args[0] + if isinstance(x,ndarray): + _d = func(nxasarray(x), **params) + _m = func(getmaskarray(x), **params) + return masked_array(_d, mask=_m) + elif isinstance(x, tuple) or isinstance(x, list): + _d = func(tuple([nxasarray(a) for a in x]), **params) + _m = func(tuple([getmaskarray(a) for a in x]), **params) + return masked_array(_d, mask=_m) + else: + arrays = [] + args = list(args) + while len(args)>0 and issequence(args[0]): + arrays.append(args.pop(0)) + res = [] + for x in arrays: + _d = func(nxasarray(x), *args, **params) + _m = func(getmaskarray(x), *args, **params) + res.append(masked_array(_d, mask=_m)) + return res + +atleast_1d = _fromnxfunction('atleast_1d') +atleast_2d = _fromnxfunction('atleast_2d') +atleast_3d = _fromnxfunction('atleast_3d') + +vstack = row_stack = _fromnxfunction('vstack') +hstack = _fromnxfunction('hstack') +column_stack = _fromnxfunction('column_stack') +dstack = _fromnxfunction('dstack') + +hsplit = _fromnxfunction('hsplit') + +#####-------------------------------------------------------------------------- +#---- +#####-------------------------------------------------------------------------- +def flatten_inplace(seq): + """Flattens a sequence in place.""" + k = 0 + while (k != len(seq)): + while hasattr(seq[k],'__iter__'): + seq[k:(k+1)] = seq[k] + k += 1 + return seq + + +def apply_along_axis(func1d,axis,arr,*args,**kwargs): + """ Execute func1d(arr[i],*args) where func1d takes 1-D arrays + and arr is an N-d array. i varies so as to apply the function + along the given axis for each 1-d subarray in arr. + """ + arr = core.array(arr, copy=False, subok=True) + nd = arr.ndim + if axis < 0: + axis += nd + if (axis >= nd): + raise ValueError("axis must be less than arr.ndim; axis=%d, rank=%d." + % (axis,nd)) + ind = [0]*(nd-1) + i = numeric.zeros(nd,'O') + indlist = range(nd) + indlist.remove(axis) + i[axis] = slice(None,None) + outshape = numeric.asarray(arr.shape).take(indlist) + i.put(indlist, ind) + j = i.copy() + res = func1d(arr[tuple(i.tolist())],*args,**kwargs) + # if res is a number, then we have a smaller output array + asscalar = numeric.isscalar(res) + if not asscalar: + try: + len(res) + except TypeError: + asscalar = True + # Note: we shouldn't set the dtype of the output from the first result... + #...so we force the type to object, and build a list of dtypes + #...we'll just take the largest, to avoid some downcasting + dtypes = [] + if asscalar: + dtypes.append(numeric.asarray(res).dtype) + outarr = zeros(outshape, object_) + outarr[tuple(ind)] = res + Ntot = numeric.product(outshape) + k = 1 + while k < Ntot: + # increment the index + ind[-1] += 1 + n = -1 + while (ind[n] >= outshape[n]) and (n > (1-nd)): + ind[n-1] += 1 + ind[n] = 0 + n -= 1 + i.put(indlist,ind) + res = func1d(arr[tuple(i.tolist())],*args,**kwargs) + outarr[tuple(ind)] = res + dtypes.append(asarray(res).dtype) + k += 1 + else: + res = core.array(res, copy=False, subok=True) + j = i.copy() + j[axis] = ([slice(None,None)] * res.ndim) + j.put(indlist, ind) + Ntot = numeric.product(outshape) + holdshape = outshape + outshape = list(arr.shape) + outshape[axis] = res.shape + dtypes.append(asarray(res).dtype) + outshape = flatten_inplace(outshape) + outarr = zeros(outshape, object_) + outarr[tuple(flatten_inplace(j.tolist()))] = res + k = 1 + while k < Ntot: + # increment the index + ind[-1] += 1 + n = -1 + while (ind[n] >= holdshape[n]) and (n > (1-nd)): + ind[n-1] += 1 + ind[n] = 0 + n -= 1 + i.put(indlist, ind) + j.put(indlist, ind) + res = func1d(arr[tuple(i.tolist())],*args,**kwargs) + outarr[tuple(flatten_inplace(j.tolist()))] = res + dtypes.append(asarray(res).dtype) + k += 1 + max_dtypes = numeric.dtype(numeric.asarray(dtypes).max()) + if not hasattr(arr, '_mask'): + result = numeric.asarray(outarr, dtype=max_dtypes) + else: + result = core.asarray(outarr, dtype=max_dtypes) + result.fill_value = core.default_fill_value(result) + return result + +def average (a, axis=None, weights=None, returned=False): + """Averages the array over the given axis. + +*Parameters*: + axis : {integer}, optional + Axis along which to perform the operation. + If None, applies to a flattened version of the array. + weights : {sequence}, optional + Sequence of weights. + The weights must have the shape of a, or be 1D with length the size of a + along the given axis. + If no weights are given, weights are assumed to be 1. + returned : {boolean} + Flag indicating whether a tuple (result, sum of weights/counts) should be + returned as output (True), or just the result (False). + """ + a = asarray(a) + mask = a.mask + ash = a.shape + if ash == (): + ash = (1,) + if axis is None: + if mask is nomask: + if weights is None: + n = a.sum(axis=None) + d = float(a.size) + else: + w = filled(weights, 0.0).ravel() + n = umath.add.reduce(a._data.ravel() * w) + d = umath.add.reduce(w) + del w + else: + if weights is None: + n = a.filled(0).sum(axis=None) + d = umath.add.reduce((-mask).ravel().astype(int_)) + else: + w = array(filled(weights, 0.0), float, mask=mask).ravel() + n = add.reduce(a.ravel() * w) + d = add.reduce(w) + del w + else: + if mask is nomask: + if weights is None: + d = ash[axis] * 1.0 + n = add.reduce(a._data, axis, dtype=float_) + else: + w = filled(weights, 0.0) + wsh = w.shape + if wsh == (): + wsh = (1,) + if wsh == ash: + w = numeric.array(w, float_, copy=0) + n = add.reduce(a*w, axis) + d = add.reduce(w, axis) + del w + elif wsh == (ash[axis],): + ni = ash[axis] + r = [None]*len(ash) + r[axis] = slice(None, None, 1) + w = eval ("w["+ repr(tuple(r)) + "] * ones(ash, float)") + n = add.reduce(a*w, axis, dtype=float_) + d = add.reduce(w, axis, dtype=float_) + del w, r + else: + raise ValueError, 'average: weights wrong shape.' + else: + if weights is None: + n = add.reduce(a, axis, dtype=float_) + d = umath.add.reduce((-mask), axis=axis, dtype=float_) + else: + w = filled(weights, 0.0) + wsh = w.shape + if wsh == (): + wsh = (1,) + if wsh == ash: + w = array(w, dtype=float_, mask=mask, copy=0) + n = add.reduce(a*w, axis, dtype=float_) + d = add.reduce(w, axis, dtype=float_) + elif wsh == (ash[axis],): + ni = ash[axis] + r = [None]*len(ash) + r[axis] = slice(None, None, 1) + w = eval ("w["+ repr(tuple(r)) + "] * masked_array(ones(ash, float), mask)") + n = add.reduce(a*w, axis, dtype=float_) + d = add.reduce(w, axis, dtype=float_) + else: + raise ValueError, 'average: weights wrong shape.' + del w + if n is masked or d is masked: + return masked + result = n/d + del n + + if isMaskedArray(result): + if ((axis is None) or (axis==0 and a.ndim == 1)) and \ + (result.mask is nomask): + result = result._data + if returned: + if not isMaskedArray(d): + d = masked_array(d) + if isinstance(d, ndarray) and (not d.shape == result.shape): + d = ones(result.shape, dtype=float_) * d + if returned: + return result, d + else: + return result + +#.............................................................................. +def compress_rowcols(x, axis=None): + """Suppresses the rows and/or columns of a 2D array that contains masked values. + + The suppression behavior is selected with the `axis`parameter. + - If axis is None, rows and columns are suppressed. + - If axis is 0, only rows are suppressed. + - If axis is 1 or -1, only columns are suppressed. + +*Returns*: + compressed_array : a ndarray. + """ + x = asarray(x) + if x.ndim != 2: + raise NotImplementedError, "compress2d works for 2D arrays only." + m = getmask(x) + # Nothing is masked: return x + if m is nomask or not m.any(): + return x._data + # All is masked: return empty + if m.all(): + return nxarray([]) + # Builds a list of rows/columns indices + (idxr, idxc) = (range(len(x)), range(x.shape[1])) + masked = m.nonzero() + if not axis: + for i in function_base.unique(masked[0]): + idxr.remove(i) + if axis in [None, 1, -1]: + for j in function_base.unique(masked[1]): + idxc.remove(j) + return x._data[idxr][:,idxc] + +def compress_rows(a): + """Suppresses whole rows of a 2D array that contain masked values.""" + return compress_rowcols(a,0) + +def compress_cols(a): + """Suppresses whole columnss of a 2D array that contain masked values.""" + return compress_rowcols(a,1) + +def mask_rowcols(a, axis=None): + """Masks whole rows and/or columns of a 2D array that contain masked values. + The masking behavior is selected with the `axis`parameter. + - If axis is None, rows and columns are masked. + - If axis is 0, only rows are masked. + - If axis is 1 or -1, only columns are masked. + Returns a *pure* ndarray. + """ + a = asarray(a) + if a.ndim != 2: + raise NotImplementedError, "compress2d works for 2D arrays only." + m = getmask(a) + # Nothing is masked: return a + if m is nomask or not m.any(): + return a + maskedval = m.nonzero() + a._mask = a._mask.copy() + if not axis: + a[function_base.unique(maskedval[0])] = masked + if axis in [None, 1, -1]: + a[:,function_base.unique(maskedval[1])] = masked + return a + +def mask_rows(a, axis=None): + """Masks whole rows of a 2D array that contain masked values.""" + return mask_rowcols(a, 0) + +def mask_cols(a, axis=None): + """Masks whole columns of a 2D array that contain masked values.""" + return mask_rowcols(a, 1) + + +def dot(a,b, strict=False): + """Returns the dot product of two 2D masked arrays a and b. + + Like the generic numpy equivalent, the product sum is over the last dimension + of a and the second-to-last dimension of b. + If strict is True, masked values are propagated: if a masked value appears + in a row or column, the whole row or column is considered masked. + +*Parameters*: + strict : {boolean} + Whether masked data are propagated (True) or set to 0 for the computation. + +*Note*: + The first argument is not conjugated. + """ + #TODO: Works only with 2D arrays. There should be a way to get it to run with higher dimension + if strict and (a.ndim == 2) and (b.ndim == 2): + a = mask_rows(a) + b = mask_cols(b) + # + d = numpy.dot(filled(a, 0), filled(b, 0)) + # + am = (~getmaskarray(a)) + bm = (~getmaskarray(b)) + m = ~numpy.dot(am,bm) + return masked_array(d, mask=m) + +#............................................................................... +def mediff1d(array, to_end=None, to_begin=None): + """Returns the differences between consecutive elements of an array, possibly with + prefixed and/or appended values. + +*Parameters*: + array : {array} + Input array, will be flattened before the difference is taken. + to_end : {number}, optional + If provided, this number will be tacked onto the end of the returned + differences. + to_begin : {number}, optional + If provided, this number will be taked onto the beginning of the + returned differences. + +*Returns*: + ed : {array} + The differences. Loosely, this will be (ary[1:] - ary[:-1]). + """ + a = masked_array(array, copy=True) + if a.ndim > 1: + a.reshape((a.size,)) + (d, m, n) = (a._data, a._mask, a.size-1) + dd = d[1:]-d[:-1] + if m is nomask: + dm = nomask + else: + dm = m[1:]-m[:-1] + # + if to_end is not None: + to_end = asarray(to_end) + nend = to_end.size + if to_begin is not None: + to_begin = asarray(to_begin) + nbegin = to_begin.size + r_data = numeric.empty((n+nend+nbegin,), dtype=a.dtype) + r_mask = numeric.zeros((n+nend+nbegin,), dtype=bool_) + r_data[:nbegin] = to_begin._data + r_mask[:nbegin] = to_begin._mask + r_data[nbegin:-nend] = dd + r_mask[nbegin:-nend] = dm + else: + r_data = numeric.empty((n+nend,), dtype=a.dtype) + r_mask = numeric.zeros((n+nend,), dtype=bool_) + r_data[:-nend] = dd + r_mask[:-nend] = dm + r_data[-nend:] = to_end._data + r_mask[-nend:] = to_end._mask + # + elif to_begin is not None: + to_begin = asarray(to_begin) + nbegin = to_begin.size + r_data = numeric.empty((n+nbegin,), dtype=a.dtype) + r_mask = numeric.zeros((n+nbegin,), dtype=bool_) + r_data[:nbegin] = to_begin._data + r_mask[:nbegin] = to_begin._mask + r_data[nbegin:] = dd + r_mask[nbegin:] = dm + # + else: + r_data = dd + r_mask = dm + return masked_array(r_data, mask=r_mask) + + + + +#####-------------------------------------------------------------------------- +#---- --- Concatenation helpers --- +#####-------------------------------------------------------------------------- + +class mconcatenator(concatenator): + """Translates slice objects to concatenation along an axis.""" + + def __init__(self, axis=0): + concatenator.__init__(self, axis, matrix=False) + + def __getitem__(self,key): + if isinstance(key, str): + raise MAError, "Unavailable for masked array." + if type(key) is not tuple: + key = (key,) + objs = [] + scalars = [] + final_dtypedescr = None + for k in range(len(key)): + scalar = False + if type(key[k]) is slice: + step = key[k].step + start = key[k].start + stop = key[k].stop + if start is None: + start = 0 + if step is None: + step = 1 + if type(step) is type(1j): + size = int(abs(step)) + newobj = function_base.linspace(start, stop, num=size) + else: + newobj = numeric.arange(start, stop, step) + elif type(key[k]) is str: + if (key[k] in 'rc'): + self.matrix = True + self.col = (key[k] == 'c') + continue + try: + self.axis = int(key[k]) + continue + except (ValueError, TypeError): + raise ValueError, "Unknown special directive" + elif type(key[k]) in numeric.ScalarType: + newobj = asarray([key[k]]) + scalars.append(k) + scalar = True + else: + newobj = key[k] + objs.append(newobj) + if isinstance(newobj, numeric.ndarray) and not scalar: + if final_dtypedescr is None: + final_dtypedescr = newobj.dtype + elif newobj.dtype > final_dtypedescr: + final_dtypedescr = newobj.dtype + if final_dtypedescr is not None: + for k in scalars: + objs[k] = objs[k].astype(final_dtypedescr) + res = concatenate(tuple(objs),axis=self.axis) + return self._retval(res) + +class mr_class(mconcatenator): + """Translates slice objects to concatenation along the first axis. + + For example: + >>> mr_[array([1,2,3]), 0, 0, array([4,5,6])] + array([1, 2, 3, 0, 0, 4, 5, 6]) + """ + def __init__(self): + mconcatenator.__init__(self, 0) + +mr_ = mr_class() + +#####-------------------------------------------------------------------------- +#---- --- +#####-------------------------------------------------------------------------- + +def flatnotmasked_edges(a): + """Finds the indices of the first and last not masked values in a 1D masked array. + If all values are masked, returns None. + """ + m = getmask(a) + if m is nomask or not numpy.any(m): + return [0,-1] + unmasked = numeric.flatnonzero(~m) + if len(unmasked) > 0: + return unmasked[[0,-1]] + else: + return None + +def notmasked_edges(a, axis=None): + """Finds the indices of the first and last not masked values along the given + axis in a masked array. + If all values are masked, returns None. + Otherwise, returns a list of 2 tuples, corresponding to the indices of the + first and last unmasked values respectively. + """ + a = asarray(a) + if axis is None or a.ndim == 1: + return flatnotmasked_edges(a) + m = getmask(a) + idx = array(numpy.indices(a.shape), mask=nxasarray([m]*a.ndim)) + return [tuple([idx[i].min(axis).compressed() for i in range(a.ndim)]), + tuple([idx[i].max(axis).compressed() for i in range(a.ndim)]),] + +def flatnotmasked_contiguous(a): + """Finds contiguous unmasked data in a flattened masked array. + Returns a sorted sequence of slices (start index, end index). + """ + m = getmask(a) + if m is nomask: + return (a.size, [0,-1]) + unmasked = numeric.flatnonzero(~m) + if len(unmasked) == 0: + return None + result = [] + for k, group in groupby(enumerate(unmasked), lambda (i,x):i-x): + tmp = numpy.fromiter((g[1] for g in group), int_) +# result.append((tmp.size, tuple(tmp[[0,-1]]))) + result.append( slice(tmp[0],tmp[-1]) ) + result.sort() + return result + +def notmasked_contiguous(a, axis=None): + """Finds contiguous unmasked data in a masked array along the given axis. + Returns a sorted sequence of slices (start index, end index). + Note: Only accepts 2D arrays at most. + """ + a = asarray(a) + nd = a.ndim + if nd > 2: + raise NotImplementedError,"Currently limited to atmost 2D array." + if axis is None or nd == 1: + return flatnotmasked_contiguous(a) + # + result = [] + # + other = (axis+1)%2 + idx = [0,0] + idx[axis] = slice(None,None) + # + for i in range(a.shape[other]): + idx[other] = i + result.append( flatnotmasked_contiguous(a[idx]) ) + return result + +################################################################################ +if __name__ == '__main__': + # + import numpy as N + from maskedarray.testutils import assert_equal + if 1: + b = ones(5) + m = [1,0,0,0,0] + d = masked_array(b,mask=m) + c = mr_[d,0,0,d] diff --git a/numpy/ma/morestats.py b/numpy/ma/morestats.py new file mode 100644 index 000000000..ae263a835 --- /dev/null +++ b/numpy/ma/morestats.py @@ -0,0 +1,406 @@ +""" +Generic statistics functions, with support to MA. + +:author: Pierre GF Gerard-Marchant +:contact: pierregm_at_uga_edu +:date: $Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $ +:version: $Id: morestats.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + + +import numpy +from numpy import bool_, float_, int_, ndarray, \ + sqrt,\ + arange, empty,\ + r_ +from numpy import array as narray +import numpy.core.numeric as numeric +from numpy.core.numeric import concatenate + +import maskedarray as MA +from maskedarray.core import masked, nomask, MaskedArray, masked_array +from maskedarray.extras import apply_along_axis, dot +from maskedarray.mstats import trim_both, trimmed_stde, mquantiles, mmedian, stde_median + +from scipy.stats.distributions import norm, beta, t, binom +from scipy.stats.morestats import find_repeats + +__all__ = ['hdquantiles', 'hdmedian', 'hdquantiles_sd', + 'trimmed_mean_ci', 'mjci', 'rank_data'] + + +#####-------------------------------------------------------------------------- +#---- --- Quantiles --- +#####-------------------------------------------------------------------------- +def hdquantiles(data, prob=list([.25,.5,.75]), axis=None, var=False,): + """Computes quantile estimates with the Harrell-Davis method, where the estimates +are calculated as a weighted linear combination of order statistics. + +*Parameters* : + data: {ndarray} + Data array. + prob: {sequence} + Sequence of quantiles to compute. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. + var : {boolean} + Whether to return the variance of the estimate. + +*Returns* + A (p,) array of quantiles (if ``var`` is False), or a (2,p) array of quantiles + and variances (if ``var`` is True), where ``p`` is the number of quantiles. + +:Note: + The function is restricted to 2D arrays. + """ + def _hd_1D(data,prob,var): + "Computes the HD quantiles for a 1D array. Returns nan for invalid data." + xsorted = numpy.squeeze(numpy.sort(data.compressed().view(ndarray))) + # Don't use length here, in case we have a numpy scalar + n = xsorted.size + #......... + hd = empty((2,len(prob)), float_) + if n < 2: + hd.flat = numpy.nan + if var: + return hd + return hd[0] + #......... + v = arange(n+1) / float(n) + betacdf = beta.cdf + for (i,p) in enumerate(prob): + _w = betacdf(v, (n+1)*p, (n+1)*(1-p)) + w = _w[1:] - _w[:-1] + hd_mean = dot(w, xsorted) + hd[0,i] = hd_mean + # + hd[1,i] = dot(w, (xsorted-hd_mean)**2) + # + hd[0, prob == 0] = xsorted[0] + hd[0, prob == 1] = xsorted[-1] + if var: + hd[1, prob == 0] = hd[1, prob == 1] = numpy.nan + return hd + return hd[0] + # Initialization & checks --------- + data = masked_array(data, copy=False, dtype=float_) + p = numpy.array(prob, copy=False, ndmin=1) + # Computes quantiles along axis (or globally) + if (axis is None) or (data.ndim == 1): + result = _hd_1D(data, p, var) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + result = apply_along_axis(_hd_1D, axis, data, p, var) + # + return masked_array(result, mask=numpy.isnan(result)) + +#.............................................................................. +def hdmedian(data, axis=-1, var=False): + """Returns the Harrell-Davis estimate of the median along the given axis. + +*Parameters* : + data: {ndarray} + Data array. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. + var : {boolean} + Whether to return the variance of the estimate. + """ + result = hdquantiles(data,[0.5], axis=axis, var=var) + return result.squeeze() + + +#.............................................................................. +def hdquantiles_sd(data, prob=list([.25,.5,.75]), axis=None): + """Computes the standard error of the Harrell-Davis quantile estimates by jackknife. + + +*Parameters* : + data: {ndarray} + Data array. + prob: {sequence} + Sequence of quantiles to compute. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. + +*Note*: + The function is restricted to 2D arrays. + """ + def _hdsd_1D(data,prob): + "Computes the std error for 1D arrays." + xsorted = numpy.sort(data.compressed()) + n = len(xsorted) + #......... + hdsd = empty(len(prob), float_) + if n < 2: + hdsd.flat = numpy.nan + #......... + vv = arange(n) / float(n-1) + betacdf = beta.cdf + # + for (i,p) in enumerate(prob): + _w = betacdf(vv, (n+1)*p, (n+1)*(1-p)) + w = _w[1:] - _w[:-1] + mx_ = numpy.fromiter([dot(w,xsorted[r_[range(0,k), + range(k+1,n)].astype(int_)]) + for k in range(n)], dtype=float_) + mx_var = numpy.array(mx_.var(), copy=False, ndmin=1) * n / float(n-1) + hdsd[i] = float(n-1) * sqrt(numpy.diag(mx_var).diagonal() / float(n)) + return hdsd + # Initialization & checks --------- + data = masked_array(data, copy=False, dtype=float_) + p = numpy.array(prob, copy=False, ndmin=1) + # Computes quantiles along axis (or globally) + if (axis is None): + result = _hdsd_1D(data.compressed(), p) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + result = apply_along_axis(_hdsd_1D, axis, data, p) + # + return masked_array(result, mask=numpy.isnan(result)).ravel() + + +#####-------------------------------------------------------------------------- +#---- --- Confidence intervals --- +#####-------------------------------------------------------------------------- + +def trimmed_mean_ci(data, proportiontocut=0.2, alpha=0.05, axis=None): + """Returns the selected confidence interval of the trimmed mean along the +given axis. + +*Parameters* : + data : {sequence} + Input data. The data is transformed to a masked array + proportiontocut : {float} + Proportion of the data to cut from each side of the data . + As a result, (2*proportiontocut*n) values are actually trimmed. + alpha : {float} + Confidence level of the intervals. + axis : {integer} + Axis along which to cut. If None, uses a flattened version of the input. + """ + data = masked_array(data, copy=False) + trimmed = trim_both(data, proportiontocut=proportiontocut, axis=axis) + tmean = trimmed.mean(axis) + tstde = trimmed_stde(data, proportiontocut=proportiontocut, axis=axis) + df = trimmed.count(axis) - 1 + tppf = t.ppf(1-alpha/2.,df) + return numpy.array((tmean - tppf*tstde, tmean+tppf*tstde)) + +#.............................................................................. +def mjci(data, prob=[0.25,0.5,0.75], axis=None): + """Returns the Maritz-Jarrett estimators of the standard error of selected +experimental quantiles of the data. + +*Parameters* : + data: {ndarray} + Data array. + prob: {sequence} + Sequence of quantiles to compute. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. + """ + def _mjci_1D(data, p): + data = data.compressed() + sorted = numpy.sort(data) + n = data.size + prob = (numpy.array(p) * n + 0.5).astype(int_) + betacdf = beta.cdf + # + mj = empty(len(prob), float_) + x = arange(1,n+1, dtype=float_) / n + y = x - 1./n + for (i,m) in enumerate(prob): + (m1,m2) = (m-1, n-m) + W = betacdf(x,m-1,n-m) - betacdf(y,m-1,n-m) + C1 = numpy.dot(W,sorted) + C2 = numpy.dot(W,sorted**2) + mj[i] = sqrt(C2 - C1**2) + return mj + # + data = masked_array(data, copy=False) + assert data.ndim <= 2, "Array should be 2D at most !" + p = numpy.array(prob, copy=False, ndmin=1) + # Computes quantiles along axis (or globally) + if (axis is None): + return _mjci_1D(data, p) + else: + return apply_along_axis(_mjci_1D, axis, data, p) + +#.............................................................................. +def mquantiles_cimj(data, prob=[0.25,0.50,0.75], alpha=0.05, axis=None): + """Computes the alpha confidence interval for the selected quantiles of the +data, with Maritz-Jarrett estimators. + +*Parameters* : + data: {ndarray} + Data array. + prob: {sequence} + Sequence of quantiles to compute. + alpha : {float} + Confidence level of the intervals. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. + """ + alpha = min(alpha, 1-alpha) + z = norm.ppf(1-alpha/2.) + xq = mquantiles(data, prob, alphap=0, betap=0, axis=axis) + smj = mjci(data, prob, axis=axis) + return (xq - z * smj, xq + z * smj) + + +#............................................................................. +def median_cihs(data, alpha=0.05, axis=None): + """Computes the alpha-level confidence interval for the median of the data, +following the Hettmasperger-Sheather method. + +*Parameters* : + data : {sequence} + Input data. Masked values are discarded. The input should be 1D only, or + axis should be set to None. + alpha : {float} + Confidence level of the intervals. + axis : {integer} + Axis along which to compute the quantiles. If None, use a flattened array. + """ + def _cihs_1D(data, alpha): + data = numpy.sort(data.compressed()) + n = len(data) + alpha = min(alpha, 1-alpha) + k = int(binom._ppf(alpha/2., n, 0.5)) + gk = binom.cdf(n-k,n,0.5) - binom.cdf(k-1,n,0.5) + if gk < 1-alpha: + k -= 1 + gk = binom.cdf(n-k,n,0.5) - binom.cdf(k-1,n,0.5) + gkk = binom.cdf(n-k-1,n,0.5) - binom.cdf(k,n,0.5) + I = (gk - 1 + alpha)/(gk - gkk) + lambd = (n-k) * I / float(k + (n-2*k)*I) + lims = (lambd*data[k] + (1-lambd)*data[k-1], + lambd*data[n-k-1] + (1-lambd)*data[n-k]) + return lims + data = masked_array(data, copy=False) + # Computes quantiles along axis (or globally) + if (axis is None): + result = _cihs_1D(data.compressed(), p, var) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + result = apply_along_axis(_cihs_1D, axis, data, alpha) + # + return result + +#.............................................................................. +def compare_medians_ms(group_1, group_2, axis=None): + """Compares the medians from two independent groups along the given axis. + +The comparison is performed using the McKean-Schrader estimate of the standard +error of the medians. + +*Parameters* : + group_1 : {sequence} + First dataset. + group_2 : {sequence} + Second dataset. + axis : {integer} + Axis along which the medians are estimated. If None, the arrays are flattened. + +*Returns* : + A (p,) array of comparison values. + + """ + (med_1, med_2) = (mmedian(group_1, axis=axis), mmedian(group_2, axis=axis)) + (std_1, std_2) = (stde_median(group_1, axis=axis), + stde_median(group_2, axis=axis)) + W = abs(med_1 - med_2) / sqrt(std_1**2 + std_2**2) + return 1 - norm.cdf(W) + + +#####-------------------------------------------------------------------------- +#---- --- Ranking --- +#####-------------------------------------------------------------------------- + +#.............................................................................. +def rank_data(data, axis=None, use_missing=False): + """Returns the rank (also known as order statistics) of each data point +along the given axis. + +If some values are tied, their rank is averaged. +If some values are masked, their rank is set to 0 if use_missing is False, or +set to the average rank of the unmasked values if use_missing is True. + +*Parameters* : + data : {sequence} + Input data. The data is transformed to a masked array + axis : {integer} + Axis along which to perform the ranking. If None, the array is first + flattened. An exception is raised if the axis is specified for arrays + with a dimension larger than 2 + use_missing : {boolean} + Whether the masked values have a rank of 0 (False) or equal to the + average rank of the unmasked values (True). + """ + # + def _rank1d(data, use_missing=False): + n = data.count() + rk = numpy.empty(data.size, dtype=float_) + idx = data.argsort() + rk[idx[:n]] = numpy.arange(1,n+1) + # + if use_missing: + rk[idx[n:]] = (n+1)/2. + else: + rk[idx[n:]] = 0 + # + repeats = find_repeats(data) + for r in repeats[0]: + condition = (data==r).filled(False) + rk[condition] = rk[condition].mean() + return rk + # + data = masked_array(data, copy=False) + if axis is None: + if data.ndim > 1: + return _rank1d(data.ravel(), use_missing).reshape(data.shape) + else: + return _rank1d(data, use_missing) + else: + return apply_along_axis(_rank1d, axis, data, use_missing) + +############################################################################### +if __name__ == '__main__': + + if 0: + from maskedarray.testutils import assert_almost_equal + data = [0.706560797,0.727229578,0.990399276,0.927065621,0.158953014, + 0.887764025,0.239407086,0.349638551,0.972791145,0.149789972, + 0.936947700,0.132359948,0.046041972,0.641675031,0.945530547, + 0.224218684,0.771450991,0.820257774,0.336458052,0.589113496, + 0.509736129,0.696838829,0.491323573,0.622767425,0.775189248, + 0.641461450,0.118455200,0.773029450,0.319280007,0.752229111, + 0.047841438,0.466295911,0.583850781,0.840581845,0.550086491, + 0.466470062,0.504765074,0.226855960,0.362641207,0.891620942, + 0.127898691,0.490094097,0.044882048,0.041441695,0.317976349, + 0.504135618,0.567353033,0.434617473,0.636243375,0.231803616, + 0.230154113,0.160011327,0.819464108,0.854706985,0.438809221, + 0.487427267,0.786907310,0.408367937,0.405534192,0.250444460, + 0.995309248,0.144389588,0.739947527,0.953543606,0.680051621, + 0.388382017,0.863530727,0.006514031,0.118007779,0.924024803, + 0.384236354,0.893687694,0.626534881,0.473051932,0.750134705, + 0.241843555,0.432947602,0.689538104,0.136934797,0.150206859, + 0.474335206,0.907775349,0.525869295,0.189184225,0.854284286, + 0.831089744,0.251637345,0.587038213,0.254475554,0.237781276, + 0.827928620,0.480283781,0.594514455,0.213641488,0.024194386, + 0.536668589,0.699497811,0.892804071,0.093835427,0.731107772] + # + assert_almost_equal(hdquantiles(data,[0., 1.]), + [0.006514031, 0.995309248]) + hdq = hdquantiles(data,[0.25, 0.5, 0.75]) + assert_almost_equal(hdq, [0.253210762, 0.512847491, 0.762232442,]) + hdq = hdquantiles_sd(data,[0.25, 0.5, 0.75]) + assert_almost_equal(hdq, [0.03786954, 0.03805389, 0.03800152,], 4) + # + data = numpy.array(data).reshape(10,10) + hdq = hdquantiles(data,[0.25,0.5,0.75],axis=0) diff --git a/numpy/ma/mrecords.py b/numpy/ma/mrecords.py new file mode 100644 index 000000000..80b9f3434 --- /dev/null +++ b/numpy/ma/mrecords.py @@ -0,0 +1,717 @@ +"""mrecords +Defines a class of record arrays supporting masked arrays. + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: mrecords.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import sys +import types + +import numpy +from numpy import bool_, complex_, float_, int_, str_, object_ +from numpy import array as narray +import numpy.core.numeric as numeric +import numpy.core.numerictypes as ntypes +from numpy.core.defchararray import chararray +from numpy.core.records import find_duplicate + +from numpy.core.records import format_parser, record, recarray +from numpy.core.records import fromarrays as recfromarrays + +ndarray = numeric.ndarray +_byteorderconv = numpy.core.records._byteorderconv +_typestr = ntypes._typestr + +import maskedarray +from maskedarray import MaskedArray, masked, nomask, masked_array,\ + make_mask, mask_or, getmask, getmaskarray, filled +from maskedarray.core import default_fill_value, masked_print_option + +import warnings + +reserved_fields = ['_data','_mask','_fieldmask', 'dtype'] + +def _getformats(data): + "Returns the formats of each array of arraylist as a comma-separated string." + if hasattr(data,'dtype'): + return ",".join([desc[1] for desc in data.dtype.descr]) + + formats = '' + for obj in data: + obj = numeric.asarray(obj) +# if not isinstance(obj, ndarray): +## if not isinstance(obj, ndarray): +# raise ValueError, "item in the array list must be an ndarray." + formats += _typestr[obj.dtype.type] + if issubclass(obj.dtype.type, ntypes.flexible): + formats += `obj.itemsize` + formats += ',' + return formats[:-1] + +def _checknames(descr, names=None): + """Checks that the field names of the descriptor ``descr`` are not some +reserved keywords. If this is the case, a default 'f%i' is substituted. +If the argument `names` is not None, updates the field names to valid names. + """ + ndescr = len(descr) + default_names = ['f%i' % i for i in range(ndescr)] + if names is None: + new_names = default_names + else: + if isinstance(names, (tuple, list)): + new_names = names + elif isinstance(names, str): + new_names = names.split(',') + else: + raise NameError, "illegal input names %s" % `names` + nnames = len(new_names) + if nnames < ndescr: + new_names += default_names[nnames:] + ndescr = [] + for (n, d, t) in zip(new_names, default_names, descr.descr): + if n in reserved_fields: + if t[0] in reserved_fields: + ndescr.append((d,t[1])) + else: + ndescr.append(t) + else: + ndescr.append((n,t[1])) + return numeric.dtype(ndescr) + + + +class MaskedRecords(MaskedArray, object): + """ + +*IVariables*: + _data : {recarray} + Underlying data, as a record array. + _mask : {boolean array} + Mask of the records. A record is masked when all its fields are masked. + _fieldmask : {boolean recarray} + Record array of booleans, setting the mask of each individual field of each record. + _fill_value : {record} + Filling values for each field. + """ + _defaultfieldmask = nomask + _defaulthardmask = False + def __new__(cls, data, mask=nomask, dtype=None, + hard_mask=False, fill_value=None, +# offset=0, strides=None, + formats=None, names=None, titles=None, + byteorder=None, aligned=False): + # Get the new descriptor ................ + if dtype is not None: + descr = numeric.dtype(dtype) + else: + if formats is None: + formats = _getformats(data) + parsed = format_parser(formats, names, titles, aligned, byteorder) + descr = parsed._descr + if names is not None: + descr = _checknames(descr,names) + _names = descr.names + mdescr = [(n,'|b1') for n in _names] + # get the shape ......................... + try: + shape = numeric.asarray(data[0]).shape + except IndexError: + shape = len(data.dtype) + if isinstance(shape, int): + shape = (shape,) + # Construct the _data recarray .......... + if isinstance(data, record): + _data = numeric.asarray(data).view(recarray) + _fieldmask = mask + elif isinstance(data, MaskedRecords): + _data = data._data + _fieldmask = data._fieldmask + elif isinstance(data, recarray): + _data = data + if mask is nomask: + _fieldmask = data.astype(mdescr) + _fieldmask.flat = tuple([False]*len(mdescr)) + else: + _fieldmask = mask + elif (isinstance(data, (tuple, numpy.void)) or\ + hasattr(data,'__len__') and isinstance(data[0], (tuple, numpy.void))): + data = numeric.array(data, dtype=descr).view(recarray) + _data = data + if mask is nomask: + _fieldmask = data.astype(mdescr) + _fieldmask.flat = tuple([False]*len(mdescr)) + else: + _fieldmask = mask + else: + _data = recarray(shape, dtype=descr) + _fieldmask = recarray(shape, dtype=mdescr) + for (n,v) in zip(_names, data): + _data[n] = numeric.asarray(v).view(ndarray) + _fieldmask[n] = getmaskarray(v) + #........................................ + _data = _data.view(cls) + _data._fieldmask = _fieldmask + _data._hardmask = hard_mask + if fill_value is None: + _data._fill_value = [default_fill_value(numeric.dtype(d[1])) + for d in descr.descr] + else: + _data._fill_value = fill_value + return _data + + def __array_finalize__(self,obj): + if isinstance(obj, MaskedRecords): + self.__dict__.update(_fieldmask=obj._fieldmask, + _hardmask=obj._hardmask, + _fill_value=obj._fill_value + ) + else: + self.__dict__.update(_fieldmask = nomask, + _hardmask = False, + fill_value = None + ) + return + + def _getdata(self): + "Returns the data as a recarray." + return self.view(recarray) + _data = property(fget=_getdata) + + #...................................................... + def __getattribute__(self, attr): + "Returns the given attribute." + try: + # Returns a generic attribute + return object.__getattribute__(self,attr) + except AttributeError: + # OK, so attr must be a field name + pass + # Get the list of fields ...... + _names = self.dtype.names + if attr in _names: + _data = self._data + _mask = self._fieldmask +# obj = masked_array(_data.__getattribute__(attr), copy=False, +# mask=_mask.__getattribute__(attr)) + # Use a view in order to avoid the copy of the mask in MaskedArray.__new__ + obj = narray(_data.__getattribute__(attr), copy=False).view(MaskedArray) + obj._mask = _mask.__getattribute__(attr) + if not obj.ndim and obj._mask: + return masked + return obj + raise AttributeError,"No attribute '%s' !" % attr + + def __setattr__(self, attr, val): + "Sets the attribute attr to the value val." + newattr = attr not in self.__dict__ + try: + # Is attr a generic attribute ? + ret = object.__setattr__(self, attr, val) + except: + # Not a generic attribute: exit if it's not a valid field + fielddict = self.dtype.names or {} + if attr not in fielddict: + exctype, value = sys.exc_info()[:2] + raise exctype, value + else: + if attr not in list(self.dtype.names) + ['_mask','mask']: + return ret + if newattr: # We just added this one + try: # or this setattr worked on an internal + # attribute. + object.__delattr__(self, attr) + except: + return ret + # Case #1.: Basic field ............ + base_fmask = self._fieldmask + _names = self.dtype.names + if attr in _names: + fval = filled(val) + mval = getmaskarray(val) + if self._hardmask: + mval = mask_or(mval, base_fmask.__getattr__(attr)) + self._data.__setattr__(attr, fval) + base_fmask.__setattr__(attr, mval) + return + elif attr == '_mask': + self.__setmask__(val) + return + #............................................ + def __getitem__(self, indx): + """Returns all the fields sharing the same fieldname base. +The fieldname base is either `_data` or `_mask`.""" + _localdict = self.__dict__ + _data = self._data + # We want a field ........ + if isinstance(indx, str): + obj = _data[indx].view(MaskedArray) + obj._set_mask(_localdict['_fieldmask'][indx]) + # Force to nomask if the mask is empty + if not obj._mask.any(): + obj._mask = nomask + return obj + # We want some elements .. + # First, the data ........ + obj = ndarray.__getitem__(self, indx) + if isinstance(obj, numpy.void): + obj = self.__class__(obj, dtype=self.dtype) + else: + obj = obj.view(type(self)) + obj._fieldmask = numpy.asarray(_localdict['_fieldmask'][indx]).view(recarray) + return obj + #............................................ + def __setitem__(self, indx, value): + "Sets the given record to value." + MaskedArray.__setitem__(self, indx, value) + + + def __setslice__(self, i, j, value): + "Sets the slice described by [i,j] to `value`." + _localdict = self.__dict__ + d = self._data + m = _localdict['_fieldmask'] + names = self.dtype.names + if value is masked: + for n in names: + m[i:j][n] = True + elif not self._hardmask: + fval = filled(value) + mval = getmaskarray(value) + for n in names: + d[n][i:j] = fval + m[n][i:j] = mval + else: + mindx = getmaskarray(self)[i:j] + dval = numeric.asarray(value) + valmask = getmask(value) + if valmask is nomask: + for n in names: + mval = mask_or(m[n][i:j], valmask) + d[n][i:j][~mval] = value + elif valmask.size > 1: + for n in names: + mval = mask_or(m[n][i:j], valmask) + d[n][i:j][~mval] = dval[~mval] + m[n][i:j] = mask_or(m[n][i:j], mval) + self._fieldmask = m + + #..................................................... + def __setmask__(self, mask): + "Sets the mask." + names = self.dtype.names + fmask = self.__dict__['_fieldmask'] + newmask = make_mask(mask, copy=False) +# self.unshare_mask() + if self._hardmask: + for n in names: + fmask[n].__ior__(newmask) + else: + for n in names: + fmask[n].flat = newmask + return + + def _getmask(self): + """Returns the mask of the mrecord: a record is masked when all the fields +are masked.""" + if self.size > 1: + return self._fieldmask.view((bool_, len(self.dtype))).all(1) + + _setmask = __setmask__ + _mask = property(fget=_getmask, fset=_setmask) + + #...................................................... + def __str__(self): + "Calculates the string representation." + if self.size > 1: + mstr = ["(%s)" % ",".join([str(i) for i in s]) + for s in zip(*[getattr(self,f) for f in self.dtype.names])] + return "[%s]" % ", ".join(mstr) + else: + mstr = ["%s" % ",".join([str(i) for i in s]) + for s in zip([getattr(self,f) for f in self.dtype.names])] + return "(%s)" % ", ".join(mstr) + + def __repr__(self): + "Calculates the repr representation." + _names = self.dtype.names + fmt = "%%%is : %%s" % (max([len(n) for n in _names])+4,) + reprstr = [fmt % (f,getattr(self,f)) for f in self.dtype.names] + reprstr.insert(0,'masked_records(') + reprstr.extend([fmt % (' fill_value', self._fill_value), + ' )']) + return str("\n".join(reprstr)) + #...................................................... + def view(self, obj): + """Returns a view of the mrecarray.""" + try: + if issubclass(obj, ndarray): + return ndarray.view(self, obj) + except TypeError: + pass + dtype = numeric.dtype(obj) + if dtype.fields is None: + return self.__array__().view(dtype) + return ndarray.view(self, obj) + #...................................................... + def filled(self, fill_value=None): + """Returns an array of the same class as ``_data``, with masked values +filled with ``fill_value``. If ``fill_value`` is None, ``self.fill_value`` is +used instead. + +Subclassing is preserved. + + """ + _localdict = self.__dict__ + d = self._data + fm = _localdict['_fieldmask'] + if not numeric.asarray(fm, dtype=bool_).any(): + return d + # + if fill_value is None: + value = _localdict['_fill_value'] + else: + value = fill_value + if numeric.size(value) == 1: + value = [value,] * len(self.dtype) + # + if self is masked: + result = numeric.asanyarray(value) + else: + result = d.copy() + for (n, v) in zip(d.dtype.names, value): + numpy.putmask(numeric.asarray(result[n]), + numeric.asarray(fm[n]), v) + return result + #............................................ + def harden_mask(self): + "Forces the mask to hard" + self._hardmask = True + def soften_mask(self): + "Forces the mask to soft" + self._hardmask = False + #............................................. + def copy(self): + """Returns a copy of the masked record.""" + _localdict = self.__dict__ + return MaskedRecords(self._data.copy(), + mask=_localdict['_fieldmask'].copy(), + dtype=self.dtype) + #............................................. + + +#####--------------------------------------------------------------------------- +#---- --- Constructors --- +#####--------------------------------------------------------------------------- + +def fromarrays(arraylist, dtype=None, shape=None, formats=None, + names=None, titles=None, aligned=False, byteorder=None): + """Creates a mrecarray from a (flat) list of masked arrays. + +*Parameters*: + arraylist : {sequence} + A list of (masked) arrays. Each element of the sequence is first converted + to a masked array if needed. If a 2D array is passed as argument, it is + processed line by line + dtype : {numeric.dtype} + Data type descriptor. + {shape} : {integer} + Number of records. If None, ``shape`` is defined from the shape of the + first array in the list. + formats : {sequence} + Sequence of formats for each individual field. If None, the formats will + be autodetected by inspecting the fields and selecting the highest dtype + possible. + names : {sequence} + Sequence of the names of each field. + -titles : {sequence} + (Description to write) + aligned : {boolean} + (Description to write, not used anyway) + byteorder: {boolean} + (Description to write, not used anyway) + +*Notes*: + Lists of tuples should be preferred over lists of lists for faster processing. + """ + arraylist = [masked_array(x) for x in arraylist] + # Define/check the shape..................... + if shape is None or shape == 0: + shape = arraylist[0].shape + if isinstance(shape, int): + shape = (shape,) + # Define formats from scratch ............... + if formats is None and dtype is None: + formats = _getformats(arraylist) + # Define the dtype .......................... + if dtype is not None: + descr = numeric.dtype(dtype) + _names = descr.names + else: + parsed = format_parser(formats, names, titles, aligned, byteorder) + _names = parsed._names + descr = parsed._descr + # Determine shape from data-type............. + if len(descr) != len(arraylist): + msg = "Mismatch between the number of fields (%i) and the number of "\ + "arrays (%i)" + raise ValueError, msg % (len(descr), len(arraylist)) + d0 = descr[0].shape + nn = len(d0) + if nn > 0: + shape = shape[:-nn] + # Make sure the shape is the correct one .... + for k, obj in enumerate(arraylist): + nn = len(descr[k].shape) + testshape = obj.shape[:len(obj.shape)-nn] + if testshape != shape: + raise ValueError, "Array-shape mismatch in array %d" % k + # Reconstruct the descriptor, by creating a _data and _mask version + return MaskedRecords(arraylist, dtype=descr) +#.............................................................................. +def fromrecords(reclist, dtype=None, shape=None, formats=None, names=None, + titles=None, aligned=False, byteorder=None): + """Creates a MaskedRecords from a list of records. + +*Parameters*: + arraylist : {sequence} + A list of (masked) arrays. Each element of the sequence is first converted + to a masked array if needed. If a 2D array is passed as argument, it is + processed line by line + dtype : {numeric.dtype} + Data type descriptor. + {shape} : {integer} + Number of records. If None, ``shape`` is defined from the shape of the + first array in the list. + formats : {sequence} + Sequence of formats for each individual field. If None, the formats will + be autodetected by inspecting the fields and selecting the highest dtype + possible. + names : {sequence} + Sequence of the names of each field. + -titles : {sequence} + (Description to write) + aligned : {boolean} + (Description to write, not used anyway) + byteorder: {boolean} + (Description to write, not used anyway) + +*Notes*: + Lists of tuples should be preferred over lists of lists for faster processing. + """ + # reclist is in fact a mrecarray ................. + if isinstance(reclist, MaskedRecords): + mdescr = reclist.dtype + shape = reclist.shape + return MaskedRecords(reclist, dtype=mdescr) + # No format, no dtype: create from to arrays ..... + nfields = len(reclist[0]) + if formats is None and dtype is None: # slower + if isinstance(reclist, recarray): + arrlist = [reclist.field(i) for i in range(len(reclist.dtype))] + if names is None: + names = reclist.dtype.names + else: + obj = numeric.array(reclist,dtype=object) + arrlist = [numeric.array(obj[...,i].tolist()) + for i in xrange(nfields)] + return MaskedRecords(arrlist, formats=formats, names=names, + titles=titles, aligned=aligned, byteorder=byteorder) + # Construct the descriptor ....................... + if dtype is not None: + descr = numeric.dtype(dtype) + _names = descr.names + else: + parsed = format_parser(formats, names, titles, aligned, byteorder) + _names = parsed._names + descr = parsed._descr + + try: + retval = numeric.array(reclist, dtype = descr).view(recarray) + except TypeError: # list of lists instead of list of tuples + if (shape is None or shape == 0): + shape = len(reclist)*2 + if isinstance(shape, (int, long)): + shape = (shape*2,) + if len(shape) > 1: + raise ValueError, "Can only deal with 1-d array." + retval = recarray(shape, mdescr) + for k in xrange(retval.size): + retval[k] = tuple(reclist[k]) + return MaskedRecords(retval, dtype=descr) + else: + if shape is not None and retval.shape != shape: + retval.shape = shape + # + return MaskedRecords(retval, dtype=descr) + +def _guessvartypes(arr): + """Tries to guess the dtypes of the str_ ndarray `arr`, by testing element-wise +conversion. Returns a list of dtypes. +The array is first converted to ndarray. If the array is 2D, the test is performed +on the first line. An exception is raised if the file is 3D or more. + """ + vartypes = [] + arr = numeric.asarray(arr) + if len(arr.shape) == 2 : + arr = arr[0] + elif len(arr.shape) > 2: + raise ValueError, "The array should be 2D at most!" + # Start the conversion loop ....... + for f in arr: + try: + val = int(f) + except ValueError: + try: + val = float(f) + except ValueError: + try: + val = complex(f) + except ValueError: + vartypes.append(arr.dtype) + else: + vartypes.append(complex_) + else: + vartypes.append(float_) + else: + vartypes.append(int_) + return vartypes + +def openfile(fname): + "Opens the file handle of file `fname`" + # A file handle ................... + if hasattr(fname, 'readline'): + return fname + # Try to open the file and guess its type + try: + f = open(fname) + except IOError: + raise IOError, "No such file: '%s'" % fname + if f.readline()[:2] != "\\x": + f.seek(0,0) + return f + raise NotImplementedError, "Wow, binary file" + + +def fromtextfile(fname, delimitor=None, commentchar='#', missingchar='', + varnames=None, vartypes=None): + """Creates a mrecarray from data stored in the file `filename`. + +*Parameters* : + filename : {file name/handle} + Handle of an opened file. + delimitor : {string} + Alphanumeric character used to separate columns in the file. + If None, any (group of) white spacestring(s) will be used. + commentchar : {string} + Alphanumeric character used to mark the start of a comment. + missingchar` : {string} + String indicating missing data, and used to create the masks. + varnames : {sequence} + Sequence of the variable names. If None, a list will be created from + the first non empty line of the file. + vartypes : {sequence} + Sequence of the variables dtypes. If None, it will be estimated from + the first non-commented line. + + + Ultra simple: the varnames are in the header, one line""" + # Try to open the file ...................... + f = openfile(fname) + # Get the first non-empty line as the varnames + while True: + line = f.readline() + firstline = line[:line.find(commentchar)].strip() + _varnames = firstline.split(delimitor) + if len(_varnames) > 1: + break + if varnames is None: + varnames = _varnames + # Get the data .............................. + _variables = masked_array([line.strip().split(delimitor) for line in f + if line[0] != commentchar and len(line) > 1]) + (_, nfields) = _variables.shape + # Try to guess the dtype .................... + if vartypes is None: + vartypes = _guessvartypes(_variables[0]) + else: + vartypes = [numeric.dtype(v) for v in vartypes] + if len(vartypes) != nfields: + msg = "Attempting to %i dtypes for %i fields!" + msg += " Reverting to default." + warnings.warn(msg % (len(vartypes), nfields)) + vartypes = _guessvartypes(_variables[0]) + # Construct the descriptor .................. + mdescr = [(n,f) for (n,f) in zip(varnames, vartypes)] + # Get the data and the mask ................. + # We just need a list of masked_arrays. It's easier to create it like that: + _mask = (_variables.T == missingchar) + _datalist = [masked_array(a,mask=m,dtype=t) + for (a,m,t) in zip(_variables.T, _mask, vartypes)] + return MaskedRecords(_datalist, dtype=mdescr) + +#.................................................................... +def addfield(mrecord, newfield, newfieldname=None): + """Adds a new field to the masked record array, using `newfield` as data +and `newfieldname` as name. If `newfieldname` is None, the new field name is +set to 'fi', where `i` is the number of existing fields. + """ + _data = mrecord._data + _mask = mrecord._fieldmask + if newfieldname is None or newfieldname in reserved_fields: + newfieldname = 'f%i' % len(_data.dtype) + newfield = masked_array(newfield) + # Get the new data ............ + # Create a new empty recarray + newdtype = numeric.dtype(_data.dtype.descr + \ + [(newfieldname, newfield.dtype)]) + newdata = recarray(_data.shape, newdtype) + # Add the exisintg field + [newdata.setfield(_data.getfield(*f),*f) + for f in _data.dtype.fields.values()] + # Add the new field + newdata.setfield(newfield._data, *newdata.dtype.fields[newfieldname]) + newdata = newdata.view(MaskedRecords) + # Get the new mask ............. + # Create a new empty recarray + newmdtype = numeric.dtype([(n,bool_) for n in newdtype.names]) + newmask = recarray(_data.shape, newmdtype) + # Add the old masks + [newmask.setfield(_mask.getfield(*f),*f) + for f in _mask.dtype.fields.values()] + # Add the mask of the new field + newmask.setfield(getmaskarray(newfield), + *newmask.dtype.fields[newfieldname]) + newdata._fieldmask = newmask + return newdata + +################################################################################ +if __name__ == '__main__': + import numpy as N + from maskedarray.testutils import assert_equal + if 1: + d = N.arange(5) + m = maskedarray.make_mask([1,0,0,1,1]) + base_d = N.r_[d,d[::-1]].reshape(2,-1).T + base_m = N.r_[[m, m[::-1]]].T + base = masked_array(base_d, mask=base_m).T + mrecord = fromarrays(base,dtype=[('a',N.float_),('b',N.float_)]) + mrec = MaskedRecords(mrecord.copy()) + # + if 1: + mrec = mrec.copy() + mrec.harden_mask() + assert(mrec._hardmask) + mrec._mask = nomask + assert_equal(mrec._mask, N.r_[[m,m[::-1]]].all(0)) + mrec.soften_mask() + assert(not mrec._hardmask) + mrec.mask = nomask + tmp = mrec['b']._mask + assert(mrec['b']._mask is nomask) + assert_equal(mrec['a']._mask,mrec['b']._mask) diff --git a/numpy/ma/mstats.py b/numpy/ma/mstats.py new file mode 100644 index 000000000..f965b291d --- /dev/null +++ b/numpy/ma/mstats.py @@ -0,0 +1,433 @@ +""" +Generic statistics functions, with support to MA. + +:author: Pierre GF Gerard-Marchant +:contact: pierregm_at_uga_edu +:date: $Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $ +:version: $Id: mstats.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + + +import numpy +from numpy import bool_, float_, int_, \ + sqrt +from numpy import array as narray +import numpy.core.numeric as numeric +from numpy.core.numeric import concatenate + +import maskedarray +from maskedarray.core import masked, nomask, MaskedArray, masked_array +from maskedarray.extras import apply_along_axis, dot + +__all__ = ['cov','meppf','plotting_positions','meppf','mmedian','mquantiles', + 'stde_median','trim_tail','trim_both','trimmed_mean','trimmed_stde', + 'winsorize'] + +#####-------------------------------------------------------------------------- +#---- -- Trimming --- +#####-------------------------------------------------------------------------- + +def winsorize(data, alpha=0.2): + """Returns a Winsorized version of the input array. + +The (alpha/2.) lowest values are set to the (alpha/2.)th percentile, and +the (alpha/2.) highest values are set to the (1-alpha/2.)th percentile +Masked values are skipped. + +*Parameters*: + data : {ndarray} + Input data to Winsorize. The data is first flattened. + alpha : {float}, optional + Percentage of total Winsorization : alpha/2. on the left, alpha/2. on the right + """ + data = masked_array(data, copy=False).ravel() + idxsort = data.argsort() + (nsize, ncounts) = (data.size, data.count()) + ntrim = int(alpha * ncounts) + (xmin,xmax) = data[idxsort[[ntrim, ncounts-nsize-ntrim-1]]] + return masked_array(numpy.clip(data, xmin, xmax), mask=data._mask) + +#.............................................................................. +def trim_both(data, proportiontocut=0.2, axis=None): + """Trims the data by masking the int(trim*n) smallest and int(trim*n) largest +values of data along the given axis, where n is the number of unmasked values. + +*Parameters*: + data : {ndarray} + Data to trim. + proportiontocut : {float} + Percentage of trimming. If n is the number of unmasked values before trimming, + the number of values after trimming is (1-2*trim)*n. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. + """ + #................... + def _trim_1D(data, trim): + "Private function: return a trimmed 1D array." + nsize = data.size + ncounts = data.count() + ntrim = int(trim * ncounts) + idxsort = data.argsort() + data[idxsort[:ntrim]] = masked + data[idxsort[ncounts-nsize-ntrim:]] = masked + return data + #................... + data = masked_array(data, copy=False, subok=True) + data.unshare_mask() + if (axis is None): + return _trim_1D(data.ravel(), proportiontocut) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + return apply_along_axis(_trim_1D, axis, data, proportiontocut) + +#.............................................................................. +def trim_tail(data, proportiontocut=0.2, tail='left', axis=None): + """Trims the data by masking int(trim*n) values from ONE tail of the data +along the given axis, where n is the number of unmasked values. + +*Parameters*: + data : {ndarray} + Data to trim. + proportiontocut : {float} + Percentage of trimming. If n is the number of unmasked values before trimming, + the number of values after trimming is (1-trim)*n. + tail : {string} + Trimming direction, in ('left', 'right'). If left, the proportiontocut + lowest values are set to the corresponding percentile. If right, the + proportiontocut highest values are used instead. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. + """ + #................... + def _trim_1D(data, trim, left): + "Private function: return a trimmed 1D array." + nsize = data.size + ncounts = data.count() + ntrim = int(trim * ncounts) + idxsort = data.argsort() + if left: + data[idxsort[:ntrim]] = masked + else: + data[idxsort[ncounts-nsize-ntrim:]] = masked + return data + #................... + data = masked_array(data, copy=False, subok=True) + data.unshare_mask() + # + if not isinstance(tail, str): + raise TypeError("The tail argument should be in ('left','right')") + tail = tail.lower()[0] + if tail == 'l': + left = True + elif tail == 'r': + left=False + else: + raise ValueError("The tail argument should be in ('left','right')") + # + if (axis is None): + return _trim_1D(data.ravel(), proportiontocut, left) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + return apply_along_axis(_trim_1D, axis, data, proportiontocut, left) + +#.............................................................................. +def trimmed_mean(data, proportiontocut=0.2, axis=None): + """Returns the trimmed mean of the data along the given axis. Trimming is +performed on both ends of the distribution. + +*Parameters*: + data : {ndarray} + Data to trim. + proportiontocut : {float} + Proportion of the data to cut from each side of the data . + As a result, (2*proportiontocut*n) values are actually trimmed. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. + """ + return trim_both(data, proportiontocut=proportiontocut, axis=axis).mean(axis=axis) + +#.............................................................................. +def trimmed_stde(data, proportiontocut=0.2, axis=None): + """Returns the standard error of the trimmed mean for the input data, +along the given axis. Trimming is performed on both ends of the distribution. + +*Parameters*: + data : {ndarray} + Data to trim. + proportiontocut : {float} + Proportion of the data to cut from each side of the data . + As a result, (2*proportiontocut*n) values are actually trimmed. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. + """ + #........................ + def _trimmed_stde_1D(data, trim=0.2): + "Returns the standard error of the trimmed mean for a 1D input data." + winsorized = winsorize(data) + nsize = winsorized.count() + winstd = winsorized.stdu() + return winstd / ((1-2*trim) * numpy.sqrt(nsize)) + #........................ + data = masked_array(data, copy=False, subok=True) + data.unshare_mask() + if (axis is None): + return _trimmed_stde_1D(data.ravel(), proportiontocut) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + return apply_along_axis(_trimmed_stde_1D, axis, data, proportiontocut) + +#............................................................................. +def stde_median(data, axis=None): + """Returns the McKean-Schrader estimate of the standard error of the sample +median along the given axis. + + +*Parameters*: + data : {ndarray} + Data to trim. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. + """ + def _stdemed_1D(data): + sorted = numpy.sort(data.compressed()) + n = len(sorted) + z = 2.5758293035489004 + k = int(round((n+1)/2. - z * sqrt(n/4.),0)) + return ((sorted[n-k] - sorted[k-1])/(2.*z)) + # + data = masked_array(data, copy=False, subok=True) + if (axis is None): + return _stdemed_1D(data) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + return apply_along_axis(_stdemed_1D, axis, data) + + +#####-------------------------------------------------------------------------- +#---- --- Quantiles --- +#####-------------------------------------------------------------------------- + + +def mquantiles(data, prob=list([.25,.5,.75]), alphap=.4, betap=.4, axis=None): + """Computes empirical quantiles for a *1xN* data array. +Samples quantile are defined by: +*Q(p) = (1-g).x[i] +g.x[i+1]* +where *x[j]* is the jth order statistic, +with *i = (floor(n*p+m))*, *m=alpha+p*(1-alpha-beta)* and *g = n*p + m - i)*. + +Typical values of (alpha,beta) are: + + - (0,1) : *p(k) = k/n* : linear interpolation of cdf (R, type 4) + - (.5,.5) : *p(k) = (k+1/2.)/n* : piecewise linear function (R, type 5) + - (0,0) : *p(k) = k/(n+1)* : (R type 6) + - (1,1) : *p(k) = (k-1)/(n-1)*. In this case, p(k) = mode[F(x[k])]. + That's R default (R type 7) + - (1/3,1/3): *p(k) = (k-1/3)/(n+1/3)*. Then p(k) ~ median[F(x[k])]. + The resulting quantile estimates are approximately median-unbiased + regardless of the distribution of x. (R type 8) + - (3/8,3/8): *p(k) = (k-3/8)/(n+1/4)*. Blom. + The resulting quantile estimates are approximately unbiased + if x is normally distributed (R type 9) + - (.4,.4) : approximately quantile unbiased (Cunnane) + - (.35,.35): APL, used with PWM + +*Parameters*: + x : {sequence} + Input data, as a sequence or array of dimension at most 2. + prob : {sequence} + List of quantiles to compute. + alpha : {float} + Plotting positions parameter. + beta : {float} + Plotting positions parameter. + axis : {integer} + Axis along which to perform the trimming. If None, the input array is first + flattened. + """ + def _quantiles1D(data,m,p): + x = numpy.sort(data.compressed()) + n = len(x) + if n == 0: + return masked_array(numpy.empty(len(p), dtype=float_), mask=True) + elif n == 1: + return masked_array(numpy.resize(x, p.shape), mask=nomask) + aleph = (n*p + m) + k = numpy.floor(aleph.clip(1, n-1)).astype(int_) + gamma = (aleph-k).clip(0,1) + return (1.-gamma)*x[(k-1).tolist()] + gamma*x[k.tolist()] + + # Initialization & checks --------- + data = masked_array(data, copy=False) + p = narray(prob, copy=False, ndmin=1) + m = alphap + p*(1.-alphap-betap) + # Computes quantiles along axis (or globally) + if (axis is None): + return _quantiles1D(data, m, p) + else: + assert data.ndim <= 2, "Array should be 2D at most !" + return apply_along_axis(_quantiles1D, axis, data, m, p) + + +def plotting_positions(data, alpha=0.4, beta=0.4): + """Returns the plotting positions (or empirical percentile points) for the + data. + Plotting positions are defined as (i-alpha)/(n-alpha-beta), where: + - i is the rank order statistics + - n is the number of unmasked values along the given axis + - alpha and beta are two parameters. + + Typical values for alpha and beta are: + - (0,1) : *p(k) = k/n* : linear interpolation of cdf (R, type 4) + - (.5,.5) : *p(k) = (k-1/2.)/n* : piecewise linear function (R, type 5) + - (0,0) : *p(k) = k/(n+1)* : Weibull (R type 6) + - (1,1) : *p(k) = (k-1)/(n-1)*. In this case, p(k) = mode[F(x[k])]. + That's R default (R type 7) + - (1/3,1/3): *p(k) = (k-1/3)/(n+1/3)*. Then p(k) ~ median[F(x[k])]. + The resulting quantile estimates are approximately median-unbiased + regardless of the distribution of x. (R type 8) + - (3/8,3/8): *p(k) = (k-3/8)/(n+1/4)*. Blom. + The resulting quantile estimates are approximately unbiased + if x is normally distributed (R type 9) + - (.4,.4) : approximately quantile unbiased (Cunnane) + - (.35,.35): APL, used with PWM + """ + data = masked_array(data, copy=False).reshape(1,-1) + n = data.count() + plpos = numpy.empty(data.size, dtype=float_) + plpos[n:] = 0 + plpos[data.argsort()[:n]] = (numpy.arange(1,n+1) - alpha)/(n+1-alpha-beta) + return masked_array(plpos, mask=data._mask) + +meppf = plotting_positions + + +def mmedian(data, axis=None): + """Returns the median of data along the given axis. Missing data are discarded.""" + def _median1D(data): + x = numpy.sort(data.compressed()) + if x.size == 0: + return masked + return numpy.median(x) + data = masked_array(data, subok=True, copy=True) + if axis is None: + return _median1D(data) + else: + return apply_along_axis(_median1D, axis, data) + + +def cov(x, y=None, rowvar=True, bias=False, strict=False): + """Estimates the covariance matrix. + + +Normalization is by (N-1) where N is the number of observations (unbiased +estimate). If bias is True then normalization is by N. + +*Parameters*: + x : {ndarray} + Input data. If x is a 1D array, returns the variance. If x is a 2D array, + returns the covariance matrix. + y : {ndarray}, optional + Optional set of variables. + rowvar : {boolean} + If rowvar is true, then each row is a variable with obersvations in columns. + If rowvar is False, each column is a variable and the observations are in + the rows. + bias : {boolean} + Whether to use a biased or unbiased estimate of the covariance. + If bias is True, then the normalization is by N, the number of observations. + Otherwise, the normalization is by (N-1) + strict : {boolean} + If strict is True, masked values are propagated: if a masked value appears in + a row or column, the whole row or column is considered masked. + """ + X = narray(x, ndmin=2, subok=True, dtype=float) + if X.shape[0] == 1: + rowvar = True + if rowvar: + axis = 0 + tup = (slice(None),None) + else: + axis = 1 + tup = (None, slice(None)) + # + if y is not None: + y = narray(y, copy=False, ndmin=2, subok=True, dtype=float) + X = concatenate((X,y),axis) + # + X -= X.mean(axis=1-axis)[tup] + n = X.count(1-axis) + # + if bias: + fact = n*1.0 + else: + fact = n-1.0 + # + if not rowvar: + return (dot(X.T, X.conj(), strict=False) / fact).squeeze() + else: + return (dot(X, X.T.conj(), strict=False) / fact).squeeze() + + +def idealfourths(data, axis=None): + """Returns an estimate of the interquartile range of the data along the given +axis, as computed with the ideal fourths. + """ + def _idf(data): + x = numpy.sort(data.compressed()) + n = len(x) + (j,h) = divmod(n/4. + 5/12.,1) + qlo = (1-h)*x[j] + h*x[j+1] + k = n - j + qup = (1-h)*x[k] + h*x[k-1] + return qup - qlo + data = masked_array(data, copy=False) + if (axis is None): + return _idf(data) + else: + return apply_along_axis(_idf, axis, data) + + +def rsh(data, points=None): + """Evalutates Rosenblatt's shifted histogram estimators for each point +on the dataset 'data'. + +*Parameters* : + data : {sequence} + Input data. Masked values are ignored. + points : {sequence} + Sequence of points where to evaluate Rosenblatt shifted histogram. + If None, use the data. + """ + data = masked_array(data, copy=False) + if points is None: + points = data + else: + points = numpy.array(points, copy=False, ndmin=1) + if data.ndim != 1: + raise AttributeError("The input array should be 1D only !") + n = data.count() + h = 1.2 * idealfourths(data) / n**(1./5) + nhi = (data[:,None] <= points[None,:] + h).sum(0) + nlo = (data[:,None] < points[None,:] - h).sum(0) + return (nhi-nlo) / (2.*n*h) + +################################################################################ +if __name__ == '__main__': + from maskedarray.testutils import assert_almost_equal + if 1: + a = maskedarray.arange(1,101) + a[1::2] = masked + b = maskedarray.resize(a, (100,100)) + assert_almost_equal(mquantiles(b), [25., 50., 75.]) + assert_almost_equal(mquantiles(b, axis=0), maskedarray.resize(a,(3,100))) + assert_almost_equal(mquantiles(b, axis=1), + maskedarray.resize([24.9, 50., 75.1], (100,3))) diff --git a/numpy/ma/setup.py b/numpy/ma/setup.py new file mode 100644 index 000000000..9d8d400f4 --- /dev/null +++ b/numpy/ma/setup.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import os + +def configuration(parent_package='',top_path=None): + from numpy.distutils.misc_util import Configuration + config = Configuration('ma',parent_package,top_path) + config.add_data_dir('tests') + return config + +if __name__ == "__main__": + from numpy.distutils.core import setup + #setup.update(nmasetup) + config = configuration(top_path='').todict() + setup(**config) diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py new file mode 100644 index 000000000..0849be135 --- /dev/null +++ b/numpy/ma/tests/test_core.py @@ -0,0 +1,1305 @@ +# pylint: disable-msg=W0611, W0612, W0511,R0201 +"""Tests suite for MaskedArray & subclassing. + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: test_core.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import types + +import numpy +import numpy.core.fromnumeric as fromnumeric +from numpy.testing import NumpyTest, NumpyTestCase +from numpy.testing.utils import build_err_msg +from numpy import array as narray + +import numpy.ma.testutils +from numpy.ma.testutils import * + +import numpy.ma.core as coremodule +from numpy.ma.core import * + +pi = numpy.pi + +#.............................................................................. +class TestMA(NumpyTestCase): + "Base test class for MaskedArrays." + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + self.setUp() + + def setUp (self): + "Base data definition." + x = narray([1.,1.,1.,-2., pi/2.0, 4., 5., -10., 10., 1., 2., 3.]) + y = narray([5.,0.,3., 2., -1., -4., 0., -10., 10., 1., 0., 3.]) + a10 = 10. + m1 = [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + m2 = [0, 0, 1, 0, 0, 1, 1, 0, 0, 0 ,0, 1] + xm = masked_array(x, mask=m1) + ym = masked_array(y, mask=m2) + z = narray([-.5, 0., .5, .8]) + zm = masked_array(z, mask=[0,1,0,0]) + xf = numpy.where(m1, 1.e+20, x) + xm.set_fill_value(1.e+20) + self.d = (x, y, a10, m1, m2, xm, ym, z, zm, xf) + #........................ + def check_basic1d(self): + "Test of basic array creation and properties in 1 dimension." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + assert(not isMaskedArray(x)) + assert(isMaskedArray(xm)) + assert((xm-ym).filled(0).any()) + fail_if_equal(xm.mask.astype(int_), ym.mask.astype(int_)) + s = x.shape + assert_equal(numpy.shape(xm), s) + assert_equal(xm.shape, s) + assert_equal(xm.dtype, x.dtype) + assert_equal(zm.dtype, z.dtype) + assert_equal(xm.size , reduce(lambda x,y:x*y, s)) + assert_equal(count(xm) , len(m1) - reduce(lambda x,y:x+y, m1)) + assert_array_equal(xm, xf) + assert_array_equal(filled(xm, 1.e20), xf) + assert_array_equal(x, xm) + #........................ + def check_basic2d(self): + "Test of basic array creation and properties in 2 dimensions." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + for s in [(4,3), (6,2)]: + x.shape = s + y.shape = s + xm.shape = s + ym.shape = s + xf.shape = s + + assert(not isMaskedArray(x)) + assert(isMaskedArray(xm)) + assert_equal(shape(xm), s) + assert_equal(xm.shape, s) + assert_equal( xm.size , reduce(lambda x,y:x*y, s)) + assert_equal( count(xm) , len(m1) - reduce(lambda x,y:x+y, m1)) + assert_equal(xm, xf) + assert_equal(filled(xm, 1.e20), xf) + assert_equal(x, xm) + #........................ + def check_basic_arithmetic (self): + "Test of basic arithmetic." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + a2d = array([[1,2],[0,4]]) + a2dm = masked_array(a2d, [[0,0],[1,0]]) + assert_equal(a2d * a2d, a2d * a2dm) + assert_equal(a2d + a2d, a2d + a2dm) + assert_equal(a2d - a2d, a2d - a2dm) + for s in [(12,), (4,3), (2,6)]: + x = x.reshape(s) + y = y.reshape(s) + xm = xm.reshape(s) + ym = ym.reshape(s) + xf = xf.reshape(s) + assert_equal(-x, -xm) + assert_equal(x + y, xm + ym) + assert_equal(x - y, xm - ym) + assert_equal(x * y, xm * ym) + assert_equal(x / y, xm / ym) + assert_equal(a10 + y, a10 + ym) + assert_equal(a10 - y, a10 - ym) + assert_equal(a10 * y, a10 * ym) + assert_equal(a10 / y, a10 / ym) + assert_equal(x + a10, xm + a10) + assert_equal(x - a10, xm - a10) + assert_equal(x * a10, xm * a10) + assert_equal(x / a10, xm / a10) + assert_equal(x**2, xm**2) + assert_equal(abs(x)**2.5, abs(xm) **2.5) + assert_equal(x**y, xm**ym) + assert_equal(numpy.add(x,y), add(xm, ym)) + assert_equal(numpy.subtract(x,y), subtract(xm, ym)) + assert_equal(numpy.multiply(x,y), multiply(xm, ym)) + assert_equal(numpy.divide(x,y), divide(xm, ym)) + #........................ + def check_mixed_arithmetic(self): + "Tests mixed arithmetics." + na = narray([1]) + ma = array([1]) + self.failUnless(isinstance(na + ma, MaskedArray)) + self.failUnless(isinstance(ma + na, MaskedArray)) + #........................ + def check_inplace_arithmetic(self): + """Test of inplace operations and rich comparisons""" + # addition + x = arange(10) + y = arange(10) + xm = arange(10) + xm[2] = masked + x += 1 + assert_equal(x, y+1) + xm += 1 + assert_equal(xm, y+1) + # subtraction + x = arange(10) + xm = arange(10) + xm[2] = masked + x -= 1 + assert_equal(x, y-1) + xm -= 1 + assert_equal(xm, y-1) + # multiplication + x = arange(10)*1.0 + xm = arange(10)*1.0 + xm[2] = masked + x *= 2.0 + assert_equal(x, y*2) + xm *= 2.0 + assert_equal(xm, y*2) + # division + x = arange(10)*2 + xm = arange(10)*2 + xm[2] = masked + x /= 2 + assert_equal(x, y) + xm /= 2 + assert_equal(xm, y) + # division, pt 2 + x = arange(10)*1.0 + xm = arange(10)*1.0 + xm[2] = masked + x /= 2.0 + assert_equal(x, y/2.0) + xm /= arange(10) + assert_equal(xm, ones((10,))) + + x = arange(10).astype(float_) + xm = arange(10) + xm[2] = masked +# id1 = id(x.raw_data()) + id1 = x.raw_data().ctypes.data + x += 1. +# assert id1 == id(x.raw_data()) + assert (id1 == x.raw_data().ctypes.data) + assert_equal(x, y+1.) + # addition w/ array + x = arange(10, dtype=float_) + xm = arange(10, dtype=float_) + xm[2] = masked + m = xm.mask + a = arange(10, dtype=float_) + a[-1] = masked + x += a + xm += a + assert_equal(x,y+a) + assert_equal(xm,y+a) + assert_equal(xm.mask, mask_or(m,a.mask)) + # subtraction w/ array + x = arange(10, dtype=float_) + xm = arange(10, dtype=float_) + xm[2] = masked + m = xm.mask + a = arange(10, dtype=float_) + a[-1] = masked + x -= a + xm -= a + assert_equal(x,y-a) + assert_equal(xm,y-a) + assert_equal(xm.mask, mask_or(m,a.mask)) + # multiplication w/ array + x = arange(10, dtype=float_) + xm = arange(10, dtype=float_) + xm[2] = masked + m = xm.mask + a = arange(10, dtype=float_) + a[-1] = masked + x *= a + xm *= a + assert_equal(x,y*a) + assert_equal(xm,y*a) + assert_equal(xm.mask, mask_or(m,a.mask)) + # division w/ array + x = arange(10, dtype=float_) + xm = arange(10, dtype=float_) + xm[2] = masked + m = xm.mask + a = arange(10, dtype=float_) + a[-1] = masked + x /= a + xm /= a + assert_equal(x,y/a) + assert_equal(xm,y/a) + assert_equal(xm.mask, mask_or(mask_or(m,a.mask), (a==0))) + # + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + z = xm/ym + assert_equal(z._mask, [1,1,1,0,0,1,1,0,0,0,1,1]) + assert_equal(z._data, [0.2,1.,1./3.,-1.,-pi/2.,-1.,5.,1.,1.,1.,2.,1.]) + xm = xm.copy() + xm /= ym + assert_equal(xm._mask, [1,1,1,0,0,1,1,0,0,0,1,1]) + assert_equal(xm._data, [1/5.,1.,1./3.,-1.,-pi/2.,-1.,5.,1.,1.,1.,2.,1.]) + + + #.......................... + def check_scalararithmetic(self): + "Tests some scalar arithmetics on MaskedArrays." + xm = array(0, mask=1) + assert((1/array(0)).mask) + assert((1 + xm).mask) + assert((-xm).mask) + assert((-xm).mask) + assert(maximum(xm, xm).mask) + assert(minimum(xm, xm).mask) + assert(xm.filled().dtype is xm.data.dtype) + x = array(0, mask=0) + assert_equal(x.filled().ctypes.data, x.ctypes.data) + assert_equal(str(xm), str(masked_print_option)) + #......................... + def check_basic_ufuncs (self): + "Test various functions such as sin, cos." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + assert_equal(numpy.cos(x), cos(xm)) + assert_equal(numpy.cosh(x), cosh(xm)) + assert_equal(numpy.sin(x), sin(xm)) + assert_equal(numpy.sinh(x), sinh(xm)) + assert_equal(numpy.tan(x), tan(xm)) + assert_equal(numpy.tanh(x), tanh(xm)) + assert_equal(numpy.sqrt(abs(x)), sqrt(xm)) + assert_equal(numpy.log(abs(x)), log(xm)) + assert_equal(numpy.log10(abs(x)), log10(xm)) + assert_equal(numpy.exp(x), exp(xm)) + assert_equal(numpy.arcsin(z), arcsin(zm)) + assert_equal(numpy.arccos(z), arccos(zm)) + assert_equal(numpy.arctan(z), arctan(zm)) + assert_equal(numpy.arctan2(x, y), arctan2(xm, ym)) + assert_equal(numpy.absolute(x), absolute(xm)) + assert_equal(numpy.equal(x,y), equal(xm, ym)) + assert_equal(numpy.not_equal(x,y), not_equal(xm, ym)) + assert_equal(numpy.less(x,y), less(xm, ym)) + assert_equal(numpy.greater(x,y), greater(xm, ym)) + assert_equal(numpy.less_equal(x,y), less_equal(xm, ym)) + assert_equal(numpy.greater_equal(x,y), greater_equal(xm, ym)) + assert_equal(numpy.conjugate(x), conjugate(xm)) + #........................ + def check_count_func (self): + "Tests count" + ott = array([0.,1.,2.,3.], mask=[1,0,0,0]) + assert( isinstance(count(ott), int)) + assert_equal(3, count(ott)) + assert_equal(1, count(1)) + assert_equal(0, array(1,mask=[1])) + ott = ott.reshape((2,2)) + assert isMaskedArray(count(ott,0)) + assert isinstance(count(ott), types.IntType) + assert_equal(3, count(ott)) + assert getmask(count(ott,0)) is nomask + assert_equal([1,2],count(ott,0)) + #........................ + def check_minmax_func (self): + "Tests minimum and maximum." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + xr = numpy.ravel(x) #max doesn't work if shaped + xmr = ravel(xm) + assert_equal(max(xr), maximum(xmr)) #true because of careful selection of data + assert_equal(min(xr), minimum(xmr)) #true because of careful selection of data + # + assert_equal(minimum([1,2,3],[4,0,9]), [1,0,3]) + assert_equal(maximum([1,2,3],[4,0,9]), [4,2,9]) + x = arange(5) + y = arange(5) - 2 + x[3] = masked + y[0] = masked + assert_equal(minimum(x,y), where(less(x,y), x, y)) + assert_equal(maximum(x,y), where(greater(x,y), x, y)) + assert minimum(x) == 0 + assert maximum(x) == 4 + # + x = arange(4).reshape(2,2) + x[-1,-1] = masked + assert_equal(maximum(x), 2) + + def check_minmax_methods(self): + "Additional tests on max/min" + (_, _, _, _, _, xm, _, _, _, _) = self.d + xm.shape = (xm.size,) + assert_equal(xm.max(), 10) + assert(xm[0].max() is masked) + assert(xm[0].max(0) is masked) + assert(xm[0].max(-1) is masked) + assert_equal(xm.min(), -10.) + assert(xm[0].min() is masked) + assert(xm[0].min(0) is masked) + assert(xm[0].min(-1) is masked) + assert_equal(xm.ptp(), 20.) + assert(xm[0].ptp() is masked) + assert(xm[0].ptp(0) is masked) + assert(xm[0].ptp(-1) is masked) + #........................ + def check_addsumprod (self): + "Tests add, sum, product." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + assert_equal(numpy.add.reduce(x), add.reduce(x)) + assert_equal(numpy.add.accumulate(x), add.accumulate(x)) + assert_equal(4, sum(array(4),axis=0)) + assert_equal(4, sum(array(4), axis=0)) + assert_equal(numpy.sum(x,axis=0), sum(x,axis=0)) + assert_equal(numpy.sum(filled(xm,0),axis=0), sum(xm,axis=0)) + assert_equal(numpy.sum(x,0), sum(x,0)) + assert_equal(numpy.product(x,axis=0), product(x,axis=0)) + assert_equal(numpy.product(x,0), product(x,0)) + assert_equal(numpy.product(filled(xm,1),axis=0), product(xm,axis=0)) + s = (3,4) + x.shape = y.shape = xm.shape = ym.shape = s + if len(s) > 1: + assert_equal(numpy.concatenate((x,y),1), concatenate((xm,ym),1)) + assert_equal(numpy.add.reduce(x,1), add.reduce(x,1)) + assert_equal(numpy.sum(x,1), sum(x,1)) + assert_equal(numpy.product(x,1), product(x,1)) + #......................... + def check_concat(self): + "Tests concatenations." + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + # basic concatenation + assert_equal(numpy.concatenate((x,y)), concatenate((xm,ym))) + assert_equal(numpy.concatenate((x,y)), concatenate((x,y))) + assert_equal(numpy.concatenate((x,y)), concatenate((xm,y))) + assert_equal(numpy.concatenate((x,y,x)), concatenate((x,ym,x))) + # Concatenation along an axis + s = (3,4) + x.shape = y.shape = xm.shape = ym.shape = s + assert_equal(xm.mask, numpy.reshape(m1, s)) + assert_equal(ym.mask, numpy.reshape(m2, s)) + xmym = concatenate((xm,ym),1) + assert_equal(numpy.concatenate((x,y),1), xmym) + assert_equal(numpy.concatenate((xm.mask,ym.mask),1), xmym._mask) + #........................ + def check_indexing(self): + "Tests conversions and indexing" + x1 = numpy.array([1,2,4,3]) + x2 = array(x1, mask=[1,0,0,0]) + x3 = array(x1, mask=[0,1,0,1]) + x4 = array(x1) + # test conversion to strings + junk, garbage = str(x2), repr(x2) + assert_equal(numpy.sort(x1),sort(x2,endwith=False)) + # tests of indexing + assert type(x2[1]) is type(x1[1]) + assert x1[1] == x2[1] + assert x2[0] is masked + assert_equal(x1[2],x2[2]) + assert_equal(x1[2:5],x2[2:5]) + assert_equal(x1[:],x2[:]) + assert_equal(x1[1:], x3[1:]) + x1[2] = 9 + x2[2] = 9 + assert_equal(x1,x2) + x1[1:3] = 99 + x2[1:3] = 99 + assert_equal(x1,x2) + x2[1] = masked + assert_equal(x1,x2) + x2[1:3] = masked + assert_equal(x1,x2) + x2[:] = x1 + x2[1] = masked + assert allequal(getmask(x2),array([0,1,0,0])) + x3[:] = masked_array([1,2,3,4],[0,1,1,0]) + assert allequal(getmask(x3), array([0,1,1,0])) + x4[:] = masked_array([1,2,3,4],[0,1,1,0]) + assert allequal(getmask(x4), array([0,1,1,0])) + assert allequal(x4, array([1,2,3,4])) + x1 = numpy.arange(5)*1.0 + x2 = masked_values(x1, 3.0) + assert_equal(x1,x2) + assert allequal(array([0,0,0,1,0],MaskType), x2.mask) +#FIXME: Well, eh, fill_value is now a property assert_equal(3.0, x2.fill_value()) + assert_equal(3.0, x2.fill_value) + x1 = array([1,'hello',2,3],object) + x2 = numpy.array([1,'hello',2,3],object) + s1 = x1[1] + s2 = x2[1] + assert_equal(type(s2), str) + assert_equal(type(s1), str) + assert_equal(s1, s2) + assert x1[1:1].shape == (0,) + #........................ + def check_copy(self): + "Tests of some subtle points of copying and sizing." + n = [0,0,1,0,0] + m = make_mask(n) + m2 = make_mask(m) + assert(m is m2) + m3 = make_mask(m, copy=1) + assert(m is not m3) + + x1 = numpy.arange(5) + y1 = array(x1, mask=m) + #assert( y1._data is x1) + assert_equal(y1._data.__array_interface__, x1.__array_interface__) + assert( allequal(x1,y1.raw_data())) + #assert( y1.mask is m) + assert_equal(y1._mask.__array_interface__, m.__array_interface__) + + y1a = array(y1) + #assert( y1a.raw_data() is y1.raw_data()) + assert( y1a._data.__array_interface__ == y1._data.__array_interface__) + assert( y1a.mask is y1.mask) + + y2 = array(x1, mask=m) + #assert( y2.raw_data() is x1) + assert (y2._data.__array_interface__ == x1.__array_interface__) + #assert( y2.mask is m) + assert (y2._mask.__array_interface__ == m.__array_interface__) + assert( y2[2] is masked) + y2[2] = 9 + assert( y2[2] is not masked) + #assert( y2.mask is not m) + assert (y2._mask.__array_interface__ != m.__array_interface__) + assert( allequal(y2.mask, 0)) + + y3 = array(x1*1.0, mask=m) + assert(filled(y3).dtype is (x1*1.0).dtype) + + x4 = arange(4) + x4[2] = masked + y4 = resize(x4, (8,)) + assert_equal(concatenate([x4,x4]), y4) + assert_equal(getmask(y4),[0,0,1,0,0,0,1,0]) + y5 = repeat(x4, (2,2,2,2), axis=0) + assert_equal(y5, [0,0,1,1,2,2,3,3]) + y6 = repeat(x4, 2, axis=0) + assert_equal(y5, y6) + y7 = x4.repeat((2,2,2,2), axis=0) + assert_equal(y5,y7) + y8 = x4.repeat(2,0) + assert_equal(y5,y8) + + y9 = x4.copy() + assert_equal(y9._data, x4._data) + assert_equal(y9._mask, x4._mask) + # + x = masked_array([1,2,3], mask=[0,1,0]) + # Copy is False by default + y = masked_array(x) +# assert_equal(id(y._data), id(x._data)) +# assert_equal(id(y._mask), id(x._mask)) + assert_equal(y._data.ctypes.data, x._data.ctypes.data) + assert_equal(y._mask.ctypes.data, x._mask.ctypes.data) + y = masked_array(x, copy=True) +# assert_not_equal(id(y._data), id(x._data)) +# assert_not_equal(id(y._mask), id(x._mask)) + assert_not_equal(y._data.ctypes.data, x._data.ctypes.data) + assert_not_equal(y._mask.ctypes.data, x._mask.ctypes.data) + #........................ + def check_where(self): + "Test the where function" + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + d = where(xm>2,xm,-9) + assert_equal(d, [-9.,-9.,-9.,-9., -9., 4., -9., -9., 10., -9., -9., 3.]) + assert_equal(d._mask, xm._mask) + d = where(xm>2,-9,ym) + assert_equal(d, [5.,0.,3., 2., -1.,-9.,-9., -10., -9., 1., 0., -9.]) + assert_equal(d._mask, [1,0,1,0,0,0,1,0,0,0,0,0]) + d = where(xm>2, xm, masked) + assert_equal(d, [-9.,-9.,-9.,-9., -9., 4., -9., -9., 10., -9., -9., 3.]) + tmp = xm._mask.copy() + tmp[(xm<=2).filled(True)] = True + assert_equal(d._mask, tmp) + # + ixm = xm.astype(int_) + d = where(ixm>2, ixm, masked) + assert_equal(d, [-9,-9,-9,-9, -9, 4, -9, -9, 10, -9, -9, 3]) + assert_equal(d.dtype, ixm.dtype) + # + x = arange(10) + x[3] = masked + c = x >= 8 + z = where(c , x, masked) + assert z.dtype is x.dtype + assert z[3] is masked + assert z[4] is masked + assert z[7] is masked + assert z[8] is not masked + assert z[9] is not masked + assert_equal(x,z) + # + z = where(c , masked, x) + assert z.dtype is x.dtype + assert z[3] is masked + assert z[4] is not masked + assert z[7] is not masked + assert z[8] is masked + assert z[9] is masked + + #........................ + def check_oddfeatures_1(self): + "Test of other odd features" + x = arange(20) + x = x.reshape(4,5) + x.flat[5] = 12 + assert x[1,0] == 12 + z = x + 10j * x + assert_equal(z.real, x) + assert_equal(z.imag, 10*x) + assert_equal((z*conjugate(z)).real, 101*x*x) + z.imag[...] = 0.0 + + x = arange(10) + x[3] = masked + assert str(x[3]) == str(masked) + c = x >= 8 + assert count(where(c,masked,masked)) == 0 + assert shape(where(c,masked,masked)) == c.shape + # + z = masked_where(c, x) + assert z.dtype is x.dtype + assert z[3] is masked + assert z[4] is not masked + assert z[7] is not masked + assert z[8] is masked + assert z[9] is masked + assert_equal(x,z) + # + #........................ + def check_oddfeatures_2(self): + "Tests some more features." + x = array([1.,2.,3.,4.,5.]) + c = array([1,1,1,0,0]) + x[2] = masked + z = where(c, x, -x) + assert_equal(z, [1.,2.,0., -4., -5]) + c[0] = masked + z = where(c, x, -x) + assert_equal(z, [1.,2.,0., -4., -5]) + assert z[0] is masked + assert z[1] is not masked + assert z[2] is masked + # + x = arange(6) + x[5] = masked + y = arange(6)*10 + y[2] = masked + c = array([1,1,1,0,0,0], mask=[1,0,0,0,0,0]) + cm = c.filled(1) + z = where(c,x,y) + zm = where(cm,x,y) + assert_equal(z, zm) + assert getmask(zm) is nomask + assert_equal(zm, [0,1,2,30,40,50]) + z = where(c, masked, 1) + assert_equal(z, [99,99,99,1,1,1]) + z = where(c, 1, masked) + assert_equal(z, [99, 1, 1, 99, 99, 99]) + #........................ + def check_oddfeatures_3(self): + """Tests some generic features.""" + atest = ones((10,10,10), dtype=float_) + btest = zeros(atest.shape, MaskType) + ctest = masked_where(btest,atest) + assert_equal(atest,ctest) + #........................ + def check_maskingfunctions(self): + "Tests masking functions." + x = array([1.,2.,3.,4.,5.]) + x[2] = masked + assert_equal(masked_where(greater(x, 2), x), masked_greater(x,2)) + assert_equal(masked_where(greater_equal(x, 2), x), masked_greater_equal(x,2)) + assert_equal(masked_where(less(x, 2), x), masked_less(x,2)) + assert_equal(masked_where(less_equal(x, 2), x), masked_less_equal(x,2)) + assert_equal(masked_where(not_equal(x, 2), x), masked_not_equal(x,2)) + assert_equal(masked_where(equal(x, 2), x), masked_equal(x,2)) + assert_equal(masked_where(not_equal(x,2), x), masked_not_equal(x,2)) + assert_equal(masked_inside(range(5), 1, 3), [0, 199, 199, 199, 4]) + assert_equal(masked_outside(range(5), 1, 3),[199,1,2,3,199]) + assert_equal(masked_inside(array(range(5), mask=[1,0,0,0,0]), 1, 3).mask, [1,1,1,1,0]) + assert_equal(masked_outside(array(range(5), mask=[0,1,0,0,0]), 1, 3).mask, [1,1,0,0,1]) + assert_equal(masked_equal(array(range(5), mask=[1,0,0,0,0]), 2).mask, [1,0,1,0,0]) + assert_equal(masked_not_equal(array([2,2,1,2,1], mask=[1,0,0,0,0]), 2).mask, [1,0,1,0,1]) + assert_equal(masked_where([1,1,0,0,0], [1,2,3,4,5]), [99,99,3,4,5]) + #........................ + def check_TakeTransposeInnerOuter(self): + "Test of take, transpose, inner, outer products" + x = arange(24) + y = numpy.arange(24) + x[5:6] = masked + x = x.reshape(2,3,4) + y = y.reshape(2,3,4) + assert_equal(numpy.transpose(y,(2,0,1)), transpose(x,(2,0,1))) + assert_equal(numpy.take(y, (2,0,1), 1), take(x, (2,0,1), 1)) + assert_equal(numpy.inner(filled(x,0),filled(y,0)), + inner(x, y)) + assert_equal(numpy.outer(filled(x,0),filled(y,0)), + outer(x, y)) + y = array(['abc', 1, 'def', 2, 3], object) + y[2] = masked + t = take(y,[0,3,4]) + assert t[0] == 'abc' + assert t[1] == 2 + assert t[2] == 3 + #....................... + def check_maskedelement(self): + "Test of masked element" + x = arange(6) + x[1] = masked + assert(str(masked) == '--') + assert(x[1] is masked) + assert_equal(filled(x[1], 0), 0) + # don't know why these should raise an exception... + #self.failUnlessRaises(Exception, lambda x,y: x+y, masked, masked) + #self.failUnlessRaises(Exception, lambda x,y: x+y, masked, 2) + #self.failUnlessRaises(Exception, lambda x,y: x+y, masked, xx) + #self.failUnlessRaises(Exception, lambda x,y: x+y, xx, masked) + #........................ + def check_scalar(self): + "Checks masking a scalar" + x = masked_array(0) + assert_equal(str(x), '0') + x = masked_array(0,mask=True) + assert_equal(str(x), str(masked_print_option)) + x = masked_array(0, mask=False) + assert_equal(str(x), '0') + #........................ + def check_usingmasked(self): + "Checks that there's no collapsing to masked" + x = masked_array([1,2]) + y = x * masked + assert_equal(y.shape, x.shape) + assert_equal(y._mask, [True, True]) + y = x[0] * masked + assert y is masked + y = x + masked + assert_equal(y.shape, x.shape) + assert_equal(y._mask, [True, True]) + + #........................ + def check_topython(self): + "Tests some communication issues with Python." + assert_equal(1, int(array(1))) + assert_equal(1.0, float(array(1))) + assert_equal(1, int(array([[[1]]]))) + assert_equal(1.0, float(array([[1]]))) + self.failUnlessRaises(ValueError, float, array([1,1])) + assert numpy.isnan(float(array([1],mask=[1]))) +#TODO: Check how bool works... +#TODO: self.failUnless(bool(array([0,1]))) +#TODO: self.failUnless(bool(array([0,0],mask=[0,1]))) +#TODO: self.failIf(bool(array([0,0]))) +#TODO: self.failIf(bool(array([0,0],mask=[0,0]))) + #........................ + def check_arraymethods(self): + "Tests some MaskedArray methods." + a = array([1,3,2]) + b = array([1,3,2], mask=[1,0,1]) + assert_equal(a.any(), a.data.any()) + assert_equal(a.all(), a.data.all()) + assert_equal(a.argmax(), a.data.argmax()) + assert_equal(a.argmin(), a.data.argmin()) + assert_equal(a.choose(0,1,2,3,4), a.data.choose(0,1,2,3,4)) + assert_equal(a.compress([1,0,1]), a.data.compress([1,0,1])) + assert_equal(a.conj(), a.data.conj()) + assert_equal(a.conjugate(), a.data.conjugate()) + # + m = array([[1,2],[3,4]]) + assert_equal(m.diagonal(), m.data.diagonal()) + assert_equal(a.sum(), a.data.sum()) + assert_equal(a.take([1,2]), a.data.take([1,2])) + assert_equal(m.transpose(), m.data.transpose()) + #........................ + def check_basicattributes(self): + "Tests some basic array attributes." + a = array([1,3,2]) + b = array([1,3,2], mask=[1,0,1]) + assert_equal(a.ndim, 1) + assert_equal(b.ndim, 1) + assert_equal(a.size, 3) + assert_equal(b.size, 3) + assert_equal(a.shape, (3,)) + assert_equal(b.shape, (3,)) + #........................ + def check_single_element_subscript(self): + "Tests single element subscripts of Maskedarrays." + a = array([1,3,2]) + b = array([1,3,2], mask=[1,0,1]) + assert_equal(a[0].shape, ()) + assert_equal(b[0].shape, ()) + assert_equal(b[1].shape, ()) + #........................ + def check_maskcreation(self): + "Tests how masks are initialized at the creation of Maskedarrays." + data = arange(24, dtype=float_) + data[[3,6,15]] = masked + dma_1 = MaskedArray(data) + assert_equal(dma_1.mask, data.mask) + dma_2 = MaskedArray(dma_1) + assert_equal(dma_2.mask, dma_1.mask) + dma_3 = MaskedArray(dma_1, mask=[1,0,0,0]*6) + fail_if_equal(dma_3.mask, dma_1.mask) + + def check_pickling(self): + "Tests pickling" + import cPickle + a = arange(10) + a[::3] = masked + a.fill_value = 999 + a_pickled = cPickle.loads(a.dumps()) + assert_equal(a_pickled._mask, a._mask) + assert_equal(a_pickled._data, a._data) + assert_equal(a_pickled.fill_value, 999) + # + a = array(numpy.matrix(range(10)), mask=[1,0,1,0,0]*2) + a_pickled = cPickle.loads(a.dumps()) + assert_equal(a_pickled._mask, a._mask) + assert_equal(a_pickled, a) + assert(isinstance(a_pickled._data,numpy.matrix)) + # + def check_fillvalue(self): + "Check that we don't lose the fill_value" + data = masked_array([1,2,3],fill_value=-999) + series = data[[0,2,1]] + assert_equal(series._fill_value, data._fill_value) + # + def check_asarray(self): + (x, y, a10, m1, m2, xm, ym, z, zm, xf) = self.d + xmm = asarray(xm) + assert_equal(xmm._data, xm._data) + assert_equal(xmm._mask, xm._mask) + # + def check_fix_invalid(self): + "Checks fix_invalid." + data = masked_array(numpy.sqrt([-1., 0., 1.]), mask=[0,0,1]) + data_fixed = fix_invalid(data) + assert_equal(data_fixed._data, [data.fill_value, 0., 1.]) + assert_equal(data_fixed._mask, [1., 0., 1.]) + # + def check_imag_real(self): + xx = array([1+10j,20+2j], mask=[1,0]) + assert_equal(xx.imag,[10,2]) + assert_equal(xx.imag.filled(), [1e+20,2]) + assert_equal(xx.imag.dtype, xx._data.imag.dtype) + assert_equal(xx.real,[1,20]) + assert_equal(xx.real.filled(), [1e+20,20]) + assert_equal(xx.real.dtype, xx._data.real.dtype) + +#............................................................................... + +class TestUfuncs(NumpyTestCase): + "Test class for the application of ufuncs on MaskedArrays." + def setUp(self): + "Base data definition." + self.d = (array([1.0, 0, -1, pi/2]*2, mask=[0,1]+[0]*6), + array([1.0, 0, -1, pi/2]*2, mask=[1,0]+[0]*6),) + + def check_testUfuncRegression(self): + "Tests new ufuncs on MaskedArrays." + for f in ['sqrt', 'log', 'log10', 'exp', 'conjugate', + 'sin', 'cos', 'tan', + 'arcsin', 'arccos', 'arctan', + 'sinh', 'cosh', 'tanh', + 'arcsinh', + 'arccosh', + 'arctanh', + 'absolute', 'fabs', 'negative', + # 'nonzero', 'around', + 'floor', 'ceil', + # 'sometrue', 'alltrue', + 'logical_not', + 'add', 'subtract', 'multiply', + 'divide', 'true_divide', 'floor_divide', + 'remainder', 'fmod', 'hypot', 'arctan2', + 'equal', 'not_equal', 'less_equal', 'greater_equal', + 'less', 'greater', + 'logical_and', 'logical_or', 'logical_xor', + ]: + #print f + try: + uf = getattr(umath, f) + except AttributeError: + uf = getattr(fromnumeric, f) + mf = getattr(coremodule, f) + args = self.d[:uf.nin] + ur = uf(*args) + mr = mf(*args) + assert_equal(ur.filled(0), mr.filled(0), f) + assert_mask_equal(ur.mask, mr.mask) + #........................ + def test_reduce(self): + "Tests reduce on MaskedArrays." + a = self.d[0] + assert(not alltrue(a,axis=0)) + assert(sometrue(a,axis=0)) + assert_equal(sum(a[:3],axis=0), 0) + assert_equal(product(a,axis=0), 0) + assert_equal(add.reduce(a), pi) + #........................ + def test_minmax(self): + "Tests extrema on MaskedArrays." + a = arange(1,13).reshape(3,4) + amask = masked_where(a < 5,a) + assert_equal(amask.max(), a.max()) + assert_equal(amask.min(), 5) + assert_equal(amask.max(0), a.max(0)) + assert_equal(amask.min(0), [5,6,7,8]) + assert(amask.max(1)[0].mask) + assert(amask.min(1)[0].mask) + +#............................................................................... + +class TestArrayMethods(NumpyTestCase): + "Test class for miscellaneous MaskedArrays methods." + def setUp(self): + "Base data definition." + x = numpy.array([ 8.375, 7.545, 8.828, 8.5 , 1.757, 5.928, + 8.43 , 7.78 , 9.865, 5.878, 8.979, 4.732, + 3.012, 6.022, 5.095, 3.116, 5.238, 3.957, + 6.04 , 9.63 , 7.712, 3.382, 4.489, 6.479, + 7.189, 9.645, 5.395, 4.961, 9.894, 2.893, + 7.357, 9.828, 6.272, 3.758, 6.693, 0.993]) + X = x.reshape(6,6) + XX = x.reshape(3,2,2,3) + + m = numpy.array([0, 1, 0, 1, 0, 0, + 1, 0, 1, 1, 0, 1, + 0, 0, 0, 1, 0, 1, + 0, 0, 0, 1, 1, 1, + 1, 0, 0, 1, 0, 0, + 0, 0, 1, 0, 1, 0]) + mx = array(data=x,mask=m) + mX = array(data=X,mask=m.reshape(X.shape)) + mXX = array(data=XX,mask=m.reshape(XX.shape)) + + m2 = numpy.array([1, 1, 0, 1, 0, 0, + 1, 1, 1, 1, 0, 1, + 0, 0, 1, 1, 0, 1, + 0, 0, 0, 1, 1, 1, + 1, 0, 0, 1, 1, 0, + 0, 0, 1, 0, 1, 1]) + m2x = array(data=x,mask=m2) + m2X = array(data=X,mask=m2.reshape(X.shape)) + m2XX = array(data=XX,mask=m2.reshape(XX.shape)) + self.d = (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) + + #------------------------------------------------------ + def check_trace(self): + "Tests trace on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + mXdiag = mX.diagonal() + assert_equal(mX.trace(), mX.diagonal().compressed().sum()) + assert_almost_equal(mX.trace(), + X.trace() - sum(mXdiag.mask*X.diagonal(),axis=0)) + + def check_clip(self): + "Tests clip on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + clipped = mx.clip(2,8) + assert_equal(clipped.mask,mx.mask) + assert_equal(clipped.data,x.clip(2,8)) + assert_equal(clipped.data,mx.data.clip(2,8)) + + def check_ptp(self): + "Tests ptp on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + (n,m) = X.shape + assert_equal(mx.ptp(),mx.compressed().ptp()) + rows = numpy.zeros(n,numpy.float_) + cols = numpy.zeros(m,numpy.float_) + for k in range(m): + cols[k] = mX[:,k].compressed().ptp() + for k in range(n): + rows[k] = mX[k].compressed().ptp() + assert_equal(mX.ptp(0),cols) + assert_equal(mX.ptp(1),rows) + + def check_swapaxes(self): + "Tests swapaxes on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + mXswapped = mX.swapaxes(0,1) + assert_equal(mXswapped[-1],mX[:,-1]) + mXXswapped = mXX.swapaxes(0,2) + assert_equal(mXXswapped.shape,(2,2,3,3)) + + def check_cumsumprod(self): + "Tests cumsum & cumprod on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + mXcp = mX.cumsum(0) + assert_equal(mXcp.data,mX.filled(0).cumsum(0)) + mXcp = mX.cumsum(1) + assert_equal(mXcp.data,mX.filled(0).cumsum(1)) + # + mXcp = mX.cumprod(0) + assert_equal(mXcp.data,mX.filled(1).cumprod(0)) + mXcp = mX.cumprod(1) + assert_equal(mXcp.data,mX.filled(1).cumprod(1)) + + def check_varstd(self): + "Tests var & std on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + assert_almost_equal(mX.var(axis=None),mX.compressed().var()) + assert_almost_equal(mX.std(axis=None),mX.compressed().std()) + assert_equal(mXX.var(axis=3).shape,XX.var(axis=3).shape) + assert_equal(mX.var().shape,X.var().shape) + (mXvar0,mXvar1) = (mX.var(axis=0), mX.var(axis=1)) + for k in range(6): + assert_almost_equal(mXvar1[k],mX[k].compressed().var()) + assert_almost_equal(mXvar0[k],mX[:,k].compressed().var()) + assert_almost_equal(numpy.sqrt(mXvar0[k]), mX[:,k].compressed().std()) + + def check_argmin(self): + "Tests argmin & argmax on MaskedArrays." + (x,X,XX,m,mx,mX,mXX,m2x,m2X,m2XX) = self.d + # + assert_equal(mx.argmin(),35) + assert_equal(mX.argmin(),35) + assert_equal(m2x.argmin(),4) + assert_equal(m2X.argmin(),4) + assert_equal(mx.argmax(),28) + assert_equal(mX.argmax(),28) + assert_equal(m2x.argmax(),31) + assert_equal(m2X.argmax(),31) + # + assert_equal(mX.argmin(0), [2,2,2,5,0,5]) + assert_equal(m2X.argmin(0), [2,2,4,5,0,4]) + assert_equal(mX.argmax(0), [0,5,0,5,4,0]) + assert_equal(m2X.argmax(0), [5,5,0,5,1,0]) + # + assert_equal(mX.argmin(1), [4,1,0,0,5,5,]) + assert_equal(m2X.argmin(1), [4,4,0,0,5,3]) + assert_equal(mX.argmax(1), [2,4,1,1,4,1]) + assert_equal(m2X.argmax(1), [2,4,1,1,1,1]) + + def check_put(self): + "Tests put." + d = arange(5) + n = [0,0,0,1,1] + m = make_mask(n) + x = array(d, mask = m) + assert( x[3] is masked) + assert( x[4] is masked) + x[[1,4]] = [10,40] +# assert( x.mask is not m) + assert( x[3] is masked) + assert( x[4] is not masked) + assert_equal(x, [0,10,2,-1,40]) + # + x = masked_array(arange(10), mask=[1,0,0,0,0]*2) + i = [0,2,4,6] + x.put(i, [6,4,2,0]) + assert_equal(x, asarray([6,1,4,3,2,5,0,7,8,9,])) + assert_equal(x.mask, [0,0,0,0,0,1,0,0,0,0]) + x.put(i, masked_array([0,2,4,6],[1,0,1,0])) + assert_array_equal(x, [0,1,2,3,4,5,6,7,8,9,]) + assert_equal(x.mask, [1,0,0,0,1,1,0,0,0,0]) + # + x = masked_array(arange(10), mask=[1,0,0,0,0]*2) + put(x, i, [6,4,2,0]) + assert_equal(x, asarray([6,1,4,3,2,5,0,7,8,9,])) + assert_equal(x.mask, [0,0,0,0,0,1,0,0,0,0]) + put(x, i, masked_array([0,2,4,6],[1,0,1,0])) + assert_array_equal(x, [0,1,2,3,4,5,6,7,8,9,]) + assert_equal(x.mask, [1,0,0,0,1,1,0,0,0,0]) + + def check_put_hardmask(self): + "Tests put on hardmask" + d = arange(5) + n = [0,0,0,1,1] + m = make_mask(n) + xh = array(d+1, mask = m, hard_mask=True, copy=True) + xh.put([4,2,0,1,3],[1,2,3,4,5]) + assert_equal(xh._data, [3,4,2,4,5]) + + def check_take(self): + "Tests take" + x = masked_array([10,20,30,40],[0,1,0,1]) + assert_equal(x.take([0,0,3]), masked_array([10, 10, 40], [0,0,1]) ) + assert_equal(x.take([0,0,3]), x[[0,0,3]]) + assert_equal(x.take([[0,1],[0,1]]), + masked_array([[10,20],[10,20]], [[0,1],[0,1]]) ) + # + x = array([[10,20,30],[40,50,60]], mask=[[0,0,1],[1,0,0,]]) + assert_equal(x.take([0,2], axis=1), + array([[10,30],[40,60]], mask=[[0,1],[1,0]])) + assert_equal(take(x, [0,2], axis=1), + array([[10,30],[40,60]], mask=[[0,1],[1,0]])) + #........................ + def check_anyall(self): + """Checks the any/all methods/functions.""" + x = numpy.array([[ 0.13, 0.26, 0.90], + [ 0.28, 0.33, 0.63], + [ 0.31, 0.87, 0.70]]) + m = numpy.array([[ True, False, False], + [False, False, False], + [True, True, False]], dtype=numpy.bool_) + mx = masked_array(x, mask=m) + xbig = numpy.array([[False, False, True], + [False, False, True], + [False, True, True]], dtype=numpy.bool_) + mxbig = (mx > 0.5) + mxsmall = (mx < 0.5) + # + assert (mxbig.all()==False) + assert (mxbig.any()==True) + assert_equal(mxbig.all(0),[False, False, True]) + assert_equal(mxbig.all(1), [False, False, True]) + assert_equal(mxbig.any(0),[False, False, True]) + assert_equal(mxbig.any(1), [True, True, True]) + # + assert (mxsmall.all()==False) + assert (mxsmall.any()==True) + assert_equal(mxsmall.all(0), [True, True, False]) + assert_equal(mxsmall.all(1), [False, False, False]) + assert_equal(mxsmall.any(0), [True, True, False]) + assert_equal(mxsmall.any(1), [True, True, False]) + # + X = numpy.matrix(x) + mX = masked_array(X, mask=m) + mXbig = (mX > 0.5) + mXsmall = (mX < 0.5) + # + assert (mXbig.all()==False) + assert (mXbig.any()==True) + assert_equal(mXbig.all(0), numpy.matrix([False, False, True])) + assert_equal(mXbig.all(1), numpy.matrix([False, False, True]).T) + assert_equal(mXbig.any(0), numpy.matrix([False, False, True])) + assert_equal(mXbig.any(1), numpy.matrix([ True, True, True]).T) + # + assert (mXsmall.all()==False) + assert (mXsmall.any()==True) + assert_equal(mXsmall.all(0), numpy.matrix([True, True, False])) + assert_equal(mXsmall.all(1), numpy.matrix([False, False, False]).T) + assert_equal(mXsmall.any(0), numpy.matrix([True, True, False])) + assert_equal(mXsmall.any(1), numpy.matrix([True, True, False]).T) + + def check_keepmask(self): + "Tests the keep mask flag" + x = masked_array([1,2,3], mask=[1,0,0]) + mx = masked_array(x) + assert_equal(mx.mask, x.mask) + mx = masked_array(x, mask=[0,1,0], keep_mask=False) + assert_equal(mx.mask, [0,1,0]) + mx = masked_array(x, mask=[0,1,0], keep_mask=True) + assert_equal(mx.mask, [1,1,0]) + # We default to true + mx = masked_array(x, mask=[0,1,0]) + assert_equal(mx.mask, [1,1,0]) + + def check_hardmask(self): + "Test hard_mask" + d = arange(5) + n = [0,0,0,1,1] + m = make_mask(n) + xh = array(d, mask = m, hard_mask=True) + # We need to copy, to avoid updating d in xh! + xs = array(d, mask = m, hard_mask=False, copy=True) + xh[[1,4]] = [10,40] + xs[[1,4]] = [10,40] + assert_equal(xh._data, [0,10,2,3,4]) + assert_equal(xs._data, [0,10,2,3,40]) + #assert_equal(xh.mask.ctypes.data, m.ctypes.data) + assert_equal(xs.mask, [0,0,0,1,0]) + assert(xh._hardmask) + assert(not xs._hardmask) + xh[1:4] = [10,20,30] + xs[1:4] = [10,20,30] + assert_equal(xh._data, [0,10,20,3,4]) + assert_equal(xs._data, [0,10,20,30,40]) + #assert_equal(xh.mask.ctypes.data, m.ctypes.data) + assert_equal(xs.mask, nomask) + xh[0] = masked + xs[0] = masked + assert_equal(xh.mask, [1,0,0,1,1]) + assert_equal(xs.mask, [1,0,0,0,0]) + xh[:] = 1 + xs[:] = 1 + assert_equal(xh._data, [0,1,1,3,4]) + assert_equal(xs._data, [1,1,1,1,1]) + assert_equal(xh.mask, [1,0,0,1,1]) + assert_equal(xs.mask, nomask) + # Switch to soft mask + xh.soften_mask() + xh[:] = arange(5) + assert_equal(xh._data, [0,1,2,3,4]) + assert_equal(xh.mask, nomask) + # Switch back to hard mask + xh.harden_mask() + xh[xh<3] = masked + assert_equal(xh._data, [0,1,2,3,4]) + assert_equal(xh._mask, [1,1,1,0,0]) + xh[filled(xh>1,False)] = 5 + assert_equal(xh._data, [0,1,2,5,5]) + assert_equal(xh._mask, [1,1,1,0,0]) + # + xh = array([[1,2],[3,4]], mask = [[1,0],[0,0]], hard_mask=True) + xh[0] = 0 + assert_equal(xh._data, [[1,0],[3,4]]) + assert_equal(xh._mask, [[1,0],[0,0]]) + xh[-1,-1] = 5 + assert_equal(xh._data, [[1,0],[3,5]]) + assert_equal(xh._mask, [[1,0],[0,0]]) + xh[filled(xh<5,False)] = 2 + assert_equal(xh._data, [[1,2],[2,5]]) + assert_equal(xh._mask, [[1,0],[0,0]]) + # + "Another test of hardmask" + d = arange(5) + n = [0,0,0,1,1] + m = make_mask(n) + xh = array(d, mask = m, hard_mask=True) + xh[4:5] = 999 + #assert_equal(xh.mask.ctypes.data, m.ctypes.data) + xh[0:1] = 999 + assert_equal(xh._data,[999,1,2,3,4]) + + def check_smallmask(self): + "Checks the behaviour of _smallmask" + a = arange(10) + a[1] = masked + a[1] = 1 + assert_equal(a._mask, nomask) + a = arange(10) + a._smallmask = False + a[1] = masked + a[1] = 1 + assert_equal(a._mask, zeros(10)) + + + def check_sort(self): + "Test sort" + x = array([1,4,2,3],mask=[0,1,0,0],dtype=numpy.uint8) + # + sortedx = sort(x) + assert_equal(sortedx._data,[1,2,3,4]) + assert_equal(sortedx._mask,[0,0,0,1]) + # + sortedx = sort(x, endwith=False) + assert_equal(sortedx._data, [4,1,2,3]) + assert_equal(sortedx._mask, [1,0,0,0]) + # + x.sort() + assert_equal(x._data,[1,2,3,4]) + assert_equal(x._mask,[0,0,0,1]) + # + x = array([1,4,2,3],mask=[0,1,0,0],dtype=numpy.uint8) + x.sort(endwith=False) + assert_equal(x._data, [4,1,2,3]) + assert_equal(x._mask, [1,0,0,0]) + # + x = [1,4,2,3] + sortedx = sort(x) + assert(not isinstance(sorted, MaskedArray)) + # + x = array([0,1,-1,-2,2], mask=nomask, dtype=numpy.int8) + sortedx = sort(x, endwith=False) + assert_equal(sortedx._data, [-2,-1,0,1,2]) + x = array([0,1,-1,-2,2], mask=[0,1,0,0,1], dtype=numpy.int8) + sortedx = sort(x, endwith=False) + assert_equal(sortedx._data, [1,2,-2,-1,0]) + assert_equal(sortedx._mask, [1,1,0,0,0]) + + def check_sort_2d(self): + "Check sort of 2D array." + # 2D array w/o mask + a = masked_array([[8,4,1],[2,0,9]]) + a.sort(0) + assert_equal(a, [[2,0,1],[8,4,9]]) + a = masked_array([[8,4,1],[2,0,9]]) + a.sort(1) + assert_equal(a, [[1,4,8],[0,2,9]]) + # 2D array w/mask + a = masked_array([[8,4,1],[2,0,9]], mask=[[1,0,0],[0,0,1]]) + a.sort(0) + assert_equal(a, [[2,0,1],[8,4,9]]) + assert_equal(a._mask, [[0,0,0],[1,0,1]]) + a = masked_array([[8,4,1],[2,0,9]], mask=[[1,0,0],[0,0,1]]) + a.sort(1) + assert_equal(a, [[1,4,8],[0,2,9]]) + assert_equal(a._mask, [[0,0,1],[0,0,1]]) + # 3D + a = masked_array([[[7, 8, 9],[4, 5, 6],[1, 2, 3]], + [[1, 2, 3],[7, 8, 9],[4, 5, 6]], + [[7, 8, 9],[1, 2, 3],[4, 5, 6]], + [[4, 5, 6],[1, 2, 3],[7, 8, 9]]]) + a[a%4==0] = masked + am = a.copy() + an = a.filled(99) + am.sort(0) + an.sort(0) + assert_equal(am, an) + am = a.copy() + an = a.filled(99) + am.sort(1) + an.sort(1) + assert_equal(am, an) + am = a.copy() + an = a.filled(99) + am.sort(2) + an.sort(2) + assert_equal(am, an) + + + def check_ravel(self): + "Tests ravel" + a = array([[1,2,3,4,5]], mask=[[0,1,0,0,0]]) + aravel = a.ravel() + assert_equal(a._mask.shape, a.shape) + a = array([0,0], mask=[1,1]) + aravel = a.ravel() + assert_equal(a._mask.shape, a.shape) + a = array(numpy.matrix([1,2,3,4,5]), mask=[[0,1,0,0,0]]) + aravel = a.ravel() + assert_equal(a.shape,(1,5)) + assert_equal(a._mask.shape, a.shape) + # Checs that small_mask is preserved + a = array([1,2,3,4],mask=[0,0,0,0],shrink=False) + assert_equal(a.ravel()._mask, [0,0,0,0]) + + def check_reshape(self): + "Tests reshape" + x = arange(4) + x[0] = masked + y = x.reshape(2,2) + assert_equal(y.shape, (2,2,)) + assert_equal(y._mask.shape, (2,2,)) + assert_equal(x.shape, (4,)) + assert_equal(x._mask.shape, (4,)) + + def check_compressed(self): + "Tests compressed" + a = array([1,2,3,4],mask=[0,0,0,0]) + b = a.compressed() + assert_equal(b, a) + assert_equal(b._mask, nomask) + a[0] = masked + b = a.compressed() + assert_equal(b._data, [2,3,4]) + assert_equal(b._mask, nomask) + + def check_tolist(self): + "Tests to list" + x = array(numpy.arange(12)) + x[[1,-2]] = masked + xlist = x.tolist() + assert(xlist[1] is None) + assert(xlist[-2] is None) + # + x.shape = (3,4) + xlist = x.tolist() + # + assert_equal(xlist[0],[0,None,2,3]) + assert_equal(xlist[1],[4,5,6,7]) + assert_equal(xlist[2],[8,9,None,11]) + + def check_squeeze(self): + "Check squeeze" + data = masked_array([[1,2,3]]) + assert_equal(data.squeeze(), [1,2,3]) + data = masked_array([[1,2,3]], mask=[[1,1,1]]) + assert_equal(data.squeeze(), [1,2,3]) + assert_equal(data.squeeze()._mask, [1,1,1]) + data = masked_array([[1]], mask=True) + assert(data.squeeze() is masked) + +#.............................................................................. + +############################################################################### +#------------------------------------------------------------------------------ +if __name__ == "__main__": + NumpyTest().run() diff --git a/numpy/ma/tests/test_extras.py b/numpy/ma/tests/test_extras.py new file mode 100644 index 000000000..5a52aeeee --- /dev/null +++ b/numpy/ma/tests/test_extras.py @@ -0,0 +1,331 @@ +# pylint: disable-msg=W0611, W0612, W0511 +"""Tests suite for MaskedArray. +Adapted from the original test_ma by Pierre Gerard-Marchant + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: test_extras.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import numpy as N +from numpy.testing import NumpyTest, NumpyTestCase +from numpy.testing.utils import build_err_msg + +import numpy.ma.testutils +from numpy.ma.testutils import * + +import numpy.ma.core +from numpy.ma.core import * +import numpy.ma.extras +from numpy.ma.extras import * + +class TestAverage(NumpyTestCase): + "Several tests of average. Why so many ? Good point..." + def check_testAverage1(self): + "Test of average." + ott = array([0.,1.,2.,3.], mask=[1,0,0,0]) + assert_equal(2.0, average(ott,axis=0)) + assert_equal(2.0, average(ott, weights=[1., 1., 2., 1.])) + result, wts = average(ott, weights=[1.,1.,2.,1.], returned=1) + assert_equal(2.0, result) + assert(wts == 4.0) + ott[:] = masked + assert_equal(average(ott,axis=0).mask, [True]) + ott = array([0.,1.,2.,3.], mask=[1,0,0,0]) + ott = ott.reshape(2,2) + ott[:,1] = masked + assert_equal(average(ott,axis=0), [2.0, 0.0]) + assert_equal(average(ott,axis=1).mask[0], [True]) + assert_equal([2.,0.], average(ott, axis=0)) + result, wts = average(ott, axis=0, returned=1) + assert_equal(wts, [1., 0.]) + + def check_testAverage2(self): + "More tests of average." + w1 = [0,1,1,1,1,0] + w2 = [[0,1,1,1,1,0],[1,0,0,0,0,1]] + x = arange(6, dtype=float_) + assert_equal(average(x, axis=0), 2.5) + assert_equal(average(x, axis=0, weights=w1), 2.5) + y = array([arange(6, dtype=float_), 2.0*arange(6)]) + assert_equal(average(y, None), N.add.reduce(N.arange(6))*3./12.) + assert_equal(average(y, axis=0), N.arange(6) * 3./2.) + assert_equal(average(y, axis=1), [average(x,axis=0), average(x,axis=0) * 2.0]) + assert_equal(average(y, None, weights=w2), 20./6.) + assert_equal(average(y, axis=0, weights=w2), [0.,1.,2.,3.,4.,10.]) + assert_equal(average(y, axis=1), [average(x,axis=0), average(x,axis=0) * 2.0]) + m1 = zeros(6) + m2 = [0,0,1,1,0,0] + m3 = [[0,0,1,1,0,0],[0,1,1,1,1,0]] + m4 = ones(6) + m5 = [0, 1, 1, 1, 1, 1] + assert_equal(average(masked_array(x, m1),axis=0), 2.5) + assert_equal(average(masked_array(x, m2),axis=0), 2.5) + assert_equal(average(masked_array(x, m4),axis=0).mask, [True]) + assert_equal(average(masked_array(x, m5),axis=0), 0.0) + assert_equal(count(average(masked_array(x, m4),axis=0)), 0) + z = masked_array(y, m3) + assert_equal(average(z, None), 20./6.) + assert_equal(average(z, axis=0), [0.,1.,99.,99.,4.0, 7.5]) + assert_equal(average(z, axis=1), [2.5, 5.0]) + assert_equal(average(z,axis=0, weights=w2), [0.,1., 99., 99., 4.0, 10.0]) + + def check_testAverage3(self): + "Yet more tests of average!" + a = arange(6) + b = arange(6) * 3 + r1, w1 = average([[a,b],[b,a]], axis=1, returned=1) + assert_equal(shape(r1) , shape(w1)) + assert_equal(r1.shape , w1.shape) + r2, w2 = average(ones((2,2,3)), axis=0, weights=[3,1], returned=1) + assert_equal(shape(w2) , shape(r2)) + r2, w2 = average(ones((2,2,3)), returned=1) + assert_equal(shape(w2) , shape(r2)) + r2, w2 = average(ones((2,2,3)), weights=ones((2,2,3)), returned=1) + assert_equal(shape(w2), shape(r2)) + a2d = array([[1,2],[0,4]], float) + a2dm = masked_array(a2d, [[0,0],[1,0]]) + a2da = average(a2d, axis=0) + assert_equal(a2da, [0.5, 3.0]) + a2dma = average(a2dm, axis=0) + assert_equal(a2dma, [1.0, 3.0]) + a2dma = average(a2dm, axis=None) + assert_equal(a2dma, 7./3.) + a2dma = average(a2dm, axis=1) + assert_equal(a2dma, [1.5, 4.0]) + +class TestConcatenator(NumpyTestCase): + "Tests for mr_, the equivalent of r_ for masked arrays." + def check_1d(self): + "Tests mr_ on 1D arrays." + assert_array_equal(mr_[1,2,3,4,5,6],array([1,2,3,4,5,6])) + b = ones(5) + m = [1,0,0,0,0] + d = masked_array(b,mask=m) + c = mr_[d,0,0,d] + assert(isinstance(c,MaskedArray) or isinstance(c,core.MaskedArray)) + assert_array_equal(c,[1,1,1,1,1,0,0,1,1,1,1,1]) + assert_array_equal(c.mask, mr_[m,0,0,m]) + + def check_2d(self): + "Tests mr_ on 2D arrays." + a_1 = rand(5,5) + a_2 = rand(5,5) + m_1 = N.round_(rand(5,5),0) + m_2 = N.round_(rand(5,5),0) + b_1 = masked_array(a_1,mask=m_1) + b_2 = masked_array(a_2,mask=m_2) + d = mr_['1',b_1,b_2] # append columns + assert(d.shape == (5,10)) + assert_array_equal(d[:,:5],b_1) + assert_array_equal(d[:,5:],b_2) + assert_array_equal(d.mask, N.r_['1',m_1,m_2]) + d = mr_[b_1,b_2] + assert(d.shape == (10,5)) + assert_array_equal(d[:5,:],b_1) + assert_array_equal(d[5:,:],b_2) + assert_array_equal(d.mask, N.r_[m_1,m_2]) + +class TestNotMasked(NumpyTestCase): + "Tests notmasked_edges and notmasked_contiguous." + def check_edges(self): + "Tests unmasked_edges" + a = masked_array(N.arange(24).reshape(3,8), + mask=[[0,0,0,0,1,1,1,0], + [1,1,1,1,1,1,1,1], + [0,0,0,0,0,0,1,0],]) + # + assert_equal(notmasked_edges(a, None), [0,23]) + # + tmp = notmasked_edges(a, 0) + assert_equal(tmp[0], (array([0,0,0,0,2,2,0]), array([0,1,2,3,4,5,7]))) + assert_equal(tmp[1], (array([2,2,2,2,2,2,2]), array([0,1,2,3,4,5,7]))) + # + tmp = notmasked_edges(a, 1) + assert_equal(tmp[0], (array([0,2,]), array([0,0]))) + assert_equal(tmp[1], (array([0,2,]), array([7,7]))) + + def check_contiguous(self): + "Tests notmasked_contiguous" + a = masked_array(N.arange(24).reshape(3,8), + mask=[[0,0,0,0,1,1,1,1], + [1,1,1,1,1,1,1,1], + [0,0,0,0,0,0,1,0],]) + tmp = notmasked_contiguous(a, None) + assert_equal(tmp[-1], slice(23,23,None)) + assert_equal(tmp[-2], slice(16,21,None)) + assert_equal(tmp[-3], slice(0,3,None)) + # + tmp = notmasked_contiguous(a, 0) + assert(len(tmp[-1]) == 1) + assert(tmp[-2] is None) + assert_equal(tmp[-3],tmp[-1]) + assert(len(tmp[0]) == 2) + # + tmp = notmasked_contiguous(a, 1) + assert_equal(tmp[0][-1], slice(0,3,None)) + assert(tmp[1] is None) + assert_equal(tmp[2][-1], slice(7,7,None)) + assert_equal(tmp[2][-2], slice(0,5,None)) + +class Test2DFunctions(NumpyTestCase): + "Tests 2D functions" + def check_compress2d(self): + "Tests compress2d" + x = array(N.arange(9).reshape(3,3), mask=[[1,0,0],[0,0,0],[0,0,0]]) + assert_equal(compress_rowcols(x), [[4,5],[7,8]] ) + assert_equal(compress_rowcols(x,0), [[3,4,5],[6,7,8]] ) + assert_equal(compress_rowcols(x,1), [[1,2],[4,5],[7,8]] ) + x = array(x._data, mask=[[0,0,0],[0,1,0],[0,0,0]]) + assert_equal(compress_rowcols(x), [[0,2],[6,8]] ) + assert_equal(compress_rowcols(x,0), [[0,1,2],[6,7,8]] ) + assert_equal(compress_rowcols(x,1), [[0,2],[3,5],[6,8]] ) + x = array(x._data, mask=[[1,0,0],[0,1,0],[0,0,0]]) + assert_equal(compress_rowcols(x), [[8]] ) + assert_equal(compress_rowcols(x,0), [[6,7,8]] ) + assert_equal(compress_rowcols(x,1,), [[2],[5],[8]] ) + x = array(x._data, mask=[[1,0,0],[0,1,0],[0,0,1]]) + assert_equal(compress_rowcols(x).size, 0 ) + assert_equal(compress_rowcols(x,0).size, 0 ) + assert_equal(compress_rowcols(x,1).size, 0 ) + # + def check_mask_rowcols(self): + "Tests mask_rowcols." + x = array(N.arange(9).reshape(3,3), mask=[[1,0,0],[0,0,0],[0,0,0]]) + assert_equal(mask_rowcols(x).mask, [[1,1,1],[1,0,0],[1,0,0]] ) + assert_equal(mask_rowcols(x,0).mask, [[1,1,1],[0,0,0],[0,0,0]] ) + assert_equal(mask_rowcols(x,1).mask, [[1,0,0],[1,0,0],[1,0,0]] ) + x = array(x._data, mask=[[0,0,0],[0,1,0],[0,0,0]]) + assert_equal(mask_rowcols(x).mask, [[0,1,0],[1,1,1],[0,1,0]] ) + assert_equal(mask_rowcols(x,0).mask, [[0,0,0],[1,1,1],[0,0,0]] ) + assert_equal(mask_rowcols(x,1).mask, [[0,1,0],[0,1,0],[0,1,0]] ) + x = array(x._data, mask=[[1,0,0],[0,1,0],[0,0,0]]) + assert_equal(mask_rowcols(x).mask, [[1,1,1],[1,1,1],[1,1,0]] ) + assert_equal(mask_rowcols(x,0).mask, [[1,1,1],[1,1,1],[0,0,0]] ) + assert_equal(mask_rowcols(x,1,).mask, [[1,1,0],[1,1,0],[1,1,0]] ) + x = array(x._data, mask=[[1,0,0],[0,1,0],[0,0,1]]) + assert(mask_rowcols(x).all()) + assert(mask_rowcols(x,0).all()) + assert(mask_rowcols(x,1).all()) + # + def test_dot(self): + "Tests dot product" + n = N.arange(1,7) + # + m = [1,0,0,0,0,0] + a = masked_array(n, mask=m).reshape(2,3) + b = masked_array(n, mask=m).reshape(3,2) + c = dot(a,b,True) + assert_equal(c.mask, [[1,1],[1,0]]) + c = dot(b,a,True) + assert_equal(c.mask, [[1,1,1],[1,0,0],[1,0,0]]) + c = dot(a,b,False) + assert_equal(c, N.dot(a.filled(0), b.filled(0))) + c = dot(b,a,False) + assert_equal(c, N.dot(b.filled(0), a.filled(0))) + # + m = [0,0,0,0,0,1] + a = masked_array(n, mask=m).reshape(2,3) + b = masked_array(n, mask=m).reshape(3,2) + c = dot(a,b,True) + assert_equal(c.mask,[[0,1],[1,1]]) + c = dot(b,a,True) + assert_equal(c.mask, [[0,0,1],[0,0,1],[1,1,1]]) + c = dot(a,b,False) + assert_equal(c, N.dot(a.filled(0), b.filled(0))) + assert_equal(c, dot(a,b)) + c = dot(b,a,False) + assert_equal(c, N.dot(b.filled(0), a.filled(0))) + # + m = [0,0,0,0,0,0] + a = masked_array(n, mask=m).reshape(2,3) + b = masked_array(n, mask=m).reshape(3,2) + c = dot(a,b) + assert_equal(c.mask,nomask) + c = dot(b,a) + assert_equal(c.mask,nomask) + # + a = masked_array(n, mask=[1,0,0,0,0,0]).reshape(2,3) + b = masked_array(n, mask=[0,0,0,0,0,0]).reshape(3,2) + c = dot(a,b,True) + assert_equal(c.mask,[[1,1],[0,0]]) + c = dot(a,b,False) + assert_equal(c, N.dot(a.filled(0),b.filled(0))) + c = dot(b,a,True) + assert_equal(c.mask,[[1,0,0],[1,0,0],[1,0,0]]) + c = dot(b,a,False) + assert_equal(c, N.dot(b.filled(0),a.filled(0))) + # + a = masked_array(n, mask=[0,0,0,0,0,1]).reshape(2,3) + b = masked_array(n, mask=[0,0,0,0,0,0]).reshape(3,2) + c = dot(a,b,True) + assert_equal(c.mask,[[0,0],[1,1]]) + c = dot(a,b) + assert_equal(c, N.dot(a.filled(0),b.filled(0))) + c = dot(b,a,True) + assert_equal(c.mask,[[0,0,1],[0,0,1],[0,0,1]]) + c = dot(b,a,False) + assert_equal(c, N.dot(b.filled(0), a.filled(0))) + # + a = masked_array(n, mask=[0,0,0,0,0,1]).reshape(2,3) + b = masked_array(n, mask=[0,0,1,0,0,0]).reshape(3,2) + c = dot(a,b,True) + assert_equal(c.mask,[[1,0],[1,1]]) + c = dot(a,b,False) + assert_equal(c, N.dot(a.filled(0),b.filled(0))) + c = dot(b,a,True) + assert_equal(c.mask,[[0,0,1],[1,1,1],[0,0,1]]) + c = dot(b,a,False) + assert_equal(c, N.dot(b.filled(0),a.filled(0))) + + def test_mediff1d(self): + "Tests mediff1d" + x = masked_array(N.arange(5), mask=[1,0,0,0,1]) + difx_d = (x._data[1:]-x._data[:-1]) + difx_m = (x._mask[1:]-x._mask[:-1]) + dx = mediff1d(x) + assert_equal(dx._data, difx_d) + assert_equal(dx._mask, difx_m) + # + dx = mediff1d(x, to_begin=masked) + assert_equal(dx._data, N.r_[0,difx_d]) + assert_equal(dx._mask, N.r_[1,difx_m]) + dx = mediff1d(x, to_begin=[1,2,3]) + assert_equal(dx._data, N.r_[[1,2,3],difx_d]) + assert_equal(dx._mask, N.r_[[0,0,0],difx_m]) + # + dx = mediff1d(x, to_end=masked) + assert_equal(dx._data, N.r_[difx_d,0]) + assert_equal(dx._mask, N.r_[difx_m,1]) + dx = mediff1d(x, to_end=[1,2,3]) + assert_equal(dx._data, N.r_[difx_d,[1,2,3]]) + assert_equal(dx._mask, N.r_[difx_m,[0,0,0]]) + # + dx = mediff1d(x, to_end=masked, to_begin=masked) + assert_equal(dx._data, N.r_[0,difx_d,0]) + assert_equal(dx._mask, N.r_[1,difx_m,1]) + dx = mediff1d(x, to_end=[1,2,3], to_begin=masked) + assert_equal(dx._data, N.r_[0,difx_d,[1,2,3]]) + assert_equal(dx._mask, N.r_[1,difx_m,[0,0,0]]) + # + dx = mediff1d(x._data, to_end=masked, to_begin=masked) + assert_equal(dx._data, N.r_[0,difx_d,0]) + assert_equal(dx._mask, N.r_[1,0,0,0,0,1]) + +class TestApplyAlongAxis(NumpyTestCase): + "Tests 2D functions" + def check_3d(self): + a = arange(12.).reshape(2,2,3) + def myfunc(b): + return b[1] + xa = apply_along_axis(myfunc,2,a) + assert_equal(xa,[[1,4],[7,10]]) + +############################################################################### +#------------------------------------------------------------------------------ +if __name__ == "__main__": + NumpyTest().run() diff --git a/numpy/ma/tests/test_morestats.py b/numpy/ma/tests/test_morestats.py new file mode 100644 index 000000000..933e974da --- /dev/null +++ b/numpy/ma/tests/test_morestats.py @@ -0,0 +1,114 @@ +# pylint: disable-msg=W0611, W0612, W0511,R0201 +"""Tests suite for maskedArray statistics. + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: test_morestats.py 317 2007-10-04 19:31:14Z backtopop $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: backtopop $)" +__version__ = '1.0' +__revision__ = "$Revision: 317 $" +__date__ = '$Date: 2007-10-04 15:31:14 -0400 (Thu, 04 Oct 2007) $' + +import numpy + +import numpy.ma +from numpy.ma import masked, masked_array + +import numpy.ma.mstats +from numpy.ma.mstats import * +import numpy.ma.morestats +from numpy.ma.morestats import * + +import numpy.ma.testutils +from numpy.ma.testutils import * + + +class TestMisc(NumpyTestCase): + # + def __init__(self, *args, **kwargs): + NumpyTestCase.__init__(self, *args, **kwargs) + # + def test_mjci(self): + "Tests the Marits-Jarrett estimator" + data = masked_array([ 77, 87, 88,114,151,210,219,246,253,262, + 296,299,306,376,428,515,666,1310,2611]) + assert_almost_equal(mjci(data),[55.76819,45.84028,198.8788],5) + # + def test_trimmedmeanci(self): + "Tests the confidence intervals of the trimmed mean." + data = masked_array([545,555,558,572,575,576,578,580, + 594,605,635,651,653,661,666]) + assert_almost_equal(trimmed_mean(data,0.2), 596.2, 1) + assert_equal(numpy.round(trimmed_mean_ci(data,0.2),1), [561.8, 630.6]) + +#.............................................................................. +class TestRanking(NumpyTestCase): + # + def __init__(self, *args, **kwargs): + NumpyTestCase.__init__(self, *args, **kwargs) + # + def test_ranking(self): + x = masked_array([0,1,1,1,2,3,4,5,5,6,]) + assert_almost_equal(rank_data(x),[1,3,3,3,5,6,7,8.5,8.5,10]) + x[[3,4]] = masked + assert_almost_equal(rank_data(x),[1,2.5,2.5,0,0,4,5,6.5,6.5,8]) + assert_almost_equal(rank_data(x,use_missing=True), + [1,2.5,2.5,4.5,4.5,4,5,6.5,6.5,8]) + x = masked_array([0,1,5,1,2,4,3,5,1,6,]) + assert_almost_equal(rank_data(x),[1,3,8.5,3,5,7,6,8.5,3,10]) + x = masked_array([[0,1,1,1,2], [3,4,5,5,6,]]) + assert_almost_equal(rank_data(x),[[1,3,3,3,5],[6,7,8.5,8.5,10]]) + assert_almost_equal(rank_data(x,axis=1),[[1,3,3,3,5],[1,2,3.5,3.5,5]]) + assert_almost_equal(rank_data(x,axis=0),[[1,1,1,1,1],[2,2,2,2,2,]]) + +#.............................................................................. +class TestQuantiles(NumpyTestCase): + # + def __init__(self, *args, **kwargs): + NumpyTestCase.__init__(self, *args, **kwargs) + # + def test_hdquantiles(self): + data = [0.706560797,0.727229578,0.990399276,0.927065621,0.158953014, + 0.887764025,0.239407086,0.349638551,0.972791145,0.149789972, + 0.936947700,0.132359948,0.046041972,0.641675031,0.945530547, + 0.224218684,0.771450991,0.820257774,0.336458052,0.589113496, + 0.509736129,0.696838829,0.491323573,0.622767425,0.775189248, + 0.641461450,0.118455200,0.773029450,0.319280007,0.752229111, + 0.047841438,0.466295911,0.583850781,0.840581845,0.550086491, + 0.466470062,0.504765074,0.226855960,0.362641207,0.891620942, + 0.127898691,0.490094097,0.044882048,0.041441695,0.317976349, + 0.504135618,0.567353033,0.434617473,0.636243375,0.231803616, + 0.230154113,0.160011327,0.819464108,0.854706985,0.438809221, + 0.487427267,0.786907310,0.408367937,0.405534192,0.250444460, + 0.995309248,0.144389588,0.739947527,0.953543606,0.680051621, + 0.388382017,0.863530727,0.006514031,0.118007779,0.924024803, + 0.384236354,0.893687694,0.626534881,0.473051932,0.750134705, + 0.241843555,0.432947602,0.689538104,0.136934797,0.150206859, + 0.474335206,0.907775349,0.525869295,0.189184225,0.854284286, + 0.831089744,0.251637345,0.587038213,0.254475554,0.237781276, + 0.827928620,0.480283781,0.594514455,0.213641488,0.024194386, + 0.536668589,0.699497811,0.892804071,0.093835427,0.731107772] + # + assert_almost_equal(hdquantiles(data,[0., 1.]), + [0.006514031, 0.995309248]) + hdq = hdquantiles(data,[0.25, 0.5, 0.75]) + assert_almost_equal(hdq, [0.253210762, 0.512847491, 0.762232442,]) + hdq = hdquantiles_sd(data,[0.25, 0.5, 0.75]) + assert_almost_equal(hdq, [0.03786954, 0.03805389, 0.03800152,], 4) + # + data = numpy.array(data).reshape(10,10) + hdq = hdquantiles(data,[0.25,0.5,0.75],axis=0) + assert_almost_equal(hdq[:,0], hdquantiles(data[:,0],[0.25,0.5,0.75])) + assert_almost_equal(hdq[:,-1], hdquantiles(data[:,-1],[0.25,0.5,0.75])) + hdq = hdquantiles(data,[0.25,0.5,0.75],axis=0,var=True) + assert_almost_equal(hdq[...,0], + hdquantiles(data[:,0],[0.25,0.5,0.75],var=True)) + assert_almost_equal(hdq[...,-1], + hdquantiles(data[:,-1],[0.25,0.5,0.75], var=True)) + + +############################################################################### +#------------------------------------------------------------------------------ +if __name__ == "__main__": + NumpyTest().run() diff --git a/numpy/ma/tests/test_mrecords.py b/numpy/ma/tests/test_mrecords.py new file mode 100644 index 000000000..1d7d5a966 --- /dev/null +++ b/numpy/ma/tests/test_mrecords.py @@ -0,0 +1,181 @@ +# pylint: disable-msg=W0611, W0612, W0511,R0201 +"""Tests suite for mrecarray. + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: test_mrecords.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import types + +import numpy as N +import numpy.core.fromnumeric as fromnumeric +from numpy.testing import NumpyTest, NumpyTestCase +from numpy.testing.utils import build_err_msg + +import numpy.ma.testutils +from numpy.ma.testutils import * + +import numpy.ma +from numpy.ma import masked_array, masked, nomask + +#import numpy.ma.mrecords +#from numpy.ma.mrecords import mrecarray, fromarrays, fromtextfile, fromrecords + +import numpy.ma.mrecords +from numpy.ma.mrecords import MaskedRecords, \ + fromarrays, fromtextfile, fromrecords, addfield + +#.............................................................................. +class TestMRecords(NumpyTestCase): + "Base test class for MaskedArrays." + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + self.setup() + + def setup(self): + "Generic setup" + d = N.arange(5) + m = numpy.ma.make_mask([1,0,0,1,1]) + base_d = N.r_[d,d[::-1]].reshape(2,-1).T + base_m = N.r_[[m, m[::-1]]].T + base = masked_array(base_d, mask=base_m) + mrecord = fromarrays(base.T, dtype=[('a',N.float_),('b',N.float_)]) + self.data = [d, m, mrecord] + + def test_get(self): + "Tests fields retrieval" + [d, m, mrec] = self.data + mrec = mrec.copy() + assert_equal(mrec.a, masked_array(d,mask=m)) + assert_equal(mrec.b, masked_array(d[::-1],mask=m[::-1])) + assert((mrec._fieldmask == N.core.records.fromarrays([m, m[::-1]], dtype=mrec._fieldmask.dtype)).all()) + assert_equal(mrec._mask, N.r_[[m,m[::-1]]].all(0)) + assert_equal(mrec.a[1], mrec[1].a) + # + assert(isinstance(mrec[:2], MaskedRecords)) + assert_equal(mrec[:2]['a'], d[:2]) + + def test_set(self): + "Tests setting fields/attributes." + [d, m, mrecord] = self.data + mrecord.a._data[:] = 5 + assert_equal(mrecord['a']._data, [5,5,5,5,5]) + mrecord.a = 1 + assert_equal(mrecord['a']._data, [1]*5) + assert_equal(getmaskarray(mrecord['a']), [0]*5) + mrecord.b = masked + assert_equal(mrecord.b.mask, [1]*5) + assert_equal(getmaskarray(mrecord['b']), [1]*5) + mrecord._mask = masked + assert_equal(getmaskarray(mrecord['b']), [1]*5) + assert_equal(mrecord['a']._mask, mrecord['b']._mask) + mrecord._mask = nomask + assert_equal(getmaskarray(mrecord['b']), [0]*5) + assert_equal(mrecord['a']._mask, mrecord['b']._mask) + # + def test_setfields(self): + "Tests setting fields." + [d, m, mrecord] = self.data + mrecord.a[3:] = 5 + assert_equal(mrecord.a, [0,1,2,5,5]) + assert_equal(mrecord.a._mask, [1,0,0,0,0]) + # + mrecord.b[3:] = masked + assert_equal(mrecord.b, [4,3,2,1,0]) + assert_equal(mrecord.b._mask, [1,1,0,1,1]) + + def test_setslices(self): + "Tests setting slices." + [d, m, mrec] = self.data + mrec[:2] = 5 + assert_equal(mrec.a._data, [5,5,2,3,4]) + assert_equal(mrec.b._data, [5,5,2,1,0]) + assert_equal(mrec.a._mask, [0,0,0,1,1]) + assert_equal(mrec.b._mask, [0,0,0,0,1]) + # + mrec[:2] = masked + assert_equal(mrec._mask, [1,1,0,0,1]) + mrec[-2] = masked + assert_equal(mrec._mask, [1,1,0,1,1]) + # + def test_setslices_hardmask(self): + "Tests setting slices w/ hardmask." + [d, m, mrec] = self.data + mrec.harden_mask() + mrec[-2:] = 5 + assert_equal(mrec.a._data, [0,1,2,3,4]) + assert_equal(mrec.b._data, [4,3,2,5,0]) + assert_equal(mrec.a._mask, [1,0,0,1,1]) + assert_equal(mrec.b._mask, [1,1,0,0,1]) + + def test_hardmask(self): + "Test hardmask" + [d, m, mrec] = self.data + mrec = mrec.copy() + mrec.harden_mask() + assert(mrec._hardmask) + mrec._mask = nomask + assert_equal(mrec._mask, N.r_[[m,m[::-1]]].all(0)) + mrec.soften_mask() + assert(not mrec._hardmask) + mrec._mask = nomask + assert(mrec['b']._mask is nomask) + assert_equal(mrec['a']._mask,mrec['b']._mask) + + def test_fromrecords(self): + "Test from recarray." + [d, m, mrec] = self.data + nrec = N.core.records.fromarrays(N.r_[[d,d[::-1]]], + dtype=[('a',N.float_),('b',N.float_)]) + #.................... + mrecfr = fromrecords(nrec) + assert_equal(mrecfr.a, mrec.a) + assert_equal(mrecfr.dtype, mrec.dtype) + #.................... + tmp = mrec[::-1] #.tolist() + mrecfr = fromrecords(tmp) + assert_equal(mrecfr.a, mrec.a[::-1]) + #.................... + mrecfr = fromrecords(nrec.tolist(), names=nrec.dtype.names) + assert_equal(mrecfr.a, mrec.a) + assert_equal(mrecfr.dtype, mrec.dtype) + + def test_fromtextfile(self): + "Tests reading from a text file." + fcontent = """# +'One (S)','Two (I)','Three (F)','Four (M)','Five (-)','Six (C)' +'strings',1,1.0,'mixed column',,1 +'with embedded "double quotes"',2,2.0,1.0,,1 +'strings',3,3.0E5,3,,1 +'strings',4,-1e-10,,,1 +""" + import os + from datetime import datetime + fname = 'tmp%s' % datetime.now().strftime("%y%m%d%H%M%S%s") + f = open(fname, 'w') + f.write(fcontent) + f.close() + mrectxt = fromtextfile(fname,delimitor=',',varnames='ABCDEFG') + os.unlink(fname) + # + assert(isinstance(mrectxt, MaskedRecords)) + assert_equal(mrectxt.F, [1,1,1,1]) + assert_equal(mrectxt.E._mask, [1,1,1,1]) + assert_equal(mrectxt.C, [1,2,3.e+5,-1e-10]) + + def test_addfield(self): + "Tests addfield" + [d, m, mrec] = self.data + mrec = addfield(mrec, masked_array(d+10, mask=m[::-1])) + assert_equal(mrec.f2, d+10) + assert_equal(mrec.f2._mask, m[::-1]) + +############################################################################### +#------------------------------------------------------------------------------ +if __name__ == "__main__": + NumpyTest().run() diff --git a/numpy/ma/tests/test_mstats.py b/numpy/ma/tests/test_mstats.py new file mode 100644 index 000000000..e4657a58f --- /dev/null +++ b/numpy/ma/tests/test_mstats.py @@ -0,0 +1,174 @@ +# pylint: disable-msg=W0611, W0612, W0511,R0201 +"""Tests suite for maskedArray statistics. + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: test_mstats.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import numpy + +import numpy.ma +from numpy.ma import masked, masked_array + +import numpy.ma.testutils +from numpy.ma.testutils import * + +from numpy.ma.mstats import * + +#.............................................................................. +class TestQuantiles(NumpyTestCase): + "Base test class for MaskedArrays." + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + self.a = numpy.ma.arange(1,101) + # + def test_1d_nomask(self): + "Test quantiles 1D - w/o mask." + a = self.a + assert_almost_equal(mquantiles(a, alphap=1., betap=1.), + [25.75, 50.5, 75.25]) + assert_almost_equal(mquantiles(a, alphap=0, betap=1.), + [25., 50., 75.]) + assert_almost_equal(mquantiles(a, alphap=0.5, betap=0.5), + [25.5, 50.5, 75.5]) + assert_almost_equal(mquantiles(a, alphap=0., betap=0.), + [25.25, 50.5, 75.75]) + assert_almost_equal(mquantiles(a, alphap=1./3, betap=1./3), + [25.41666667, 50.5, 75.5833333]) + assert_almost_equal(mquantiles(a, alphap=3./8, betap=3./8), + [25.4375, 50.5, 75.5625]) + assert_almost_equal(mquantiles(a), [25.45, 50.5, 75.55])# + # + def test_1d_mask(self): + "Test quantiles 1D - w/ mask." + a = self.a + a[1::2] = masked + assert_almost_equal(mquantiles(a, alphap=1., betap=1.), + [25.5, 50.0, 74.5]) + assert_almost_equal(mquantiles(a, alphap=0, betap=1.), + [24., 49., 74.]) + assert_almost_equal(mquantiles(a, alphap=0.5, betap=0.5), + [25., 50., 75.]) + assert_almost_equal(mquantiles(a, alphap=0., betap=0.), + [24.5, 50.0, 75.5]) + assert_almost_equal(mquantiles(a, alphap=1./3, betap=1./3), + [24.833333, 50.0, 75.166666]) + assert_almost_equal(mquantiles(a, alphap=3./8, betap=3./8), + [24.875, 50., 75.125]) + assert_almost_equal(mquantiles(a), [24.9, 50., 75.1]) + # + def test_2d_nomask(self): + "Test quantiles 2D - w/o mask." + a = self.a + b = numpy.ma.resize(a, (100,100)) + assert_almost_equal(mquantiles(b), [25.45, 50.5, 75.55]) + assert_almost_equal(mquantiles(b, axis=0), numpy.ma.resize(a,(3,100))) + assert_almost_equal(mquantiles(b, axis=1), + numpy.ma.resize([25.45, 50.5, 75.55], (100,3))) + # + def test_2d_mask(self): + "Test quantiles 2D - w/ mask." + a = self.a + a[1::2] = masked + b = numpy.ma.resize(a, (100,100)) + assert_almost_equal(mquantiles(b), [25., 50., 75.]) + assert_almost_equal(mquantiles(b, axis=0), numpy.ma.resize(a,(3,100))) + assert_almost_equal(mquantiles(b, axis=1), + numpy.ma.resize([24.9, 50., 75.1], (100,3))) + +class TestMedian(NumpyTestCase): + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + + def test_2d(self): + "Tests median w/ 2D" + (n,p) = (101,30) + x = masked_array(numpy.linspace(-1.,1.,n),) + x[:10] = x[-10:] = masked + z = masked_array(numpy.empty((n,p), dtype=numpy.float_)) + z[:,0] = x[:] + idx = numpy.arange(len(x)) + for i in range(1,p): + numpy.random.shuffle(idx) + z[:,i] = x[idx] + assert_equal(mmedian(z[:,0]), 0) + assert_equal(mmedian(z), numpy.zeros((p,))) + + def test_3d(self): + "Tests median w/ 3D" + x = numpy.ma.arange(24).reshape(3,4,2) + x[x%3==0] = masked + assert_equal(mmedian(x,0), [[12,9],[6,15],[12,9],[18,15]]) + x.shape = (4,3,2) + assert_equal(mmedian(x,0),[[99,10],[11,99],[13,14]]) + x = numpy.ma.arange(24).reshape(4,3,2) + x[x%5==0] = masked + assert_equal(mmedian(x,0), [[12,10],[8,9],[16,17]]) + +#.............................................................................. +class TestTrimming(NumpyTestCase): + # + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + # + def test_trim(self): + "Tests trimming." + x = numpy.ma.arange(100) + assert_equal(trim_both(x).count(), 60) + assert_equal(trim_tail(x,tail='r').count(), 80) + x[50:70] = masked + trimx = trim_both(x) + assert_equal(trimx.count(), 48) + assert_equal(trimx._mask, [1]*16 + [0]*34 + [1]*20 + [0]*14 + [1]*16) + x._mask = nomask + x.shape = (10,10) + assert_equal(trim_both(x).count(), 60) + assert_equal(trim_tail(x).count(), 80) + # + def test_trimmedmean(self): + "Tests the trimmed mean." + data = masked_array([ 77, 87, 88,114,151,210,219,246,253,262, + 296,299,306,376,428,515,666,1310,2611]) + assert_almost_equal(trimmed_mean(data,0.1), 343, 0) + assert_almost_equal(trimmed_mean(data,0.2), 283, 0) + # + def test_trimmed_stde(self): + "Tests the trimmed mean standard error." + data = masked_array([ 77, 87, 88,114,151,210,219,246,253,262, + 296,299,306,376,428,515,666,1310,2611]) + assert_almost_equal(trimmed_stde(data,0.2), 56.1, 1) + # + def test_winsorization(self): + "Tests the Winsorization of the data." + data = masked_array([ 77, 87, 88,114,151,210,219,246,253,262, + 296,299,306,376,428,515,666,1310,2611]) + assert_almost_equal(winsorize(data).varu(), 21551.4, 1) + data[5] = masked + winsorized = winsorize(data) + assert_equal(winsorized.mask, data.mask) +#.............................................................................. + +class TestMisc(NumpyTestCase): + def __init__(self, *args, **kwds): + NumpyTestCase.__init__(self, *args, **kwds) + + def check_cov(self): + "Tests the cov function." + x = masked_array([[1,2,3],[4,5,6]], mask=[[1,0,0],[0,0,0]]) + c = cov(x[0]) + assert_equal(c, (x[0].anom()**2).sum()) + c = cov(x[1]) + assert_equal(c, (x[1].anom()**2).sum()/2.) + c = cov(x) + assert_equal(c[1,0], (x[0].anom()*x[1].anom()).sum()) + + +############################################################################### +#------------------------------------------------------------------------------ +if __name__ == "__main__": + NumpyTest().run() diff --git a/numpy/ma/tests/test_subclassing.py b/numpy/ma/tests/test_subclassing.py new file mode 100644 index 000000000..331ef887d --- /dev/null +++ b/numpy/ma/tests/test_subclassing.py @@ -0,0 +1,183 @@ +# pylint: disable-msg=W0611, W0612, W0511,R0201 +"""Tests suite for MaskedArray & subclassing. + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: test_subclassing.py 3473 2007-10-29 15:18:13Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = '1.0' +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + +import numpy as N +import numpy.core.numeric as numeric + +from numpy.testing import NumpyTest, NumpyTestCase + +import numpy.ma.testutils +from numpy.ma.testutils import * + +import numpy.ma.core as coremodule +from numpy.ma.core import * + + +class SubArray(N.ndarray): + """Defines a generic N.ndarray subclass, that stores some metadata + in the dictionary `info`.""" + def __new__(cls,arr,info={}): + x = N.asanyarray(arr).view(cls) + x.info = info + return x + def __array_finalize__(self, obj): + self.info = getattr(obj,'info',{}) + return + def __add__(self, other): + result = N.ndarray.__add__(self, other) + result.info.update({'added':result.info.pop('added',0)+1}) + return result +subarray = SubArray + +class MSubArray(SubArray,MaskedArray): + def __new__(cls, data, info=None, mask=nomask): + subarr = SubArray(data, info) + _data = MaskedArray.__new__(cls, data=subarr, mask=mask) + _data.info = subarr.info + return _data + def __array_finalize__(self,obj): + MaskedArray.__array_finalize__(self,obj) + SubArray.__array_finalize__(self, obj) + return + def _get_series(self): + _view = self.view(MaskedArray) + _view._sharedmask = False + return _view + _series = property(fget=_get_series) +msubarray = MSubArray + +class MMatrix(MaskedArray, N.matrix,): + def __new__(cls, data, mask=nomask): + mat = N.matrix(data) + _data = MaskedArray.__new__(cls, data=mat, mask=mask) + return _data + def __array_finalize__(self,obj): + N.matrix.__array_finalize__(self, obj) + MaskedArray.__array_finalize__(self,obj) + return + def _get_series(self): + _view = self.view(MaskedArray) + _view._sharedmask = False + return _view + _series = property(fget=_get_series) +mmatrix = MMatrix + + + +class TestSubclassing(NumpyTestCase): + """Test suite for masked subclasses of ndarray.""" + + def check_data_subclassing(self): + "Tests whether the subclass is kept." + x = N.arange(5) + m = [0,0,1,0,0] + xsub = SubArray(x) + xmsub = masked_array(xsub, mask=m) + assert isinstance(xmsub, MaskedArray) + assert_equal(xmsub._data, xsub) + assert isinstance(xmsub._data, SubArray) + + def check_maskedarray_subclassing(self): + "Tests subclassing MaskedArray" + x = N.arange(5) + mx = mmatrix(x,mask=[0,1,0,0,0]) + assert isinstance(mx._data, N.matrix) + "Tests masked_unary_operation" + assert isinstance(add(mx,mx), mmatrix) + assert isinstance(add(mx,x), mmatrix) + assert_equal(add(mx,x), mx+x) + assert isinstance(add(mx,mx)._data, N.matrix) + assert isinstance(add.outer(mx,mx), mmatrix) + "Tests masked_binary_operation" + assert isinstance(hypot(mx,mx), mmatrix) + assert isinstance(hypot(mx,x), mmatrix) + + def check_attributepropagation(self): + x = array(arange(5), mask=[0]+[1]*4) + my = masked_array(subarray(x)) + ym = msubarray(x) + # + z = (my+1) + assert isinstance(z,MaskedArray) + assert not isinstance(z, MSubArray) + assert isinstance(z._data, SubArray) + assert_equal(z._data.info, {}) + # + z = (ym+1) + assert isinstance(z, MaskedArray) + assert isinstance(z, MSubArray) + assert isinstance(z._data, SubArray) + assert z._data.info['added'] > 0 + # + ym._set_mask([1,0,0,0,1]) + assert_equal(ym._mask, [1,0,0,0,1]) + ym._series._set_mask([0,0,0,0,1]) + assert_equal(ym._mask, [0,0,0,0,1]) + # + xsub = subarray(x, info={'name':'x'}) + mxsub = masked_array(xsub) + assert hasattr(mxsub, 'info') + assert_equal(mxsub.info, xsub.info) + + def check_subclasspreservation(self): + "Checks that masked_array(...,subok=True) preserves the class." + x = N.arange(5) + m = [0,0,1,0,0] + xinfo = [(i,j) for (i,j) in zip(x,m)] + xsub = MSubArray(x, mask=m, info={'xsub':xinfo}) + # + mxsub = masked_array(xsub, subok=False) + assert not isinstance(mxsub, MSubArray) + assert isinstance(mxsub, MaskedArray) + assert_equal(mxsub._mask, m) + # + mxsub = asarray(xsub) + assert not isinstance(mxsub, MSubArray) + assert isinstance(mxsub, MaskedArray) + assert_equal(mxsub._mask, m) + # + mxsub = masked_array(xsub, subok=True) + assert isinstance(mxsub, MSubArray) + assert_equal(mxsub.info, xsub.info) + assert_equal(mxsub._mask, xsub._mask) + # + mxsub = asanyarray(xsub) + assert isinstance(mxsub, MSubArray) + assert_equal(mxsub.info, xsub.info) + assert_equal(mxsub._mask, m) + + +################################################################################ +if __name__ == '__main__': + NumpyTest().run() + # + if 0: + x = array(arange(5), mask=[0]+[1]*4) + my = masked_array(subarray(x)) + ym = msubarray(x) + # + z = (my+1) + assert isinstance(z,MaskedArray) + assert not isinstance(z, MSubArray) + assert isinstance(z._data, SubArray) + assert_equal(z._data.info, {}) + # + z = (ym+1) + assert isinstance(z, MaskedArray) + assert isinstance(z, MSubArray) + assert isinstance(z._data, SubArray) + assert z._data.info['added'] > 0 + # + ym._set_mask([1,0,0,0,1]) + assert_equal(ym._mask, [1,0,0,0,1]) + ym._series._set_mask([0,0,0,0,1]) + assert_equal(ym._mask, [0,0,0,0,1]) diff --git a/numpy/ma/testutils.py b/numpy/ma/testutils.py new file mode 100644 index 000000000..d51e8dab1 --- /dev/null +++ b/numpy/ma/testutils.py @@ -0,0 +1,220 @@ +"""Miscellaneous functions for testing masked arrays and subclasses + +:author: Pierre Gerard-Marchant +:contact: pierregm_at_uga_dot_edu +:version: $Id: testutils.py 3529 2007-11-13 08:01:14Z jarrod.millman $ +""" +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__version__ = "1.0" +__revision__ = "$Revision: 3529 $" +__date__ = "$Date: 2007-11-13 10:01:14 +0200 (Tue, 13 Nov 2007) $" + + +import numpy as N +from numpy.core import ndarray +from numpy.core.numerictypes import float_ +import numpy.core.umath as umath +from numpy.testing import NumpyTest, NumpyTestCase +from numpy.testing.utils import build_err_msg, rand + +import core +from core import mask_or, getmask, getmaskarray, masked_array, nomask, masked +from core import filled, equal, less + +#------------------------------------------------------------------------------ +def approx (a, b, fill_value=True, rtol=1.e-5, atol=1.e-8): + """Returns true if all components of a and b are equal subject to given tolerances. + +If fill_value is True, masked values considered equal. Otherwise, masked values +are considered unequal. +The relative error rtol should be positive and << 1.0 +The absolute error atol comes into play for those elements of b that are very +small or zero; it says how small a must be also. + """ + m = mask_or(getmask(a), getmask(b)) + d1 = filled(a) + d2 = filled(b) + if d1.dtype.char == "O" or d2.dtype.char == "O": + return N.equal(d1,d2).ravel() + x = filled(masked_array(d1, copy=False, mask=m), fill_value).astype(float_) + y = filled(masked_array(d2, copy=False, mask=m), 1).astype(float_) + d = N.less_equal(umath.absolute(x-y), atol + rtol * umath.absolute(y)) + return d.ravel() +#................................................ +def _assert_equal_on_sequences(actual, desired, err_msg=''): + "Asserts the equality of two non-array sequences." + assert_equal(len(actual),len(desired),err_msg) + for k in range(len(desired)): + assert_equal(actual[k], desired[k], 'item=%r\n%s' % (k,err_msg)) + return + +def assert_equal_records(a,b): + """Asserts that two records are equal. Pretty crude for now.""" + assert_equal(a.dtype, b.dtype) + for f in a.dtype.names: + (af, bf) = (getattr(a,f), getattr(b,f)) + if not (af is masked) and not (bf is masked): + assert_equal(getattr(a,f), getattr(b,f)) + return + +def assert_equal(actual,desired,err_msg=''): + """Asserts that two items are equal. + """ + # Case #1: dictionary ..... + if isinstance(desired, dict): + assert isinstance(actual, dict), repr(type(actual)) + assert_equal(len(actual),len(desired),err_msg) + for k,i in desired.items(): + assert k in actual, repr(k) + assert_equal(actual[k], desired[k], 'key=%r\n%s' % (k,err_msg)) + return + # Case #2: lists ..... + if isinstance(desired, (list,tuple)) and isinstance(actual, (list,tuple)): + return _assert_equal_on_sequences(actual, desired, err_msg='') + if not (isinstance(actual, ndarray) or isinstance(desired, ndarray)): + msg = build_err_msg([actual, desired], err_msg,) + assert desired == actual, msg + return + # Case #4. arrays or equivalent + if ((actual is masked) and not (desired is masked)) or \ + ((desired is masked) and not (actual is masked)): + msg = build_err_msg([actual, desired], err_msg, header='', names=('x', 'y')) + raise ValueError(msg) + actual = N.array(actual, copy=False, subok=True) + desired = N.array(desired, copy=False, subok=True) + if actual.dtype.char in "OS" and desired.dtype.char in "OS": + return _assert_equal_on_sequences(actual.tolist(), + desired.tolist(), + err_msg='') + return assert_array_equal(actual, desired, err_msg) +#............................. +def fail_if_equal(actual,desired,err_msg='',): + """Raises an assertion error if two items are equal. + """ + if isinstance(desired, dict): + assert isinstance(actual, dict), repr(type(actual)) + fail_if_equal(len(actual),len(desired),err_msg) + for k,i in desired.items(): + assert k in actual, repr(k) + fail_if_equal(actual[k], desired[k], 'key=%r\n%s' % (k,err_msg)) + return + if isinstance(desired, (list,tuple)) and isinstance(actual, (list,tuple)): + fail_if_equal(len(actual),len(desired),err_msg) + for k in range(len(desired)): + fail_if_equal(actual[k], desired[k], 'item=%r\n%s' % (k,err_msg)) + return + if isinstance(actual, N.ndarray) or isinstance(desired, N.ndarray): + return fail_if_array_equal(actual, desired, err_msg) + msg = build_err_msg([actual, desired], err_msg) + assert desired != actual, msg +assert_not_equal = fail_if_equal +#............................ +def assert_almost_equal(actual,desired,decimal=7,err_msg=''): + """Asserts that two items are almost equal. + The test is equivalent to abs(desired-actual) < 0.5 * 10**(-decimal) + """ + if isinstance(actual, N.ndarray) or isinstance(desired, N.ndarray): + return assert_array_almost_equal(actual, desired, decimal, err_msg) + msg = build_err_msg([actual, desired], err_msg) + assert round(abs(desired - actual),decimal) == 0, msg +#............................ +def assert_array_compare(comparison, x, y, err_msg='', header='', + fill_value=True): + """Asserts that a comparison relation between two masked arrays is satisfied + elementwise.""" + xf = filled(x) + yf = filled(y) + m = mask_or(getmask(x), getmask(y)) + + x = masked_array(xf, copy=False, subok=False, mask=m).filled(fill_value) + y = masked_array(yf, copy=False, subok=False, mask=m).filled(fill_value) + + if ((x is masked) and not (y is masked)) or \ + ((y is masked) and not (x is masked)): + msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y')) + raise ValueError(msg) + + if (x.dtype.char != "O") and (x.dtype.char != "S"): + x = x.astype(float_) + if isinstance(x, N.ndarray) and x.size > 1: + x[N.isnan(x)] = 0 + elif N.isnan(x): + x = 0 + if (y.dtype.char != "O") and (y.dtype.char != "S"): + y = y.astype(float_) + if isinstance(y, N.ndarray) and y.size > 1: + y[N.isnan(y)] = 0 + elif N.isnan(y): + y = 0 + try: + cond = (x.shape==() or y.shape==()) or x.shape == y.shape + if not cond: + msg = build_err_msg([x, y], + err_msg + + '\n(shapes %s, %s mismatch)' % (x.shape, + y.shape), + header=header, + names=('x', 'y')) + assert cond, msg + val = comparison(x,y) + if m is not nomask and fill_value: + val = masked_array(val, mask=m, copy=False) + if isinstance(val, bool): + cond = val + reduced = [0] + else: + reduced = val.ravel() + cond = reduced.all() + reduced = reduced.tolist() + if not cond: + match = 100-100.0*reduced.count(1)/len(reduced) + msg = build_err_msg([x, y], + err_msg + + '\n(mismatch %s%%)' % (match,), + header=header, + names=('x', 'y')) + assert cond, msg + except ValueError: + msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y')) + raise ValueError(msg) +#............................ +def assert_array_equal(x, y, err_msg=''): + """Checks the elementwise equality of two masked arrays.""" + assert_array_compare(equal, x, y, err_msg=err_msg, + header='Arrays are not equal') +##............................ +def fail_if_array_equal(x, y, err_msg=''): + "Raises an assertion error if two masked arrays are not equal (elementwise)." + def compare(x,y): + + return (not N.alltrue(approx(x, y))) + assert_array_compare(compare, x, y, err_msg=err_msg, + header='Arrays are not equal') +#............................ +def assert_array_almost_equal(x, y, decimal=6, err_msg=''): + """Checks the elementwise equality of two masked arrays, up to a given + number of decimals.""" + def compare(x, y): + "Returns the result of the loose comparison between x and y)." + return approx(x,y, rtol=10.**-decimal) + assert_array_compare(compare, x, y, err_msg=err_msg, + header='Arrays are not almost equal') +#............................ +def assert_array_less(x, y, err_msg=''): + "Checks that x is smaller than y elementwise." + assert_array_compare(less, x, y, err_msg=err_msg, + header='Arrays are not less-ordered') +#............................ +assert_close = assert_almost_equal +#............................ +def assert_mask_equal(m1, m2): + """Asserts the equality of two masks.""" + if m1 is nomask: + assert(m2 is nomask) + if m2 is nomask: + assert(m1 is nomask) + assert_array_equal(m1, m2) + +if __name__ == '__main__': + a = 12 + assert_equal(a, masked) diff --git a/numpy/ma/timer_comparison.py b/numpy/ma/timer_comparison.py new file mode 100644 index 000000000..4f789ec53 --- /dev/null +++ b/numpy/ma/timer_comparison.py @@ -0,0 +1,461 @@ + +import timeit + +import numpy +from numpy import int_, float_, bool_ +import numpy.core.fromnumeric as fromnumeric + +from numpy.testing.utils import build_err_msg, rand + + +numpy.seterr(all='ignore') + +pi = numpy.pi + +class moduletester: + #----------------------------------- + def __init__(self, module): + self.module = module + self.allequal = module.allequal + self.arange = module.arange + self.array = module.array +# self.average = module.average + self.concatenate = module.concatenate + self.count = module.count + self.equal = module.equal + self.filled = module.filled + self.getmask = module.getmask + self.getmaskarray = module.getmaskarray + self.id = id + self.inner = module.inner + self.make_mask = module.make_mask + self.masked = module.masked + self.masked_array = module.masked_array + self.masked_values = module.masked_values + self.mask_or = module.mask_or + self.nomask = module.nomask + self.ones = module.ones + self.outer = module.outer + self.repeat = module.repeat + self.resize = module.resize + self.sort = module.sort + self.take = module.take + self.transpose = module.transpose + self.zeros = module.zeros + self.MaskType = module.MaskType + try: + self.umath = module.umath + except AttributeError: + self.umath = module.core.umath + self.testnames = [] + #........................ + def assert_array_compare(self, comparison, x, y, err_msg='', header='', + fill_value=True): + """Asserts that a comparison relation between two masked arrays is satisfied + elementwise.""" + xf = self.filled(x) + yf = self.filled(y) + m = self.mask_or(self.getmask(x), self.getmask(y)) + + x = self.filled(self.masked_array(xf, mask=m), fill_value) + y = self.filled(self.masked_array(yf, mask=m), fill_value) + if (x.dtype.char != "O"): + x = x.astype(float_) + if isinstance(x, numpy.ndarray) and x.size > 1: + x[numpy.isnan(x)] = 0 + elif numpy.isnan(x): + x = 0 + if (y.dtype.char != "O"): + y = y.astype(float_) + if isinstance(y, numpy.ndarray) and y.size > 1: + y[numpy.isnan(y)] = 0 + elif numpy.isnan(y): + y = 0 + try: + cond = (x.shape==() or y.shape==()) or x.shape == y.shape + if not cond: + msg = build_err_msg([x, y], + err_msg + + '\n(shapes %s, %s mismatch)' % (x.shape, + y.shape), + header=header, + names=('x', 'y')) + assert cond, msg + val = comparison(x,y) + if m is not self.nomask and fill_value: + val = self.masked_array(val, mask=m) + if isinstance(val, bool): + cond = val + reduced = [0] + else: + reduced = val.ravel() + cond = reduced.all() + reduced = reduced.tolist() + if not cond: + match = 100-100.0*reduced.count(1)/len(reduced) + msg = build_err_msg([x, y], + err_msg + + '\n(mismatch %s%%)' % (match,), + header=header, + names=('x', 'y')) + assert cond, msg + except ValueError: + msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y')) + raise ValueError(msg) + #............................ + def assert_array_equal(self, x, y, err_msg=''): + """Checks the elementwise equality of two masked arrays.""" + self.assert_array_compare(self.equal, x, y, err_msg=err_msg, + header='Arrays are not equal') + #---------------------------------- + def test_0(self): + "Tests creation" + x = numpy.array([1.,1.,1.,-2., pi/2.0, 4., 5., -10., 10., 1., 2., 3.]) + m = [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + xm = self.masked_array(x, mask=m) + xm[0] + #---------------------------------- + def test_1(self): + "Tests creation" + x = numpy.array([1.,1.,1.,-2., pi/2.0, 4., 5., -10., 10., 1., 2., 3.]) + y = numpy.array([5.,0.,3., 2., -1., -4., 0., -10., 10., 1., 0., 3.]) + a10 = 10. + m1 = [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + m2 = [0, 0, 1, 0, 0, 1, 1, 0, 0, 0 ,0, 1] + xm = self.masked_array(x, mask=m1) + ym = self.masked_array(y, mask=m2) + z = numpy.array([-.5, 0., .5, .8]) + zm = self.masked_array(z, mask=[0,1,0,0]) + xf = numpy.where(m1, 1.e+20, x) + xm.set_fill_value(1.e+20) + #..... + assert((xm-ym).filled(0).any()) + #fail_if_equal(xm.mask.astype(int_), ym.mask.astype(int_)) + s = x.shape + assert(xm.size == reduce(lambda x,y:x*y, s)) + assert(self.count(xm) == len(m1) - reduce(lambda x,y:x+y, m1)) + #..... + for s in [(4,3), (6,2)]: + x.shape = s + y.shape = s + xm.shape = s + ym.shape = s + xf.shape = s + + assert(self.count(xm) == len(m1) - reduce(lambda x,y:x+y, m1)) + #---------------------------------- + def test_2(self): + "Tests conversions and indexing" + x1 = numpy.array([1,2,4,3]) + x2 = self.array(x1, mask=[1,0,0,0]) + x3 = self.array(x1, mask=[0,1,0,1]) + x4 = self.array(x1) + # test conversion to strings + junk, garbage = str(x2), repr(x2) +# assert_equal(numpy.sort(x1), self.sort(x2, fill_value=0)) + # tests of indexing + assert type(x2[1]) is type(x1[1]) + assert x1[1] == x2[1] +# assert self.allequal(x1[2],x2[2]) +# assert self.allequal(x1[2:5],x2[2:5]) +# assert self.allequal(x1[:],x2[:]) +# assert self.allequal(x1[1:], x3[1:]) + x1[2] = 9 + x2[2] = 9 + self.assert_array_equal(x1,x2) + x1[1:3] = 99 + x2[1:3] = 99 +# assert self.allequal(x1,x2) + x2[1] = self.masked +# assert self.allequal(x1,x2) + x2[1:3] = self.masked +# assert self.allequal(x1,x2) + x2[:] = x1 + x2[1] = self.masked +# assert self.allequal(self.getmask(x2),self.array([0,1,0,0])) + x3[:] = self.masked_array([1,2,3,4],[0,1,1,0]) +# assert self.allequal(self.getmask(x3), self.array([0,1,1,0])) + x4[:] = self.masked_array([1,2,3,4],[0,1,1,0]) +# assert self.allequal(self.getmask(x4), self.array([0,1,1,0])) +# assert self.allequal(x4, self.array([1,2,3,4])) + x1 = numpy.arange(5)*1.0 + x2 = self.masked_values(x1, 3.0) +# assert self.allequal(x1,x2) +# assert self.allequal(self.array([0,0,0,1,0], self.MaskType), x2.mask) + x1 = self.array([1,'hello',2,3],object) + x2 = numpy.array([1,'hello',2,3],object) + s1 = x1[1] + s2 = x2[1] + assert x1[1:1].shape == (0,) + # Tests copy-size + n = [0,0,1,0,0] + m = self.make_mask(n) + m2 = self.make_mask(m) + assert(m is m2) + m3 = self.make_mask(m, copy=1) + assert(m is not m3) + + #---------------------------------- + def test_3(self): + "Tests resize/repeat" + x4 = self.arange(4) + x4[2] = self.masked + y4 = self.resize(x4, (8,)) + assert self.allequal(self.concatenate([x4,x4]), y4) + assert self.allequal(self.getmask(y4),[0,0,1,0,0,0,1,0]) + y5 = self.repeat(x4, (2,2,2,2), axis=0) + self.assert_array_equal(y5, [0,0,1,1,2,2,3,3]) + y6 = self.repeat(x4, 2, axis=0) + assert self.allequal(y5, y6) + y7 = x4.repeat((2,2,2,2), axis=0) + assert self.allequal(y5,y7) + y8 = x4.repeat(2,0) + assert self.allequal(y5,y8) + + #---------------------------------- + def test_4(self): + "Test of take, transpose, inner, outer products" + x = self.arange(24) + y = numpy.arange(24) + x[5:6] = self.masked + x = x.reshape(2,3,4) + y = y.reshape(2,3,4) + assert self.allequal(numpy.transpose(y,(2,0,1)), self.transpose(x,(2,0,1))) + assert self.allequal(numpy.take(y, (2,0,1), 1), self.take(x, (2,0,1), 1)) + assert self.allequal(numpy.inner(self.filled(x,0), self.filled(y,0)), + self.inner(x, y)) + assert self.allequal(numpy.outer(self.filled(x,0), self.filled(y,0)), + self.outer(x, y)) + y = self.array(['abc', 1, 'def', 2, 3], object) + y[2] = self.masked + t = self.take(y,[0,3,4]) + assert t[0] == 'abc' + assert t[1] == 2 + assert t[2] == 3 + #---------------------------------- + def test_5(self): + "Tests inplace w/ scalar" + + x = self.arange(10) + y = self.arange(10) + xm = self.arange(10) + xm[2] = self.masked + x += 1 + assert self.allequal(x, y+1) + xm += 1 + assert self.allequal(xm, y+1) + + x = self.arange(10) + xm = self.arange(10) + xm[2] = self.masked + x -= 1 + assert self.allequal(x, y-1) + xm -= 1 + assert self.allequal(xm, y-1) + + x = self.arange(10)*1.0 + xm = self.arange(10)*1.0 + xm[2] = self.masked + x *= 2.0 + assert self.allequal(x, y*2) + xm *= 2.0 + assert self.allequal(xm, y*2) + + x = self.arange(10)*2 + xm = self.arange(10)*2 + xm[2] = self.masked + x /= 2 + assert self.allequal(x, y) + xm /= 2 + assert self.allequal(xm, y) + + x = self.arange(10)*1.0 + xm = self.arange(10)*1.0 + xm[2] = self.masked + x /= 2.0 + assert self.allequal(x, y/2.0) + xm /= self.arange(10) + self.assert_array_equal(xm, self.ones((10,))) + + x = self.arange(10).astype(float_) + xm = self.arange(10) + xm[2] = self.masked + id1 = self.id(x.raw_data()) + x += 1. + #assert id1 == self.id(x.raw_data()) + assert self.allequal(x, y+1.) + + + def test_6(self): + "Tests inplace w/ array" + + x = self.arange(10, dtype=float_) + y = self.arange(10) + xm = self.arange(10, dtype=float_) + xm[2] = self.masked + m = xm.mask + a = self.arange(10, dtype=float_) + a[-1] = self.masked + x += a + xm += a + assert self.allequal(x,y+a) + assert self.allequal(xm,y+a) + assert self.allequal(xm.mask, self.mask_or(m,a.mask)) + + x = self.arange(10, dtype=float_) + xm = self.arange(10, dtype=float_) + xm[2] = self.masked + m = xm.mask + a = self.arange(10, dtype=float_) + a[-1] = self.masked + x -= a + xm -= a + assert self.allequal(x,y-a) + assert self.allequal(xm,y-a) + assert self.allequal(xm.mask, self.mask_or(m,a.mask)) + + x = self.arange(10, dtype=float_) + xm = self.arange(10, dtype=float_) + xm[2] = self.masked + m = xm.mask + a = self.arange(10, dtype=float_) + a[-1] = self.masked + x *= a + xm *= a + assert self.allequal(x,y*a) + assert self.allequal(xm,y*a) + assert self.allequal(xm.mask, self.mask_or(m,a.mask)) + + x = self.arange(10, dtype=float_) + xm = self.arange(10, dtype=float_) + xm[2] = self.masked + m = xm.mask + a = self.arange(10, dtype=float_) + a[-1] = self.masked + x /= a + xm /= a + + #---------------------------------- + def test_7(self): + "Tests ufunc" + d = (self.array([1.0, 0, -1, pi/2]*2, mask=[0,1]+[0]*6), + self.array([1.0, 0, -1, pi/2]*2, mask=[1,0]+[0]*6),) + for f in ['sqrt', 'log', 'log10', 'exp', 'conjugate', +# 'sin', 'cos', 'tan', +# 'arcsin', 'arccos', 'arctan', +# 'sinh', 'cosh', 'tanh', +# 'arcsinh', +# 'arccosh', +# 'arctanh', +# 'absolute', 'fabs', 'negative', +# # 'nonzero', 'around', +# 'floor', 'ceil', +# # 'sometrue', 'alltrue', +# 'logical_not', +# 'add', 'subtract', 'multiply', +# 'divide', 'true_divide', 'floor_divide', +# 'remainder', 'fmod', 'hypot', 'arctan2', +# 'equal', 'not_equal', 'less_equal', 'greater_equal', +# 'less', 'greater', +# 'logical_and', 'logical_or', 'logical_xor', + ]: + #print f + try: + uf = getattr(self.umath, f) + except AttributeError: + uf = getattr(fromnumeric, f) + mf = getattr(self.module, f) + args = d[:uf.nin] + ur = uf(*args) + mr = mf(*args) + self.assert_array_equal(ur.filled(0), mr.filled(0), f) + self.assert_array_equal(ur._mask, mr._mask) + + #---------------------------------- + def test_99(self): + # test average + ott = self.array([0.,1.,2.,3.], mask=[1,0,0,0]) + self.assert_array_equal(2.0, self.average(ott,axis=0)) + self.assert_array_equal(2.0, self.average(ott, weights=[1., 1., 2., 1.])) + result, wts = self.average(ott, weights=[1.,1.,2.,1.], returned=1) + self.assert_array_equal(2.0, result) + assert(wts == 4.0) + ott[:] = self.masked + assert(self.average(ott,axis=0) is self.masked) + ott = self.array([0.,1.,2.,3.], mask=[1,0,0,0]) + ott = ott.reshape(2,2) + ott[:,1] = self.masked + self.assert_array_equal(self.average(ott,axis=0), [2.0, 0.0]) + assert(self.average(ott,axis=1)[0] is self.masked) + self.assert_array_equal([2.,0.], self.average(ott, axis=0)) + result, wts = self.average(ott, axis=0, returned=1) + self.assert_array_equal(wts, [1., 0.]) + w1 = [0,1,1,1,1,0] + w2 = [[0,1,1,1,1,0],[1,0,0,0,0,1]] + x = self.arange(6) + self.assert_array_equal(self.average(x, axis=0), 2.5) + self.assert_array_equal(self.average(x, axis=0, weights=w1), 2.5) + y = self.array([self.arange(6), 2.0*self.arange(6)]) + self.assert_array_equal(self.average(y, None), numpy.add.reduce(numpy.arange(6))*3./12.) + self.assert_array_equal(self.average(y, axis=0), numpy.arange(6) * 3./2.) + self.assert_array_equal(self.average(y, axis=1), [self.average(x,axis=0), self.average(x,axis=0) * 2.0]) + self.assert_array_equal(self.average(y, None, weights=w2), 20./6.) + self.assert_array_equal(self.average(y, axis=0, weights=w2), [0.,1.,2.,3.,4.,10.]) + self.assert_array_equal(self.average(y, axis=1), [self.average(x,axis=0), self.average(x,axis=0) * 2.0]) + m1 = self.zeros(6) + m2 = [0,0,1,1,0,0] + m3 = [[0,0,1,1,0,0],[0,1,1,1,1,0]] + m4 = self.ones(6) + m5 = [0, 1, 1, 1, 1, 1] + self.assert_array_equal(self.average(self.masked_array(x, m1),axis=0), 2.5) + self.assert_array_equal(self.average(self.masked_array(x, m2),axis=0), 2.5) + # assert(self.average(masked_array(x, m4),axis=0) is masked) + self.assert_array_equal(self.average(self.masked_array(x, m5),axis=0), 0.0) + self.assert_array_equal(self.count(self.average(self.masked_array(x, m4),axis=0)), 0) + z = self.masked_array(y, m3) + self.assert_array_equal(self.average(z, None), 20./6.) + self.assert_array_equal(self.average(z, axis=0), [0.,1.,99.,99.,4.0, 7.5]) + self.assert_array_equal(self.average(z, axis=1), [2.5, 5.0]) + self.assert_array_equal(self.average(z,axis=0, weights=w2), [0.,1., 99., 99., 4.0, 10.0]) + #------------------------ + def test_A(self): + x = self.arange(24) + y = numpy.arange(24) + x[5:6] = self.masked + x = x.reshape(2,3,4) + + +################################################################################ +if __name__ == '__main__': + + setup_base = "from __main__ import moduletester \n"\ + "import numpy\n" \ + "tester = moduletester(module)\n" + setup_old = "import numpy.core.ma as module\n"+setup_base + setup_new = "import maskedarray.core_ini as module\n"+setup_base + setup_cur = "import maskedarray.core as module\n"+setup_base +# setup_alt = "import maskedarray.core_alt as module\n"+setup_base +# setup_tmp = "import maskedarray.core_tmp as module\n"+setup_base + + (nrepeat, nloop) = (10, 10) + + if 1: + for i in range(1,8): + func = 'tester.test_%i()' % i + old = timeit.Timer(func, setup_old).repeat(nrepeat, nloop*10) + new = timeit.Timer(func, setup_new).repeat(nrepeat, nloop*10) + cur = timeit.Timer(func, setup_cur).repeat(nrepeat, nloop*10) +# alt = timeit.Timer(func, setup_alt).repeat(nrepeat, nloop*10) +# tmp = timeit.Timer(func, setup_tmp).repeat(nrepeat, nloop*10) + old = numpy.sort(old) + new = numpy.sort(new) + cur = numpy.sort(cur) +# alt = numpy.sort(alt) +# tmp = numpy.sort(tmp) + print "#%i" % i +50*'.' + print eval("moduletester.test_%i.__doc__" % i) + print "numpy.core.ma: %.3f - %.3f" % (old[0], old[1]) + print "core_ini : %.3f - %.3f" % (new[0], new[1]) + print "core_current : %.3f - %.3f" % (cur[0], cur[1]) +# print "core_alt : %.3f - %.3f" % (alt[0], alt[1]) +# print "core_tmp : %.3f - %.3f" % (tmp[0], tmp[1]) diff --git a/numpy/ma/version.py b/numpy/ma/version.py new file mode 100644 index 000000000..7a925f1a8 --- /dev/null +++ b/numpy/ma/version.py @@ -0,0 +1,11 @@ +"""Version number""" + +version = '1.00' +release = False + +if not release: + import core + import extras + revision = [core.__revision__.split(':')[-1][:-1].strip(), + extras.__revision__.split(':')[-1][:-1].strip(),] + version += '.dev%04i' % max([int(rev) for rev in revision]) |