diff options
Diffstat (limited to 'numpy/lib')
| -rw-r--r-- | numpy/lib/_iotools.py | 19 | ||||
| -rw-r--r-- | numpy/lib/io.py | 18 | ||||
| -rw-r--r-- | numpy/lib/tests/test__iotools.py | 17 | ||||
| -rw-r--r-- | numpy/lib/tests/test_io.py | 29 |
4 files changed, 77 insertions, 6 deletions
diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py index 644d83d1c..23053bf4d 100644 --- a/numpy/lib/_iotools.py +++ b/numpy/lib/_iotools.py @@ -54,6 +54,16 @@ def _to_filehandle(fname, flag='r', return_opened=False): return fhd +def has_nested_fields(ndtype): + """ + Returns whether one or several fields of a structured array are nested. + """ + for name in ndtype.names or (): + if ndtype[name].names: + return True + return False + + def flatten_dtype(ndtype): """ Unpack a structured data-type. @@ -71,7 +81,6 @@ def flatten_dtype(ndtype): return types - class LineSplitter: """ Defines a function to split a string at a given delimiter or at given places. @@ -377,11 +386,17 @@ class StringConverter: default = None ttype = self._getsubdtype(default) # Set the status according to the dtype + _status = -1 for (i, (deftype, func, default_def)) in enumerate(self._mapper): if np.issubdtype(ttype, deftype): - self._status = i + _status = i self.default = default or default_def break + if _status == -1: + # We never found a match in the _mapper... + _status = 0 + self.default = default + self._status = _status # If the input was a dtype, set the function to the last we saw if self.func is None: self.func = func diff --git a/numpy/lib/io.py b/numpy/lib/io.py index 2967ba17d..be1e03d4c 100644 --- a/numpy/lib/io.py +++ b/numpy/lib/io.py @@ -17,7 +17,7 @@ from _datasource import DataSource from _compiled_base import packbits, unpackbits from _iotools import LineSplitter, NameValidator, StringConverter, \ - _is_string_like, flatten_dtype + _is_string_like, has_nested_fields, flatten_dtype _file = file _string_like = _is_string_like @@ -703,6 +703,11 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, skiprows=0, there must not be any header in the file (else a :exc:ValueError exception is raised). + Warnings + -------- + * Individual values are not stripped of spaces by default. + When using a custom converter, make sure the function does remove spaces. + See Also -------- numpy.loadtxt : equivalent function when no data is missing. @@ -918,8 +923,15 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, skiprows=0, # First, create the array using a flattened dtype: # [('a', int), ('b1', int), ('b2', float)] # Then, view the array using the specified dtype. - rows = np.array(data, dtype=[('', t) for t in flatdtypes]) - output = rows.view(dtype) + if has_nested_fields(dtype): + if 'O' in (_.char for _ in flatdtypes): + errmsg = "Nested fields involving objects "\ + "are not supported..." + raise NotImplementedError(errmsg) + rows = np.array(data, dtype=[('', t) for t in flatdtypes]) + output = rows.view(dtype) + else: + output = np.array(data, dtype=dtype) # Now, process the rowmasks the same way if usemask: rowmasks = np.array(masks, diff --git a/numpy/lib/tests/test__iotools.py b/numpy/lib/tests/test__iotools.py index 61b069659..bb1483186 100644 --- a/numpy/lib/tests/test__iotools.py +++ b/numpy/lib/tests/test__iotools.py @@ -2,7 +2,8 @@ import StringIO import numpy as np -from numpy.lib._iotools import LineSplitter, NameValidator, StringConverter +from numpy.lib._iotools import LineSplitter, NameValidator, StringConverter,\ + has_nested_fields from numpy.testing import * class TestLineSplitter(TestCase): @@ -142,3 +143,17 @@ class TestStringConverter(TestCase): test = convert('') assert_equal(test, date(2000, 01, 01)) + +#------------------------------------------------------------------------------- + +class TestMiscFunctions(TestCase): + # + def test_has_nested_dtype(self): + "Test has_nested_dtype" + ndtype = np.dtype(np.float) + assert_equal(has_nested_fields(ndtype), False) + ndtype = np.dtype([('A', '|S3'), ('B', float)]) + assert_equal(has_nested_fields(ndtype), False) + ndtype = np.dtype([('A', int), ('B', [('BA', float), ('BB', '|S1')])]) + assert_equal(has_nested_fields(ndtype), True) + diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index da91d8a8d..8c1bfbfb5 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -573,6 +573,35 @@ M 33 21.99 assert_equal(test, control) + def test_dtype_with_object(self): + "Test using an explicit dtype qith an object" + from datetime import date + import time + data = """ + 1; 2001-01-01 + 2; 2002-01-31 + """ + ndtype = [('idx', int), ('code', np.object)] + func = lambda s: date(*(time.strptime(s.strip(), "%Y-%m-%d")[:3])) + converters = {1: func} + test = np.genfromtxt(StringIO.StringIO(data), delimiter=";", dtype=ndtype, + converters=converters) + control = np.array([(1, date(2001,1,1)), (2, date(2002,1,31))], + dtype=ndtype) + assert_equal(test, control) + # + ndtype = [('nest', [('idx', int), ('code', np.object)])] + try: + test = np.genfromtxt(StringIO.StringIO(data), delimiter=";", + dtype=ndtype, converters=converters) + except NotImplementedError: + pass + else: + errmsg = "Nested dtype involving objects should be supported." + raise AssertionError(errmsg) + + + def test_spacedelimiter(self): "Test space delimiter" data = StringIO.StringIO("1 2 3 4 5\n6 7 8 9 10") |