From 09a52ed47bb26498c97a579ce1147861df696d84 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Thu, 28 Mar 2013 17:13:53 -0600 Subject: 2to3: Apply `imports` fixer. The `imports` fixer deals with the standard packages that have been renamed, removed, or methods that have moved. cPickle -- removed, use pickle commands -- removed, getoutput, getstatusoutput moved to subprocess urlparse -- removed, urlparse moved to urllib.parse cStringIO -- removed, use StringIO or io.StringIO copy_reg -- renamed copyreg _winreg -- renamed winreg ConfigParser -- renamed configparser __builtin__ -- renamed builtins In the case of `cPickle`, it is imported as `pickle` when python < 3 and performance may be a consideration, but otherwise plain old `pickle` is used. Dealing with `StringIO` is a bit tricky. There is an `io.StringIO` function in the `io` module, available since Python 2.6, but it expects unicode whereas `StringIO.StringIO` expects ascii. The Python 3 equivalent is then `io.BytesIO`. What I have done here is used BytesIO for anything that is emulating a file for testing purposes. That is more explicit than using a redefined StringIO as was done before we dropped support for Python 2.4 and 2.5. Closes #3180. --- numpy/lib/_datasource.py | 12 +- numpy/lib/_iotools.py | 6 +- numpy/lib/format.py | 11 +- numpy/lib/npyio.py | 18 +- numpy/lib/tests/test__datasource.py | 13 +- numpy/lib/tests/test__iotools.py | 11 +- numpy/lib/tests/test_format.py | 48 ++-- numpy/lib/tests/test_io.py | 461 +++++++++++++++++++----------------- numpy/lib/tests/test_regression.py | 7 +- numpy/lib/tests/test_utils.py | 6 +- numpy/lib/utils.py | 7 +- 11 files changed, 314 insertions(+), 286 deletions(-) (limited to 'numpy/lib') diff --git a/numpy/lib/_datasource.py b/numpy/lib/_datasource.py index db97c5507..9f0bb1ae8 100644 --- a/numpy/lib/_datasource.py +++ b/numpy/lib/_datasource.py @@ -36,6 +36,7 @@ from __future__ import division, absolute_import __docformat__ = "restructuredtext en" import os +import sys from shutil import rmtree, copyfile, copyfileobj _open = open @@ -252,7 +253,10 @@ class DataSource (object): """Test if path is a net location. Tests the scheme and netloc.""" # We do this here to reduce the 'import numpy' initial import time. - from urlparse import urlparse + if sys.version_info[0] >= 3: + from urllib.parse import urlparse + else: + from urlparse import urlparse # BUG : URLs require a scheme string ('http://') to be used. # www.google.com will fail. @@ -351,8 +355,10 @@ class DataSource (object): """ # We do this here to reduce the 'import numpy' initial import time. - from urlparse import urlparse - + if sys.version_info[0] >= 3: + from urllib.parse import urlparse + else: + from urlparse import urlparse # TODO: This should be more robust. Handles case where path includes # the destpath, but not other sub-paths. Failing case: diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py index 87ce9b01d..b8a01bafc 100644 --- a/numpy/lib/_iotools.py +++ b/numpy/lib/_iotools.py @@ -8,7 +8,11 @@ __docformat__ = "restructuredtext en" import sys import numpy as np import numpy.core.numeric as nx -from __builtin__ import bool, int, long, float, complex, object, unicode, str + +if sys.version_info[0] >= 3: + from builtins import bool, int, long, float, complex, object, unicode, str +else: + from __builtin__ import bool, int, long, float, complex, object, unicode, str from numpy.compat import asbytes, bytes, asbytes_nested diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 21b0607e7..dedfabfd2 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -136,13 +136,16 @@ alternatives, is described fully in the "npy-format" NEP. """ from __future__ import division, absolute_import -import cPickle - import numpy import sys from numpy.lib.utils import safe_eval from numpy.compat import asbytes, isfileobj +if sys.version_info[0] >= 3: + import pickle +else: + import cPickle as pickle + MAGIC_PREFIX = asbytes('\x93NUMPY') MAGIC_LEN = len(MAGIC_PREFIX) + 2 @@ -399,7 +402,7 @@ def write_array(fp, array, version=(1,0)): if array.dtype.hasobject: # We contain Python objects so we cannot write out the data directly. # Instead, we will pickle it out with version 2 of the pickle protocol. - cPickle.dump(array, fp, protocol=2) + pickle.dump(array, fp, protocol=2) elif array.flags.f_contiguous and not array.flags.c_contiguous: if isfileobj(fp): array.T.tofile(fp) @@ -447,7 +450,7 @@ def read_array(fp): # Now read the actual data. if dtype.hasobject: # The array contained Python objects. We need to unpickle the data. - array = cPickle.load(fp) + array = pickle.load(fp) else: if isfileobj(fp): # We can use the fast fromfile() function. diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 065a3ba46..eb4ffd4ce 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -1,9 +1,5 @@ from __future__ import division, absolute_import -__all__ = ['savetxt', 'loadtxt', 'genfromtxt', 'ndfromtxt', 'mafromtxt', - 'recfromtxt', 'recfromcsv', 'load', 'loads', 'save', 'savez', - 'savez_compressed', 'packbits', 'unpackbits', 'fromregex', 'DataSource'] - import numpy as np from . import format import sys @@ -15,7 +11,6 @@ import warnings import weakref from operator import itemgetter -from cPickle import load as _cload, loads from ._datasource import DataSource from ._compiled_base import packbits, unpackbits @@ -25,11 +20,18 @@ from ._iotools import LineSplitter, NameValidator, StringConverter, \ easy_dtype, _bytes_to_name from numpy.compat import asbytes, asstr, asbytes_nested, bytes +from io import BytesIO if sys.version_info[0] >= 3: - from io import BytesIO + import pickle else: - from cStringIO import StringIO as BytesIO + import cPickle as pickle + +loads = pickle.loads + +__all__ = ['savetxt', 'loadtxt', 'genfromtxt', 'ndfromtxt', 'mafromtxt', + 'recfromtxt', 'recfromcsv', 'load', 'loads', 'save', 'savez', + 'savez_compressed', 'packbits', 'unpackbits', 'fromregex', 'DataSource'] _string_like = _is_string_like @@ -380,7 +382,7 @@ def load(file, mmap_mode=None): return format.read_array(fid) else: # Try a pickle try: - return _cload(fid) + return pickle.load(fid) except: raise IOError( "Failed to interpret file %s as a pickle" % repr(file)) diff --git a/numpy/lib/tests/test__datasource.py b/numpy/lib/tests/test__datasource.py index e784e4296..ccd7405fe 100644 --- a/numpy/lib/tests/test__datasource.py +++ b/numpy/lib/tests/test__datasource.py @@ -1,17 +1,20 @@ from __future__ import division, absolute_import import os +import urllib2 +import sys +import numpy.lib._datasource as datasource from tempfile import mkdtemp, mkstemp, NamedTemporaryFile from shutil import rmtree -from urlparse import urlparse from urllib2 import URLError -import urllib2 - +from numpy.compat import asbytes from numpy.testing import * -from numpy.compat import asbytes -import numpy.lib._datasource as datasource +if sys.version_info[0] >= 3: + from urllib.parse import urlparse +else: + from urlparse import urlparse def urlopen_stub(url, data=None): '''Stub to replace urlopen for testing.''' diff --git a/numpy/lib/tests/test__iotools.py b/numpy/lib/tests/test__iotools.py index ebb01ad3a..1cd9a25da 100644 --- a/numpy/lib/tests/test__iotools.py +++ b/numpy/lib/tests/test__iotools.py @@ -1,22 +1,13 @@ from __future__ import division, absolute_import import sys - -if sys.version_info[0] >= 3: - from io import BytesIO - def StringIO(s=""): - return BytesIO(asbytes(s)) -else: - from StringIO import StringIO - -from datetime import date import time +from datetime import date import numpy as np from numpy.lib._iotools import LineSplitter, NameValidator, StringConverter, \ has_nested_fields, easy_dtype, flatten_dtype from numpy.testing import * - from numpy.compat import asbytes, asbytes_nested class TestLineSplitter(TestCase): diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index 5c07da1c3..aae3b1ba8 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -5,10 +5,7 @@ r''' Test the .npy file format. Set up: >>> import sys - >>> if sys.version_info[0] >= 3: - ... from io import BytesIO as StringIO - ... else: - ... from cStringIO import StringIO + >>> from io import BytesIO >>> from numpy.lib import format >>> >>> scalars = [ @@ -101,19 +98,19 @@ Test the magic string writing. Test the magic string reading. - >>> format.read_magic(StringIO(format.magic(1, 0))) + >>> format.read_magic(BytesIO(format.magic(1, 0))) (1, 0) - >>> format.read_magic(StringIO(format.magic(0, 0))) + >>> format.read_magic(BytesIO(format.magic(0, 0))) (0, 0) - >>> format.read_magic(StringIO(format.magic(255, 255))) + >>> format.read_magic(BytesIO(format.magic(255, 255))) (255, 255) - >>> format.read_magic(StringIO(format.magic(2, 5))) + >>> format.read_magic(BytesIO(format.magic(2, 5))) (2, 5) Test the header writing. >>> for arr in basic_arrays + record_arrays: - ... f = StringIO() + ... f = BytesIO() ... format.write_array_header_1_0(f, arr) # XXX: arr is not a dict, items gets called on it ... print repr(f.getvalue()) ... @@ -279,22 +276,15 @@ Test the header writing. "\x16\x02{'descr': [('x', '>i4', (2,)),\n ('Info',\n [('value', '>c16'),\n ('y2', '>f8'),\n ('Info2',\n [('name', '|S2'),\n ('value', '>c16', (2,)),\n ('y3', '>f8', (2,)),\n ('z3', '>u4', (2,))]),\n ('name', '|S2'),\n ('z2', '|b1')]),\n ('color', '|S2'),\n ('info', [('Name', '>U8'), ('Value', '>c16')]),\n ('y', '>f8', (2, 2)),\n ('z', '|u1')],\n 'fortran_order': False,\n 'shape': (2,)} \n" ''' - import sys import os import shutil import tempfile - -if sys.version_info[0] >= 3: - from io import BytesIO as StringIO -else: - from cStringIO import StringIO +from io import BytesIO import numpy as np from numpy.testing import * - from numpy.lib import format - from numpy.compat import asbytes, asbytes_nested @@ -416,9 +406,9 @@ record_arrays = [ ] def roundtrip(arr): - f = StringIO() + f = BytesIO() format.write_array(f, arr) - f2 = StringIO(f.getvalue()) + f2 = BytesIO(f.getvalue()) arr2 = format.read_array(f2) return arr2 @@ -469,7 +459,7 @@ def test_memmap_roundtrip(): def test_write_version_1_0(): - f = StringIO() + f = BytesIO() arr = np.arange(1) # These should pass. format.write_array(f, arr, version=(1, 0)) @@ -513,12 +503,12 @@ malformed_magic = asbytes_nested([ def test_read_magic_bad_magic(): for magic in malformed_magic: - f = StringIO(magic) + f = BytesIO(magic) yield raises(ValueError)(format.read_magic), f def test_read_version_1_0_bad_magic(): for magic in bad_version_magic + malformed_magic: - f = StringIO(magic) + f = BytesIO(magic) yield raises(ValueError)(format.read_array), f def test_bad_magic_args(): @@ -528,29 +518,29 @@ def test_bad_magic_args(): assert_raises(ValueError, format.magic, 1, 256) def test_large_header(): - s = StringIO() + s = BytesIO() d = {'a':1,'b':2} format.write_array_header_1_0(s,d) - s = StringIO() + s = BytesIO() d = {'a':1,'b':2,'c':'x'*256*256} assert_raises(ValueError, format.write_array_header_1_0, s, d) def test_bad_header(): # header of length less than 2 should fail - s = StringIO() + s = BytesIO() assert_raises(ValueError, format.read_array_header_1_0, s) - s = StringIO(asbytes('1')) + s = BytesIO(asbytes('1')) assert_raises(ValueError, format.read_array_header_1_0, s) # header shorter than indicated size should fail - s = StringIO(asbytes('\x01\x00')) + s = BytesIO(asbytes('\x01\x00')) assert_raises(ValueError, format.read_array_header_1_0, s) # headers without the exact keys required should fail d = {"shape":(1,2), "descr":"x"} - s = StringIO() + s = BytesIO() format.write_array_header_1_0(s,d) assert_raises(ValueError, format.read_array_header_1_0, s) @@ -558,7 +548,7 @@ def test_bad_header(): "fortran_order":False, "descr":"x", "extrakey":-1} - s = StringIO() + s = BytesIO() format.write_array_header_1_0(s,d) assert_raises(ValueError, format.read_array_header_1_0, s) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index ab990976e..6b5173890 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -4,31 +4,43 @@ import sys import gzip import os import threading -from tempfile import mkstemp, NamedTemporaryFile import time -from datetime import datetime import warnings import gc +from tempfile import mkstemp, NamedTemporaryFile +from io import BytesIO +from datetime import datetime from numpy.testing.utils import WarningManager import numpy as np import numpy.ma as ma from numpy.lib._iotools import ConverterError, ConverterLockError, \ ConversionWarning -from numpy.compat import asbytes, asbytes_nested, bytes - +from numpy.compat import asbytes, asbytes_nested, bytes, asstr from nose import SkipTest from numpy.ma.testutils import (TestCase, assert_equal, assert_array_equal, assert_raises, run_module_suite) from numpy.testing import assert_warns, assert_, build_err_msg -if sys.version_info[0] >= 3: - from io import BytesIO - def StringIO(s=""): - return BytesIO(asbytes(s)) -else: - from StringIO import StringIO - BytesIO = StringIO + +class TextIO(BytesIO): + """Helper IO class. + + Writes encode strings to bytes if needed, reads return bytes. + This makes it easier to emulate files opened in binary mode + without needing to explicitly convert strings to bytes in + setting up the test data. + + """ + def __init__(self, s=""): + BytesIO.__init__(self, asbytes(s)) + + def write(self, s): + BytesIO.write(self, asbytes(s)) + + def writelines(self, lines): + BytesIO.writelines(self, [asbytes(s) for s in lines]) + MAJVER, MINVER = sys.version_info[:2] @@ -73,7 +85,7 @@ class RoundtripTest(object): target_file = NamedTemporaryFile() load_file = target_file.name else: - target_file = StringIO() + target_file = BytesIO() load_file = target_file arr = args @@ -135,7 +147,7 @@ class TestSavezLoad(RoundtripTest, TestCase): def test_named_arrays(self): a = np.array([[1, 2], [3, 4]], float) b = np.array([[1 + 2j, 2 + 7j], [3 - 6j, 4 + 12j]], complex) - c = StringIO() + c = BytesIO() np.savez(c, file_a=a, file_b=b) c.seek(0) l = np.load(c) @@ -228,95 +240,94 @@ class TestSaveTxt(TestCase): def test_array(self): a = np.array([[1, 2], [3, 4]], float) fmt = "%.18e" - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt=fmt) c.seek(0) assert_equal(c.readlines(), - asbytes_nested( - [(fmt + ' ' + fmt + '\n') % (1, 2), - (fmt + ' ' + fmt + '\n') % (3, 4)])) + [asbytes((fmt + ' ' + fmt + '\n') % (1, 2)), + asbytes((fmt + ' ' + fmt + '\n') % (3, 4))]) a = np.array([[1, 2], [3, 4]], int) - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt='%d') c.seek(0) - assert_equal(c.readlines(), asbytes_nested(['1 2\n', '3 4\n'])) + assert_equal(c.readlines(), [b'1 2\n', b'3 4\n']) def test_1D(self): a = np.array([1, 2, 3, 4], int) - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt='%d') c.seek(0) lines = c.readlines() - assert_equal(lines, asbytes_nested(['1\n', '2\n', '3\n', '4\n'])) + assert_equal(lines, [b'1\n', b'2\n', b'3\n', b'4\n']) def test_record(self): a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt='%d') c.seek(0) - assert_equal(c.readlines(), asbytes_nested(['1 2\n', '3 4\n'])) + assert_equal(c.readlines(), [b'1 2\n', b'3 4\n']) def test_delimiter(self): a = np.array([[1., 2.], [3., 4.]]) - c = StringIO() - np.savetxt(c, a, delimiter=asbytes(','), fmt='%d') + c = BytesIO() + np.savetxt(c, a, delimiter=',', fmt='%d') c.seek(0) - assert_equal(c.readlines(), asbytes_nested(['1,2\n', '3,4\n'])) + assert_equal(c.readlines(), [b'1,2\n', b'3,4\n']) def test_format(self): a = np.array([(1, 2), (3, 4)]) - c = StringIO() + c = BytesIO() # Sequence of formats np.savetxt(c, a, fmt=['%02d', '%3.1f']) c.seek(0) - assert_equal(c.readlines(), asbytes_nested(['01 2.0\n', '03 4.0\n'])) + assert_equal(c.readlines(), [b'01 2.0\n', b'03 4.0\n']) # A single multiformat string - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt='%02d : %3.1f') c.seek(0) lines = c.readlines() - assert_equal(lines, asbytes_nested(['01 : 2.0\n', '03 : 4.0\n'])) + assert_equal(lines, [b'01 : 2.0\n', b'03 : 4.0\n']) # Specify delimiter, should be overiden - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt='%02d : %3.1f', delimiter=',') c.seek(0) lines = c.readlines() - assert_equal(lines, asbytes_nested(['01 : 2.0\n', '03 : 4.0\n'])) + assert_equal(lines, [b'01 : 2.0\n', b'03 : 4.0\n']) def test_header_footer(self): """ Test the functionality of the header and footer keyword argument. """ - c = StringIO() + c = BytesIO() a = np.array([(1, 2), (3, 4)], dtype=np.int) test_header_footer = 'Test header / footer' # Test the header keyword argument np.savetxt(c, a, fmt='%1d', header=test_header_footer) c.seek(0) assert_equal(c.read(), - asbytes('# ' + test_header_footer +'\n1 2\n3 4\n' )) + asbytes('# ' + test_header_footer + '\n1 2\n3 4\n')) # Test the footer keyword argument - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt='%1d', footer=test_header_footer) c.seek(0) assert_equal(c.read(), asbytes('1 2\n3 4\n# ' + test_header_footer + '\n')) # Test the commentstr keyword argument used on the header - c = StringIO() + c = BytesIO() commentstr = '% ' - np.savetxt(c, a, fmt='%1d', header=test_header_footer, - comments=commentstr) + np.savetxt(c, a, fmt='%1d', + header=test_header_footer, comments=commentstr) c.seek(0) assert_equal(c.read(), asbytes(commentstr + test_header_footer + '\n' + '1 2\n3 4\n')) # Test the commentstr keyword argument used on the footer - c = StringIO() + c = BytesIO() commentstr = '% ' - np.savetxt(c, a, fmt='%1d', footer=test_header_footer, - comments=commentstr) + np.savetxt(c, a, fmt='%1d', + footer=test_header_footer, comments=commentstr) c.seek(0) assert_equal(c.read(), asbytes('1 2\n3 4\n' + commentstr + test_header_footer + '\n')) @@ -340,29 +351,29 @@ class TestSaveTxt(TestCase): im = np.e a[:] = re + 1.0j * im # One format only - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt=' %+.3e') c.seek(0) lines = c.readlines() - _assert_floatstr_lines_equal(lines, asbytes_nested([ - ' ( +3.142e+00+ +2.718e+00j) ( +3.142e+00+ +2.718e+00j)\n', - ' ( +3.142e+00+ +2.718e+00j) ( +3.142e+00+ +2.718e+00j)\n'])) + _assert_floatstr_lines_equal(lines, + [b' ( +3.142e+00+ +2.718e+00j) ( +3.142e+00+ +2.718e+00j)\n', + b' ( +3.142e+00+ +2.718e+00j) ( +3.142e+00+ +2.718e+00j)\n']) # One format for each real and imaginary part - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt=' %+.3e' * 2 * ncols) c.seek(0) lines = c.readlines() - _assert_floatstr_lines_equal(lines, asbytes_nested([ - ' +3.142e+00 +2.718e+00 +3.142e+00 +2.718e+00\n', - ' +3.142e+00 +2.718e+00 +3.142e+00 +2.718e+00\n'])) + _assert_floatstr_lines_equal(lines, + [b' +3.142e+00 +2.718e+00 +3.142e+00 +2.718e+00\n', + b' +3.142e+00 +2.718e+00 +3.142e+00 +2.718e+00\n']) # One format for each complex number - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt=['(%.3e%+.3ej)'] * ncols) c.seek(0) lines = c.readlines() - _assert_floatstr_lines_equal(lines, asbytes_nested([ - '(3.142e+00+2.718e+00j) (3.142e+00+2.718e+00j)\n', - '(3.142e+00+2.718e+00j) (3.142e+00+2.718e+00j)\n'])) + _assert_floatstr_lines_equal(lines, + [b'(3.142e+00+2.718e+00j) (3.142e+00+2.718e+00j)\n', + b'(3.142e+00+2.718e+00j) (3.142e+00+2.718e+00j)\n']) def _assert_floatstr_lines_equal(actual_lines, expected_lines): @@ -387,15 +398,15 @@ def _assert_floatstr_lines_equal(actual_lines, expected_lines): class TestLoadTxt(TestCase): def test_record(self): - c = StringIO() - c.write(asbytes('1 2\n3 4')) + c = TextIO() + c.write('1 2\n3 4') c.seek(0) x = np.loadtxt(c, dtype=[('x', np.int32), ('y', np.int32)]) a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) assert_array_equal(x, a) - d = StringIO() - d.write(asbytes('M 64.0 75.0\nF 25.0 60.0')) + d = TextIO() + d.write('M 64.0 75.0\nF 25.0 60.0') d.seek(0) mydescriptor = {'names': ('gender', 'age', 'weight'), 'formats': ('S1', @@ -406,8 +417,8 @@ class TestLoadTxt(TestCase): assert_array_equal(y, b) def test_array(self): - c = StringIO() - c.write(asbytes('1 2\n3 4')) + c = TextIO() + c.write('1 2\n3 4') c.seek(0) x = np.loadtxt(c, dtype=int) @@ -420,23 +431,23 @@ class TestLoadTxt(TestCase): assert_array_equal(x, a) def test_1D(self): - c = StringIO() - c.write(asbytes('1\n2\n3\n4\n')) + c = TextIO() + c.write('1\n2\n3\n4\n') c.seek(0) x = np.loadtxt(c, dtype=int) a = np.array([1, 2, 3, 4], int) assert_array_equal(x, a) - c = StringIO() - c.write(asbytes('1,2,3,4\n')) + c = TextIO() + c.write('1,2,3,4\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',') a = np.array([1, 2, 3, 4], int) assert_array_equal(x, a) def test_missing(self): - c = StringIO() - c.write(asbytes('1,2,3,,5\n')) + c = TextIO() + c.write('1,2,3,,5\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',', \ converters={3:lambda s: int(s or - 999)}) @@ -444,8 +455,8 @@ class TestLoadTxt(TestCase): assert_array_equal(x, a) def test_converters_with_usecols(self): - c = StringIO() - c.write(asbytes('1,2,3,,5\n6,7,8,9,10\n')) + c = TextIO() + c.write('1,2,3,,5\n6,7,8,9,10\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',', \ converters={3:lambda s: int(s or - 999)}, \ @@ -454,8 +465,8 @@ class TestLoadTxt(TestCase): assert_array_equal(x, a) def test_comments(self): - c = StringIO() - c.write(asbytes('# comment\n1,2,3,5\n')) + c = TextIO() + c.write('# comment\n1,2,3,5\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',', \ comments='#') @@ -463,16 +474,16 @@ class TestLoadTxt(TestCase): assert_array_equal(x, a) def test_skiprows(self): - c = StringIO() - c.write(asbytes('comment\n1,2,3,5\n')) + c = TextIO() + c.write('comment\n1,2,3,5\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',', \ skiprows=1) a = np.array([1, 2, 3, 5], int) assert_array_equal(x, a) - c = StringIO() - c.write(asbytes('# comment\n1,2,3,5\n')) + c = TextIO() + c.write('# comment\n1,2,3,5\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',', \ skiprows=1) @@ -481,14 +492,14 @@ class TestLoadTxt(TestCase): def test_usecols(self): a = np.array([[1, 2], [3, 4]], float) - c = StringIO() + c = BytesIO() np.savetxt(c, a) c.seek(0) x = np.loadtxt(c, dtype=float, usecols=(1,)) assert_array_equal(x, a[:, 1]) a = np.array([[1, 2, 3], [3, 4, 5]], float) - c = StringIO() + c = BytesIO() np.savetxt(c, a) c.seek(0) x = np.loadtxt(c, dtype=float, usecols=(1, 2)) @@ -503,16 +514,16 @@ class TestLoadTxt(TestCase): data = '''JOE 70.1 25.3 BOB 60.5 27.9 ''' - c = StringIO(data) + c = TextIO(data) names = ['stid', 'temp'] dtypes = ['S4', 'f8'] arr = np.loadtxt(c, usecols=(0, 2), dtype=zip(names, dtypes)) - assert_equal(arr['stid'], asbytes_nested(["JOE", "BOB"])) + assert_equal(arr['stid'], [b"JOE", b"BOB"]) assert_equal(arr['temp'], [25.3, 27.9]) def test_fancy_dtype(self): - c = StringIO() - c.write(asbytes('1,2,3.0\n4,5,6.0\n')) + c = TextIO() + c.write('1,2,3.0\n4,5,6.0\n') c.seek(0) dt = np.dtype([('x', int), ('y', [('t', int), ('s', float)])]) x = np.loadtxt(c, dtype=dt, delimiter=',') @@ -520,7 +531,7 @@ class TestLoadTxt(TestCase): assert_array_equal(x, a) def test_shaped_dtype(self): - c = StringIO("aaaa 1.0 8.0 1 2 3 4 5 6") + c = TextIO("aaaa 1.0 8.0 1 2 3 4 5 6") dt = np.dtype([('name', 'S4'), ('x', float), ('y', float), ('block', int, (2, 3))]) x = np.loadtxt(c, dtype=dt) @@ -529,7 +540,7 @@ class TestLoadTxt(TestCase): assert_array_equal(x, a) def test_3d_shaped_dtype(self): - c = StringIO("aaaa 1.0 8.0 1 2 3 4 5 6 7 8 9 10 11 12") + c = TextIO("aaaa 1.0 8.0 1 2 3 4 5 6 7 8 9 10 11 12") dt = np.dtype([('name', 'S4'), ('x', float), ('y', float), ('block', int, (2, 2, 3))]) x = np.loadtxt(c, dtype=dt) @@ -543,7 +554,7 @@ class TestLoadTxt(TestCase): try: warnings.filterwarnings("ignore", message="loadtxt: Empty input file:") - c = StringIO() + c = TextIO() x = np.loadtxt(c) assert_equal(x.shape, (0,)) x = np.loadtxt(c, dtype=np.int64) @@ -554,8 +565,8 @@ class TestLoadTxt(TestCase): def test_unused_converter(self): - c = StringIO() - c.writelines([asbytes('1 21\n'), asbytes('3 42\n')]) + c = TextIO() + c.writelines(['1 21\n', '3 42\n']) c.seek(0) data = np.loadtxt(c, usecols=(1,), converters={0: lambda s: int(s, 16)}) @@ -570,12 +581,12 @@ class TestLoadTxt(TestCase): "Test using an explicit dtype with an object" from datetime import date import time - data = asbytes(""" 1; 2001-01-01 - 2; 2002-01-31 """) + data = """ 1; 2001-01-01 + 2; 2002-01-31 """ ndtype = [('idx', int), ('code', np.object)] func = lambda s: strptime(s.strip(), "%Y-%m-%d") converters = {1: func} - test = np.loadtxt(StringIO(data), delimiter=";", dtype=ndtype, + test = np.loadtxt(TextIO(data), delimiter=";", dtype=ndtype, converters=converters) control = np.array([(1, datetime(2001, 1, 1)), (2, datetime(2002, 1, 31))], dtype=ndtype) @@ -583,23 +594,23 @@ class TestLoadTxt(TestCase): def test_uint64_type(self): tgt = (9223372043271415339, 9223372043271415853) - c = StringIO() - c.write(asbytes("%s %s" % tgt)) + c = TextIO() + c.write("%s %s" % tgt) c.seek(0) res = np.loadtxt(c, dtype=np.uint64) assert_equal(res, tgt) def test_int64_type(self): tgt = (-9223372036854775807, 9223372036854775807) - c = StringIO() - c.write(asbytes("%s %s" % tgt)) + c = TextIO() + c.write("%s %s" % tgt) c.seek(0) res = np.loadtxt(c, dtype=np.int64) assert_equal(res, tgt) def test_universal_newline(self): f, name = mkstemp() - os.write(f, asbytes('1 21\r3 42\r')) + os.write(f, b'1 21\r3 42\r') os.close(f) try: @@ -609,29 +620,29 @@ class TestLoadTxt(TestCase): os.unlink(name) def test_empty_field_after_tab(self): - c = StringIO() - c.write(asbytes('1 \t2 \t3\tstart \n4\t5\t6\t \n7\t8\t9.5\t')) + c = TextIO() + c.write('1 \t2 \t3\tstart \n4\t5\t6\t \n7\t8\t9.5\t') c.seek(0) dt = { 'names': ('x', 'y', 'z', 'comment'), 'formats': ('= 3: + from io import StringIO + else: + from StringIO import StringIO + dt = [("a", 'u1', 2), ("b", 'u1', 2)] x = np.loadtxt(StringIO("0 1 2 3"), dtype=dt) assert_equal(x, np.array([((0, 1), (2, 3))], dtype=dt)) diff --git a/numpy/lib/tests/test_utils.py b/numpy/lib/tests/test_utils.py index 4b386e0b1..8cec1887b 100644 --- a/numpy/lib/tests/test_utils.py +++ b/numpy/lib/tests/test_utils.py @@ -1,10 +1,14 @@ from __future__ import division, absolute_import +import sys from numpy.testing import * import numpy.lib.utils as utils from numpy.lib import deprecate -from StringIO import StringIO +if sys.version_info[0] >= 3: + from io import StringIO +else: + from StringIO import StringIO def test_lookfor(): out = StringIO() diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index 9a8776098..b81db681e 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -848,7 +848,12 @@ def _lookfor_generate_cache(module, import_modules, regenerate): global _lookfor_caches # Local import to speed up numpy's import time. import inspect - from cStringIO import StringIO + + if sys.version_info[0] >= 3: + # In Python3 stderr, stdout are text files. + from io import StringIO + else: + from StringIO import StringIO if module is None: module = "numpy" -- cgit v1.2.1