diff options
author | Julian Taylor <juliantaylor108@gmail.com> | 2017-03-25 14:47:04 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-03-25 14:47:04 +0100 |
commit | ab49be1d5a6f1a7c51298a8fe46a9f43bb2bb33b (patch) | |
tree | 434f751384e02a2d386dda491527c50fed070625 /numpy/lib | |
parent | a7ed83f6d9e5e8f5a024dfe2d42ad290d79422b1 (diff) | |
parent | 1dc20cba80660ee9f2b16e26c0d6e7f23bf9e08f (diff) | |
download | numpy-ab49be1d5a6f1a7c51298a8fe46a9f43bb2bb33b.tar.gz |
Merge pull request #8832 from eric-wieser/avoid-as-bytes
MAINT: Remove python <2.7,<3.3 string/unicode workarounds
Diffstat (limited to 'numpy/lib')
-rw-r--r-- | numpy/lib/_iotools.py | 26 | ||||
-rw-r--r-- | numpy/lib/format.py | 2 | ||||
-rw-r--r-- | numpy/lib/npyio.py | 18 | ||||
-rw-r--r-- | numpy/lib/tests/test__datasource.py | 3 | ||||
-rw-r--r-- | numpy/lib/tests/test__iotools.py | 111 | ||||
-rw-r--r-- | numpy/lib/tests/test_arraypad.py | 5 | ||||
-rw-r--r-- | numpy/lib/tests/test_format.py | 43 | ||||
-rw-r--r-- | numpy/lib/tests/test_io.py | 2 | ||||
-rw-r--r-- | numpy/lib/tests/test_twodim_base.py | 3 |
9 files changed, 103 insertions, 110 deletions
diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py index 312a9f02a..304bba3d3 100644 --- a/numpy/lib/_iotools.py +++ b/numpy/lib/_iotools.py @@ -44,7 +44,7 @@ def _is_bytes_like(obj): Check whether obj behaves like a bytes object. """ try: - obj + asbytes('') + obj + b'' except (TypeError, ValueError): return False return True @@ -189,7 +189,7 @@ class LineSplitter(object): return lambda input: [_.strip() for _ in method(input)] # - def __init__(self, delimiter=None, comments=asbytes('#'), autostrip=True): + def __init__(self, delimiter=None, comments=b'#', autostrip=True): self.comments = comments # Delimiter is a character if isinstance(delimiter, unicode): @@ -218,7 +218,7 @@ class LineSplitter(object): def _delimited_splitter(self, line): if self.comments is not None: line = line.split(self.comments)[0] - line = line.strip(asbytes(" \r\n")) + line = line.strip(b" \r\n") if not line: return [] return line.split(self.delimiter) @@ -227,7 +227,7 @@ class LineSplitter(object): def _fixedwidth_splitter(self, line): if self.comments is not None: line = line.split(self.comments)[0] - line = line.strip(asbytes("\r\n")) + line = line.strip(b"\r\n") if not line: return [] fixed = self.delimiter @@ -434,9 +434,9 @@ def str2bool(value): """ value = value.upper() - if value == asbytes('TRUE'): + if value == b'TRUE': return True - elif value == asbytes('FALSE'): + elif value == b'FALSE': return False else: raise ValueError("Invalid boolean") @@ -529,7 +529,7 @@ class StringConverter(object): _mapper.extend([(nx.floating, float, nx.nan), (complex, _bytes_to_complex, nx.nan + 0j), (nx.longdouble, nx.longdouble, nx.nan), - (nx.string_, bytes, asbytes('???'))]) + (nx.string_, bytes, b'???')]) (_defaulttype, _defaultfunc, _defaultfill) = zip(*_mapper) @@ -631,7 +631,7 @@ class StringConverter(object): # None if default is None: try: - default = self.func(asbytes('0')) + default = self.func(b'0') except ValueError: default = None dtype = self._getdtype(default) @@ -676,11 +676,11 @@ class StringConverter(object): self.func = lambda x: int(float(x)) # Store the list of strings corresponding to missing values. if missing_values is None: - self.missing_values = set([asbytes('')]) + self.missing_values = set([b'']) else: if isinstance(missing_values, bytes): - missing_values = missing_values.split(asbytes(",")) - self.missing_values = set(list(missing_values) + [asbytes('')]) + missing_values = missing_values.split(b",") + self.missing_values = set(list(missing_values) + [b'']) # self._callingfunction = self._strict_call self.type = self._dtypeortype(dtype) @@ -801,7 +801,7 @@ class StringConverter(object): self.iterupgrade(value) def update(self, func, default=None, testing_value=None, - missing_values=asbytes(''), locked=False): + missing_values=b'', locked=False): """ Set StringConverter attributes directly. @@ -838,7 +838,7 @@ class StringConverter(object): self.type = self._dtypeortype(self._getdtype(default)) else: try: - tester = func(testing_value or asbytes('1')) + tester = func(testing_value or b'1') except (TypeError, ValueError): tester = None self.type = self._dtypeortype(self._getdtype(tester)) diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 633aee675..14dec01d5 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -161,7 +161,7 @@ if sys.version_info[0] >= 3: else: import cPickle as pickle -MAGIC_PREFIX = asbytes('\x93NUMPY') +MAGIC_PREFIX = b'\x93NUMPY' MAGIC_LEN = len(MAGIC_PREFIX) + 2 BUFFER_SIZE = 2**18 # size of buffer for reading npz files in bytes diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 54a37fbad..dc1c951e7 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -397,7 +397,7 @@ def load(file, mmap_mode=None, allow_pickle=True, fix_imports=True, try: # Code to distinguish from NumPy binary files and pickles. - _ZIP_PREFIX = asbytes('PK\x03\x04') + _ZIP_PREFIX = b'PK\x03\x04' N = len(format.MAGIC_PREFIX) magic = fid.read(N) # If the file size is less than N, we need to make sure not @@ -856,7 +856,7 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, # Compile regex for comments beforehand comments = (re.escape(comment) for comment in comments) - regex_comments = re.compile(asbytes('|').join(comments)) + regex_comments = re.compile(b'|'.join(comments)) user_converters = converters if delimiter is not None: delimiter = asbytes(delimiter) @@ -958,7 +958,7 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, line = asbytes(line) if comments is not None: line = regex_comments.split(asbytes(line), maxsplit=1)[0] - line = line.strip(asbytes('\r\n')) + line = line.strip(b'\r\n') if line: return line.split(delimiter) else: @@ -1576,11 +1576,11 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, if names is True: if comments in first_line: first_line = ( - asbytes('').join(first_line.split(comments)[1:])) + b''.join(first_line.split(comments)[1:])) first_values = split_line(first_line) except StopIteration: # return an empty array if the datafile is empty - first_line = asbytes('') + first_line = b'' first_values = [] warnings.warn('genfromtxt: Empty input file: "%s"' % fname, stacklevel=2) @@ -1605,7 +1605,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, if names is True: names = validate_names([_bytes_to_name(_.strip()) for _ in first_values]) - first_line = asbytes('') + first_line = b'' elif _is_string_like(names): names = validate_names([_.strip() for _ in names.split(',')]) elif names: @@ -1644,7 +1644,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, user_missing_values = missing_values or () # Define the list of missing_values (one column: one list) - missing_values = [list([asbytes('')]) for _ in range(nbcols)] + missing_values = [list([b'']) for _ in range(nbcols)] # We have a dictionary: process it field by field if isinstance(user_missing_values, dict): @@ -1684,7 +1684,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, entry.append(value) # We have a string : apply it to all entries elif isinstance(user_missing_values, bytes): - user_value = user_missing_values.split(asbytes(",")) + user_value = user_missing_values.split(b",") for entry in missing_values: entry.extend(user_value) # We have something else: apply it to all entries @@ -1977,7 +1977,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, if usemask and names: for (name, conv) in zip(names or (), converters): missing_values = [conv(_) for _ in conv.missing_values - if _ != asbytes('')] + if _ != b''] for mval in missing_values: outputmask[name] |= (output[name] == mval) # Construct the final array diff --git a/numpy/lib/tests/test__datasource.py b/numpy/lib/tests/test__datasource.py index f4bece352..f2ad0344a 100644 --- a/numpy/lib/tests/test__datasource.py +++ b/numpy/lib/tests/test__datasource.py @@ -5,7 +5,6 @@ import sys from tempfile import mkdtemp, mkstemp, NamedTemporaryFile from shutil import rmtree -from numpy.compat import asbytes from numpy.testing import ( run_module_suite, TestCase, assert_, SkipTest ) @@ -53,7 +52,7 @@ http_fakefile = 'fake.txt' malicious_files = ['/etc/shadow', '../../shadow', '..\\system.dat', 'c:\\windows\\system.dat'] -magic_line = asbytes('three is the magic number') +magic_line = b'three is the magic number' # Utility functions used by many TestCases diff --git a/numpy/lib/tests/test__iotools.py b/numpy/lib/tests/test__iotools.py index e0a917a21..6c0b2c6db 100644 --- a/numpy/lib/tests/test__iotools.py +++ b/numpy/lib/tests/test__iotools.py @@ -5,7 +5,6 @@ import time from datetime import date import numpy as np -from numpy.compat import asbytes, asbytes_nested from numpy.testing import ( run_module_suite, TestCase, assert_, assert_equal, assert_allclose, assert_raises @@ -21,61 +20,61 @@ class TestLineSplitter(TestCase): def test_no_delimiter(self): "Test LineSplitter w/o delimiter" - strg = asbytes(" 1 2 3 4 5 # test") + strg = b" 1 2 3 4 5 # test" test = LineSplitter()(strg) - assert_equal(test, asbytes_nested(['1', '2', '3', '4', '5'])) + assert_equal(test, [b'1', b'2', b'3', b'4', b'5']) test = LineSplitter('')(strg) - assert_equal(test, asbytes_nested(['1', '2', '3', '4', '5'])) + assert_equal(test, [b'1', b'2', b'3', b'4', b'5']) def test_space_delimiter(self): "Test space delimiter" - strg = asbytes(" 1 2 3 4 5 # test") - test = LineSplitter(asbytes(' '))(strg) - assert_equal(test, asbytes_nested(['1', '2', '3', '4', '', '5'])) - test = LineSplitter(asbytes(' '))(strg) - assert_equal(test, asbytes_nested(['1 2 3 4', '5'])) + strg = b" 1 2 3 4 5 # test" + test = LineSplitter(b' ')(strg) + assert_equal(test, [b'1', b'2', b'3', b'4', b'', b'5']) + test = LineSplitter(b' ')(strg) + assert_equal(test, [b'1 2 3 4', b'5']) def test_tab_delimiter(self): "Test tab delimiter" - strg = asbytes(" 1\t 2\t 3\t 4\t 5 6") - test = LineSplitter(asbytes('\t'))(strg) - assert_equal(test, asbytes_nested(['1', '2', '3', '4', '5 6'])) - strg = asbytes(" 1 2\t 3 4\t 5 6") - test = LineSplitter(asbytes('\t'))(strg) - assert_equal(test, asbytes_nested(['1 2', '3 4', '5 6'])) + strg = b" 1\t 2\t 3\t 4\t 5 6" + test = LineSplitter(b'\t')(strg) + assert_equal(test, [b'1', b'2', b'3', b'4', b'5 6']) + strg = b" 1 2\t 3 4\t 5 6" + test = LineSplitter(b'\t')(strg) + assert_equal(test, [b'1 2', b'3 4', b'5 6']) def test_other_delimiter(self): "Test LineSplitter on delimiter" - strg = asbytes("1,2,3,4,,5") - test = LineSplitter(asbytes(','))(strg) - assert_equal(test, asbytes_nested(['1', '2', '3', '4', '', '5'])) + strg = b"1,2,3,4,,5" + test = LineSplitter(b',')(strg) + assert_equal(test, [b'1', b'2', b'3', b'4', b'', b'5']) # - strg = asbytes(" 1,2,3,4,,5 # test") - test = LineSplitter(asbytes(','))(strg) - assert_equal(test, asbytes_nested(['1', '2', '3', '4', '', '5'])) + strg = b" 1,2,3,4,,5 # test" + test = LineSplitter(b',')(strg) + assert_equal(test, [b'1', b'2', b'3', b'4', b'', b'5']) def test_constant_fixed_width(self): "Test LineSplitter w/ fixed-width fields" - strg = asbytes(" 1 2 3 4 5 # test") + strg = b" 1 2 3 4 5 # test" test = LineSplitter(3)(strg) - assert_equal(test, asbytes_nested(['1', '2', '3', '4', '', '5', ''])) + assert_equal(test, [b'1', b'2', b'3', b'4', b'', b'5', b'']) # - strg = asbytes(" 1 3 4 5 6# test") + strg = b" 1 3 4 5 6# test" test = LineSplitter(20)(strg) - assert_equal(test, asbytes_nested(['1 3 4 5 6'])) + assert_equal(test, [b'1 3 4 5 6']) # - strg = asbytes(" 1 3 4 5 6# test") + strg = b" 1 3 4 5 6# test" test = LineSplitter(30)(strg) - assert_equal(test, asbytes_nested(['1 3 4 5 6'])) + assert_equal(test, [b'1 3 4 5 6']) def test_variable_fixed_width(self): - strg = asbytes(" 1 3 4 5 6# test") + strg = b" 1 3 4 5 6# test" test = LineSplitter((3, 6, 6, 3))(strg) - assert_equal(test, asbytes_nested(['1', '3', '4 5', '6'])) + assert_equal(test, [b'1', b'3', b'4 5', b'6']) # - strg = asbytes(" 1 3 4 5 6# test") + strg = b" 1 3 4 5 6# test" test = LineSplitter((6, 6, 9))(strg) - assert_equal(test, asbytes_nested(['1', '3 4', '5 6'])) + assert_equal(test, [b'1', b'3 4', b'5 6']) # ----------------------------------------------------------------------------- @@ -157,7 +156,7 @@ class TestStringConverter(TestCase): assert_equal(converter._status, 0) # test int - assert_equal(converter.upgrade(asbytes('0')), 0) + assert_equal(converter.upgrade(b'0'), 0) assert_equal(converter._status, 1) # On systems where integer defaults to 32-bit, the statuses will be @@ -166,30 +165,30 @@ class TestStringConverter(TestCase): status_offset = int(nx.dtype(nx.integer).itemsize < nx.dtype(nx.int64).itemsize) # test int > 2**32 - assert_equal(converter.upgrade(asbytes('17179869184')), 17179869184) + assert_equal(converter.upgrade(b'17179869184'), 17179869184) assert_equal(converter._status, 1 + status_offset) # test float - assert_allclose(converter.upgrade(asbytes('0.')), 0.0) + assert_allclose(converter.upgrade(b'0.'), 0.0) assert_equal(converter._status, 2 + status_offset) # test complex - assert_equal(converter.upgrade(asbytes('0j')), complex('0j')) + assert_equal(converter.upgrade(b'0j'), complex('0j')) assert_equal(converter._status, 3 + status_offset) # test str - assert_equal(converter.upgrade(asbytes('a')), asbytes('a')) + assert_equal(converter.upgrade(b'a'), b'a') assert_equal(converter._status, len(converter._mapper) - 1) def test_missing(self): "Tests the use of missing values." - converter = StringConverter(missing_values=(asbytes('missing'), - asbytes('missed'))) - converter.upgrade(asbytes('0')) - assert_equal(converter(asbytes('0')), 0) - assert_equal(converter(asbytes('')), converter.default) - assert_equal(converter(asbytes('missing')), converter.default) - assert_equal(converter(asbytes('missed')), converter.default) + converter = StringConverter(missing_values=(b'missing', + b'missed')) + converter.upgrade(b'0') + assert_equal(converter(b'0'), 0) + assert_equal(converter(b''), converter.default) + assert_equal(converter(b'missing'), converter.default) + assert_equal(converter(b'missed'), converter.default) try: converter('miss') except ValueError: @@ -200,11 +199,11 @@ class TestStringConverter(TestCase): dateparser = _bytes_to_date StringConverter.upgrade_mapper(dateparser, date(2000, 1, 1)) convert = StringConverter(dateparser, date(2000, 1, 1)) - test = convert(asbytes('2001-01-01')) + test = convert(b'2001-01-01') assert_equal(test, date(2001, 1, 1)) - test = convert(asbytes('2009-01-01')) + test = convert(b'2009-01-01') assert_equal(test, date(2009, 1, 1)) - test = convert(asbytes('')) + test = convert(b'') assert_equal(test, date(2000, 1, 1)) def test_string_to_object(self): @@ -215,43 +214,43 @@ class TestStringConverter(TestCase): def test_keep_default(self): "Make sure we don't lose an explicit default" - converter = StringConverter(None, missing_values=asbytes(''), + converter = StringConverter(None, missing_values=b'', default=-999) - converter.upgrade(asbytes('3.14159265')) + converter.upgrade(b'3.14159265') assert_equal(converter.default, -999) assert_equal(converter.type, np.dtype(float)) # converter = StringConverter( - None, missing_values=asbytes(''), default=0) - converter.upgrade(asbytes('3.14159265')) + None, missing_values=b'', default=0) + converter.upgrade(b'3.14159265') assert_equal(converter.default, 0) assert_equal(converter.type, np.dtype(float)) def test_keep_default_zero(self): "Check that we don't lose a default of 0" converter = StringConverter(int, default=0, - missing_values=asbytes("N/A")) + missing_values=b"N/A") assert_equal(converter.default, 0) def test_keep_missing_values(self): "Check that we're not losing missing values" converter = StringConverter(int, default=0, - missing_values=asbytes("N/A")) + missing_values=b"N/A") assert_equal( - converter.missing_values, set(asbytes_nested(['', 'N/A']))) + converter.missing_values, set([b'', b'N/A'])) def test_int64_dtype(self): "Check that int64 integer types can be specified" converter = StringConverter(np.int64, default=0) - val = asbytes("-9223372036854775807") + val = b"-9223372036854775807" assert_(converter(val) == -9223372036854775807) - val = asbytes("9223372036854775807") + val = b"9223372036854775807" assert_(converter(val) == 9223372036854775807) def test_uint64_dtype(self): "Check that uint64 integer types can be specified" converter = StringConverter(np.uint64, default=0) - val = asbytes("9223372043271415339") + val = b"9223372043271415339" assert_(converter(val) == 9223372043271415339) diff --git a/numpy/lib/tests/test_arraypad.py b/numpy/lib/tests/test_arraypad.py index d037962e6..056aa4582 100644 --- a/numpy/lib/tests/test_arraypad.py +++ b/numpy/lib/tests/test_arraypad.py @@ -986,10 +986,7 @@ class TestNdarrayPadWidth(TestCase): class TestUnicodeInput(TestCase): def test_unicode_mode(self): - try: - constant_mode = unicode('constant') - except NameError: - constant_mode = 'constant' + constant_mode = u'constant' a = np.pad([1], 2, mode=constant_mode) b = np.array([0, 0, 1, 0, 0]) assert_array_equal(a, b) diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index 7cc72e775..93727ef0c 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -284,7 +284,6 @@ import warnings from io import BytesIO import numpy as np -from numpy.compat import asbytes, asbytes_nested, sixu from numpy.testing import ( run_module_suite, assert_, assert_array_equal, assert_raises, raises, dec, SkipTest @@ -545,8 +544,8 @@ def test_pickle_python2_python3(): import __builtin__ xrange = __builtin__.xrange - expected = np.array([None, xrange, sixu('\u512a\u826f'), - asbytes('\xe4\xb8\x8d\xe8\x89\xaf')], + expected = np.array([None, xrange, u'\u512a\u826f', + b'\xe4\xb8\x8d\xe8\x89\xaf'], dtype=object) for fname in ['py2-objarr.npy', 'py2-objarr.npz', @@ -682,23 +681,23 @@ def test_write_version(): raise AssertionError("we should have raised a ValueError for the bad version %r" % (version,)) -bad_version_magic = asbytes_nested([ - '\x93NUMPY\x01\x01', - '\x93NUMPY\x00\x00', - '\x93NUMPY\x00\x01', - '\x93NUMPY\x02\x00', - '\x93NUMPY\x02\x02', - '\x93NUMPY\xff\xff', -]) -malformed_magic = asbytes_nested([ - '\x92NUMPY\x01\x00', - '\x00NUMPY\x01\x00', - '\x93numpy\x01\x00', - '\x93MATLB\x01\x00', - '\x93NUMPY\x01', - '\x93NUMPY', - '', -]) +bad_version_magic = [ + b'\x93NUMPY\x01\x01', + b'\x93NUMPY\x00\x00', + b'\x93NUMPY\x00\x01', + b'\x93NUMPY\x02\x00', + b'\x93NUMPY\x02\x02', + b'\x93NUMPY\xff\xff', +] +malformed_magic = [ + b'\x92NUMPY\x01\x00', + b'\x00NUMPY\x01\x00', + b'\x93numpy\x01\x00', + b'\x93MATLB\x01\x00', + b'\x93NUMPY\x01', + b'\x93NUMPY', + b'', +] def test_read_magic(): s1 = BytesIO() @@ -778,11 +777,11 @@ def test_bad_header(): # header of length less than 2 should fail s = BytesIO() assert_raises(ValueError, format.read_array_header_1_0, s) - s = BytesIO(asbytes('1')) + s = BytesIO(b'1') assert_raises(ValueError, format.read_array_header_1_0, s) # header shorter than indicated size should fail - s = BytesIO(asbytes('\x01\x00')) + s = BytesIO(b'\x01\x00') assert_raises(ValueError, format.read_array_header_1_0, s) # headers without the exact keys required should fail diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index 83fca5b91..2a494fe08 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -532,7 +532,7 @@ class TestLoadTxt(TestCase): c.write('# comment\n1,2,3,5\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',', - comments=unicode('#')) + comments=u'#') a = np.array([1, 2, 3, 5], int) assert_array_equal(x, a) diff --git a/numpy/lib/tests/test_twodim_base.py b/numpy/lib/tests/test_twodim_base.py index 98b8aa39c..d57791e34 100644 --- a/numpy/lib/tests/test_twodim_base.py +++ b/numpy/lib/tests/test_twodim_base.py @@ -15,7 +15,6 @@ from numpy import ( ) import numpy as np -from numpy.compat import asbytes_nested def get_mat(n): @@ -91,7 +90,7 @@ class TestEye(TestCase): def test_strings(self): assert_equal(eye(2, 2, dtype='S3'), - asbytes_nested([['1', ''], ['', '1']])) + [[b'1', b''], [b'', b'1']]) def test_bool(self): assert_equal(eye(2, 2, dtype=bool), [[True, False], [False, True]]) |