natsort release version 3.5.4.3.5.4

  - Fixed bug where a 'TypeError' was raised if a string containing a
    leading number was sorted with alpha-only strings when 'LOCALE' is
    used.

11 files changed, 85 insertions, 28 deletions

diff --git a/MANIFEST.in b/MANIFEST.in
index ad8cd5a..aeffe3f 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -7,12 +7,14 @@ include natsort/__init__.py
 include natsort/py23compat.py
 include natsort/locale_help.py
 include natsort/fake_fastnumbers.py
+include natsort/utils.py
 include test_natsort/profile_natsorted.py
 include test_natsort/stress_natsort.py
 include test_natsort/test_natsort.py
 include test_natsort/test_locale_help.py
 include test_natsort/test_fake_fastnumbers.py
 include test_natsort/test_main.py
+include test_natsort/test_utils.py
 include setup.py
 include setup.cfg
 prune natsort/__pycache__
diff --git a/README.rst b/README.rst
index 1822adc..d933201 100644
--- a/README.rst
+++ b/README.rst
@@ -92,6 +92,8 @@ when you sort:
     >>> # On Python 2, sorted(a) would return [2.0, 6, '4.5', '5', 'a']
     >>> # On Python 3, sorted(a) would raise an "unorderable types" TypeError
 
+You cannot mix and match ``str`` and ``bytes`` objects on Python 3.
+
 The natsort algorithm does other fancy things like 
 
  - recursively descend into lists of lists
@@ -176,10 +178,16 @@ History
 These are the last three entries of the changelog.  See the package documentation
 for the complete `changelog <http://pythonhosted.org//natsort/changelog.html>`_.
 
+04-02-2015 v. 3.5.4
+'''''''''''''''''''
+
+    - Fixed bug where a 'TypeError' was raised if a string containing a leading
+      number was sorted with alpha-only strings when 'LOCALE' is used.
+
 03-26-2015 v. 3.5.3
 '''''''''''''''''''
 
-    - Fixed bug where ``--reverse-filter`` option in shell script was not
+    - Fixed bug where '--reverse-filter; option in shell script was not
       getting checked for correctness.
     - Documentation updates to better describe locale bug, and illustrate
       upcoming default behavior change.
@@ -190,12 +198,3 @@ for the complete `changelog <http://pythonhosted.org//natsort/changelog.html>`_.
 
     - Enhancement that will convert a 'pathlib.Path' object to a 'str' if
       'ns.PATH' is enabled.
-
-09-25-2014 v. 3.5.1
-'''''''''''''''''''
-
-    - Fixed bug that caused list/tuples to fail when using 'ns.LOWECASEFIRST'
-      or 'ns.IGNORECASE'.
-    - Refactored modules so that only the public API was in natsort.py and
-      ns_enum.py.
-    - Refactored all import statements to be absolute, not relative.
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index a57532c..a99687d 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -3,10 +3,16 @@
 Changelog
 ---------
 
+04-02-2015 v. 3.5.4
+'''''''''''''''''''
+
+    - Fixed bug where a 'TypeError' was raised if a string containing a leading
+      number was sorted with alpha-only strings when 'LOCALE' is used.
+
 03-26-2015 v. 3.5.3
 '''''''''''''''''''
 
-    - Fixed bug where ``--reverse-filter`` option in shell script was not
+    - Fixed bug where '--reverse-filter' option in shell script was not
       getting checked for correctness.
     - Documentation updates to better describe locale bug, and illustrate
       upcoming default behavior change.
diff --git a/docs/source/intro.rst b/docs/source/intro.rst
index 94fec70..5e6ef14 100644
--- a/docs/source/intro.rst
+++ b/docs/source/intro.rst
@@ -87,6 +87,8 @@ when you sort::
     >>> # On Python 2, sorted(a) would return [2.0, 6, '4.5', '5', 'a']
     >>> # On Python 3, sorted(a) would raise an "unorderable types" TypeError
 
+You cannot mix and match ``str`` and ``bytes`` objects on Python 3.
+
 The natsort algorithm does other fancy things like 
 
  - recursively descend into lists of lists
diff --git a/natsort/_version.py b/natsort/_version.py
index 0047fef..5c785cd 100644
--- a/natsort/_version.py
+++ b/natsort/_version.py
@@ -2,4 +2,4 @@
 from __future__ import (print_function, division,
                         unicode_literals, absolute_import)
 
-__version__ = '3.5.3'
+__version__ = '3.5.4'
diff --git a/natsort/locale_help.py b/natsort/locale_help.py
index 8a5cf4f..2278601 100644
--- a/natsort/locale_help.py
+++ b/natsort/locale_help.py
@@ -15,6 +15,13 @@ from locale import localeconv
 # Local imports.
 from natsort.py23compat import py23_zip
 
+# If the user has fastnumbers installed, they will get great speed
+# benefits. If not, we simulate the functions here.
+try:
+    from fastnumbers import isreal
+except ImportError:
+    from natsort.fake_fastnumbers import isreal
+
 # We need cmp_to_key for Python2 because strxfrm is broken for unicode.
 if sys.version[:3] == '2.7':
     from functools import cmp_to_key
@@ -70,12 +77,15 @@ try:
             _d[l] = c.getSortKey
         return _d[l]
     use_pyicu = True
+    null_string = b''
 except ImportError:
     if sys.version[0] == '2':
         from locale import strcoll
         strxfrm = cmp_to_key(strcoll)
+        null_string = strxfrm('')
     else:
         from locale import strxfrm
+        null_string = ''
     use_pyicu = False
 
 
@@ -119,12 +129,12 @@ def locale_convert(val, func, group):
     if group:
         if use_pyicu:
             xfrm = get_pyicu_transform(getlocale())
-            return xfrm(groupletters(val)) if s is t else t
+            return xfrm(groupletters(val)) if not isreal(t) else t
         else:
-            return strxfrm(groupletters(val)) if s is t else t
+            return strxfrm(groupletters(val)) if not isreal(t) else t
     else:
         if use_pyicu:
             xfrm = get_pyicu_transform(getlocale())
-            return xfrm(val) if s is t else t
+            return xfrm(val) if not isreal(t) else t
         else:
-            return strxfrm(val) if s is t else t
+            return strxfrm(val) if not isreal(t) else t
diff --git a/natsort/natsort.py b/natsort/natsort.py
index b136f91..05ccc64 100644
--- a/natsort/natsort.py
+++ b/natsort/natsort.py
@@ -16,6 +16,7 @@ from __future__ import (print_function, division,
                         unicode_literals, absolute_import)
 
 # Std lib. imports.
+import re
 from operator import itemgetter
 from functools import partial
 from warnings import warn
@@ -328,8 +329,10 @@ def natsorted(seq, key=None, number_type=float, signed=None, exp=None,
                       key=natsort_keygen(key, alg=alg))
     except TypeError as e:  # pragma: no cover
         # In the event of an unresolved "unorderable types" error
+        # for string to number type comparisons (not str/bytes),
         # attempt to sort again, being careful to prevent this error.
-        if 'unorderable types' in str(e):
+        r = re.compile(r'(?:str|bytes)\(\) [<>] (?:str|bytes)\(\)')
+        if 'unorderable types' in str(e) and not r.search(str(e)):
             return sorted(seq, reverse=reverse,
                           key=natsort_keygen(key,
                                              alg=alg | ns.TYPESAFE))
diff --git a/natsort/ns_enum.py b/natsort/ns_enum.py
index 6f042f9..162653f 100644
--- a/natsort/ns_enum.py
+++ b/natsort/ns_enum.py
@@ -83,6 +83,8 @@ class ns(object):
     TYPESAFE, T
         Try hard to avoid "unorderable types" error on Python 3. It
         is the same as setting the old `py3_safe` option to `True`.
+        You shouldn't need to use this unless you are using
+        ``natsort_keygen``.
 
     Notes
     -----
diff --git a/natsort/utils.py b/natsort/utils.py
index 0bd8309..71292c5 100644
--- a/natsort/utils.py
+++ b/natsort/utils.py
@@ -17,7 +17,7 @@ from itertools import islice
 from locale import localeconv
 
 # Local imports.
-from natsort.locale_help import locale_convert, grouper
+from natsort.locale_help import locale_convert, grouper, null_string
 from natsort.py23compat import py23_str, py23_zip
 from natsort.ns_enum import ns, _ns
 
@@ -132,13 +132,13 @@ def _number_extracter(s, regex, numconv, py3_safe, use_locale, group_letters):
     if not s:  # Return empty tuple for empty results.
         return ()
     elif isreal(s[0]):
-        s = [''] + s
+        s = [null_string if use_locale else ''] + s
 
     # The _py3_safe function inserts "" between numbers in the list,
     # and is used to get around "unorderable types" in complex cases.
     # It is a separate function that needs to be requested specifically
     # because it is expensive to call.
-    return _py3_safe(s) if py3_safe else s
+    return _py3_safe(s, use_locale) if py3_safe else s
 
 
 def _path_splitter(s, _d_match=re.compile(r'\.\d').match):
@@ -189,7 +189,7 @@ def _path_splitter(s, _d_match=re.compile(r'\.\d').match):
     return path_parts + base_parts
 
 
-def _py3_safe(parsed_list):
+def _py3_safe(parsed_list, use_locale):
     """Insert '' between two numbers."""
     length = len(parsed_list)
     if length < 2:
@@ -200,7 +200,7 @@ def _py3_safe(parsed_list):
         for before, after in py23_zip(islice(parsed_list, 0, length-1),
                                       islice(parsed_list, 1, None)):
             if isreal(before) and isreal(after):
-                nl_append("")
+                nl_append(null_string if use_locale else '')
             nl_append(after)
         return new_list
 
@@ -277,6 +277,10 @@ def _natsort_key(val, key, alg):
                                            use_locale,
                                            alg & _ns['GROUPLETTERS']))
         except (TypeError, AttributeError):
+            # Check if it is a bytes type, and if so return as a
+            # one element tuple.
+            if isinstance(val, bytes):
+                return (val,)
             # If not strings, assume it is an iterable that must
             # be parsed recursively. Do not apply the key recursively.
             # If this string was split as a path, turn off 'PATH'.
@@ -288,4 +292,5 @@ def _natsort_key(val, key, alg):
             # If there is still an error, it must be a number.
             # Return as-is, with a leading empty string.
             except TypeError:
-                return (('', val,),) if alg & _ns['PATH'] else ('', val,)
+                n = null_string if use_locale else ''
+                return ((n, val,),) if alg & _ns['PATH'] else (n, val,)
diff --git a/test_natsort/test_natsort.py b/test_natsort/test_natsort.py
index cb81663..a311e3f 100644
--- a/test_natsort/test_natsort.py
+++ b/test_natsort/test_natsort.py
@@ -4,6 +4,7 @@ Here are a collection of examples of how this module can be used.
 See the README or the natsort homepage for more details.
 """
 from __future__ import unicode_literals, print_function
+import sys
 import warnings
 import locale
 from operator import itemgetter
@@ -105,6 +106,20 @@ def test_natsorted_returns_sorted_list_with_mixed_type_input_and_does_not_raise_
     assert natsorted(a) == ['5a5-4', '5a5b2', 46, 'af5']
 
 
+def test_natsorted_with_mixed_input_returns_sorted_results_without_error():
+    a = ['2', 'ä', 'b', 1.5, 3]
+    assert natsorted(a) == [1.5, '2', 3, 'b', 'ä']
+
+
+def test_natsorted_with_mixed_input_raises_TypeError_if_bytes_type_is_involved_on_Python3():
+    if sys.version[0] == '3':
+        with raises(TypeError) as e:
+            assert natsorted(['ä', b'b'])
+        assert 'bytes' in str(e.value)
+    else:
+        assert True
+
+
 def test_natsorted_raises_ValueError_for_non_iterable_input():
     with raises(TypeError) as err:
         natsorted(100)
@@ -215,6 +230,15 @@ def test_natsorted_with_LOCALE_and_de_setting_returns_results_sorted_by_de_langu
     locale.setlocale(locale.LC_ALL, str(''))
 
 
+def test_natsorted_with_LOCALE_and_mixed_input_returns_sorted_results_without_error():
+    locale.setlocale(locale.LC_ALL, str('en_US.UTF-8'))
+    a = ['0', 'Á', '2', 'Z']
+    assert natsorted(a) == ['0', '2', 'Z', 'Á']
+    a = ['2', 'ä', 'b', 1.5, 3]
+    assert natsorted(a, alg=ns.LOCALE) == [1.5, '2', 3, 'ä', 'b']
+    locale.setlocale(locale.LC_ALL, str(''))
+
+
 def test_versorted_returns_results_identical_to_natsorted_with_VERSION():
     a = ['1.9.9a', '1.11', '1.9.9b', '1.11.4', '1.10.1']
     assert versorted(a) == natsorted(a, alg=ns.VERSION)
diff --git a/test_natsort/test_utils.py b/test_natsort/test_utils.py
index 8d61c1a..9f7c930 100644
--- a/test_natsort/test_utils.py
+++ b/test_natsort/test_utils.py
@@ -8,7 +8,7 @@ from natsort.ns_enum import ns
 from natsort.utils import _number_extracter, _py3_safe, _natsort_key, _args_to_enum
 from natsort.utils import _float_sign_exp_re, _float_nosign_exp_re, _float_sign_noexp_re
 from natsort.utils import _float_nosign_noexp_re, _int_nosign_re, _int_sign_re
-from natsort.locale_help import use_pyicu
+from natsort.locale_help import use_pyicu, null_string
 
 try:
     from fastnumbers import fast_float, fast_int
@@ -162,16 +162,20 @@ def test_number_extracter_extracts_numbers_and_strxfrms_letter_doubled_strings_w
 
 
 def test_py3_safe_does_nothing_if_no_numbers():
-    assert _py3_safe(['a', 'b', 'c']) == ['a', 'b', 'c']
-    assert _py3_safe(['a']) == ['a']
+    assert _py3_safe(['a', 'b', 'c'], False) == ['a', 'b', 'c']
+    assert _py3_safe(['a'], False) == ['a']
 
 
 def test_py3_safe_does_nothing_if_only_one_number():
-    assert _py3_safe(['a', 5]) == ['a', 5]
+    assert _py3_safe(['a', 5], False) == ['a', 5]
 
 
 def test_py3_safe_inserts_empty_string_between_two_numbers():
-    assert _py3_safe([5, 9]) == [5, '', 9]
+    assert _py3_safe([5, 9], False) == [5, '', 9]
+
+
+def test_py3_safe_with_use_locale_inserts_null_string_between_two_numbers():
+    assert _py3_safe([5, 9], True) == [5, null_string, 9]
 
 
 def test__natsort_key_with_float_splits_input_into_string_and_signed_float_with_exponent():