passlib.ext.django: Updated UTs to work with latest django release

(should fix long-standing issue 98)

* test_ext_django:

    - Simplified "stock config" setup code.  It now gets it's "sha_rounds" value
      from the django source, so we don't have to manually update it every time
      django changes their default.  This should require less maintenance across
      minor django releases.  (Should fix issue 98, and prevent recurrence)

    - Updated tests to account for quirks in how encoded hashes are handled.
      Specifically: None, "", and invalid hashes all cause subtly different
      behaviors across django versions.  tests pass against django 1.8 - 3.1.

    - split "empty hash" test out from the loop it shared with "null hash" test,
      since the two behave differently.

* tox: expanded envlist to explicitly test a bunch more django versions
  (1.8 - 3.1); and remove some needless "django 2.x + py2" tests

* passlib.apps: reformatted django CryptContext declarations;
  added one for django 2.1 (which dropped "django_bcrypt" it's default list)

* passlib.ext.django:

    - added internal "quirks" helper as central place to track
      minor edge-case changes between django versions.

    - passlib_to_django() helper now falls back to searching hasher classes
      directly, even if patch isn't installed.  this allows it to work
      for django hashers that have been removed from django's default list.

6 files changed, 257 insertions, 80 deletions

diff --git a/docs/history/1.7.rst b/docs/history/1.7.rst
index 95d57f5..3a8250f 100644
--- a/docs/history/1.7.rst
+++ b/docs/history/1.7.rst
@@ -22,6 +22,10 @@ Bugfixes
 
 Other Changes
 -------------
+
+* :mod:`passlib.ext.django` -- updated tests to pass for Django 1.8 - 3.1 (:issue:`98`);
+  along with some internal refactoring of the test classes.
+
 * .. py:currentmodule:: passlib.context
 
   :class:`CryptContext` will now throw :exc:`~passlib.exc.UnknownHashError` when it can't identify
@@ -29,6 +33,12 @@ Other Changes
   Previously it would throw a generic :exc:`ValueError`.
 
 
+Deprecations
+------------
+
+* :mod:`passlib.ext.django`: This extension will require Django 2.2 or newer as of Passlib 1.8.
+
+
 **1.7.3** (2020-10-06)
 ======================
 
diff --git a/docs/lib/passlib.ext.django.rst b/docs/lib/passlib.ext.django.rst
index 51097cc..747c5e7 100644
--- a/docs/lib/passlib.ext.django.rst
+++ b/docs/lib/passlib.ext.django.rst
@@ -6,12 +6,6 @@
 :mod:`passlib.ext.django` - Django Password Hashing Plugin
 ==========================================================
 
-.. versionadded:: 1.6
-
-.. versionchanged:: 1.7
-
-    As of Passlib 1.7, this module requires Django 1.8 or newer.
-
 .. rst-class:: float-center without-title
 
 .. warning::
@@ -49,9 +43,19 @@ of uses:
     This plugin should be considered "release candidate" quality.
     It works, and has good unittest coverage, but has seen only
     limited real-world use. Please report any issues.
-    It has been tested with Django 1.8 - 1.9.
+    It has been tested with Django 1.8 - 3.1.
+
+.. versionadded:: 1.6
+
+.. versionchanged:: 1.7
+
+    Support for Django 1.0 - 1.7 was dropped; now requires Django 1.8 or newer.
+
+.. rst-class:: without-title
+
+.. warning::
 
-    (Support for Django 1.0 - 1.7 was dropped after Passlib 1.6).
+    As of Passlib 1.8, this module will require Django 2.2 or newer.
 
 Installation
 =============
diff --git a/passlib/apps.py b/passlib/apps.py
index 30cd6af..682bbff 100644
--- a/passlib/apps.py
+++ b/passlib/apps.py
@@ -87,9 +87,17 @@ custom_app_context = LazyCryptContext(
 #=============================================================================
 # django
 #=============================================================================
+
+#-----------------------------------------------------------------------
+# 1.0
+#-----------------------------------------------------------------------
+
 _django10_schemes = [
-        "django_salted_sha1", "django_salted_md5", "django_des_crypt",
-        "hex_md5", "django_disabled",
+    "django_salted_sha1",
+    "django_salted_md5",
+    "django_des_crypt",
+    "hex_md5",
+    "django_disabled",
 ]
 
 django10_context = LazyCryptContext(
@@ -98,28 +106,60 @@ django10_context = LazyCryptContext(
     deprecated=["hex_md5"],
 )
 
-_django14_schemes = ["django_pbkdf2_sha256", "django_pbkdf2_sha1",
-                     "django_bcrypt"] + _django10_schemes
+#-----------------------------------------------------------------------
+# 1.4
+#-----------------------------------------------------------------------
+
+_django14_schemes = [
+    "django_pbkdf2_sha256",
+    "django_pbkdf2_sha1",
+    "django_bcrypt"
+] + _django10_schemes
+
 django14_context = LazyCryptContext(
     schemes=_django14_schemes,
     deprecated=_django10_schemes,
 )
 
-_django16_schemes = _django14_schemes[:]
+#-----------------------------------------------------------------------
+# 1.6
+#-----------------------------------------------------------------------
+
+_django16_schemes = list(_django14_schemes)
 _django16_schemes.insert(1, "django_bcrypt_sha256")
 django16_context = LazyCryptContext(
     schemes=_django16_schemes,
     deprecated=_django10_schemes,
 )
 
-django110_context = LazyCryptContext(
-    schemes=["django_pbkdf2_sha256", "django_pbkdf2_sha1",
-             "django_argon2", "django_bcrypt", "django_bcrypt_sha256",
-             "django_disabled"],
-)
+#-----------------------------------------------------------------------
+# 1.10
+#-----------------------------------------------------------------------
+
+_django_110_schemes = [
+    "django_pbkdf2_sha256",
+    "django_pbkdf2_sha1",
+    "django_argon2",
+    "django_bcrypt",
+    "django_bcrypt_sha256",
+    "django_disabled",
+]
+django110_context = LazyCryptContext(schemes=_django_110_schemes)
+
+#-----------------------------------------------------------------------
+# 2.1
+#-----------------------------------------------------------------------
+
+_django21_schemes = list(_django_110_schemes)
+_django21_schemes.remove("django_bcrypt")
+django21_context = LazyCryptContext(schemes=_django21_schemes)
+
+#-----------------------------------------------------------------------
+# latest
+#-----------------------------------------------------------------------
 
-# this will always point to latest version
-django_context = django110_context
+# this will always point to latest version in passlib
+django_context = django21_context
 
 #=============================================================================
 # ldap
diff --git a/passlib/ext/django/utils.py b/passlib/ext/django/utils.py
index 1d1bd67..2f8a2ef 100644
--- a/passlib/ext/django/utils.py
+++ b/passlib/ext/django/utils.py
@@ -26,13 +26,29 @@ __all__ = [
     "DJANGO_VERSION",
     "MIN_DJANGO_VERSION",
     "get_preset_config",
-    "get_django_hasher",
+    "quirks",
 ]
 
 #: minimum version supported by passlib.ext.django
 MIN_DJANGO_VERSION = (1, 8)
 
 #=============================================================================
+# quirk detection
+#=============================================================================
+
+class quirks:
+
+    #: django check_password() started throwing error on encoded=None
+    #: (really identify_hasher did)
+    none_causes_check_password_error = DJANGO_VERSION >= (2, 1)
+
+    #: django is_usable_password() started returning True for password = {None, ""} values.
+    empty_is_usable_password = DJANGO_VERSION >= (2, 1)
+
+    #: django is_usable_password() started returning True for non-hash strings in 2.1
+    invalid_is_usable_password = DJANGO_VERSION >= (2, 1)
+
+#=============================================================================
 # default policies
 #=============================================================================
 
@@ -235,6 +251,13 @@ class DjangoTranslator(object):
         md5="MD5PasswordHasher",
     )
 
+    if DJANGO_VERSION > (2, 1):
+        # present but disabled by default as of django 2.1; not sure when added,
+        # so not listing it by default.
+        _builtin_django_hashers.update(
+            bcrypt="BCryptPasswordHasher",
+        )
+
     def _create_django_hasher(self, django_name):
         """
         helper to create new django hasher by name.
@@ -244,17 +267,22 @@ class DjangoTranslator(object):
         module = sys.modules.get("passlib.ext.django.models")
         if module is None or not module.adapter.patched:
             from django.contrib.auth.hashers import get_hasher
-            return get_hasher(django_name)
-
-        # We've patched django's get_hashers(), so calling django's get_hasher()
-        # or get_hashers_by_algorithm() would only land us back here.
-        # As non-ideal workaround, have to use original get_hashers(),
-        get_hashers = module.adapter._manager.getorig("django.contrib.auth.hashers:get_hashers").__wrapped__
-        for hasher in get_hashers():
-            if hasher.algorithm == django_name:
-                return hasher
-
-        # hardcode a few for cases where get_hashers() look won't work.
+            try:
+                return get_hasher(django_name)
+            except ValueError as err:
+                if not str(err).startswith("Unknown password hashing algorithm"):
+                    raise
+        else:
+            # We've patched django's get_hashers(), so calling django's get_hasher()
+            # or get_hashers_by_algorithm() would only land us back here.
+            # As non-ideal workaround, have to use original get_hashers(),
+            get_hashers = module.adapter._manager.getorig("django.contrib.auth.hashers:get_hashers").__wrapped__
+            for hasher in get_hashers():
+                if hasher.algorithm == django_name:
+                    return hasher
+
+        # hardcode a few for cases where get_hashers() lookup won't work
+        # (mainly, hashers that are present in django, but disabled by their default config)
         path = self._builtin_django_hashers.get(django_name)
         if path:
             if "." not in path:
@@ -545,12 +573,20 @@ class DjangoContextAdapter(DjangoTranslator):
         """
         # XXX: this currently ignores "preferred" keyword, since its purpose
         #      was for hash migration, and that's handled by the context.
+        # XXX: honor "none_causes_check_password_error" quirk for django 2.2+?
+        #      seems safer to return False.
         if password is None or not self.is_password_usable(encoded):
             return False
 
         # verify password
         context = self.context
-        correct = context.verify(password, encoded)
+        try:
+            correct = context.verify(password, encoded)
+        except exc.UnknownHashError:
+            # As of django 1.5, unidentifiable hashes returns False
+            # (side-effect of django issue 18453)
+            return False
+
         if not (correct and setter):
             return correct
 
@@ -591,8 +627,12 @@ class DjangoContextAdapter(DjangoTranslator):
         if not self.is_password_usable(hash):
             return False
         cat = self.get_user_category(user)
-        ok, new_hash = self.context.verify_and_update(password, hash,
-                                                      category=cat)
+        try:
+            ok, new_hash = self.context.verify_and_update(password, hash, category=cat)
+        except exc.UnknownHashError:
+            # As of django 1.5, unidentifiable hashes returns False
+            # (side-effect of django issue 18453)
+            return False
         if ok and new_hash is not None:
             # migrate to new hash if needed.
             user.password = new_hash
diff --git a/passlib/tests/test_ext_django.py b/passlib/tests/test_ext_django.py
index f827a36..8464fa0 100644
--- a/passlib/tests/test_ext_django.py
+++ b/passlib/tests/test_ext_django.py
@@ -13,7 +13,7 @@ from passlib import apps as _apps, exc, registry
 from passlib.apps import django10_context, django14_context, django16_context
 from passlib.context import CryptContext
 from passlib.ext.django.utils import (
-    DJANGO_VERSION, MIN_DJANGO_VERSION, DjangoTranslator,
+    DJANGO_VERSION, MIN_DJANGO_VERSION, DjangoTranslator, quirks,
 )
 from passlib.utils.compat import iteritems, get_method_function, u
 from passlib.utils.decor import memoized_property
@@ -60,6 +60,11 @@ if has_min_django:
     from django.apps import apps
     apps.populate(["django.contrib.contenttypes", "django.contrib.auth"])
 
+# log a warning if tested w/ newer version.
+# NOTE: this is mainly here as place to mark what version it was run against before release.
+if DJANGO_VERSION >= (3, 2):
+    log.info("this release hasn't been tested against Django %r", DJANGO_VERSION)
+
 #=============================================================================
 # support funcs
 #=============================================================================
@@ -133,28 +138,58 @@ def check_django_hasher_has_backend(name):
 # work up stock django config
 #=============================================================================
 
+def _modify_django_config(kwds, sha_rounds=None):
+    """
+    helper to build django CryptContext config matching expected setup for stock django deploy.
+    :param kwds:
+    :param sha_rounds:
+    :return:
+    """
+    # make sure we have dict
+    if hasattr(kwds, "to_dict"):
+        # type: CryptContext
+        kwds = kwds.to_dict()
+
+    # update defaults
+    kwds.update(
+        # TODO: push this to passlib.apps django contexts
+        deprecated="auto",
+    )
+
+    # fill in default rounds for current django version, so our sample hashes come back
+    # unchanged, instead of being upgraded in-place by check_password().
+    if sha_rounds is None and has_min_django:
+        from django.contrib.auth.hashers import PBKDF2PasswordHasher
+        sha_rounds = PBKDF2PasswordHasher.iterations
+
+    # modify rounds
+    if sha_rounds:
+        kwds.update(
+            django_pbkdf2_sha1__default_rounds=sha_rounds,
+            django_pbkdf2_sha256__default_rounds=sha_rounds,
+        )
+
+    return kwds
+
+#----------------------------------------------------
 # build config dict that matches stock django
-# XXX: move these to passlib.apps?
-if DJANGO_VERSION >= (1, 11):
-    stock_config = _apps.django110_context.to_dict()
-    stock_rounds = 36000
+#----------------------------------------------------
+
+# XXX: replace this with code that interrogates default django config directly?
+#      could then separate out "validation of djangoXX_context objects"
+#      and "validation that individual hashers match django".
+#      or maybe add a "get_django_context(django_version)" helper to passlib.apps?
+if DJANGO_VERSION >= (2, 1):
+    stock_config = _modify_django_config(_apps.django21_context)
 elif DJANGO_VERSION >= (1, 10):
-    stock_config = _apps.django110_context.to_dict()
-    stock_rounds = 30000
-elif DJANGO_VERSION >= (1, 9):
-    stock_config = _apps.django16_context.to_dict()
-    stock_rounds = 24000
-else:  # 1.8
-    stock_config = _apps.django16_context.to_dict()
-    stock_rounds = 20000
-
-stock_config.update(
-    deprecated="auto",
-    django_pbkdf2_sha1__default_rounds=stock_rounds,
-    django_pbkdf2_sha256__default_rounds=stock_rounds,
-)
+    stock_config = _modify_django_config(_apps.django110_context)
+else:
+    # assert DJANGO_VERSION >= (1, 8)
+    stock_config = _modify_django_config(_apps.django16_context)
 
+#----------------------------------------------------
 # override sample hashes used in test cases
+#----------------------------------------------------
 from passlib.hash import django_pbkdf2_sha256
 sample_hashes = dict(
     django_pbkdf2_sha256=("not a password", django_pbkdf2_sha256
@@ -459,34 +494,65 @@ class DjangoBehaviorTest(_ExtensionTest):
         # User.set_password() - n/a
 
         # User.check_password() - returns False
-        # FIXME: at some point past 1.8, some of these django started handler None differently;
-        #        and/or throwing TypeError.  need to investigate when that change occurred;
-        #        update these tests, and maybe passlib.ext.django as well.
         user = FakeUser()
         user.password = None
-        self.assertFalse(user.check_password(PASS1))
-        self.assertFalse(user.has_usable_password())
+        if quirks.none_causes_check_password_error and not patched:
+            # django 2.1+
+            self.assertRaises(TypeError, user.check_password, PASS1)
+        else:
+            self.assertFalse(user.check_password(PASS1))
+
+        self.assertEqual(user.has_usable_password(),
+                         quirks.empty_is_usable_password)
 
         # make_password() - n/a
 
         # check_password() - error
-        self.assertFalse(check_password(PASS1, None))
+        if quirks.none_causes_check_password_error and not patched:
+            self.assertRaises(TypeError, check_password, PASS1, None)
+        else:
+            self.assertFalse(check_password(PASS1, None))
 
         # identify_hasher() - error
         self.assertRaises(TypeError, identify_hasher, None)
 
         #=======================================================
-        # empty & invalid hash values
-        # NOTE: django 1.5 behavior change due to django ticket 18453
-        # NOTE: passlib integration tries to match current django version
+        # empty hash values
+        #=======================================================
+
+        # User.set_password() - n/a
+
+        # User.check_password()
+        # As of django 1.5, blank hash returns False (django issue 18453)
+        user = FakeUser()
+        user.password = ""
+        self.assertFalse(user.check_password(PASS1))
+
+        # verify hash wasn't changed/upgraded during check_password() call
+        self.assertEqual(user.password, "")
+        self.assertEqual(user.pop_saved_passwords(), [])
+
+        # User.has_usable_password()
+        self.assertEqual(user.has_usable_password(), quirks.empty_is_usable_password)
+
+        # make_password() - n/a
+
+        # check_password()
+        self.assertFalse(check_password(PASS1, ""))
+
+        # identify_hasher() - throws error
+        self.assertRaises(ValueError, identify_hasher, "")
+
+        #=======================================================
+        # invalid hash values
         #=======================================================
-        for hash in ("", # empty hash
-                     "$789$foo", # empty identifier
-                     ):
+        for hash in [
+            "$789$foo",  # empty identifier
+        ]:
             # User.set_password() - n/a
 
             # User.check_password()
-            # As of django 1.5, blank OR invalid hash returns False
+            # As of django 1.5, invalid hash returns False (side effect of django issue 18453)
             user = FakeUser()
             user.password = hash
             self.assertFalse(user.check_password(PASS1))
@@ -496,7 +562,7 @@ class DjangoBehaviorTest(_ExtensionTest):
             self.assertEqual(user.pop_saved_passwords(), [])
 
             # User.has_usable_password()
-            self.assertFalse(user.has_usable_password())
+            self.assertEqual(user.has_usable_password(), quirks.invalid_is_usable_password)
 
             # make_password() - n/a
 
@@ -701,12 +767,15 @@ class DjangoExtensionTest(_ExtensionTest):
         passlib_to_django = DjangoTranslator().passlib_to_django
 
         # should return native django hasher if available
-        if DJANGO_VERSION > (1,10):
+        if DJANGO_VERSION > (1, 10):
             self.assertRaises(ValueError, passlib_to_django, "hex_md5")
         else:
             hasher = passlib_to_django("hex_md5")
             self.assertIsInstance(hasher, hashers.UnsaltedMD5PasswordHasher)
 
+        # should return native django hasher
+        # NOTE: present but not enabled by default in django as of 2.1
+        #       (see _builtin_django_hashers)
         hasher = passlib_to_django("django_bcrypt")
         self.assertIsInstance(hasher, hashers.BCryptPasswordHasher)
 
@@ -732,6 +801,10 @@ class DjangoExtensionTest(_ExtensionTest):
              'hash': u('v2RWkZ*************************************'),
              })
 
+        # made up name should throw error
+        # XXX: should this throw ValueError instead, to match django?
+        self.assertRaises(KeyError, passlib_to_django, "does_not_exist")
+
     #===================================================================
     # PASSLIB_CONFIG settings
     #===================================================================
diff --git a/tox.ini b/tox.ini
index 80d4981..43ef667 100644
--- a/tox.ini
+++ b/tox.ini
@@ -25,7 +25,7 @@
 #
 #    "full"
 #        extra regression and internal tests are enabled, hash algorithms are tested
-#        against all available backends, unavailable ones are mocked whre possible,
+#        against all available backends, unavailable ones are mocked where possible,
 #        additional time is devoted to fuzz testing.
 #
 #-----------------------------------------------------------------------
@@ -76,11 +76,16 @@ envlist =
     argon2hash-argon2pure-py{2,3,py,py3},
 
     # django tests
-    # NOTE: django >= 1.7 distributes tests as part of source, not the package, so for full
+    # NOTE: django distributes it'a tests as part of source, not the package, so for full
     #       integration tests to run, caller must provide a copy of the latest django source,
     #       and set the env var PASSLIB_TESTS_DJANGO_SOURCE_PATH to point to it.
-    django{18,1x}-wdeps-py{2,3},
-    django-{wdeps,nodeps}-py{2,3},
+    # NOTE: django 2.0 dropped python 2 support, so not including that in matrix.
+    # django support roadmap -- https://www.djangoproject.com/download/
+    # django python versions -- https://docs.djangoproject.com/en/3.1/faq/install/#what-python-version-can-i-use-with-django
+    django-dj{1x,18}-wdeps-py{2,3},
+    django-dj{Latest,31,30,22,21,20}-wdeps-py3,
+    django-dj{1x}-nodeps-py2,
+    django-dj{Latest}-nodeps-py3,
 
     # other tests
     docs
@@ -141,7 +146,7 @@ commands =
     argon2hash: nosetests {posargs:{env:TEST_OPTS} passlib.tests.test_handlers_argon2}
 
     # django tests
-    django{,1x,18}: nosetests {posargs:{env:TEST_OPTS} passlib.tests.test_ext_django passlib.tests.test_handlers_django}
+    django: nosetests {posargs:{env:TEST_OPTS} passlib.tests.test_ext_django passlib.tests.test_handlers_django}
 
 deps =
     # common
@@ -183,14 +188,19 @@ deps =
     argon2hash-argon2pure: argon2pure
 
     # django extension tests
-    django18: django>=1.8,<1.9
-    django1x: django<2
-    django: django
-    django{,1x,18}-wdeps: bcrypt
-    # django{,18}-nodeps -- would like to use this as negative dependancy for 'bcrypt' instead
+    dj18: django<1.9
+    dj1x: django<2.0
+    dj20: django<2.1
+    dj21: django<2.2
+    dj22: django<3.0
+    dj30: django<3.1
+    dj31: django<3.2
+    djLatest: django
+    django-wdeps: bcrypt
+    # django-nodeps -- would like to use this as negative dependancy for 'bcrypt' instead
 
     # needed by django's internal tests
-    django{,1x,18}: mock
+    django: mock
 
 #===========================================================================
 # build documentation