misc test bugfixes

* test_handlers: fix py3 u() compat issue

* test_totp: clean norm_hash_name() caches so warnings repeat per-test,
             added/fixed some warnings checks.

* HandlerCase: HasRounds.using() test: hack so bsdi_crypt can pass
  (the 'odd rounds only' was playing havoc w/ the test's expectations)

* HandlerCase: effective_rounds() / effective_ident() helpers
  now unwrap PrefixWrappers first; wrappers aren't callable like classes.

* HandlerCase: HasRounds.using() test: don't check min_rounds-1
  if min_rounds is 0.

* HandlerCase: multithreaded fuzz test -- detect & log errors if stalled thread,
  rather than main thread stalling forever. reduced thread count down to 10.

* reset_warning_filter() context manager -- simplified __exit__() cleanup code

4 files changed, 86 insertions, 46 deletions

diff --git a/passlib/tests/test_handlers.py b/passlib/tests/test_handlers.py
index 167402b..7992f82 100644
--- a/passlib/tests/test_handlers.py
+++ b/passlib/tests/test_handlers.py
@@ -304,10 +304,10 @@ class cisco_asa_test(UserHandlerMixin, HandlerCase):
         (('0123456789abcdefqwertyuiopasdfghj', 'user1234'), '5hPT/iC6DnoBxo6a'),
 
         # unicode password -- assumes cisco will use utf-8 encoding
-        ((u't\xe1ble', ''), 'xQXX755BKYRl0ZpQ'),
-        ((u't\xe1ble', '36'), 'Q/43xXKmIaKLycSj'),
-        ((u't\xe1ble', 'user'), 'Og8fB4NyF0m5Ed9c'),
-        ((u't\xe1ble', 'user1234'), 'Og8fB4NyF0m5Ed9c'),
+        ((u('t\xe1ble'), ''), 'xQXX755BKYRl0ZpQ'),
+        ((u('t\xe1ble'), '36'), 'Q/43xXKmIaKLycSj'),
+        ((u('t\xe1ble'), 'user'), 'Og8fB4NyF0m5Ed9c'),
+        ((u('t\xe1ble'), 'user1234'), 'Og8fB4NyF0m5Ed9c'),
     ]
 
     # append all the cisco_pix hashes w/ password < 13 chars ... those should be the same.
diff --git a/passlib/tests/test_totp.py b/passlib/tests/test_totp.py
index 924f6c1..0b8647d 100644
--- a/passlib/tests/test_totp.py
+++ b/passlib/tests/test_totp.py
@@ -15,6 +15,7 @@ import time as _time
 # pkg
 from passlib import exc
 from passlib.utils import to_bytes, to_unicode
+from passlib.utils.pbkdf2 import _clear_caches
 from passlib.utils.compat import unicode, u
 from passlib.tests.utils import TestCase
 # local
@@ -260,6 +261,15 @@ class _BaseOTPTest(TestCase):
     OtpType = None
 
     #=============================================================================
+    # setup
+    #=============================================================================
+    def setUp(self):
+        super(_BaseOTPTest, self).setUp()
+
+        # clear norm_hash_name() cache so 'unknown hash' warnings get emitted each time
+        _clear_caches()
+
+    #=============================================================================
     # subclass utils
     #=============================================================================
     def randotp(self, **kwds):
@@ -373,7 +383,7 @@ class _BaseOTPTest(TestCase):
 
         # invalid alg
         with self.assertWarningList([
-            dict(category=exc.PasslibRuntimeWarning, message_re="unknown hash.*SHA333")
+            dict(category=exc.PasslibRuntimeWarning, message_re="unknown hash.*SHA-333")
         ]):
             self.assertRaises(ValueError, OTP, KEY1, alg="SHA-333")
 
@@ -819,7 +829,7 @@ class TotpTest(_BaseOTPTest):
         otp.last_counter = counter + 1
         otp.now = lambda : time
         with self.assertWarningList([
-                dict(message_re=".*earlier than last time.*", category=PasslibSecurityWarning),
+                dict(message_re=".*earlier than last-used time.*", category=PasslibSecurityWarning),
                 ]):
             self.assertTrue(otp.generate_next().token)
         self.assertEqual(otp.last_counter, counter)
@@ -1255,7 +1265,9 @@ class TotpTest(_BaseOTPTest):
         self.assertEqual(otp.alg, "sha256")
 
         # unknown alg
-        self.assertRaises(ValueError, from_uri, "otpauth://totp/Example:alice@google.com?secret=JBSWY3DPEHPK3PXP&algorithm=SHA333")
+        with self.assertWarningList([exc.PasslibRuntimeWarning]):
+            self.assertRaises(ValueError, from_uri, "otpauth://totp/Example:alice@google.com?"
+                                                    "secret=JBSWY3DPEHPK3PXP&algorithm=SHA333")
 
         #--------------------------------------------------------------------------------
         # digit param
diff --git a/passlib/tests/utils.py b/passlib/tests/utils.py
index 353bf39..21e954a 100644
--- a/passlib/tests/utils.py
+++ b/passlib/tests/utils.py
@@ -154,6 +154,12 @@ def get_alt_backend(*args, **kwds):
         return backend
     return None
 
+def unwrap_handler(handler):
+    """return original handler, removing any wrapper objects"""
+    while hasattr(handler, "wrapped"):
+        handler = handler.wrapped
+    return handler
+
 #=============================================================================
 # misc helpers
 #=============================================================================
@@ -368,7 +374,7 @@ class TestCase(_TestCase):
 
         def __exit__(self, *exc_info):
             self.__super.__exit__(*exc_info)
-            if not exc_info or exc_info[0] is None:
+            if exc_info[0] is None:
                 self.case.assertWarningList(self.log, **self.kwds)
 
     def assertWarningList(self, wlist=None, desc=None, msg=None):
@@ -1278,7 +1284,15 @@ class HandlerCase(TestCase):
         # helpers
         #-------------------------------------
         handler = self.handler
+
+        if handler.name == "bsdi_crypt":
+            # hack to bypass bsdi-crypt's "odd rounds only" behavior, messes up this test
+            orig_handler = handler
+            handler = handler.using()
+            handler._generate_rounds = lambda self: super(orig_handler, self)._generate_rounds()
+
         def effective_rounds(cls, rounds=None):
+            cls = unwrap_handler(cls)
             return cls(rounds=rounds, use_defaults=True).rounds
 
         # create some fake values to test with
@@ -1322,9 +1336,10 @@ class HandlerCase(TestCase):
         #-------------------------------------
 
         # .using() should clip values below valid minimum, w/ warning
-        with self.assertWarningList([PasslibHashWarning]):
-            temp = handler.using(min_desired_rounds=orig_min_rounds - 1)
-        self.assertEqual(temp.min_desired_rounds, orig_min_rounds)
+        if orig_min_rounds > 0:
+            with self.assertWarningList([PasslibHashWarning]):
+                temp = handler.using(min_desired_rounds=orig_min_rounds - 1)
+            self.assertEqual(temp.min_desired_rounds, orig_min_rounds)
 
         # .using() should clip values above valid maximum, w/ warning
         if orig_max_rounds:
@@ -1360,9 +1375,10 @@ class HandlerCase(TestCase):
         #-------------------------------------
 
         # .using() should clip values below valid minimum w/ warning
-        with self.assertWarningList([PasslibHashWarning]):
-            temp = handler.using(max_desired_rounds=orig_min_rounds - 1)
-        self.assertEqual(temp.max_desired_rounds, orig_min_rounds)
+        if orig_min_rounds > 0:
+            with self.assertWarningList([PasslibHashWarning]):
+                temp = handler.using(max_desired_rounds=orig_min_rounds - 1)
+            self.assertEqual(temp.max_desired_rounds, orig_min_rounds)
 
         # .using() should clip values above valid maximum, w/ warning
         if orig_max_rounds:
@@ -1431,6 +1447,8 @@ class HandlerCase(TestCase):
     # TODO: HasRounds -- using() -- linear & log vary_rounds.
     #       borrow code from CryptContext's test_51_linear_vary_rounds & friends
 
+    # TODO: HasRounds.needs_update() -- min_desired_rounds / max_desired_rounds checks.
+
     #===================================================================
     # idents
     #===================================================================
@@ -1498,12 +1516,14 @@ class HandlerCase(TestCase):
         handler = self.handler
         orig_ident = handler.default_ident
         for alt_ident in handler.ident_values:
-            if alt_ident != handler.default_ident:
+            if alt_ident != orig_ident:
                 break
         else:
-            raise AssertionError("expected to find alternate ident")
+            raise AssertionError("expected to find alternate ident: default=%r values=%r" %
+                                 (orig_ident, handler.ident_values))
 
         def effective_ident(cls):
+            cls = unwrap_handler(cls)
             return cls(use_defaults=True).ident
 
         # keep default if nothing else specified
@@ -1898,6 +1918,10 @@ class HandlerCase(TestCase):
         * runs output of selected backend against other available backends
           (if any) to detect errors occurring between different backends.
         * runs output against other "external" verifiers such as OS crypt()
+
+        :param report_thread_state:
+            if true, writes state of loop to current_thread().passlib_fuzz_state.
+            used to help debug multi-threaded fuzz test issues (below)
         """
         if self.is_disabled_handler:
             raise self.skipTest("not applicable")
@@ -1993,14 +2017,23 @@ class HandlerCase(TestCase):
         threads = [launch(n) for n in irange(thread_count)]
 
         # wait until all threads exit
-        # XXX: should this have a maximum timeout set?
+        timeout = self.max_fuzz_time * 4
+        stalled = 0
         for thread in threads:
-            thread.join()
+            thread.join(timeout)
+            if not thread.is_alive():
+                continue
+            # XXX: not sure why this is happening, main one seems 1/4 times for sun_md5_crypt
+            log.error("%s timed out after %f seconds", thread.name, timeout)
+            stalled += 1
 
         # if any thread threw an error, raise one ourselves.
         if failed[0]:
             raise self.fail("%d/%d threads failed concurrent fuzz testing "
                       "(see error log for details)" % (failed[0], thread_count))
+        if stalled:
+            raise self.fail("%d/%d threads stalled during concurrent fuzz testing "
+                      "(see error log for details)" % (stalled, thread_count))
 
     #---------------------------------------------------------------
     # fuzz constants & helpers
@@ -2033,10 +2066,8 @@ class HandlerCase(TestCase):
             return value
         elif TEST_MODE(max="quick"):
             return 0
-        elif TEST_MODE(max="default"):
-            return 10
         else:
-            return 20
+            return 10
 
     #---------------------------------------------------------------
     # fuzz verifiers
@@ -2226,9 +2257,7 @@ class OsCryptMixin(HandlerCase):
         # find handler that generates safe_crypt() compatible hash
         handler = cls.alt_safe_crypt_handler
         if not handler:
-            handler = cls.handler
-            while hasattr(handler, "wrapped"):
-                handler = handler.wrapped
+            handler = unwrap_handler(cls.handler)
 
         # hack to prevent recursion issue when .has_backend() is called
         handler.get_backend()
@@ -2536,6 +2565,7 @@ class EncodingHandlerMixin(HandlerCase):
 #=============================================================================
 class reset_warnings(warnings.catch_warnings):
     """catch_warnings() wrapper which clears warning registry & filters"""
+
     def __init__(self, reset_filter="always", reset_registry=".*", **kwds):
         super(reset_warnings, self).__init__(**kwds)
         self._reset_filter = reset_filter
@@ -2554,37 +2584,34 @@ class reset_warnings(warnings.catch_warnings):
         # that match the 'reset' pattern.
         pattern = self._reset_registry
         if pattern:
-            orig = self._orig_registry = {}
-            for name, mod in sys.modules.items():
-                if pattern.match(name):
-                    reg = getattr(mod, "__warningregistry__", None)
-                    if reg:
-                        orig[name] = reg.copy()
-                        reg.clear()
+            backup = self._orig_registry = {}
+            for name, mod in list(sys.modules.items()):
+                if mod is None or not pattern.match(name):
+                    continue
+                reg = getattr(mod, "__warningregistry__", None)
+                if reg:
+                    backup[name] = reg.copy()
+                    reg.clear()
         return ret
 
     def __exit__(self, *exc_info):
         # restore warning registry for all modules
         pattern = self._reset_registry
         if pattern:
-            # restore archived registry data
-            orig = self._orig_registry
-            for name, content in iteritems(orig):
-                mod = sys.modules.get(name)
-                if mod is None:
+            # restore registry backup, clearing all registry entries that we didn't archive
+            backup = self._orig_registry
+            for name, mod in list(sys.modules.items()):
+                if mod is None or not pattern.match(name):
                     continue
                 reg = getattr(mod, "__warningregistry__", None)
-                if reg is None:
-                    setattr(mod, "__warningregistry__", content)
-                else:
+                if reg:
                     reg.clear()
-                    reg.update(content)
-            # clear all registry entries that we didn't archive
-            for name, mod in sys.modules.items():
-                if pattern.match(name) and name not in orig:
-                    reg = getattr(mod, "__warningregistry__", None)
-                    if reg:
-                        reg.clear()
+                orig = backup.get(name)
+                if orig:
+                    if reg is None:
+                        setattr(mod, "__warningregistry__", orig)
+                    else:
+                        reg.update(orig)
         super(reset_warnings, self).__exit__(*exc_info)
 
 #=============================================================================
diff --git a/passlib/utils/pbkdf2.py b/passlib/utils/pbkdf2.py
index 07471fb..74dbc0c 100644
--- a/passlib/utils/pbkdf2.py
+++ b/passlib/utils/pbkdf2.py
@@ -41,6 +41,7 @@ def _clear_caches():
     """unittest helper -- clears get_hash_info() / get_prf() caches"""
     _ghi_cache.clear()
     _prf_cache.clear()
+    _nhn_cache.clear()
 
 #=============================================================================
 # hash helpers