Merge "Cache tokens using memorycache from oslo."0.2.3

3 files changed, 53 insertions, 107 deletions

diff --git a/doc/source/middlewarearchitecture.rst b/doc/source/middlewarearchitecture.rst
index 407a5d2..803fbd9 100644
--- a/doc/source/middlewarearchitecture.rst
+++ b/doc/source/middlewarearchitecture.rst
@@ -196,15 +196,16 @@ Configuration Options
 Caching for improved response
 -----------------------------
 
-In order to prevent every service request, the middleware may be configured
-to utilize a cache, and the keystone API returns the tokens with an
-expiration (configurable in duration on the keystone service). The middleware
-supports memcache based caching.
+In order to prevent excessive requests and validations, the middleware uses an
+in-memory cache for the tokens the keystone API returns. Keep in mind that
+invalidated tokens may continue to work if they are still in the token cache,
+so token_cache_time is configurable. For larger deployments, the middleware
+also supports memcache based caching.
 
 * ``memcache_servers``: (optonal) if defined, the memcache server(s) to use for
   cacheing. It will be ignored if Swift MemcacheRing is used instead.
-* ``token_cache_time``: (optional, default 300 seconds) Only valid if
-  memcache_servers is defined.
+* ``token_cache_time``: (optional, default 300 seconds) Set to -1 to disable
+  caching completely.
 
 When deploying auth_token middleware with Swift, user may elect
 to use Swift MemcacheRing instead of the local Keystone memcache.
diff --git a/keystoneclient/middleware/auth_token.py b/keystoneclient/middleware/auth_token.py
index 2fb6216..98a427b 100644
--- a/keystoneclient/middleware/auth_token.py
+++ b/keystoneclient/middleware/auth_token.py
@@ -158,6 +158,7 @@ from keystoneclient.openstack.common import jsonutils
 from keystoneclient.common import cms
 from keystoneclient import utils
 from keystoneclient.middleware import memcache_crypt
+from keystoneclient.openstack.common import memorycache
 from keystoneclient.openstack.common import timeutils
 
 CONF = None
@@ -361,16 +362,8 @@ class AuthProtocol(object):
             self._cache = env.get(cache)
         else:
             # use Keystone memcache
-            memcache_servers = self._conf_get('memcache_servers')
-            if memcache_servers:
-                try:
-                    import memcache
-                    self.LOG.info('Using Keystone memcache for caching token')
-                    self._cache = memcache.Client(memcache_servers)
-                    self._use_keystone_cache = True
-                except ImportError as e:
-                    msg = 'disabled caching due to missing libraries %s' % (e)
-                    self.LOG.warn(msg)
+            self._cache = memorycache.get_client(memcache_servers)
+            self._use_keystone_cache = True
         self._cache_initialized = True
 
     def _conf_get(self, name):
@@ -989,12 +982,8 @@ class AuthProtocol(object):
                 additional_headers=headers)
 
         if response.status == 200:
-            self._cache_put(user_token, data)
             return data
         if response.status == 404:
-            # FIXME(ja): I'm assuming the 404 status means that user_token is
-            #            invalid - not that the admin_token is invalid
-            self._cache_store_invalid(user_token)
             self.LOG.warn("Authorization failed for token %s", user_token)
             raise InvalidUserToken('Token authorization failed')
         if response.status == 401:
diff --git a/tests/test_auth_token_middleware.py b/tests/test_auth_token_middleware.py
index 350a091..59f3d22 100644
--- a/tests/test_auth_token_middleware.py
+++ b/tests/test_auth_token_middleware.py
@@ -28,6 +28,7 @@ from keystoneclient.common import cms
 from keystoneclient import utils
 from keystoneclient.middleware import auth_token
 from keystoneclient.middleware import memcache_crypt
+from keystoneclient.openstack.common import memorycache
 from keystoneclient.openstack.common import jsonutils
 from keystoneclient.openstack.common import timeutils
 from keystoneclient.middleware import test
@@ -361,58 +362,11 @@ VERSION_LIST_v2 = {
 }
 
 
-class FakeMemcache(object):
-    def __init__(self):
-        self.set_key = None
-        self.set_value = None
-        self.token_expiration = None
-        self.token_key = SIGNED_TOKEN_SCOPED_KEY
-
-    def get(self, key):
-        data = TOKEN_RESPONSES[self.token_key].copy()
-        if not data or key != "tokens/%s" % self.token_key:
-            return
-        if not self.token_expiration:
-            dt = datetime.datetime.now() + datetime.timedelta(minutes=5)
-            self.token_expiration = dt.strftime("%s")
-        return (data, str(self.token_expiration))
-
-    def set(self, key, value, time=None):
-        self.set_value = value
-        self.set_key = key
-
-
-class v3FakeMemcache(FakeMemcache):
-    def __init__(self):
-        super(v3FakeMemcache, self).__init__()
-        self.token_key = SIGNED_v3_TOKEN_SCOPED_KEY
-
-
-class FakeSwiftMemcacheRing(object):
-    def __init__(self):
-        self.set_key = None
-        self.set_value = None
-        self.token_expiration = None
-        self.token_key = SIGNED_TOKEN_SCOPED_KEY
-
-    def get(self, key):
-        data = TOKEN_RESPONSES[self.token_key].copy()
-        if not data or key != "tokens/%s" % self.token_key:
-            return
-        if not self.token_expiration:
-            dt = datetime.datetime.now() + datetime.timedelta(minutes=5)
-            self.token_expiration = dt.strftime("%s")
-        return (data, str(self.token_expiration))
-
-    def set(self, key, value, serialize=True, timeout=0):
-        self.set_value = value
-        self.set_key = key
-
-
-class v3FakeSwiftMemcacheRing(FakeSwiftMemcacheRing):
-    def __init__(self):
-        super(v3FakeSwiftMemcacheRing, self).__init__()
-        self.token_key = SIGNED_v3_TOKEN_SCOPED_KEY
+class FakeSwiftMemcacheRing(memorycache.Client):
+    # NOTE(vish): swift memcache uses param timeout instead of time
+    def set(self, key, value, timeout=0, min_compress_len=0):
+        sup = super(FakeSwiftMemcacheRing, self)
+        sup.set(key, value, timeout, min_compress_len)
 
 
 class FakeHTTPResponse(object):
@@ -606,8 +560,7 @@ class BaseAuthTokenMiddlewareTest(testtools.TestCase):
 
     """
     def setUp(self, expected_env=None, auth_version=None,
-              fake_app=None, fake_http=None, token_dict=None,
-              fake_memcache=None, fake_memcache_ring=None):
+              fake_app=None, fake_http=None, token_dict=None):
         testtools.TestCase.setUp(self)
         expected_env = expected_env or {}
 
@@ -640,9 +593,6 @@ class BaseAuthTokenMiddlewareTest(testtools.TestCase):
         self.response_status = None
         self.response_headers = None
 
-        self.fake_memcache = fake_memcache or FakeMemcache
-        self.fake_memcache_ring = fake_memcache_ring or FakeSwiftMemcacheRing
-
         signed_list = 'SIGNED_REVOCATION_LIST'
         valid_signed_list = 'VALID_SIGNED_REVOCATION_LIST'
         globals()[signed_list] = globals()[valid_signed_list]
@@ -978,44 +928,53 @@ class AuthTokenMiddlewareTest(test.NoModule, BaseAuthTokenMiddlewareTest):
         self.assertEqual(self.response_headers['WWW-Authenticate'],
                          "Keystone uri='https://keystone.example.com:1234'")
 
+    def _get_cached_token(self, token):
+        token_id = cms.cms_hash_token(token)
+        # NOTE(vish): example tokens are expired so skip the expiration check.
+        key = self.middleware._get_cache_key(token_id)
+        cached = self.middleware._cache.get(key)
+        return self.middleware._unprotect_cache_value(token, cached)
+
     def test_memcache(self):
         req = webob.Request.blank('/')
-        req.headers['X-Auth-Token'] = (
-            self.token_dict['signed_token_scoped'])
-        self.middleware._cache = self.fake_memcache()
-        self.middleware._use_keystone_cache = True
+        token = self.token_dict['signed_token_scoped']
+        req.headers['X-Auth-Token'] = token
         self.middleware(req.environ, self.start_fake_response)
-        self.assertEqual(self.middleware._cache.set_value, None)
+        self.assertNotEqual(self._get_cached_token(token), None)
 
     def test_memcache_set_invalid(self):
         req = webob.Request.blank('/')
-        req.headers['X-Auth-Token'] = 'invalid-token'
-        self.middleware._cache = self.fake_memcache()
-        self.middleware._use_keystone_cache = True
+        token = 'invalid-token'
+        req.headers['X-Auth-Token'] = token
         self.middleware(req.environ, self.start_fake_response)
-        self.assertEqual(self.middleware._cache.set_value, "invalid")
+        self.assertEqual(self._get_cached_token(token), "invalid")
 
     def test_memcache_set_expired(self):
+        token_cache_time = 10
+        conf = {
+            'token_cache_time': token_cache_time,
+            'signing_dir': CERTDIR,
+        }
+        self.set_middleware(conf=conf)
         req = webob.Request.blank('/')
-        req.headers['X-Auth-Token'] = (
-            self.token_dict['signed_token_scoped'])
-        self.middleware._cache = self.fake_memcache()
-        self.middleware._use_keystone_cache = True
-        expired = datetime.datetime.now() - datetime.timedelta(minutes=1)
-        self.middleware._cache.token_expiration = float(expired.strftime("%s"))
-        self.middleware(req.environ, self.start_fake_response)
-        self.assertEqual(len(self.middleware._cache.set_value), 2)
+        token = self.token_dict['signed_token_scoped']
+        req.headers['X-Auth-Token'] = token
+        try:
+            now = datetime.datetime.utcnow()
+            timeutils.set_time_override(now)
+            self.middleware(req.environ, self.start_fake_response)
+            self.assertNotEqual(self._get_cached_token(token), None)
+            expired = now + datetime.timedelta(seconds=token_cache_time)
+            timeutils.set_time_override(expired)
+            self.assertEqual(self._get_cached_token(token), None)
+        finally:
+            timeutils.clear_time_override()
 
     def test_swift_memcache_set_expired(self):
-        req = webob.Request.blank('/')
-        req.headers['X-Auth-Token'] = (
-            self.token_dict['signed_token_scoped'])
-        self.middleware._cache = self.fake_memcache_ring()
+        self.middleware._cache = FakeSwiftMemcacheRing()
         self.middleware._use_keystone_cache = False
-        expired = datetime.datetime.now() - datetime.timedelta(minutes=1)
-        self.middleware._cache.token_expiration = float(expired.strftime("%s"))
-        self.middleware(req.environ, self.start_fake_response)
-        self.assertEqual(len(self.middleware._cache.set_value), 2)
+        self.middleware._cache_initialized = True
+        self.test_memcache_set_expired()
 
     def test_nomemcache(self):
         self.disable_module('memcache')
@@ -1042,7 +1001,6 @@ class AuthTokenMiddlewareTest(test.NoModule, BaseAuthTokenMiddlewareTest):
         self.assertEqual(self.middleware._cache, 'CACHE_TEST')
 
     def test_not_use_cache_from_env(self):
-        self.disable_module('memcache')
         env = {'swift.cache': 'CACHE_TEST'}
         conf = {
             'auth_host': 'keystone.example.com',
@@ -1052,7 +1010,7 @@ class AuthTokenMiddlewareTest(test.NoModule, BaseAuthTokenMiddlewareTest):
         }
         self.set_middleware(conf=conf)
         self.middleware._init_cache(env)
-        self.assertEqual(self.middleware._cache, None)
+        self.assertNotEqual(self.middleware._cache, 'CACHE_TEST')
 
     def test_will_expire_soon(self):
         tenseconds = datetime.datetime.utcnow() + datetime.timedelta(
@@ -1341,9 +1299,7 @@ class v3AuthTokenMiddlewareTest(AuthTokenMiddlewareTest):
                 auth_version='v3.0',
                 fake_app=v3FakeApp,
                 fake_http=v3FakeHTTPConnection,
-                token_dict=token_dict,
-                fake_memcache=v3FakeMemcache,
-                fake_memcache_ring=v3FakeSwiftMemcacheRing)
+                token_dict=token_dict)
 
     def assert_valid_last_url(self, token_id):
         # Token ID is not part of the url in v3, so override