Migrate stored credentials to keystone trusts

Migrate the stored user_creds, which currently only supports
storing username/password credentials to use the keystone v3
API OS-TRUST extension, which allows explicit impersonation of
users calling heat (trustors) by the heat service user (the
trustee)

Note this feature is made optional via a new config option,
defaulted to off, and it requires the following patches to
keystoneclient (in 0.3.2 release) and keystone to work:

https://review.openstack.org/#/c/39899/
https://review.openstack.org/#/c/42456/

Also note that if the feature is enabled, by setting
deferred_auth_method=trusts in heat.conf, you must add
a keystone_authtoken section, which is also used by the
keystoneclient auth_token middleware.

blueprint heat-trusts

Change-Id: I288114d827481bc0a24eba4556400d98b1a44c09

12 files changed, 634 insertions, 71 deletions

diff --git a/etc/heat/heat.conf.sample b/etc/heat/heat.conf.sample
index 90e608f5a..d8f238fd9 100644
--- a/etc/heat/heat.conf.sample
+++ b/etc/heat/heat.conf.sample
@@ -24,6 +24,13 @@
 # The directory to search for environment files (string value)
 #environment_dir=/etc/heat/environment.d
 
+# Select deferred auth method, stored password or trusts
+# (string value)
+#deferred_auth_method=password
+
+# Subset of trustor roles to be delegated to heat (list value)
+#trusts_delegated_roles=heat_stack_owner
+
 # Name of the engine node. This can be an opaque identifier.It
 # is not necessarily a hostname, FQDN, or IP address. (string
 # value)
@@ -86,6 +93,17 @@
 
 
 #
+# Options defined in heat.openstack.common.db.sqlalchemy.session
+#
+
+# the filename to use with sqlite (string value)
+#sqlite_db=heat.sqlite
+
+# If true, use synchronous mode for sqlite (boolean value)
+#sqlite_synchronous=true
+
+
+#
 # Options defined in heat.openstack.common.eventlet_backdoor
 #
 
@@ -460,6 +478,55 @@
 #use_tpool=false
 
 
+#
+# Options defined in heat.openstack.common.db.sqlalchemy.session
+#
+
+# The SQLAlchemy connection string used to connect to the
+# database (string value)
+#connection=sqlite:////heat/openstack/common/db/$sqlite_db
+
+# The SQLAlchemy connection string used to connect to the
+# slave database (string value)
+#slave_connection=
+
+# timeout before idle sql connections are reaped (integer
+# value)
+#idle_timeout=3600
+
+# Minimum number of SQL connections to keep open in a pool
+# (integer value)
+#min_pool_size=1
+
+# Maximum number of SQL connections to keep open in a pool
+# (integer value)
+#max_pool_size=<None>
+
+# maximum db connection retries during startup. (setting -1
+# implies an infinite retry count) (integer value)
+#max_retries=10
+
+# interval between retries of opening a sql connection
+# (integer value)
+#retry_interval=10
+
+# If set, use this value for max_overflow with sqlalchemy
+# (integer value)
+#max_overflow=<None>
+
+# Verbosity of SQL debugging information. 0=None,
+# 100=Everything (integer value)
+#connection_debug=0
+
+# Add python stack traces to SQL as comment strings (boolean
+# value)
+#connection_trace=false
+
+# If set, use this value for pool_timeout with sqlalchemy
+# (integer value)
+#pool_timeout=<None>
+
+
 [paste_deploy]
 
 #
diff --git a/heat/common/config.py b/heat/common/config.py
index c428bb637..53d26b51d 100644
--- a/heat/common/config.py
+++ b/heat/common/config.py
@@ -89,7 +89,16 @@ engine_opts = [
                 help='List of directories to search for Plugins'),
     cfg.StrOpt('environment_dir',
                default='/etc/heat/environment.d',
-               help='The directory to search for environment files')]
+               help='The directory to search for environment files'),
+    cfg.StrOpt('deferred_auth_method',
+               choices=['password', 'trusts'],
+               default='password',
+               help=_('Select deferred auth method, '
+                      'stored password or trusts')),
+    cfg.ListOpt('trusts_delegated_roles',
+                default=['heat_stack_owner'],
+                help=_('Subset of trustor roles to be delegated to heat'))]
+
 
 rpc_opts = [
     cfg.StrOpt('host',
diff --git a/heat/common/heat_keystoneclient.py b/heat/common/heat_keystoneclient.py
index 978b3ce90..ec463d676 100644
--- a/heat/common/heat_keystoneclient.py
+++ b/heat/common/heat_keystoneclient.py
@@ -13,12 +13,16 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from heat.openstack.common import exception
+from heat.common import exception
 
 import eventlet
+import hashlib
+
 from keystoneclient.v2_0 import client as kc
+from keystoneclient.v3 import client as kc_v3
 from oslo.config import cfg
 
+from heat.openstack.common import importutils
 from heat.openstack.common import log as logging
 
 logger = logging.getLogger('heat.common.keystoneclient')
@@ -35,24 +39,162 @@ class KeystoneClient(object):
     """
     def __init__(self, context):
         self.context = context
+        # We have to maintain two clients authenticated with keystone:
+        # - ec2 interface is v2.0 only
+        # - trusts is v3 only
+        # - passing a v2 auth_token to the v3 client won't work until lp bug
+        #   #1212778 is fixed
+        # - passing a v3 token to the v2 client works but we have to either
+        #   md5sum it or use the nocatalog option to auth/tokens (not yet
+        #   supported by keystoneclient), or we hit the v2 8192byte size limit
+        # - context.auth_url is expected to contain the v2.0 keystone endpoint
+        if cfg.CONF.deferred_auth_method == 'trusts':
+            # Create connection to v3 API
+            self.client_v3 = self._v3_client_init()
+
+            # Set context auth_token to md5sum of v3 token
+            auth_token = self.client_v3.auth_ref.get('auth_token')
+            self.context.auth_token = self._md5_token(auth_token)
+
+            # Create the connection to the v2 API, reusing the md5-ified token
+            self.client_v2 = self._v2_client_init()
+        else:
+            # Create the connection to the v2 API, using the context creds
+            self.client_v2 = self._v2_client_init()
+            self.client_v3 = None
+
+    def _md5_token(self, auth_token):
+        # Get the md5sum of the v3 token, which we can pass instead of the
+        # actual token to avoid v2 8192byte size limit on the v2 token API
+        m_enc = hashlib.md5()
+        m_enc.update(auth_token)
+        return m_enc.hexdigest()
+
+    def _v2_client_init(self):
         kwargs = {
-            'auth_url': context.auth_url,
+            'auth_url': self.context.auth_url
         }
+        # Note check for auth_token first so we use existing token if
+        # available from v3 auth
+        if self.context.auth_token is not None:
+            kwargs['tenant_name'] = self.context.tenant
+            kwargs['token'] = self.context.auth_token
+        elif self.context.password is not None:
+            kwargs['username'] = self.context.username
+            kwargs['password'] = self.context.password
+            kwargs['tenant_name'] = self.context.tenant
+            kwargs['tenant_id'] = self.context.tenant_id
+        else:
+            logger.error("Keystone v2 API connection failed, no password or "
+                         "auth_token!")
+            raise exception.AuthorizationFailure()
+        client_v2 = kc.Client(**kwargs)
+        if not client_v2.authenticate():
+            logger.error("Keystone v2 API authentication failed")
+            raise exception.AuthorizationFailure()
+        return client_v2
 
-        if context.password is not None:
-            kwargs['username'] = context.username
-            kwargs['password'] = context.password
-            kwargs['tenant_name'] = context.tenant
-            kwargs['tenant_id'] = context.tenant_id
-        elif context.auth_token is not None:
-            kwargs['tenant_name'] = context.tenant
-            kwargs['token'] = context.auth_token
+    @staticmethod
+    def _service_admin_creds(api_version=2):
+        # Import auth_token to have keystone_authtoken settings setup.
+        importutils.import_module('keystoneclient.middleware.auth_token')
+
+        creds = {
+            'username': cfg.CONF.keystone_authtoken.admin_user,
+            'password': cfg.CONF.keystone_authtoken.admin_password,
+        }
+        if api_version >= 3:
+            creds['auth_url'] =\
+                cfg.CONF.keystone_authtoken.auth_uri.replace('v2.0', 'v3')
+            creds['project_name'] =\
+                cfg.CONF.keystone_authtoken.admin_tenant_name
         else:
-            logger.error("Keystone connection failed, no password or " +
+            creds['auth_url'] = cfg.CONF.keystone_authtoken.auth_uri
+            creds['tenant_name'] =\
+                cfg.CONF.keystone_authtoken.admin_tenant_name
+
+        return creds
+
+    def _v3_client_init(self):
+        kwargs = {}
+        if self.context.auth_token is not None:
+            kwargs['project_name'] = self.context.tenant
+            kwargs['token'] = self.context.auth_token
+            kwargs['auth_url'] = self.context.auth_url.replace('v2.0', 'v3')
+            kwargs['endpoint'] = kwargs['auth_url']
+        elif self.context.trust_id is not None:
+            # We got a trust_id, so we use the admin credentials and get a
+            # Token back impersonating the trustor user
+            kwargs.update(self._service_admin_creds(api_version=3))
+            kwargs['trust_id'] = self.context.trust_id
+        elif self.context.password is not None:
+            kwargs['username'] = self.context.username
+            kwargs['password'] = self.context.password
+            kwargs['project_name'] = self.context.tenant
+            kwargs['project_id'] = self.context.tenant_id
+            kwargs['auth_url'] = self.context.auth_url.replace('v2.0', 'v3')
+            kwargs['endpoint'] = kwargs['auth_url']
+        else:
+            logger.error("Keystone v3 API connection failed, no password or "
                          "auth_token!")
+            raise exception.AuthorizationFailure()
+
+        client_v3 = kc_v3.Client(**kwargs)
+        if not client_v3.authenticate():
+            logger.error("Keystone v3 API authentication failed")
+            raise exception.AuthorizationFailure()
+        return client_v3
+
+    def create_trust_context(self):
+        """
+        If cfg.CONF.deferred_auth_method is trusts, we create a
+        trust using the trustor identity in the current context, with the
+        trustee as the heat service user
+
+        If deferred_auth_method != trusts, we do nothing
+
+        If the current context already contains a trust_id, we do nothing
+        """
+        if cfg.CONF.deferred_auth_method != 'trusts':
             return
-        self.client = kc.Client(**kwargs)
-        self.client.authenticate()
+
+        if self.context.trust_id:
+            return
+
+        # We need the service admin user ID (not name), as the trustor user
+        # can't lookup the ID in keystoneclient unless they're admin
+        # workaround this by creating a temporary admin client connection
+        # then getting the user ID from the auth_ref
+        admin_creds = self._service_admin_creds()
+        admin_client = kc.Client(**admin_creds)
+        if not admin_client.authenticate():
+            logger.error("Keystone v2 API admin authentication failed")
+            raise exception.AuthorizationFailure()
+
+        trustee_user_id = admin_client.auth_ref['user']['id']
+        trustor_user_id = self.client_v3.auth_ref['user']['id']
+        trustor_project_id = self.client_v3.auth_ref['project']['id']
+        roles = cfg.CONF.trusts_delegated_roles
+        trust = self.client_v3.trusts.create(trustor_user=trustor_user_id,
+                                             trustee_user=trustee_user_id,
+                                             project=trustor_project_id,
+                                             impersonation=True,
+                                             role_names=roles)
+        self.context.trust_id = trust.id
+        self.context.trustor_user_id = trustor_user_id
+
+    def delete_trust_context(self):
+        """
+        If a trust_id exists in the context, we delete it
+
+        """
+        if not self.context.trust_id:
+            return
+
+        self.client_v3.trusts.delete(self.context.trust_id)
+
+        self.context.trust_id = None
+        self.context.trustor_user_id = None
 
     def create_stack_user(self, username, password=''):
         """
@@ -66,35 +208,34 @@ class KeystoneClient(object):
                            "characters." % username)
             #get the last 64 characters of the username
             username = username[-64:]
-        user = self.client.users.create(username,
-                                        password,
-                                        '%s@heat-api.org' %
-                                        username,
-                                        tenant_id=self.context.tenant_id,
-                                        enabled=True)
+        user = self.client_v2.users.create(username,
+                                           password,
+                                           '%s@heat-api.org' %
+                                           username,
+                                           tenant_id=self.context.tenant_id,
+                                           enabled=True)
 
         # We add the new user to a special keystone role
         # This role is designed to allow easier differentiation of the
         # heat-generated "stack users" which will generally have credentials
         # deployed on an instance (hence are implicitly untrusted)
-        roles = self.client.roles.list()
+        roles = self.client_v2.roles.list()
         stack_user_role = [r.id for r in roles
                            if r.name == cfg.CONF.heat_stack_user_role]
         if len(stack_user_role) == 1:
             role_id = stack_user_role[0]
             logger.debug("Adding user %s to role %s" % (user.id, role_id))
-            self.client.roles.add_user_role(user.id, role_id,
-                                            self.context.tenant_id)
+            self.client_v2.roles.add_user_role(user.id, role_id,
+                                               self.context.tenant_id)
         else:
             logger.error("Failed to add user %s to role %s, check role exists!"
-                         % (username,
-                            cfg.CONF.heat_stack_user_role))
+                         % (username, cfg.CONF.heat_stack_user_role))
 
         return user.id
 
     def delete_stack_user(self, user_id):
 
-        user = self.client.users.get(user_id)
+        user = self.client_v2.users.get(user_id)
 
         # FIXME (shardy) : need to test, do we still need this retry logic?
         # Copied from user.py, but seems like something we really shouldn't
@@ -128,16 +269,16 @@ class KeystoneClient(object):
             raise exception.Error(reason)
 
     def delete_ec2_keypair(self, user_id, accesskey):
-        self.client.ec2.delete(user_id, accesskey)
+        self.client_v2.ec2.delete(user_id, accesskey)
 
     def get_ec2_keypair(self, user_id):
         # We make the assumption that each user will only have one
         # ec2 keypair, it's not clear if AWS allow multiple AccessKey resources
         # to be associated with a single User resource, but for simplicity
         # we assume that here for now
-        cred = self.client.ec2.list(user_id)
+        cred = self.client_v2.ec2.list(user_id)
         if len(cred) == 0:
-            return self.client.ec2.create(user_id, self.context.tenant_id)
+            return self.client_v2.ec2.create(user_id, self.context.tenant_id)
         if len(cred) == 1:
             return cred[0]
         else:
@@ -146,15 +287,15 @@ class KeystoneClient(object):
 
     def disable_stack_user(self, user_id):
         # FIXME : This won't work with the v3 keystone API
-        self.client.users.update_enabled(user_id, False)
+        self.client_v2.users.update_enabled(user_id, False)
 
     def enable_stack_user(self, user_id):
         # FIXME : This won't work with the v3 keystone API
-        self.client.users.update_enabled(user_id, True)
+        self.client_v2.users.update_enabled(user_id, True)
 
     def url_for(self, **kwargs):
-        return self.client.service_catalog.url_for(**kwargs)
+        return self.client_v2.service_catalog.url_for(**kwargs)
 
     @property
     def auth_token(self):
-        return self.client.auth_token
+        return self.client_v2.auth_token
diff --git a/heat/engine/service.py b/heat/engine/service.py
index 8e258c9fd..5b35bdcb0 100644
--- a/heat/engine/service.py
+++ b/heat/engine/service.py
@@ -30,6 +30,7 @@ from heat.engine.event import Event
 from heat.engine import environment
 from heat.common import exception
 from heat.common import identifier
+from heat.common import heat_keystoneclient as hkc
 from heat.engine import parameters
 from heat.engine import parser
 from heat.engine import properties
@@ -252,6 +253,11 @@ class EngineService(service.Service):
 
         stack.validate()
 
+        # Creates a trust and sets the trust_id and trustor_user_id in
+        # the current context, before we store it in stack.store()
+        # Does nothing if deferred_auth_method is 'password'
+        stack.clients.keystone().create_trust_context()
+
         stack_id = stack.store()
 
         self._start_in_thread(stack_id, _stack_create, stack)
@@ -384,6 +390,13 @@ class EngineService(service.Service):
 
         stack = parser.Stack.load(cnxt, stack=st)
 
+        # If we created a trust, delete it
+        # Note this is using the current request context, not the stored
+        # context, as it seems it's not possible to delete a trust with
+        # a token obtained via that trust.  This means that only the user
+        # who created the stack can delete it when using trusts atm.
+        stack.clients.keystone().delete_trust_context()
+
         # Kill any pending threads by calling ThreadGroup.stop()
         if st.id in self.stg:
             self.stg[st.id].stop()
@@ -529,8 +542,7 @@ class EngineService(service.Service):
         # but this happens because the keystone user associated with the
         # signal doesn't have permission to read the secret key of
         # the user associated with the cfn-credentials file
-        user_creds = db_api.user_creds_get(s.user_creds_id)
-        stack_context = context.RequestContext.from_dict(user_creds)
+        stack_context = self._load_user_creds(s.user_creds_id)
         stack = parser.Stack.load(stack_context, stack=s)
 
         if resource_name not in stack:
@@ -614,6 +626,15 @@ class EngineService(service.Service):
         stack = parser.Stack.load(cnxt, stack=s)
         self._start_in_thread(stack.id, _stack_resume, stack)
 
+    def _load_user_creds(self, creds_id):
+        user_creds = db_api.user_creds_get(creds_id)
+        stored_context = context.RequestContext.from_dict(user_creds)
+        # heat_keystoneclient populates the context with an auth_token
+        # either via the stored user/password or trust_id, depending
+        # on how deferred_auth_method is configured in the conf file
+        kc = hkc.KeystoneClient(stored_context)
+        return stored_context
+
     @request_context
     def metadata_update(self, cnxt, stack_identity,
                         resource_name, metadata):
@@ -634,8 +655,7 @@ class EngineService(service.Service):
         # but this happens because the keystone user associated with the
         # WaitCondition doesn't have permission to read the secret key of
         # the user associated with the cfn-credentials file
-        user_creds = db_api.user_creds_get(s.user_creds_id)
-        stack_context = context.RequestContext.from_dict(user_creds)
+        stack_context = self._load_user_creds(s.user_creds_id)
         refresh_stack = parser.Stack.load(stack_context, stack=s)
 
         # Refresh the metadata for all other resources, since we expect
@@ -664,8 +684,7 @@ class EngineService(service.Service):
             logger.error("Unable to retrieve stack %s for periodic task" %
                          sid)
             return
-        user_creds = db_api.user_creds_get(stack.user_creds_id)
-        stack_context = context.RequestContext.from_dict(user_creds)
+        stack_context = self._load_user_creds(stack.user_creds_id)
 
         # Get all watchrules for this stack and evaluate them
         try:
diff --git a/heat/tests/fakes.py b/heat/tests/fakes.py
index 6e77c7031..dd0427955 100644
--- a/heat/tests/fakes.py
+++ b/heat/tests/fakes.py
@@ -137,3 +137,9 @@ class FakeKeystoneClient(object):
 
     def url_for(self, **kwargs):
         return 'http://example.com:1234/v1'
+
+    def create_trust_context(self):
+        pass
+
+    def delete_trust_context(self):
+        pass
diff --git a/heat/tests/test_ceilometer_alarm.py b/heat/tests/test_ceilometer_alarm.py
index a07f0ecdc..a7fb4c7b0 100644
--- a/heat/tests/test_ceilometer_alarm.py
+++ b/heat/tests/test_ceilometer_alarm.py
@@ -25,7 +25,6 @@ from heat.tests import generic_resource
 from heat.tests.common import HeatTestCase
 from heat.tests import utils
 
-from heat.common import context
 from heat.common import template_format
 
 from heat.openstack.common.importutils import try_import
@@ -105,7 +104,7 @@ class CeilometerAlarmTest(HeatTestCase):
             template = alarm_template
         temp = template_format.parse(template)
         template = parser.Template(temp)
-        ctx = context.get_admin_context()
+        ctx = utils.dummy_context()
         ctx.tenant_id = 'test_tenant'
         stack = parser.Stack(ctx, utils.random_name(), template,
                              disable_rollback=True)
diff --git a/heat/tests/test_engine_service.py b/heat/tests/test_engine_service.py
index 0959c49e0..bd60bbd3c 100644
--- a/heat/tests/test_engine_service.py
+++ b/heat/tests/test_engine_service.py
@@ -23,6 +23,7 @@ from testtools import matchers
 from oslo.config import cfg
 
 from heat.engine import environment
+from heat.common import heat_keystoneclient as hkc
 from heat.common import exception
 from heat.tests.v1_1 import fakes
 import heat.rpc.api as engine_api
@@ -298,6 +299,16 @@ class StackServiceCreateUpdateDeleteTest(HeatTestCase):
         self.m.StubOutWithMock(stack, 'validate')
         stack.validate().AndReturn(None)
 
+        self.m.StubOutClassWithMocks(hkc.kc, "Client")
+        mock_ks_client = hkc.kc.Client(
+            auth_url=mox.IgnoreArg(),
+            tenant_name='test_tenant',
+            token='abcd1234')
+        mock_ks_client.authenticate().AndReturn(True)
+
+        self.m.StubOutWithMock(hkc.KeystoneClient, 'create_trust_context')
+        hkc.KeystoneClient.create_trust_context().AndReturn(None)
+
         self.m.StubOutWithMock(threadgroup, 'ThreadGroup')
         threadgroup.ThreadGroup().AndReturn(DummyThreadGroup())
 
@@ -413,6 +424,16 @@ class StackServiceCreateUpdateDeleteTest(HeatTestCase):
 
         parser.Stack.load(self.ctx, stack=s).AndReturn(stack)
 
+        self.m.StubOutClassWithMocks(hkc.kc, "Client")
+        mock_ks_client = hkc.kc.Client(
+            auth_url=mox.IgnoreArg(),
+            tenant_name='test_tenant',
+            token='abcd1234')
+        mock_ks_client.authenticate().AndReturn(True)
+
+        self.m.StubOutWithMock(hkc.KeystoneClient, 'delete_trust_context')
+        hkc.KeystoneClient.delete_trust_context().AndReturn(None)
+
         self.man.tg = DummyThreadGroup()
 
         self.m.ReplayAll()
@@ -1185,9 +1206,9 @@ class StackServiceTest(HeatTestCase):
         service.EngineService._get_stack(self.ctx,
                                          self.stack.identifier()).AndReturn(s)
 
-        self.m.StubOutWithMock(db_api, 'user_creds_get')
-        db_api.user_creds_get(mox.IgnoreArg()).MultipleTimes().AndReturn(
-            self.ctx.to_dict())
+        self.m.StubOutWithMock(service.EngineService, '_load_user_creds')
+        service.EngineService._load_user_creds(
+            mox.IgnoreArg()).AndReturn(self.ctx)
 
         self.m.StubOutWithMock(rsrs.Resource, 'signal')
         rsrs.Resource.signal(mox.IgnoreArg()).AndReturn(None)
@@ -1215,9 +1236,9 @@ class StackServiceTest(HeatTestCase):
         service.EngineService._get_stack(self.ctx,
                                          self.stack.identifier()).AndReturn(s)
 
-        self.m.StubOutWithMock(db_api, 'user_creds_get')
-        db_api.user_creds_get(mox.IgnoreArg()).MultipleTimes().AndReturn(
-            self.ctx.to_dict())
+        self.m.StubOutWithMock(service.EngineService, '_load_user_creds')
+        service.EngineService._load_user_creds(
+            mox.IgnoreArg()).AndReturn(self.ctx)
         self.m.ReplayAll()
 
         self.assertRaises(exception.ResourceNotFound,
@@ -1238,10 +1259,10 @@ class StackServiceTest(HeatTestCase):
         service.EngineService._get_stack(self.ctx,
                                          self.stack.identifier()).AndReturn(s)
         self.m.StubOutWithMock(instances.Instance, 'metadata_update')
-        self.m.StubOutWithMock(db_api, 'user_creds_get')
         instances.Instance.metadata_update(new_metadata=test_metadata)
-        db_api.user_creds_get(mox.IgnoreArg()).MultipleTimes().AndReturn(
-            self.ctx.to_dict())
+        self.m.StubOutWithMock(service.EngineService, '_load_user_creds')
+        service.EngineService._load_user_creds(
+            mox.IgnoreArg()).AndReturn(self.ctx)
         self.m.ReplayAll()
 
         result = self.eng.metadata_update(self.ctx,
diff --git a/heat/tests/test_heatclient.py b/heat/tests/test_heatclient.py
index fb655a6bb..77755a4e6 100644
--- a/heat/tests/test_heatclient.py
+++ b/heat/tests/test_heatclient.py
@@ -14,32 +14,103 @@
 
 import mox
 
+from oslo.config import cfg
+
+from heat.common import exception
 from heat.common import heat_keystoneclient
 from heat.tests.common import HeatTestCase
 from heat.tests import utils
 
+from heat.openstack.common import importutils
+
 
 class KeystoneClientTest(HeatTestCase):
     """Test cases for heat.common.heat_keystoneclient."""
 
     def setUp(self):
         super(KeystoneClientTest, self).setUp()
-        # load config so role checking doesn't barf
-        # mock the internal keystone client and its authentication
-        self.m.StubOutClassWithMocks(heat_keystoneclient.kc, "Client")
-        self.mock_ks_client = heat_keystoneclient.kc.Client(
-            auth_url=mox.IgnoreArg(),
-            password=mox.IgnoreArg(),
-            tenant_id=mox.IgnoreArg(),
-            tenant_name=mox.IgnoreArg(),
-            username=mox.IgnoreArg())
-        self.mock_ks_client.authenticate().AndReturn(True)
-        # verify all the things
+
+        # Import auth_token to have keystone_authtoken settings setup.
+        importutils.import_module('keystoneclient.middleware.auth_token')
+
+        dummy_url = 'http://_testnoexisthost_:5000/v2.0'
+        cfg.CONF.set_override('auth_uri', dummy_url,
+                              group='keystone_authtoken')
+        cfg.CONF.set_override('admin_user', 'heat',
+                              group='keystone_authtoken')
+        cfg.CONF.set_override('admin_password', 'verybadpass',
+                              group='keystone_authtoken')
+        cfg.CONF.set_override('admin_tenant_name', 'service',
+                              group='keystone_authtoken')
         self.addCleanup(self.m.VerifyAll)
 
+    def tearDown(self):
+        super(KeystoneClientTest, self).tearDown()
+        cfg.CONF.clear_override('deferred_auth_method')
+        cfg.CONF.clear_override('auth_uri', group='keystone_authtoken')
+        cfg.CONF.clear_override('admin_user', group='keystone_authtoken')
+        cfg.CONF.clear_override('admin_password', group='keystone_authtoken')
+        cfg.CONF.clear_override('admin_tenant_name',
+                                group='keystone_authtoken')
+
+    def _stubs_v2(self, method='token', auth_ok=True):
+        self.m.StubOutClassWithMocks(heat_keystoneclient.kc, "Client")
+        if method == 'token':
+            self.mock_ks_client = heat_keystoneclient.kc.Client(
+                auth_url=mox.IgnoreArg(),
+                tenant_name='test_tenant',
+                token='abcd1234')
+            self.mock_ks_client.authenticate().AndReturn(auth_ok)
+        elif method == 'password':
+            self.mock_ks_client = heat_keystoneclient.kc.Client(
+                auth_url=mox.IgnoreArg(),
+                tenant_name='test_tenant',
+                tenant_id='test_tenant_id',
+                username='test_username',
+                password='password')
+            self.mock_ks_client.authenticate().AndReturn(auth_ok)
+
+    def _stubs_v3(self, method='token', auth_ok=True):
+        self.m.StubOutClassWithMocks(heat_keystoneclient.kc, "Client")
+        self.m.StubOutClassWithMocks(heat_keystoneclient.kc_v3, "Client")
+
+        if method == 'token':
+            self.mock_ks_v3_client = heat_keystoneclient.kc_v3.Client(
+                token='abcd1234', project_name='test_tenant',
+                auth_url='http://_testnoexisthost_:5000/v3',
+                endpoint='http://_testnoexisthost_:5000/v3')
+        elif method == 'password':
+            self.mock_ks_v3_client = heat_keystoneclient.kc_v3.Client(
+                username='test_username',
+                password='password',
+                project_name='test_tenant',
+                project_id='test_tenant_id',
+                auth_url='http://_testnoexisthost_:5000/v3',
+                endpoint='http://_testnoexisthost_:5000/v3')
+        elif method == 'trust':
+            self.mock_ks_v3_client = heat_keystoneclient.kc_v3.Client(
+                username='heat',
+                password='verybadpass',
+                project_name='service',
+                auth_url='http://_testnoexisthost_:5000/v3',
+                trust_id='atrust123')
+
+        self.mock_ks_v3_client.authenticate().AndReturn(auth_ok)
+        if auth_ok:
+            self.mock_ks_v3_client.auth_ref = self.m.CreateMockAnything()
+            self.mock_ks_v3_client.auth_ref.get('auth_token').AndReturn(
+                'av3token')
+            self.mock_ks_client = heat_keystoneclient.kc.Client(
+                auth_url=mox.IgnoreArg(),
+                tenant_name='test_tenant',
+                token='4b97cc1b2454e137ee2e8261e115bbe8')
+            self.mock_ks_client.authenticate().AndReturn(auth_ok)
+
     def test_username_length(self):
         """Test that user names >64 characters are properly truncated."""
 
+        self._stubs_v2()
+
         # a >64 character user name and the expected version
         long_user_name = 'U' * 64 + 'S'
         good_user_name = long_user_name[-64:]
@@ -64,3 +135,230 @@ class KeystoneClientTest(HeatTestCase):
         heat_ks_client = heat_keystoneclient.KeystoneClient(
             utils.dummy_context())
         heat_ks_client.create_stack_user(long_user_name, password='password')
+
+    def test_init_v2_password(self):
+
+        """Test creating the client without trusts, user/password context."""
+
+        self._stubs_v2(method='password')
+        self.m.ReplayAll()
+
+        ctx = utils.dummy_context()
+        ctx.auth_token = None
+        heat_ks_client = heat_keystoneclient.KeystoneClient(ctx)
+        self.assertIsNotNone(heat_ks_client.client_v2)
+        self.assertIsNone(heat_ks_client.client_v3)
+
+    def test_init_v2_bad_nocreds(self):
+
+        """Test creating the client without trusts, no credentials."""
+
+        ctx = utils.dummy_context()
+        ctx.auth_token = None
+        ctx.username = None
+        ctx.password = None
+        self.assertRaises(exception.AuthorizationFailure,
+                          heat_keystoneclient.KeystoneClient, ctx)
+
+    def test_init_v2_bad_denied(self):
+
+        """Test creating the client without trusts, auth failure."""
+
+        self._stubs_v2(method='password', auth_ok=False)
+        self.m.ReplayAll()
+
+        ctx = utils.dummy_context()
+        ctx.auth_token = None
+        self.assertRaises(exception.AuthorizationFailure,
+                          heat_keystoneclient.KeystoneClient, ctx)
+
+    def test_init_v3_token(self):
+
+        """Test creating the client with trusts, token auth."""
+
+        cfg.CONF.set_override('deferred_auth_method', 'trusts')
+
+        self._stubs_v3()
+        self.m.ReplayAll()
+
+        ctx = utils.dummy_context()
+        ctx.username = None
+        ctx.password = None
+        heat_ks_client = heat_keystoneclient.KeystoneClient(ctx)
+        self.assertIsNotNone(heat_ks_client.client_v2)
+        self.assertIsNotNone(heat_ks_client.client_v3)
+
+    def test_init_v3_password(self):
+
+        """Test creating the client with trusts, password auth."""
+
+        cfg.CONF.set_override('deferred_auth_method', 'trusts')
+
+        self._stubs_v3(method='password')
+        self.m.ReplayAll()
+
+        ctx = utils.dummy_context()
+        ctx.auth_token = None
+        ctx.trust_id = None
+        heat_ks_client = heat_keystoneclient.KeystoneClient(ctx)
+        self.assertIsNotNone(heat_ks_client.client_v2)
+        self.assertIsNotNone(heat_ks_client.client_v3)
+
+    def test_init_v3_bad_nocreds(self):
+
+        """Test creating the client with trusts, no credentials."""
+
+        cfg.CONF.set_override('deferred_auth_method', 'trusts')
+
+        ctx = utils.dummy_context()
+        ctx.auth_token = None
+        ctx.trust_id = None
+        ctx.username = None
+        ctx.password = None
+        self.assertRaises(exception.AuthorizationFailure,
+                          heat_keystoneclient.KeystoneClient, ctx)
+
+    def test_init_v3_bad_denied(self):
+
+        """Test creating the client with trusts, auth failure."""
+
+        cfg.CONF.set_override('deferred_auth_method', 'trusts')
+
+        self._stubs_v3(method='password', auth_ok=False)
+        self.m.ReplayAll()
+
+        ctx = utils.dummy_context()
+        ctx.auth_token = None
+        ctx.trust_id = None
+        self.assertRaises(exception.AuthorizationFailure,
+                          heat_keystoneclient.KeystoneClient, ctx)
+
+    def test_create_trust_context_notrust(self):
+
+        """Test create_trust_context with trusts disabled."""
+
+        self._stubs_v2(method='password')
+        self.m.ReplayAll()
+
+        ctx = utils.dummy_context()
+        ctx.auth_token = None
+        ctx.trust_id = None
+        heat_ks_client = heat_keystoneclient.KeystoneClient(ctx)
+        self.assertIsNone(heat_ks_client.create_trust_context())
+
+    def test_create_trust_context_trust_id(self):
+
+        """Test create_trust_context with existing trust_id."""
+
+        cfg.CONF.set_override('deferred_auth_method', 'trusts')
+
+        self._stubs_v3()
+        self.m.ReplayAll()
+
+        ctx = utils.dummy_context()
+        heat_ks_client = heat_keystoneclient.KeystoneClient(ctx)
+        self.assertIsNone(heat_ks_client.create_trust_context())
+
+    def test_create_trust_context_trust_create(self):
+
+        """Test create_trust_context when creating a new trust."""
+
+        cfg.CONF.set_override('deferred_auth_method', 'trusts')
+
+        class MockTrust(object):
+            id = 'atrust123'
+
+        self._stubs_v3()
+        mock_admin_client = heat_keystoneclient.kc.Client(
+            auth_url=mox.IgnoreArg(),
+            username='heat',
+            password='verybadpass',
+            tenant_name='service')
+        mock_admin_client.authenticate().AndReturn(True)
+        mock_admin_client.auth_ref = self.m.CreateMockAnything()
+        mock_admin_client.auth_ref.__getitem__('user').AndReturn(
+            {'id': '1234'})
+        self.mock_ks_v3_client.auth_ref.__getitem__('user').AndReturn(
+            {'id': '5678'})
+        self.mock_ks_v3_client.auth_ref.__getitem__('project').AndReturn(
+            {'id': '42'})
+        self.mock_ks_v3_client.trusts = self.m.CreateMockAnything()
+        self.mock_ks_v3_client.trusts.create(
+            trustor_user='5678',
+            trustee_user='1234',
+            project='42',
+            impersonation=True,
+            role_names=['heat_stack_owner']).AndReturn(MockTrust())
+
+        self.m.ReplayAll()
+
+        ctx = utils.dummy_context()
+        ctx.trust_id = None
+        heat_ks_client = heat_keystoneclient.KeystoneClient(ctx)
+        self.assertIsNone(heat_ks_client.create_trust_context())
+        self.assertEqual(ctx.trust_id, 'atrust123')
+        self.assertEqual(ctx.trustor_user_id, '5678')
+
+    def test_create_trust_context_denied(self):
+
+        """Test create_trust_context when creating admin auth fails."""
+
+        cfg.CONF.set_override('deferred_auth_method', 'trusts')
+
+        self._stubs_v3()
+        mock_admin_client = heat_keystoneclient.kc.Client(
+            auth_url=mox.IgnoreArg(),
+            username='heat',
+            password='verybadpass',
+            tenant_name='service')
+        mock_admin_client.authenticate().AndReturn(False)
+        self.m.ReplayAll()
+
+        ctx = utils.dummy_context()
+        ctx.trust_id = None
+        heat_ks_client = heat_keystoneclient.KeystoneClient(ctx)
+        self.assertRaises(exception.AuthorizationFailure,
+                          heat_ks_client.create_trust_context)
+
+    def test_trust_init(self):
+
+        """Test consuming a trust when initializing."""
+
+        cfg.CONF.set_override('deferred_auth_method', 'trusts')
+
+        self._stubs_v3(method='trust')
+        self.m.ReplayAll()
+
+        ctx = utils.dummy_context()
+        ctx.username = None
+        ctx.password = None
+        ctx.auth_token = None
+        heat_ks_client = heat_keystoneclient.KeystoneClient(ctx)
+
+    def test_delete_trust_context(self):
+
+        """Test delete_trust_context when deleting trust."""
+
+        cfg.CONF.set_override('deferred_auth_method', 'trusts')
+
+        self._stubs_v3()
+        self.mock_ks_v3_client.trusts = self.m.CreateMockAnything()
+        self.mock_ks_v3_client.trusts.delete('atrust123').AndReturn(None)
+
+        self.m.ReplayAll()
+        ctx = utils.dummy_context()
+        heat_ks_client = heat_keystoneclient.KeystoneClient(ctx)
+        self.assertIsNone(heat_ks_client.delete_trust_context())
+
+    def test_delete_trust_context_notrust(self):
+
+        """Test delete_trust_context no trust_id specified."""
+
+        cfg.CONF.set_override('deferred_auth_method', 'trusts')
+
+        self._stubs_v3()
+        self.m.ReplayAll()
+        ctx = utils.dummy_context()
+        ctx.trust_id = None
+        heat_ks_client = heat_keystoneclient.KeystoneClient(ctx)
+        self.assertIsNone(heat_ks_client.delete_trust_context())
diff --git a/heat/tests/test_metadata_refresh.py b/heat/tests/test_metadata_refresh.py
index a3f0598ad..5221ca9d6 100644
--- a/heat/tests/test_metadata_refresh.py
+++ b/heat/tests/test_metadata_refresh.py
@@ -20,7 +20,6 @@ from heat.tests import fakes
 from heat.tests.common import HeatTestCase
 from heat.tests import utils
 
-from heat.db import api as db_api
 from heat.engine import environment
 from heat.common import identifier
 from heat.common import template_format
@@ -203,8 +202,8 @@ class WaitCondMetadataUpdateTest(HeatTestCase):
     def create_stack(self, stack_name='test_stack'):
         temp = template_format.parse(test_template_waitcondition)
         template = parser.Template(temp)
-        stack = parser.Stack(utils.dummy_context(), stack_name, template,
-                             disable_rollback=True)
+        ctx = utils.dummy_context()
+        stack = parser.Stack(ctx, stack_name, template, disable_rollback=True)
 
         self.stack_id = stack.store()
 
@@ -223,7 +222,9 @@ class WaitCondMetadataUpdateTest(HeatTestCase):
         wc.WaitConditionHandle.identifier().MultipleTimes().AndReturn(id)
 
         self.m.StubOutWithMock(scheduler.TaskRunner, '_sleep')
-        self.m.StubOutWithMock(db_api, 'user_creds_get')
+        self.m.StubOutWithMock(service.EngineService, '_load_user_creds')
+        service.EngineService._load_user_creds(
+            mox.IgnoreArg()).MultipleTimes().AndReturn(ctx)
 
         return stack
 
@@ -258,8 +259,6 @@ class WaitCondMetadataUpdateTest(HeatTestCase):
 
         scheduler.TaskRunner._sleep(mox.IsA(int)).WithSideEffects(check_empty)
         scheduler.TaskRunner._sleep(mox.IsA(int)).WithSideEffects(post_success)
-        db_api.user_creds_get(mox.IgnoreArg()).MultipleTimes().AndReturn(
-            self.stack.context.to_dict())
         scheduler.TaskRunner._sleep(mox.IsA(int)).AndReturn(None)
 
         self.m.ReplayAll()
diff --git a/heat/tests/test_signal.py b/heat/tests/test_signal.py
index 68d42835a..5cbbf9cbb 100644
--- a/heat/tests/test_signal.py
+++ b/heat/tests/test_signal.py
@@ -21,7 +21,6 @@ from heat.tests import fakes
 from heat.tests.common import HeatTestCase
 from heat.tests import utils
 
-from heat.common import context
 from heat.common import exception
 from heat.common import template_format
 
@@ -72,7 +71,7 @@ class SignalTest(HeatTestCase):
     def create_stack(self, stack_name='test_stack', stub=True):
         temp = template_format.parse(test_template_signal)
         template = parser.Template(temp)
-        ctx = context.get_admin_context()
+        ctx = utils.dummy_context()
         ctx.tenant_id = 'test_tenant'
         stack = parser.Stack(ctx, stack_name, template,
                              disable_rollback=True)
@@ -85,6 +84,9 @@ class SignalTest(HeatTestCase):
             self.m.StubOutWithMock(sr.SignalResponder, 'keystone')
             sr.SignalResponder.keystone().MultipleTimes().AndReturn(
                 self.fc)
+
+        self.m.ReplayAll()
+
         return stack
 
     @utils.stack_delete_after
diff --git a/heat/tests/utils.py b/heat/tests/utils.py
index 01a285ed4..f5b0c2c2d 100644
--- a/heat/tests/utils.py
+++ b/heat/tests/utils.py
@@ -136,6 +136,7 @@ def dummy_context(user='test_username', tenant_id='test_tenant_id',
         'username': user,
         'password': password,
         'roles': roles,
+        'trust_id': 'atrust123',
         'auth_url': 'http://_testnoexisthost_:5000/v2.0',
         'auth_token': 'abcd1234'
     })
diff --git a/requirements.txt b/requirements.txt
index c9f0f9cd5..6fc84de28 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,12 +15,13 @@ PasteDeploy>=1.5.0
 Routes>=1.12.3
 SQLAlchemy>=0.7.8,<=0.7.99
 WebOb>=1.2.3,<1.3
-python-keystoneclient>=0.3.0
+python-keystoneclient>=0.3.2
 python-swiftclient>=1.2
 python-neutronclient>=2.2.3,<3
 python-ceilometerclient>=1.0.2
 python-cinderclient>=1.0.4
 PyYAML>=3.1.0
-oslo.config>=1.1.0
 paramiko>=1.8.0
 Babel>=0.9.6
+-f http://tarballs.openstack.org/oslo.config/oslo.config-1.2.0a3.tar.gz#egg=oslo.config-1.2.0a3
+oslo.config>=1.2.0a3