Accommodate v2 and v3 auth for integration tests

This patch has two reviews:

1.  devstack has moved default Keystone API version to v3[1].
[1] https://github.com/openstack-dev/devstack/commit/f4ce44bf3fbf06e53c2ae3ec6aa4996831cf4605
Though the above patch has been reverted, this would help
if devstack removes v2 support in the future.

2. use domain env variables as defaults.
Use OS_USER_DOMAIN_NAME and OS_PROJECT_DOMAIN_NAME
env variables as the defaults for keystone v3 in
integration tests.

Squash of commit e80d27213cf4489aeeda48e2bdb5a890d8b56433
and commit 55713b94bf31409b4b0f8d3ba9ea9bc3b598f34a

Closes-Bug: #1539692
Closes-Bug: #1560032

Change-Id: I393750d00b3712a015e48a3cf38ab5f95bb61dae

6 files changed, 102 insertions, 45 deletions

diff --git a/heat_integrationtests/common/clients.py b/heat_integrationtests/common/clients.py
index e89fb3a6d..b68d7edf3 100644
--- a/heat_integrationtests/common/clients.py
+++ b/heat_integrationtests/common/clients.py
@@ -15,13 +15,49 @@ import os
 import ceilometerclient.client
 import cinderclient.client
 import heatclient.client
+from keystoneclient.auth.identity.generic import password
+import keystoneclient.client
 import keystoneclient.exceptions
-import keystoneclient.v2_0.client
+from keystoneclient import session
 import neutronclient.v2_0.client
 import novaclient.client
 import swiftclient
 
 
+class KeystoneWrapperClient(object):
+    """Wrapper object for keystone client
+
+    This Wraps keystone client,so we can encpasulate certain
+    added properties like auth_token, project_id etc.
+    """
+    def __init__(self, auth_plugin, verify=True):
+        self.auth_plugin = auth_plugin
+        self.session = session.Session(
+            auth=auth_plugin,
+            verify=verify)
+
+    @property
+    def auth_token(self):
+        return self.auth_plugin.get_token(self.session)
+
+    @property
+    def auth_ref(self):
+        return self.auth_plugin.get_access(self.session)
+
+    @property
+    def project_id(self):
+        return self.auth_plugin.get_project_id(self.session)
+
+    def get_endpoint_url(self, service_type, region=None):
+        kwargs = {
+            'service_type': service_type,
+            'endpoint_type': 'publicURL'}
+        if region:
+            kwargs.update({'attr': 'region',
+                           'filter_value': region})
+        return self.auth_ref.service_catalog.url_for(**kwargs)
+
+
 class ClientManager(object):
     """
     Manager that provides access to the official python clients for
@@ -37,6 +73,8 @@ class ClientManager(object):
         self.conf = conf
         self.insecure = self.conf.disable_ssl_certificate_validation
         self.ca_file = self.conf.ca_file
+        self.v2_auth_url = self.conf.auth_url.replace('/v3', '/v2.0')
+        self.auth_version = self.conf.auth_url.split('/v')[1]
         self.identity_client = self._get_identity_client()
         self.orchestration_client = self._get_orchestration_client()
         self.compute_client = self._get_compute_client()
@@ -46,20 +84,15 @@ class ClientManager(object):
         self.metering_client = self._get_metering_client()
 
     def _get_orchestration_client(self):
-        region = self.conf.region
         endpoint = os.environ.get('HEAT_URL')
         if os.environ.get('OS_NO_CLIENT_AUTH') == 'True':
             token = None
         else:
-            keystone = self._get_identity_client()
-            token = keystone.auth_token
+            token = self.identity_client.auth_token
         try:
             if endpoint is None:
-                endpoint = keystone.service_catalog.url_for(
-                    attr='region',
-                    filter_value=region,
-                    service_type='orchestration',
-                    endpoint_type='publicURL')
+                endpoint = self.identity_client.get_endpoint_url(
+                    'orchestration', self.conf.region)
         except keystoneclient.exceptions.EndpointNotFound:
             return None
         else:
@@ -71,13 +104,25 @@ class ClientManager(object):
                 password=self.conf.password)
 
     def _get_identity_client(self):
-        return keystoneclient.v2_0.client.Client(
-            username=self.conf.username,
-            password=self.conf.password,
-            tenant_name=self.conf.tenant_name,
-            auth_url=self.conf.auth_url,
-            insecure=self.insecure,
-            cacert=self.ca_file)
+        user_domain_name = self.conf.user_domain_name
+        project_domain_name = self.conf.project_domain_name
+        kwargs = {
+            'username': self.conf.username,
+            'password': self.conf.password,
+            'tenant_name': self.conf.tenant_name,
+            'auth_url': self.conf.auth_url
+        }
+        # keystone v2 can't ignore domain details
+        if self.auth_version == '3':
+            kwargs.update({
+                'user_domain_name': user_domain_name,
+                'project_domain_name': project_domain_name})
+        auth = password.Password(**kwargs)
+        if self.insecure:
+            verify = False
+        else:
+            verify = self.ca_file or True
+        return KeystoneWrapperClient(auth, verify=verify)
 
     def _get_compute_client(self):
 
@@ -87,7 +132,8 @@ class ClientManager(object):
             self.conf.username,
             self.conf.password,
             self.conf.tenant_name,
-            self.conf.auth_url
+            # novaclient can not use v3 url
+            self.v2_auth_url
         )
 
         # Create our default Nova client to use in testing
@@ -103,19 +149,18 @@ class ClientManager(object):
             http_log_debug=True)
 
     def _get_network_client(self):
-        auth_url = self.conf.auth_url
 
         return neutronclient.v2_0.client.Client(
             username=self.conf.username,
             password=self.conf.password,
             tenant_name=self.conf.tenant_name,
             endpoint_type='publicURL',
-            auth_url=auth_url,
+            # neutronclient can not use v3 url
+            auth_url=self.v2_auth_url,
             insecure=self.insecure,
             ca_cert=self.ca_file)
 
     def _get_volume_client(self):
-        auth_url = self.conf.auth_url
         region = self.conf.region
         endpoint_type = 'publicURL'
         return cinderclient.client.Client(
@@ -123,7 +168,8 @@ class ClientManager(object):
             self.conf.username,
             self.conf.password,
             self.conf.tenant_name,
-            auth_url,
+            # cinderclient can not use v3 url
+            self.v2_auth_url,
             region_name=region,
             endpoint_type=endpoint_type,
             insecure=self.insecure,
@@ -132,7 +178,7 @@ class ClientManager(object):
 
     def _get_object_client(self):
         args = {
-            'auth_version': '2.0',
+            'auth_version': self.auth_version,
             'tenant_name': self.conf.tenant_name,
             'user': self.conf.username,
             'key': self.conf.password,
@@ -144,15 +190,11 @@ class ClientManager(object):
         return swiftclient.client.Connection(**args)
 
     def _get_metering_client(self):
-
-        keystone = self._get_identity_client()
+        user_domain_name = self.conf.user_domain_name
+        project_domain_name = self.conf.project_domain_name
         try:
-            endpoint = keystone.service_catalog.url_for(
-                attr='region',
-                filter_value=self.conf.region,
-                service_type='metering',
-                endpoint_type='publicURL')
-
+            endpoint = self.identity_client.get_endpoint_url('metering',
+                                                             self.conf.region)
         except keystoneclient.exceptions.EndpointNotFound:
             return None
         else:
@@ -167,6 +209,12 @@ class ClientManager(object):
                 'endpoint_type': 'publicURL',
                 'service_type': 'metering',
             }
+            # ceilometerclient can't ignore domain details for
+            # v2 auth_url
+            if self.auth_version == '3':
+                args.update(
+                    {'user_domain_name': user_domain_name,
+                     'project_domain_name': project_domain_name})
 
             return ceilometerclient.client.Client(self.CEILOMETER_VERSION,
                                                   endpoint, **args)
diff --git a/heat_integrationtests/common/config.py b/heat_integrationtests/common/config.py
index a6014dadf..b8a089a56 100644
--- a/heat_integrationtests/common/config.py
+++ b/heat_integrationtests/common/config.py
@@ -37,10 +37,18 @@ IntegrationTestGroup = [
                help="Tenant name to use for API requests."),
     cfg.StrOpt('auth_url',
                default=os.environ.get('OS_AUTH_URL'),
-               help="Full URI of the OpenStack Identity API (Keystone), v2"),
+               help="Full URI of the OpenStack Identity API (Keystone)"),
+    cfg.StrOpt('user_domain_name',
+               default=os.environ.get('OS_USER_DOMAIN_NAME'),
+               help="User domain name, if keystone v3 auth_url"
+                    "is used"),
+    cfg.StrOpt('project_domain_name',
+               default=os.environ.get('OS_PROJECT_DOMAIN_NAME'),
+               help="Project domain name, if keystone v3 auth_url"
+                    "is used"),
     cfg.StrOpt('region',
                default=os.environ.get('OS_REGION_NAME'),
-               help="The region name to us"),
+               help="The region name to use"),
     cfg.StrOpt('instance_type',
                help="Instance type for tests. Needs to be big enough for a "
                     "full OS plus the test workload"),
@@ -54,10 +62,6 @@ IntegrationTestGroup = [
     cfg.StrOpt('minimal_image_ref',
                help="Name of minimal (e.g cirros) image to use when "
                     "launching test instances."),
-    cfg.StrOpt('auth_version',
-               default='v2',
-               help="Identity API version to be used for authentication "
-                    "for API tests."),
     cfg.BoolOpt('disable_ssl_certificate_validation',
                 default=False,
                 help="Set to True if using self-signed SSL certificates."),
diff --git a/heat_integrationtests/functional/test_aws_stack.py b/heat_integrationtests/functional/test_aws_stack.py
index 9fb2197b0..9c52320da 100644
--- a/heat_integrationtests/functional/test_aws_stack.py
+++ b/heat_integrationtests/functional/test_aws_stack.py
@@ -77,7 +77,7 @@ Outputs:
     def setUp(self):
         super(AwsStackTest, self).setUp()
         self.object_container_name = AwsStackTest.__name__
-        self.project_id = self.identity_client.auth_ref.project_id
+        self.project_id = self.identity_client.project_id
         self.object_client.put_container(self.object_container_name)
         self.nested_name = '%s.yaml' % test.rand_name()
 
diff --git a/heat_integrationtests/functional/test_conditional_exposure.py b/heat_integrationtests/functional/test_conditional_exposure.py
index d03771235..6899a8838 100644
--- a/heat_integrationtests/functional/test_conditional_exposure.py
+++ b/heat_integrationtests/functional/test_conditional_exposure.py
@@ -42,13 +42,9 @@ resources:
                           "Sahara resources availability.")
 
     def _is_sahara_deployed(self):
-        keystone = self.identity_client
         try:
-            keystone.service_catalog.url_for(
-                attr='region',
-                filter_value=self.conf.region,
-                service_type='data-processing',
-                endpoint_type='publicURL')
+            self.identity_client.get_endpoint_url('data-processing',
+                                                  self.conf.region)
         except keystoneclient.exceptions.EndpointNotFound:
             return False
         return True
diff --git a/heat_integrationtests/functional/test_preview.py b/heat_integrationtests/functional/test_preview.py
index 19da01bc2..ace0b4122 100644
--- a/heat_integrationtests/functional/test_preview.py
+++ b/heat_integrationtests/functional/test_preview.py
@@ -43,7 +43,7 @@ parameters:
     def setUp(self):
         super(StackPreviewTest, self).setUp()
         self.client = self.orchestration_client
-        self.project_id = self.identity_client.auth_ref.project_id
+        self.project_id = self.identity_client.project_id
 
     def _assert_resource(self, res, stack_name):
         self.assertEqual(stack_name, res['stack_name'])
diff --git a/heat_integrationtests/heat_integrationtests.conf.sample b/heat_integrationtests/heat_integrationtests.conf.sample
index 2da9af2c9..41fd7f9d4 100644
--- a/heat_integrationtests/heat_integrationtests.conf.sample
+++ b/heat_integrationtests/heat_integrationtests.conf.sample
@@ -22,7 +22,13 @@
 # Full URI of the OpenStack Identity API (Keystone), v2 (string value)
 #auth_url = <None>
 
-# The region name to us (string value)
+# User domain name, if keystone v3 auth_urlis used (string value)
+#user_domain_name = <None>
+
+# Project domain name, if keystone v3 auth_urlis used (string value)
+#project_domain_name = <None>
+
+# The region name to use (string value)
 #region = <None>
 
 # Instance type for tests. Needs to be big enough for a full OS plus the test
@@ -49,6 +55,9 @@
 # Set to True if using self-signed SSL certificates. (boolean value)
 #disable_ssl_certificate_validation = false
 
+# CA certificate to pass for servers that have https endpoint. (string value)
+#ca_file = <None>
+
 # Time in seconds between build status checks. (integer value)
 #build_interval = 4