Allow for invalid packet sequence in keepalive

In the SQLAlchemy keep_alive class, MariaDB is failing
as pymysql reports an invalid packet sequence.
MariaDB seems to timeout the client in a different
way than MySQL and PXC, which manifests itself as the
aforementioned invalid sequence.  It is now handled
as a special-case exception.

With this fix, the MariaDB scenario tests now pass.

The scenario tests were also tweaked a bit, which aided
in the testing of the fix.  'group=instance' was created,
plus instance_error properly interleaved with
instance_create.  _has_status now calls get_instance with
the admin client so that any faults are accompanied by
a relevant stack trace.  Cases where the result code
was being checked out-of-sequence were removed, and explicit
calls to check the http code for the right client were added.

The replication error messages for promote and eject were
enhanced as well to attempt to debug spurious failures.
One of those failures was 'Replication is not on after 60 seconds.'
This was fixed by setting 'MASTER_CONNECT_RETRY' in the mariadb
gtid replication strategy as was done in:
https://review.openstack.org/#/c/188933

Finally, backup_incremental was added to MariaDB supported
groups and cleaned up elsewhere.

Closes-Bug: #1621702
Change-Id: Id6bde5a34e1d79eece3084f761dcd153c38ccbad

18 files changed, 240 insertions, 152 deletions

diff --git a/trove/guestagent/datastore/mysql_common/service.py b/trove/guestagent/datastore/mysql_common/service.py
index da04d278..96c9330d 100644
--- a/trove/guestagent/datastore/mysql_common/service.py
+++ b/trove/guestagent/datastore/mysql_common/service.py
@@ -25,6 +25,7 @@ import uuid
 
 from oslo_log import log as logging
 from oslo_utils import encodeutils
+from pymysql import err as pymysql_err
 from six.moves import urllib
 import sqlalchemy
 from sqlalchemy import exc
@@ -568,6 +569,14 @@ class BaseKeepAliveConnection(interfaces.PoolListener):
                 raise exc.DisconnectionError()
             else:
                 raise
+        # MariaDB seems to timeout the client in a different
+        # way than MySQL and PXC, which manifests itself as
+        # an invalid packet sequence.  Handle it as well.
+        except pymysql_err.InternalError as ex:
+            if "Packet sequence number wrong" in ex.message:
+                raise exc.DisconnectionError()
+            else:
+                raise
 
 
 @six.add_metaclass(abc.ABCMeta)
diff --git a/trove/guestagent/strategies/replication/experimental/mariadb_gtid.py b/trove/guestagent/strategies/replication/experimental/mariadb_gtid.py
index 89446ed5..c73d9315 100644
--- a/trove/guestagent/strategies/replication/experimental/mariadb_gtid.py
+++ b/trove/guestagent/strategies/replication/experimental/mariadb_gtid.py
@@ -55,6 +55,7 @@ class MariaDBGTIDReplication(mysql_base.MysqlReplicationBase):
             "MASTER_PORT=%(port)s, "
             "MASTER_USER='%(user)s', "
             "MASTER_PASSWORD='%(password)s', "
+            "MASTER_CONNECT_RETRY=15, "
             "MASTER_USE_GTID=slave_pos" %
             {
                 'host': snapshot['master']['host'],
diff --git a/trove/taskmanager/manager.py b/trove/taskmanager/manager.py
index d375b19a..a70872b7 100644
--- a/trove/taskmanager/manager.py
+++ b/trove/taskmanager/manager.py
@@ -120,41 +120,43 @@ class Manager(periodic_task.PeriodicTasks):
             # and possibly some number of "orphaned" slaves
 
             exception_replicas = []
+            error_messages = ""
             for replica in replica_models:
                 try:
                     if replica.id != master_candidate.id:
                         replica.detach_replica(old_master, for_failover=True)
                         replica.attach_replica(master_candidate)
-                except exception.TroveError:
-                    msg = _("promote-to-replica-source: Unable to migrate "
-                            "replica %(slave)s from old replica source "
-                            "%(old_master)s to new source %(new_master)s.")
-                    msg_values = {
-                        "slave": replica.id,
-                        "old_master": old_master.id,
-                        "new_master": master_candidate.id
-                    }
-                    LOG.exception(msg % msg_values)
+                except exception.TroveError as ex:
+                    msg = (_("Unable to migrate replica %(slave)s from "
+                             "old replica source %(old_master)s to "
+                             "new source %(new_master)s on promote.") %
+                           {"slave": replica.id,
+                            "old_master": old_master.id,
+                            "new_master": master_candidate.id})
+                    LOG.exception(msg)
                     exception_replicas.append(replica)
+                    error_messages += "%s (%s)\n" % (msg, ex)
 
             try:
                 old_master.demote_replication_master()
-            except Exception:
-                LOG.exception(_("Exception demoting old replica source"))
+            except Exception as ex:
+                msg = (_("Exception demoting old replica source %s.") %
+                       old_master.id)
+                LOG.exception(msg)
                 exception_replicas.append(old_master)
+                error_messages += "%s (%s)\n" % (msg, ex)
 
             self._set_task_status([old_master] + replica_models,
                                   InstanceTasks.NONE)
             if exception_replicas:
                 self._set_task_status(exception_replicas,
                                       InstanceTasks.PROMOTION_ERROR)
-                msg = _("promote-to-replica-source %(id)s: The following "
-                        "replicas may not have been switched: %(replicas)s")
-                msg_values = {
-                    "id": master_candidate.id,
-                    "replicas": exception_replicas
-                }
-                raise ReplicationSlaveAttachError(msg % msg_values)
+                msg = (_("promote-to-replica-source %(id)s: The following "
+                         "replicas may not have been switched: %(replicas)s") %
+                       {"id": master_candidate.id,
+                        "replicas": [repl.id for repl in exception_replicas]})
+                raise ReplicationSlaveAttachError("%s:\n%s" %
+                                                  (msg, error_messages))
 
         with EndNotification(context):
             master_candidate = BuiltInstanceTasks.load(context, instance_id)
@@ -207,35 +209,34 @@ class Manager(periodic_task.PeriodicTasks):
             old_master.attach_public_ips(slave_ips)
 
             exception_replicas = []
+            error_messages = ""
             for replica in replica_models:
                 try:
                     if replica.id != master_candidate.id:
                         replica.detach_replica(old_master, for_failover=True)
                         replica.attach_replica(master_candidate)
-                except exception.TroveError:
-                    msg = _("eject-replica-source: Unable to migrate "
-                            "replica %(slave)s from old replica source "
-                            "%(old_master)s to new source %(new_master)s.")
-                    msg_values = {
-                        "slave": replica.id,
-                        "old_master": old_master.id,
-                        "new_master": master_candidate.id
-                    }
-                    LOG.exception(msg % msg_values)
-                    exception_replicas.append(replica.id)
+                except exception.TroveError as ex:
+                    msg = (_("Unable to migrate replica %(slave)s from "
+                             "old replica source %(old_master)s to "
+                             "new source %(new_master)s on eject.") %
+                           {"slave": replica.id,
+                            "old_master": old_master.id,
+                            "new_master": master_candidate.id})
+                    LOG.exception(msg)
+                    exception_replicas.append(replica)
+                    error_messages += "%s (%s)\n" % (msg, ex)
 
             self._set_task_status([old_master] + replica_models,
                                   InstanceTasks.NONE)
             if exception_replicas:
                 self._set_task_status(exception_replicas,
                                       InstanceTasks.EJECTION_ERROR)
-                msg = _("eject-replica-source %(id)s: The following "
-                        "replicas may not have been switched: %(replicas)s")
-                msg_values = {
-                    "id": master_candidate.id,
-                    "replicas": exception_replicas
-                }
-                raise ReplicationSlaveAttachError(msg % msg_values)
+                msg = (_("eject-replica-source %(id)s: The following "
+                         "replicas may not have been switched: %(replicas)s") %
+                       {"id": master_candidate.id,
+                        "replicas": [repl.id for repl in exception_replicas]})
+                raise ReplicationSlaveAttachError("%s:\n%s" %
+                                                  (msg, error_messages))
 
         with EndNotification(context):
             master = BuiltInstanceTasks.load(context, instance_id)
diff --git a/trove/tests/int_tests.py b/trove/tests/int_tests.py
index 18bfe5c9..519db3ec 100644
--- a/trove/tests/int_tests.py
+++ b/trove/tests/int_tests.py
@@ -177,6 +177,10 @@ guest_log_groups.extend([guest_log_group.GROUP])
 instance_actions_groups = list(instance_create_groups)
 instance_actions_groups.extend([instance_actions_group.GROUP])
 
+instance_groups = list(instance_actions_groups)
+instance_groups.extend([instance_error_create_group.GROUP,
+                        instance_force_delete_group.GROUP])
+
 module_groups = list(instance_create_groups)
 module_groups.extend([module_group.GROUP])
 
@@ -197,9 +201,8 @@ user_actions_groups = list(instance_create_groups)
 user_actions_groups.extend([user_actions_group.GROUP])
 
 # groups common to all datastores
-common_groups = list(instance_actions_groups)
-common_groups.extend([guest_log_groups, instance_error_create_groups,
-                      instance_force_delete_groups, module_groups])
+common_groups = list(instance_groups)
+common_groups.extend([guest_log_groups, module_groups])
 
 # Register: Component based groups
 register(["backup"], backup_groups)
@@ -210,11 +213,12 @@ register(["configuration"], configuration_groups)
 register(["configuration_create"], configuration_create_groups)
 register(["database"], database_actions_groups)
 register(["guest_log"], guest_log_groups)
-register(["instance", "instance_actions"], instance_actions_groups)
+register(["instance"], instance_groups)
+register(["instance_actions"], instance_actions_groups)
 register(["instance_create"], instance_create_groups)
-register(["instance_error_create"], instance_error_create_groups)
-register(["instance_upgrade"], instance_upgrade_groups)
+register(["instance_error"], instance_error_create_groups)
 register(["instance_force_delete"], instance_force_delete_groups)
+register(["instance_upgrade"], instance_upgrade_groups)
 register(["module"], module_groups)
 register(["module_create"], module_create_groups)
 register(["replication"], replication_groups)
@@ -223,35 +227,100 @@ register(["root"], root_actions_groups)
 register(["user"], user_actions_groups)
 
 # Register: Datastore based groups
-# These should contain all functionality currently supported by the datastore
-register(["db2_supported"], common_groups,
-         database_actions_groups, user_actions_groups, configuration_groups)
-register(["cassandra_supported"], common_groups,
-         user_actions_groups, database_actions_groups,
-         backup_groups, configuration_groups, cluster_actions_groups)
-register(["couchbase_supported"], common_groups, backup_groups,
-         root_actions_groups)
-register(["couchdb_supported"], common_groups, backup_groups,
-         user_actions_groups, database_actions_groups, root_actions_groups)
-register(["postgresql_supported"], common_groups,
-         backup_groups, database_actions_groups, configuration_groups,
-         root_actions_groups, user_actions_groups,
-         backup_incremental_groups, replication_groups)
-register(["mysql_supported", "percona_supported"], common_groups,
-         backup_groups, configuration_groups, database_actions_groups,
-         replication_promote_groups, instance_upgrade_groups,
-         root_actions_groups, user_actions_groups, backup_incremental_groups)
-register(["mariadb_supported"], common_groups,
-         backup_groups, cluster_actions_groups, configuration_groups,
-         database_actions_groups, replication_promote_groups,
-         root_actions_groups, user_actions_groups)
-register(["mongodb_supported"], common_groups,
-         backup_groups, cluster_actions_groups, configuration_groups,
-         database_actions_groups, root_actions_groups, user_actions_groups)
-register(["pxc_supported"], common_groups,
-         backup_groups, configuration_groups, database_actions_groups,
-         cluster_actions_groups, root_actions_groups, user_actions_groups)
-register(["redis_supported"], common_groups,
-         backup_groups, replication_promote_groups, cluster_actions_groups)
-register(["vertica_supported"], common_groups,
-         cluster_actions_groups, root_actions_groups, configuration_groups)
+# These should contain all functionality currently supported by the datastore.
+# Keeping them in alphabetical order may reduce the number of merge conflicts.
+register(
+    ["db2_supported"], common_groups,
+    configuration_groups,
+    database_actions_groups,
+    user_actions_groups,
+)
+
+register(
+    ["cassandra_supported"], common_groups,
+    backup_groups,
+    database_actions_groups,
+    cluster_actions_groups,
+    configuration_groups,
+    user_actions_groups,
+)
+
+register(
+    ["couchbase_supported"], common_groups,
+    backup_groups,
+    root_actions_groups,
+)
+
+register(
+    ["couchdb_supported"], common_groups,
+    backup_groups,
+    database_actions_groups,
+    root_actions_groups,
+    user_actions_groups,
+)
+
+register(
+    ["postgresql_supported"], common_groups,
+    backup_incremental_groups,
+    database_actions_groups,
+    configuration_groups,
+    replication_groups,
+    root_actions_groups,
+    user_actions_groups,
+)
+
+register(
+    ["mysql_supported", "percona_supported"], common_groups,
+    backup_incremental_groups,
+    configuration_groups,
+    database_actions_groups,
+    instance_upgrade_groups,
+    replication_promote_groups,
+    root_actions_groups,
+    user_actions_groups,
+)
+
+register(
+    ["mariadb_supported"], common_groups,
+    backup_incremental_groups,
+    cluster_actions_groups,
+    configuration_groups,
+    database_actions_groups,
+    replication_promote_groups,
+    root_actions_groups,
+    user_actions_groups,
+)
+
+register(
+    ["mongodb_supported"], common_groups,
+    backup_groups,
+    cluster_actions_groups,
+    configuration_groups,
+    database_actions_groups,
+    root_actions_groups,
+    user_actions_groups,
+)
+
+register(
+    ["pxc_supported"], common_groups,
+    backup_incremental_groups,
+    cluster_actions_groups,
+    configuration_groups,
+    database_actions_groups,
+    root_actions_groups,
+    user_actions_groups,
+)
+
+register(
+    ["redis_supported"], common_groups,
+    backup_groups,
+    cluster_actions_groups,
+    replication_promote_groups,
+)
+
+register(
+    ["vertica_supported"], common_groups,
+    cluster_actions_groups,
+    configuration_groups,
+    root_actions_groups,
+)
diff --git a/trove/tests/scenario/groups/instance_create_group.py b/trove/tests/scenario/groups/instance_create_group.py
index 46a0356b..228be246 100644
--- a/trove/tests/scenario/groups/instance_create_group.py
+++ b/trove/tests/scenario/groups/instance_create_group.py
@@ -58,7 +58,8 @@ class InstanceCreateGroup(TestGroup):
 
 @test(depends_on_groups=[groups.INST_CREATE],
       groups=[GROUP, groups.INST_CREATE_WAIT],
-      runs_after_groups=[groups.MODULE_CREATE, groups.CFGGRP_CREATE])
+      runs_after_groups=[groups.MODULE_CREATE, groups.CFGGRP_CREATE,
+                         groups.INST_ERROR_CREATE_WAIT])
 class InstanceCreateWaitGroup(TestGroup):
     """Test that Instance Create Completes."""
 
diff --git a/trove/tests/scenario/runners/backup_runners.py b/trove/tests/scenario/runners/backup_runners.py
index 3833806c..1c2f26fd 100644
--- a/trove/tests/scenario/runners/backup_runners.py
+++ b/trove/tests/scenario/runners/backup_runners.py
@@ -260,8 +260,7 @@ class BackupRunner(TestRunner):
             self.unauth_client.backups.get, self.backup_info.id)
         # we're using a different client, so we'll check the return code
         # on it explicitly, instead of depending on 'assert_raises'
-        self.assert_client_code(expected_http_code=expected_http_code,
-                                client=self.unauth_client)
+        self.assert_client_code(expected_http_code, client=self.unauth_client)
 
     def run_add_data_for_inc_backup_1(self):
         self.backup_host = self.get_instance_host()
@@ -304,7 +303,7 @@ class BackupRunner(TestRunner):
     def assert_restore_from_backup(self, backup_ref, suffix='',
                                    expected_http_code=200):
         result = self._restore_from_backup(backup_ref, suffix=suffix)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         self.assert_equal('BUILD', result.status,
                           'Unexpected instance status')
         return result.id
@@ -374,7 +373,7 @@ class BackupRunner(TestRunner):
     def assert_delete_restored_instance(
             self, instance_id, expected_http_code):
         self.auth_client.instances.delete(instance_id)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
     def run_delete_restored_inc_1_instance(self, expected_http_code=202):
         self.assert_delete_restored_instance(
@@ -412,8 +411,7 @@ class BackupRunner(TestRunner):
             self.unauth_client.backups.delete, self.backup_info.id)
         # we're using a different client, so we'll check the return code
         # on it explicitly, instead of depending on 'assert_raises'
-        self.assert_client_code(expected_http_code=expected_http_code,
-                                client=self.unauth_client)
+        self.assert_client_code(expected_http_code, client=self.unauth_client)
 
     def run_delete_inc_2_backup(self, expected_http_code=202):
         self.assert_delete_backup(
@@ -423,7 +421,7 @@ class BackupRunner(TestRunner):
     def assert_delete_backup(
             self, backup_id, expected_http_code):
         self.auth_client.backups.delete(backup_id)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         self._wait_until_backup_is_gone(backup_id)
 
     def _wait_until_backup_is_gone(self, backup_id):
diff --git a/trove/tests/scenario/runners/cluster_actions_runners.py b/trove/tests/scenario/runners/cluster_actions_runners.py
index 5c08d846..61b2b24a 100644
--- a/trove/tests/scenario/runners/cluster_actions_runners.py
+++ b/trove/tests/scenario/runners/cluster_actions_runners.py
@@ -139,7 +139,7 @@ class ClusterActionsRunner(TestRunner):
     def assert_cluster_list(self, expected_count,
                             expected_http_code):
         count = len(self.auth_client.clusters.list())
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         self.assert_equal(expected_count, count, "Unexpected cluster count")
 
     def run_cluster_show(self, expected_http_code=200,
@@ -328,7 +328,8 @@ class ClusterActionsRunner(TestRunner):
             self, cluster_id, expected_task_name, expected_http_code,
             check_locality=True):
         if expected_http_code is not None:
-            self.assert_client_code(expected_http_code)
+            self.assert_client_code(expected_http_code,
+                                    client=self.auth_client)
         if expected_task_name:
             self._assert_cluster_response(cluster_id, expected_task_name,
                                           check_locality=check_locality)
@@ -365,7 +366,7 @@ class ClusterActionsRunner(TestRunner):
     def _assert_cluster_response(self, cluster_id, expected_task_name,
                                  expected_http_code=200, check_locality=True):
         cluster = self.auth_client.clusters.get(cluster_id)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         self._assert_cluster_values(cluster, expected_task_name,
                                     check_locality=check_locality)
 
@@ -401,7 +402,7 @@ class ClusterActionsRunner(TestRunner):
                 "Cluster '%s' still existed after %s seconds."
                 % (cluster_id, self._time_since(t0)))
         except exceptions.NotFound:
-            self.assert_client_code(404)
+            self.assert_client_code(404, client=self.auth_client)
 
 
 class CassandraClusterActionsRunner(ClusterActionsRunner):
diff --git a/trove/tests/scenario/runners/configuration_runners.py b/trove/tests/scenario/runners/configuration_runners.py
index 6d0a8101..53584e3c 100644
--- a/trove/tests/scenario/runners/configuration_runners.py
+++ b/trove/tests/scenario/runners/configuration_runners.py
@@ -134,7 +134,7 @@ class ConfigurationRunner(TestRunner):
             description,
             datastore=self.instance_info.dbaas_datastore,
             datastore_version=self.instance_info.dbaas_datastore_version)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
         with TypeCheck('Configuration', result) as configuration:
             configuration.has_field('name', basestring)
@@ -473,7 +473,7 @@ class ConfigurationRunner(TestRunner):
 
     def assert_group_delete(self, group_id, expected_http_code):
         self.auth_client.configurations.delete(group_id)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
     def run_delete_non_dynamic_group(self, expected_http_code=202):
         if self.non_dynamic_group_id:
@@ -514,15 +514,14 @@ class ConfigurationRunner(TestRunner):
             datastore_version=self.instance_info.dbaas_datastore_version,
             availability_zone="nova",
             configuration=config_id)
-        self.assert_client_code(200)
+        self.assert_client_code(200, client=self.auth_client)
         self.assert_equal("BUILD", result.status, 'Unexpected inst status')
         return result.id
 
     def run_wait_for_conf_instance(
-            self, expected_states=['BUILD', 'ACTIVE'], expected_http_code=200):
+            self, expected_states=['BUILD', 'ACTIVE']):
         if self.config_inst_id:
-            self.assert_instance_action(self.config_inst_id, expected_states,
-                                        expected_http_code)
+            self.assert_instance_action(self.config_inst_id, expected_states)
             self.create_test_helper_on_instance(self.config_inst_id)
             inst = self.auth_client.instances.get(self.config_inst_id)
             self.assert_equal(self.config_id_for_inst,
@@ -546,7 +545,7 @@ class ConfigurationRunner(TestRunner):
 
     def assert_delete_conf_instance(self, instance_id, expected_http_code):
         self.auth_client.instances.delete(instance_id)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
     def run_wait_for_delete_conf_instance(
             self, expected_last_state=['SHUTDOWN']):
diff --git a/trove/tests/scenario/runners/database_actions_runners.py b/trove/tests/scenario/runners/database_actions_runners.py
index b2f890a3..52829ee9 100644
--- a/trove/tests/scenario/runners/database_actions_runners.py
+++ b/trove/tests/scenario/runners/database_actions_runners.py
@@ -53,7 +53,7 @@ class DatabaseActionsRunner(TestRunner):
     def assert_databases_create(self, instance_id, serial_databases_def,
                                 expected_http_code):
         self.auth_client.databases.create(instance_id, serial_databases_def)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         self.wait_for_database_create(instance_id, serial_databases_def)
         return serial_databases_def
 
@@ -64,7 +64,7 @@ class DatabaseActionsRunner(TestRunner):
     def assert_databases_list(self, instance_id, expected_database_defs,
                               expected_http_code, limit=2):
         full_list = self.auth_client.databases.list(instance_id)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         listed_databases = {database.name: database for database in full_list}
         self.assert_is_none(full_list.next,
                             "Unexpected pagination in the list.")
@@ -86,7 +86,7 @@ class DatabaseActionsRunner(TestRunner):
 
         # Test list pagination.
         list_page = self.auth_client.databases.list(instance_id, limit=limit)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
         self.assert_true(len(list_page) <= limit)
         if len(full_list) > limit:
@@ -104,7 +104,8 @@ class DatabaseActionsRunner(TestRunner):
                               "in the page.")
             list_page = self.auth_client.databases.list(
                 instance_id, marker=marker)
-            self.assert_client_code(expected_http_code)
+            self.assert_client_code(expected_http_code,
+                                    client=self.auth_client)
             self.assert_pagination_match(
                 list_page, full_list, limit, len(full_list))
 
@@ -163,7 +164,7 @@ class DatabaseActionsRunner(TestRunner):
             database_name,
             expected_http_code):
         self.auth_client.databases.delete(instance_id, database_name)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         self._wait_for_database_delete(instance_id, database_name)
 
     def _wait_for_database_delete(self, instance_id, deleted_database_name):
diff --git a/trove/tests/scenario/runners/guest_log_runners.py b/trove/tests/scenario/runners/guest_log_runners.py
index 87a9a35e..3c7d299e 100644
--- a/trove/tests/scenario/runners/guest_log_runners.py
+++ b/trove/tests/scenario/runners/guest_log_runners.py
@@ -100,7 +100,7 @@ class GuestLogRunner(TestRunner):
         self.report.log("Executing log_show for log '%s'" % log_name)
         log_details = client.instances.log_show(
             self.instance_info.id, log_name)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=client)
         self.assert_log_details(
             log_details, log_name,
             expected_type=expected_type,
@@ -183,7 +183,7 @@ class GuestLogRunner(TestRunner):
         self.report.log("Executing log_enable for log '%s'" % log_name)
         log_details = client.instances.log_enable(
             self.instance_info.id, log_name)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=client)
         self.assert_log_details(
             log_details, log_name,
             expected_type=expected_type,
@@ -200,7 +200,7 @@ class GuestLogRunner(TestRunner):
                         (log_name, discard))
         log_details = client.instances.log_disable(
             self.instance_info.id, log_name, discard=discard)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=client)
         self.assert_log_details(
             log_details, log_name,
             expected_type=expected_type,
@@ -218,7 +218,7 @@ class GuestLogRunner(TestRunner):
                         (log_name, disable, discard))
         log_details = client.instances.log_publish(
             self.instance_info.id, log_name, disable=disable, discard=discard)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=client)
         self.assert_log_details(
             log_details, log_name,
             expected_type=expected_type,
@@ -234,7 +234,7 @@ class GuestLogRunner(TestRunner):
         self.report.log("Executing log_discard for log '%s'" % log_name)
         log_details = client.instances.log_discard(
             self.instance_info.id, log_name)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=client)
         self.assert_log_details(
             log_details, log_name,
             expected_type=expected_type,
@@ -302,7 +302,7 @@ class GuestLogRunner(TestRunner):
                            client.instances.log_enable,
                            self.instance_info.id, log_name)
         # we may not be using the main client, so check explicitly here
-        self.assert_client_code(expected_http_code, client)
+        self.assert_client_code(expected_http_code, client=client)
 
     def run_test_log_disable_sys(self,
                                  expected_exception=exceptions.BadRequest,
@@ -320,7 +320,7 @@ class GuestLogRunner(TestRunner):
                            self.instance_info.id, log_name,
                            discard=discard)
         # we may not be using the main client, so check explicitly here
-        self.assert_client_code(expected_http_code, client)
+        self.assert_client_code(expected_http_code, client=client)
 
     def run_test_log_show_unauth_user(self,
                                       expected_exception=exceptions.NotFound,
@@ -337,7 +337,7 @@ class GuestLogRunner(TestRunner):
                            client.instances.log_show,
                            self.instance_info.id, log_name)
         # we may not be using the main client, so check explicitly here
-        self.assert_client_code(expected_http_code, client)
+        self.assert_client_code(expected_http_code, client=client)
 
     def run_test_log_list_unauth_user(self,
                                       expected_exception=exceptions.NotFound,
@@ -346,7 +346,7 @@ class GuestLogRunner(TestRunner):
                            self.unauth_client.instances.log_list,
                            self.instance_info.id)
         # we're not using the main client, so check explicitly here
-        self.assert_client_code(expected_http_code, self.unauth_client)
+        self.assert_client_code(expected_http_code, client=self.unauth_client)
 
     def run_test_log_generator_unauth_user(self):
         self.assert_log_generator_unauth_user(
@@ -406,7 +406,7 @@ class GuestLogRunner(TestRunner):
                            self.instance_info.id, log_name,
                            disable=disable, discard=discard)
         # we may not be using the main client, so check explicitly here
-        self.assert_client_code(expected_http_code, client)
+        self.assert_client_code(expected_http_code, client=client)
 
     def run_test_log_discard_unexposed_user(
             self, expected_exception=exceptions.BadRequest,
@@ -423,7 +423,7 @@ class GuestLogRunner(TestRunner):
                            client.instances.log_discard,
                            self.instance_info.id, log_name)
         # we may not be using the main client, so check explicitly here
-        self.assert_client_code(expected_http_code, client)
+        self.assert_client_code(expected_http_code, client=client)
 
     def run_test_log_enable_user(self):
         expected_status = guest_log.LogStatus.Ready.name
@@ -468,7 +468,8 @@ class GuestLogRunner(TestRunner):
             expected_states = ['RESTART_REQUIRED', 'ACTIVE']
             self.assert_instance_action(instance_id, expected_states, None)
             self.auth_client.instances.restart(instance_id)
-            self.assert_client_code(expected_http_code)
+            self.assert_client_code(expected_http_code,
+                                    client=self.auth_client)
 
     def run_test_wait_for_restart(self, expected_states=['REBOOT', 'ACTIVE']):
         if self.test_helper.log_enable_requires_restart():
diff --git a/trove/tests/scenario/runners/instance_actions_runners.py b/trove/tests/scenario/runners/instance_actions_runners.py
index 17f738a0..6b182046 100644
--- a/trove/tests/scenario/runners/instance_actions_runners.py
+++ b/trove/tests/scenario/runners/instance_actions_runners.py
@@ -88,7 +88,7 @@ class InstanceActionsRunner(TestRunner):
                         (resize_flavor_id, instance_id))
         self.auth_client.instances.resize_instance(
             instance_id, resize_flavor_id)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
     def run_wait_for_instance_resize_flavor(
             self, expected_states=['RESIZE', 'ACTIVE']):
diff --git a/trove/tests/scenario/runners/instance_create_runners.py b/trove/tests/scenario/runners/instance_create_runners.py
index bfb3ad8e..5eb6a837 100644
--- a/trove/tests/scenario/runners/instance_create_runners.py
+++ b/trove/tests/scenario/runners/instance_create_runners.py
@@ -73,7 +73,8 @@ class InstanceCreateRunner(TestRunner):
                 "Configuration group used by instance create tests.",
                 datastore=self.instance_info.dbaas_datastore,
                 datastore_version=self.instance_info.dbaas_datastore_version)
-            self.assert_client_code(expected_http_code)
+            self.assert_client_code(expected_http_code,
+                                    client=self.auth_client)
 
             self.config_group_id = result.id
         else:
@@ -309,7 +310,8 @@ class InstanceCreateRunner(TestRunner):
     def run_initialized_instance_delete(self, expected_http_code=202):
         if self.init_inst_id:
             self.auth_client.instances.delete(self.init_inst_id)
-            self.assert_client_code(expected_http_code)
+            self.assert_client_code(expected_http_code,
+                                    client=self.auth_client)
         else:
             raise SkipTest("Cleanup is not required.")
 
@@ -331,7 +333,8 @@ class InstanceCreateRunner(TestRunner):
     def run_initial_configuration_delete(self, expected_http_code=202):
         if self.config_group_id:
             self.auth_client.configurations.delete(self.config_group_id)
-            self.assert_client_code(expected_http_code)
+            self.assert_client_code(expected_http_code,
+                                    client=self.auth_client)
         else:
             raise SkipTest("Cleanup is not required.")
         self.config_group_id = None
diff --git a/trove/tests/scenario/runners/instance_delete_runners.py b/trove/tests/scenario/runners/instance_delete_runners.py
index 09003db5..f5d1bb3e 100644
--- a/trove/tests/scenario/runners/instance_delete_runners.py
+++ b/trove/tests/scenario/runners/instance_delete_runners.py
@@ -36,7 +36,7 @@ class InstanceDeleteRunner(TestRunner):
         self.report.log("Testing delete on instance: %s" % instance_id)
 
         self.auth_client.instances.delete(instance_id)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
     def run_instance_delete_wait(self, expected_states=['SHUTDOWN']):
         if self.has_do_not_delete_instance:
diff --git a/trove/tests/scenario/runners/instance_error_create_runners.py b/trove/tests/scenario/runners/instance_error_create_runners.py
index 279763a0..c0f3c4df 100644
--- a/trove/tests/scenario/runners/instance_error_create_runners.py
+++ b/trove/tests/scenario/runners/instance_error_create_runners.py
@@ -40,7 +40,7 @@ class InstanceErrorCreateRunner(TestRunner):
             nics=self.instance_info.nics,
             datastore=self.instance_info.dbaas_datastore,
             datastore_version=self.instance_info.dbaas_datastore_version)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         self.error_inst_id = inst.id
 
     def run_create_error2_instance(self, expected_http_code=200):
@@ -57,7 +57,7 @@ class InstanceErrorCreateRunner(TestRunner):
             nics=self.instance_info.nics,
             datastore=self.instance_info.dbaas_datastore,
             datastore_version=self.instance_info.dbaas_datastore_version)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         self.error2_inst_id = inst.id
 
     def run_wait_for_error_instances(self, expected_states=['ERROR']):
@@ -75,7 +75,8 @@ class InstanceErrorCreateRunner(TestRunner):
         if not self.error_inst_id:
             raise SkipTest("No error instance created.")
 
-        instance = self.get_instance(self.error_inst_id)
+        instance = self.get_instance(
+            self.error_inst_id, self.auth_client)
         with CheckInstance(instance._info) as check:
             check.fault()
 
@@ -101,10 +102,12 @@ class InstanceErrorCreateRunner(TestRunner):
     def run_delete_error_instances(self, expected_http_code=202):
         if self.error_inst_id:
             self.auth_client.instances.delete(self.error_inst_id)
-            self.assert_client_code(expected_http_code)
+            self.assert_client_code(expected_http_code,
+                                    client=self.auth_client)
         if self.error2_inst_id:
             self.auth_client.instances.delete(self.error2_inst_id)
-            self.assert_client_code(expected_http_code)
+            self.assert_client_code(expected_http_code,
+                                    client=self.auth_client)
 
     def run_wait_for_error_delete(self, expected_states=['SHUTDOWN']):
         delete_ids = []
diff --git a/trove/tests/scenario/runners/module_runners.py b/trove/tests/scenario/runners/module_runners.py
index df9cb22f..a7c76756 100644
--- a/trove/tests/scenario/runners/module_runners.py
+++ b/trove/tests/scenario/runners/module_runners.py
@@ -386,8 +386,7 @@ class ModuleRunner(TestRunner):
             self.unauth_client.modules.get, self.main_test_module.id)
         # we're using a different client, so we'll check the return code
         # on it explicitly, instead of depending on 'assert_raises'
-        self.assert_client_code(expected_http_code=expected_http_code,
-                                client=self.unauth_client)
+        self.assert_client_code(expected_http_code, client=self.unauth_client)
 
     def run_module_list(self):
         self.assert_module_list(
@@ -686,7 +685,7 @@ class ModuleRunner(TestRunner):
     def assert_module_list_instance(self, client, instance_id, expected_count,
                                     expected_http_code=200):
         module_list = client.instances.modules(instance_id)
-        self.assert_client_code(expected_http_code, client)
+        self.assert_client_code(expected_http_code, client=client)
         count = len(module_list)
         self.assert_equal(expected_count, count,
                           "Wrong number of modules from list instance")
@@ -701,7 +700,7 @@ class ModuleRunner(TestRunner):
     def assert_module_instances(self, client, module_id, expected_count,
                                 expected_http_code=200):
         instance_list = client.modules.instances(module_id)
-        self.assert_client_code(expected_http_code, client)
+        self.assert_client_code(expected_http_code, client=client)
         count = len(instance_list)
         self.assert_equal(expected_count, count,
                           "Wrong number of instances applied from module")
@@ -713,7 +712,7 @@ class ModuleRunner(TestRunner):
     def assert_module_query(self, client, instance_id, expected_count,
                             expected_http_code=200, expected_results=None):
         modquery_list = client.instances.module_query(instance_id)
-        self.assert_client_code(expected_http_code, client)
+        self.assert_client_code(expected_http_code, client=client)
         count = len(modquery_list)
         self.assert_equal(expected_count, count,
                           "Wrong number of modules from query")
@@ -736,7 +735,7 @@ class ModuleRunner(TestRunner):
                             expected_http_code=200):
         module_apply_list = client.instances.module_apply(
             instance_id, [module.id])
-        self.assert_client_code(expected_http_code, client)
+        self.assert_client_code(expected_http_code, client=client)
         admin_only = (not module.visible or module.auto_apply or
                       not module.tenant_id)
         expected_status = expected_status or 'OK'
@@ -856,7 +855,7 @@ class ModuleRunner(TestRunner):
             nics=self.instance_info.nics,
             modules=[module_id],
         )
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         return inst.id
 
     def run_module_delete_applied(
@@ -873,7 +872,7 @@ class ModuleRunner(TestRunner):
     def assert_module_remove(self, client, instance_id, module_id,
                              expected_http_code=200):
         client.instances.module_remove(instance_id, module_id)
-        self.assert_client_code(expected_http_code, client)
+        self.assert_client_code(expected_http_code, client=client)
 
     def run_wait_for_inst_with_mods(self, expected_states=['BUILD', 'ACTIVE']):
         self.assert_instance_action(self.mod_inst_id, expected_states, None)
@@ -903,7 +902,7 @@ class ModuleRunner(TestRunner):
             prefix = 'contents'
             modretrieve_list = client.instances.module_retrieve(
                 instance_id, directory=temp_dir, prefix=prefix)
-            self.assert_client_code(expected_http_code, client)
+            self.assert_client_code(expected_http_code, client=client)
             count = len(modretrieve_list)
             self.assert_equal(expected_count, count,
                               "Wrong number of modules from retrieve")
@@ -964,7 +963,7 @@ class ModuleRunner(TestRunner):
 
     def assert_delete_instance(self, instance_id, expected_http_code):
         self.auth_client.instances.delete(instance_id)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
     def run_wait_for_delete_inst_with_mods(
             self, expected_last_state=['SHUTDOWN']):
diff --git a/trove/tests/scenario/runners/replication_runners.py b/trove/tests/scenario/runners/replication_runners.py
index 99e21437..2d402321 100644
--- a/trove/tests/scenario/runners/replication_runners.py
+++ b/trove/tests/scenario/runners/replication_runners.py
@@ -69,7 +69,7 @@ class ReplicationRunner(TestRunner):
             datastore=self.instance_info.dbaas_datastore,
             datastore_version=self.instance_info.dbaas_datastore_version,
             locality='anti-affinity').id
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
     def run_create_single_replica(self, expected_http_code=200):
         self.master_backup_count = len(
@@ -87,7 +87,7 @@ class ReplicationRunner(TestRunner):
             datastore_version=self.instance_info.dbaas_datastore_version,
             nics=self.instance_info.nics,
             replica_count=replica_count)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         return replica.id
 
     def run_wait_for_single_replica(self, expected_states=['BUILD', 'ACTIVE']):
@@ -98,8 +98,8 @@ class ReplicationRunner(TestRunner):
         self.replica_1_host = self.get_instance_host(self.replica_1_id)
 
     def _assert_is_master(self, instance_id, replica_ids):
-        instance = self.get_instance(instance_id)
-        self.assert_client_code(200)
+        instance = self.get_instance(instance_id, client=self.admin_client)
+        self.assert_client_code(200, client=self.admin_client)
         CheckInstance(instance._info).slaves()
         self.assert_true(
             set(replica_ids).issubset(self._get_replica_set(instance_id)))
@@ -110,8 +110,8 @@ class ReplicationRunner(TestRunner):
         return set([replica['id'] for replica in instance._info['replicas']])
 
     def _assert_is_replica(self, instance_id, master_id):
-        instance = self.get_instance(instance_id)
-        self.assert_client_code(200)
+        instance = self.get_instance(instance_id, client=self.admin_client)
+        self.assert_client_code(200, client=self.admin_client)
         CheckInstance(instance._info).replica_of()
         self.assert_equal(master_id, instance._info['replica_of']['id'],
                           'Unexpected replication master ID')
@@ -145,7 +145,7 @@ class ReplicationRunner(TestRunner):
             datastore_version=self.instance_info.dbaas_datastore_version,
             replica_of=self.non_affinity_master_id,
             replica_count=1).id
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
     def run_create_multiple_replicas(self, expected_http_code=200):
         self.replica_2_id = self.assert_replica_create(
@@ -176,7 +176,8 @@ class ReplicationRunner(TestRunner):
                         else [instance_ids])
         for instance_id in instance_ids:
             self.auth_client.instances.delete(instance_id)
-            self.assert_client_code(expected_http_code)
+            self.assert_client_code(expected_http_code,
+                                    client=self.auth_client)
 
     def run_wait_for_delete_non_affinity_repl(
             self, expected_last_status=['SHUTDOWN']):
@@ -344,8 +345,8 @@ class ReplicationRunner(TestRunner):
             replica_id, expected_states, expected_http_code)
 
     def _assert_is_not_replica(self, instance_id):
-        instance = self.get_instance(instance_id)
-        self.assert_client_code(200)
+        instance = self.get_instance(instance_id, client=self.admin_client)
+        self.assert_client_code(200, client=self.admin_client)
 
         if 'replica_of' not in instance._info:
             try:
diff --git a/trove/tests/scenario/runners/test_runners.py b/trove/tests/scenario/runners/test_runners.py
index 1ef0f5aa..ebf1ecb4 100644
--- a/trove/tests/scenario/runners/test_runners.py
+++ b/trove/tests/scenario/runners/test_runners.py
@@ -572,7 +572,7 @@ class TestRunner(object):
 
     def _has_status(self, instance_id, status, fast_fail_status=None):
         fast_fail_status = fast_fail_status or []
-        instance = self.get_instance(instance_id)
+        instance = self.get_instance(instance_id, self.admin_client)
         self.report.log("Polling instance '%s' for state '%s', was '%s'."
                         % (instance_id, status, instance.status))
         if instance.status in fast_fail_status:
diff --git a/trove/tests/scenario/runners/user_actions_runners.py b/trove/tests/scenario/runners/user_actions_runners.py
index 30bd19b6..d822b1bb 100644
--- a/trove/tests/scenario/runners/user_actions_runners.py
+++ b/trove/tests/scenario/runners/user_actions_runners.py
@@ -66,7 +66,7 @@ class UserActionsRunner(TestRunner):
     def assert_users_create(self, instance_id, serial_users_def,
                             expected_http_code):
         self.auth_client.users.create(instance_id, serial_users_def)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         self.wait_for_user_create(instance_id, serial_users_def)
         return serial_users_def
 
@@ -82,7 +82,7 @@ class UserActionsRunner(TestRunner):
 
         queried_user = self.auth_client.users.get(
             instance_id, user_name, user_host)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         self._assert_user_matches(queried_user, expected_user_def)
 
     def _assert_user_matches(self, user, expected_user_def):
@@ -100,7 +100,7 @@ class UserActionsRunner(TestRunner):
     def assert_users_list(self, instance_id, expected_user_defs,
                           expected_http_code, limit=2):
         full_list = self.auth_client.users.list(instance_id)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         listed_users = {user.name: user for user in full_list}
         self.assert_is_none(full_list.next,
                             "Unexpected pagination in the list.")
@@ -121,7 +121,7 @@ class UserActionsRunner(TestRunner):
 
         # Test list pagination.
         list_page = self.auth_client.users.list(instance_id, limit=limit)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
         self.assert_true(len(list_page) <= limit)
         if len(full_list) > limit:
@@ -138,7 +138,8 @@ class UserActionsRunner(TestRunner):
                               "Pagination marker should be the last element "
                               "in the page.")
             list_page = self.auth_client.users.list(instance_id, marker=marker)
-            self.assert_client_code(expected_http_code)
+            self.assert_client_code(expected_http_code,
+                                    client=self.auth_client)
             self.assert_pagination_match(
                 list_page, full_list, limit, len(full_list))
 
@@ -155,7 +156,7 @@ class UserActionsRunner(TestRunner):
         user_name, user_host = self._get_user_name_host_pair(user_def)
         user_dbs = self.auth_client.users.list_access(instance_id, user_name,
                                                       hostname=user_host)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
         expected_dbs = {db_def['name'] for db_def in user_def['databases']}
         listed_dbs = [db.name for db in user_dbs]
@@ -190,7 +191,7 @@ class UserActionsRunner(TestRunner):
                                   database, expected_http_code):
         self.auth_client.users.revoke(
             instance_id, user_name, database, hostname=user_host)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         user_dbs = self.auth_client.users.list_access(
             instance_id, user_name, hostname=user_host)
         self.assert_false(any(db.name == database for db in user_dbs),
@@ -206,7 +207,7 @@ class UserActionsRunner(TestRunner):
                                  database, expected_http_code):
         self.auth_client.users.grant(
             instance_id, user_name, [database], hostname=user_host)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         user_dbs = self.auth_client.users.list_access(
             instance_id, user_name, hostname=user_host)
         self.assert_true(any(db.name == database for db in user_dbs),
@@ -339,7 +340,7 @@ class UserActionsRunner(TestRunner):
 
         self.auth_client.users.update_attributes(
             instance_id, user_name, update_attribites, user_host)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
 
         # Update the stored definitions with the new value.
         expected_def = None
@@ -388,7 +389,7 @@ class UserActionsRunner(TestRunner):
         user_name, user_host = self._get_user_name_host_pair(user_def)
 
         self.auth_client.users.delete(instance_id, user_name, user_host)
-        self.assert_client_code(expected_http_code)
+        self.assert_client_code(expected_http_code, client=self.auth_client)
         self._wait_for_user_delete(instance_id, user_name)
 
     def _wait_for_user_delete(self, instance_id, deleted_user_name):