Execute functional test jobs running under python3

Functional tests are executed by trovestack script, which uses python
in default OS version which is 2.7.

This change wraps python in tox and executes trovestack int_tests in
virtual enviroment, starting them in python3.

Any future python version change could be managed by tox framework.

Change-Id: I3a849978241d3d0669ef9e1c802ff504ac3c32cb
Signed-off-by: Marcin Piwowarczyk <m.piwowarczy@samsung.com>

13 files changed, 50 insertions, 30 deletions

diff --git a/integration/tests/integration/tests/util/services.py b/integration/tests/integration/tests/util/services.py
index f5f8cbf1..6e8f1da8 100644
--- a/integration/tests/integration/tests/util/services.py
+++ b/integration/tests/integration/tests/util/services.py
@@ -189,12 +189,12 @@ class Service(object):
         # as python /path/to/executable args, so the entry is
         # /path/to/executable
         actual_command = self.cmd[proc_name_index].split("/")[-1]
-        print actual_command
+        print(actual_command)
         proc_command = ["/usr/bin/pgrep", "-f", actual_command]
-        print proc_command
+        print(proc_command)
         proc = start_proc(proc_command, shell=False)
         line = proc.stdout.readline()
-        print line
+        print(line)
         # pgrep only returns a pid. if there is no pid, it'll return nothing
         return len(line) != 0
 
diff --git a/roles/trove-devstack/tasks/main.yml b/roles/trove-devstack/tasks/main.yml
index 7ee91646..01132837 100644
--- a/roles/trove-devstack/tasks/main.yml
+++ b/roles/trove-devstack/tasks/main.yml
@@ -5,5 +5,5 @@
     export PATH_DEVSTACK_SRC=$DEST/devstack
     export TROVE_RESIZE_TIME_OUT={{trove_resize_time_out}}
 
-    cd $DEST/trove/integration/scripts
-    ./trovestack gate-tests {{trove_test_datastore}} {{trove_test_group}}
+    cd $DEST/trove
+    tox -etrovestack -vv -- gate-tests {{trove_test_datastore}} {{trove_test_group}}
diff --git a/tox.ini b/tox.ini
index 5a7d7caf..1ced348d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -123,3 +123,14 @@ deps =
   -c{toxinidir}/lower-constraints.txt
   -r{toxinidir}/test-requirements.txt
   -r{toxinidir}/requirements.txt
+
+[testenv:trovestack]
+basepython = python3
+skip_install = True
+changedir = {toxinidir}/integration/scripts
+passenv = *
+commands =
+    pip install --no-binary :all: {toxinidir} \
+      -c/opt/stack/trove/test-upper-constraints.txt \
+      -chttps://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt
+    ./trovestack {posargs}
\ No newline at end of file
diff --git a/trove/common/utils.py b/trove/common/utils.py
index 484c044e..7710923c 100644
--- a/trove/common/utils.py
+++ b/trove/common/utils.py
@@ -185,14 +185,14 @@ class MethodInspector(object):
 
 
 def build_polling_task(retriever, condition=lambda value: value,
-                       sleep_time=1, time_out=None):
+                       sleep_time=1, time_out=0):
     start_time = time.time()
 
     def poll_and_check():
         obj = retriever()
         if condition(obj):
             raise loopingcall.LoopingCallDone(retvalue=obj)
-        if time_out is not None and time.time() - start_time > time_out:
+        if time_out > 0 and time.time() - start_time > time_out:
             raise exception.PollTimeOut
 
     return loopingcall.BackOffLoopingCall(
@@ -202,7 +202,7 @@ def build_polling_task(retriever, condition=lambda value: value,
 
 
 def poll_until(retriever, condition=lambda value: value,
-               sleep_time=1, time_out=None):
+               sleep_time=1, time_out=0):
     """Retrieves object until it passes condition, then returns it.
 
     If time_out_limit is passed in, PollTimeOut will be raised once that
diff --git a/trove/tests/api/configurations.py b/trove/tests/api/configurations.py
index 1a7dbaa8..c2dc968c 100644
--- a/trove/tests/api/configurations.py
+++ b/trove/tests/api/configurations.py
@@ -138,7 +138,7 @@ def _test_configuration_is_applied_to_instance(instance, configuration_id):
         resp, body = instance_info.dbaas.client.last_response
         print(resp)
         print(body)
-        return json.loads(body)['type']
+        return json.loads(body.decode())['type']
 
     # check the config values are correct
     for key, value in actual_values:
diff --git a/trove/tests/api/replication.py b/trove/tests/api/replication.py
index 57b0662c..9a16a2ec 100644
--- a/trove/tests/api/replication.py
+++ b/trove/tests/api/replication.py
@@ -53,7 +53,7 @@ def _get_user_count(server_info):
            ' where user like \\\"slave_%\\\"\\\'')
     server = create_server_connection(server_info.id)
     stdout, stderr = server.execute(cmd)
-    return int(stdout.rstrip())
+    return int(stdout)
 
 
 def slave_is_running(running=True):
@@ -364,10 +364,7 @@ class DetachReplica(object):
             cmd = "mysql -BNq -e \\\'select @@read_only\\\'"
             server = create_server_connection(slave_instance.id)
             stdout, stderr = server.execute(cmd)
-            if (stdout.rstrip() != "0"):
-                return False
-            else:
-                return True
+            return stdout.rstrip() == "0"
         poll_until(check_not_read_only)
 
 
diff --git a/trove/tests/scenario/helpers/mysql_helper.py b/trove/tests/scenario/helpers/mysql_helper.py
index 7fada681..60f202c6 100644
--- a/trove/tests/scenario/helpers/mysql_helper.py
+++ b/trove/tests/scenario/helpers/mysql_helper.py
@@ -20,7 +20,7 @@ class MysqlHelper(SqlHelper):
 
     def __init__(self, expected_override_name, report):
         super(MysqlHelper, self).__init__(expected_override_name, report,
-                                          'mysql')
+                                          'mysql+pymysql')
 
     def get_helper_credentials(self):
         return {'name': 'lite', 'password': 'litepass', 'database': 'firstdb'}
diff --git a/trove/tests/scenario/helpers/sql_helper.py b/trove/tests/scenario/helpers/sql_helper.py
index 325fb03e..a158d963 100644
--- a/trove/tests/scenario/helpers/sql_helper.py
+++ b/trove/tests/scenario/helpers/sql_helper.py
@@ -27,7 +27,8 @@ class SqlHelper(TestHelper):
 
     DATA_COLUMN_NAME = 'value'
 
-    def __init__(self, expected_override_name, report, protocol, port=None):
+    def __init__(self, expected_override_name, report,
+                 protocol="mysql+pymysql", port=None):
         super(SqlHelper, self).__init__(expected_override_name, report)
 
         self.protocol = protocol
diff --git a/trove/tests/scenario/runners/configuration_runners.py b/trove/tests/scenario/runners/configuration_runners.py
index 40fba068..a2c9c5f5 100644
--- a/trove/tests/scenario/runners/configuration_runners.py
+++ b/trove/tests/scenario/runners/configuration_runners.py
@@ -144,7 +144,7 @@ class ConfigurationRunner(TestRunner):
             configuration.has_field('description', six.string_types)
             configuration.has_field('values', dict)
             configuration.has_field('datastore_name', six.string_types)
-            configuration.has_field('datastore_version_id', unicode)
+            configuration.has_field('datastore_version_id', six.text_type)
             configuration.has_field('datastore_version_name', six.string_types)
 
             self.assert_equal(name, result.name)
diff --git a/trove/tests/scenario/runners/module_runners.py b/trove/tests/scenario/runners/module_runners.py
index c6483e31..e7806516 100644
--- a/trove/tests/scenario/runners/module_runners.py
+++ b/trove/tests/scenario/runners/module_runners.py
@@ -17,6 +17,7 @@
 import os
 from proboscis import SkipTest
 import re
+import six
 import tempfile
 import time
 
@@ -43,7 +44,7 @@ class ModuleRunner(TestRunner):
         self.MODULE_BINARY_SUFFIX = '_bin_auto'
         self.MODULE_BINARY_SUFFIX2 = self.MODULE_BINARY_SUFFIX + '_2'
         self.MODULE_BINARY_CONTENTS = os.urandom(20)
-        self.MODULE_BINARY_CONTENTS2 = '\x00\xFF\xea\x9c\x11\xfeok\xb1\x8ax'
+        self.MODULE_BINARY_CONTENTS2 = b'\x00\xFF\xea\x9c\x11\xfeok\xb1\x8ax'
 
         self.module_name_order = [
             {'suffix': self.MODULE_BINARY_SUFFIX,
@@ -1283,12 +1284,12 @@ class ModuleRunner(TestRunner):
                     if 'contents' in expected and expected['contents']:
                         with open(filename, 'rb') as fh:
                             contents = fh.read()
-                        # convert contents into bytearray to work with py27
-                        # and py34
-                        contents = bytes([ord(item) for item in contents])
-                        expected_contents = bytes(
-                            [ord(item) for item in expected['contents']])
-                        self.assert_equal(expected_contents, contents,
+
+                        expected = expected['contents']
+                        if isinstance(expected, six.string_types):
+                            expected = expected.encode()
+
+                        self.assert_equal(expected, contents,
                                           "Unexpected contents for %s" %
                                           module_name)
         finally:
diff --git a/trove/tests/scenario/runners/test_runners.py b/trove/tests/scenario/runners/test_runners.py
index 58e079a6..43a4fecd 100644
--- a/trove/tests/scenario/runners/test_runners.py
+++ b/trove/tests/scenario/runners/test_runners.py
@@ -145,7 +145,7 @@ class RunnerFactory(object):
                 # Only fail silently if it's something we expect,
                 # such as a missing override class.  Anything else
                 # shouldn't be suppressed.
-                l_msg = ie.message.lower()
+                l_msg = str(ie).lower()
                 if (load_type and load_type not in l_msg) or (
                         'no module named' not in l_msg and
                         'cannot be found' not in l_msg):
@@ -401,7 +401,16 @@ class TestRunner(object):
         """Assert that two lists contain same elements
         (with same multiplicities) ignoring the element order.
         """
-        return cls.assert_equal(sorted(expected), sorted(actual), message)
+        # Sorts the elements of a given list, including dictionaries.
+        # For dictionaries sorts based on dictionary key.
+        # example:
+        # [1, 3, 2]             -> [1, 2, 3]
+        # ["b", "a", "c"]       -> ["a", "b", "c"]
+        # [{'b':'y'},{'a':'x'}] -> [{'a':'x'},{'b':'y'}]
+        sort = lambda object: sorted(object, key=lambda e: sorted(e.keys())
+                                     if isinstance(e, dict) else e)
+
+        return cls.assert_equal(sort(expected), sort(actual), message)
 
     @classmethod
     def assert_equal(cls, expected, actual, message=None):
@@ -506,7 +515,7 @@ class TestRunner(object):
         if client:
             # Make sure that the client_cmd comes from the same client that
             # was passed in, otherwise asserting the client code may fail.
-            cmd_clz = client_cmd.im_self
+            cmd_clz = client_cmd.__self__
             cmd_clz_name = cmd_clz.__class__.__name__
             client_attrs = [attr[0] for attr in inspect.getmembers(
                             client.real_client)
@@ -700,7 +709,7 @@ class TestRunner(object):
         return False
 
     def _poll_while(self, instance_id, expected_status,
-                    sleep_time=1, time_out=None):
+                    sleep_time=1, time_out=0):
         poll_until(lambda: not self._has_status(instance_id, expected_status),
                    sleep_time=sleep_time, time_out=time_out)
 
diff --git a/trove/tests/util/event_simulator.py b/trove/tests/util/event_simulator.py
index d21254de..79f5a4ba 100644
--- a/trove/tests/util/event_simulator.py
+++ b/trove/tests/util/event_simulator.py
@@ -227,7 +227,7 @@ def fake_sleep(time_to_sleep):
 
 
 def fake_poll_until(retriever, condition=lambda value: value,
-                    sleep_time=1, time_out=None):
+                    sleep_time=1, time_out=0):
     """Fakes out poll until."""
     from trove.common import exception
     slept_time = 0
diff --git a/trove/tests/util/server_connection.py b/trove/tests/util/server_connection.py
index 6eed3663..156b8c98 100644
--- a/trove/tests/util/server_connection.py
+++ b/trove/tests/util/server_connection.py
@@ -48,7 +48,8 @@ class ServerSSHConnection(object):
     def execute(self, cmd):
         exe_cmd = "%s %s %s" % (tests.SSH_CMD, self.ip_address, cmd)
         print("RUNNING COMMAND: %s" % exe_cmd)
-        return util.process(exe_cmd)
+        stdout, stderr = util.process(exe_cmd)
+        return (stdout.decode(), stderr.decode())
 
 
 class OpenVZServerConnection(object):