Merge branch 'sam/functional-tests-shared' into 'master'

functional-tests: Improve helper code and share it with tracker-miners

See merge request GNOME/tracker!117

40 files changed, 799 insertions, 510 deletions

diff --git a/tests/functional-tests/01-insertion.py b/tests/functional-tests/01-insertion.py
index 180bca5d3..0bed7aeb3 100755
--- a/tests/functional-tests/01-insertion.py
+++ b/tests/functional-tests/01-insertion.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -28,7 +29,7 @@ import random
 import datetime
 
 import unittest as ut
-from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
+from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
 
 
 class TrackerStoreInsertionTests (CommonTrackerStoreTest):
diff --git a/tests/functional-tests/02-sparql-bugs.py b/tests/functional-tests/02-sparql-bugs.py
index c5f23674f..4305ea0a9 100755
--- a/tests/functional-tests/02-sparql-bugs.py
+++ b/tests/functional-tests/02-sparql-bugs.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -24,7 +25,7 @@ Peculiar Sparql behavour reported in bugs
 from gi.repository import GLib
 
 import unittest as ut
-from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
+from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
 
 
 class TrackerStoreSparqlBugsTests (CommonTrackerStoreTest):
diff --git a/tests/functional-tests/03-fts-functions.py b/tests/functional-tests/03-fts-functions.py
index df2668b22..46c43f368 100755
--- a/tests/functional-tests/03-fts-functions.py
+++ b/tests/functional-tests/03-fts-functions.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -23,7 +24,7 @@ These tests use only the store. They insert instances with known text
 and run sparql with fts functions to check the results.
 """
 import unittest as ut
-from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
+from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
 
 
 class TestFTSFunctions (CommonTrackerStoreTest):
diff --git a/tests/functional-tests/04-group-concat.py b/tests/functional-tests/04-group-concat.py
index 3ae53239a..a8064a828 100755
--- a/tests/functional-tests/04-group-concat.py
+++ b/tests/functional-tests/04-group-concat.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -21,7 +22,7 @@
 Test the GROUP_CONCAT function in Sparql. Only requires the store.
 """
 import unittest as ut
-from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
+from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
 
 
 class TestGroupConcat (CommonTrackerStoreTest):
diff --git a/tests/functional-tests/05-coalesce.py b/tests/functional-tests/05-coalesce.py
index a9ef15aab..48d8e6eb6 100755
--- a/tests/functional-tests/05-coalesce.py
+++ b/tests/functional-tests/05-coalesce.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -21,7 +22,7 @@
 Test tracker:coalesce function in Sparql. Only uses the Store
 """
 import unittest as ut
-from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
+from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
 
 
 class TestCoalesce (CommonTrackerStoreTest):
diff --git a/tests/functional-tests/06-distance.py b/tests/functional-tests/06-distance.py
index 52191559f..80d35dfb9 100755
--- a/tests/functional-tests/06-distance.py
+++ b/tests/functional-tests/06-distance.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -21,7 +22,7 @@
 Test the distance-calculation functions in Sparql. Only requires the Store
 """
 import unittest as ut
-from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
+from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
 
 POINT_COORDS = [
     (0, 0), (1, 1), (2, 2), (3, 3), (4, 4)
diff --git a/tests/functional-tests/07-graph.py b/tests/functional-tests/07-graph.py
index 03366fd30..aad935f77 100755
--- a/tests/functional-tests/07-graph.py
+++ b/tests/functional-tests/07-graph.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -21,7 +22,7 @@
 Tests graphs in Sparql. Only requires the store.
 """
 import unittest as ut
-from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
+from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
 
 
 class TestGraphs (CommonTrackerStoreTest):
diff --git a/tests/functional-tests/08-unique-insertions.py b/tests/functional-tests/08-unique-insertions.py
index 23650231f..9c9578a5c 100755
--- a/tests/functional-tests/08-unique-insertions.py
+++ b/tests/functional-tests/08-unique-insertions.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -21,7 +22,7 @@
 Replicate the behaviour of the miner inserting information in the store.
 """
 import unittest as ut
-from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
+from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
 
 
 class TestMinerInsertBehaviour (CommonTrackerStoreTest):
diff --git a/tests/functional-tests/09-concurrent-query.py b/tests/functional-tests/09-concurrent-query.py
index f362dc98d..b24fdcc40 100755
--- a/tests/functional-tests/09-concurrent-query.py
+++ b/tests/functional-tests/09-concurrent-query.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -23,7 +24,7 @@ Send concurrent inserts and queries to the daemon to check the concurrency.
 from gi.repository import GLib
 
 import unittest as ut
-from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
+from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
 
 AMOUNT_OF_TEST_INSTANCES = 100
 AMOUNT_OF_QUERIES = 10
diff --git a/tests/functional-tests/14-signals.py b/tests/functional-tests/14-signals.py
index 44806e8a2..242ae8480 100755
--- a/tests/functional-tests/14-signals.py
+++ b/tests/functional-tests/14-signals.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -23,18 +24,16 @@ are emitted. Theses tests are not extensive (only few selected signals
 are tested)
 """
 
-import unittest as ut
-from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
-from common.utils import configuration as cfg
-
 from gi.repository import Gio
 from gi.repository import GLib
+
 import time
+import unittest as ut
+
+from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
 
-GRAPH_UPDATED_SIGNAL = "GraphUpdated"
 
-SIGNALS_PATH = "/org/freedesktop/Tracker1/Resources"
-SIGNALS_IFACE = "org.freedesktop.Tracker1.Resources"
+GRAPH_UPDATED_SIGNAL = "GraphUpdated"
 
 CONTACT_CLASS_URI = "http://www.semanticdesktop.org/ontologies/2007/03/22/nco#PersonContact"
 
@@ -70,10 +69,10 @@ class TrackerStoreSignalsTests (CommonTrackerStoreTest):
         After connecting to the signal, call self.__wait_for_signal.
         """
         self.cb_id = self.bus.signal_subscribe(
-            sender=cfg.TRACKER_BUSNAME,
-            interface_name=SIGNALS_IFACE,
+            sender=self.tracker.TRACKER_BUSNAME,
+            interface_name=self.tracker.RESOURCES_IFACE,
             member=GRAPH_UPDATED_SIGNAL,
-            object_path=SIGNALS_PATH,
+            object_path=self.tracker.TRACKER_OBJ_PATH,
             arg0=CONTACT_CLASS_URI,
             flags=Gio.DBusSignalFlags.NONE,
             callback=self.__signal_received_cb)
diff --git a/tests/functional-tests/15-statistics.py b/tests/functional-tests/15-statistics.py
index ac6a2f210..6f6ca3014 100755
--- a/tests/functional-tests/15-statistics.py
+++ b/tests/functional-tests/15-statistics.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -25,7 +26,7 @@ are updated when different operations are executed on the store
 import time
 
 import unittest as ut
-from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
+from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
 
 RDFS_RESOURCE = "rdfs:Resource"
 NIE_IE = "nie:InformationElement"
diff --git a/tests/functional-tests/16-collation.py b/tests/functional-tests/16-collation.py
index 36a1fbbc7..40a993d82 100755
--- a/tests/functional-tests/16-collation.py
+++ b/tests/functional-tests/16-collation.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -27,7 +28,7 @@ import random
 import locale
 
 import unittest as ut
-from common.utils.storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
+from storetest import CommonTrackerStoreTest as CommonTrackerStoreTest
 
 
 class TrackerStoreCollationTests (CommonTrackerStoreTest):
diff --git a/tests/functional-tests/17-ontology-changes.py b/tests/functional-tests/17-ontology-changes.py
index 687757fe1..8cf7db220 100755
--- a/tests/functional-tests/17-ontology-changes.py
+++ b/tests/functional-tests/17-ontology-changes.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -25,18 +26,20 @@ changes and checking if the data is still there.
 
 from gi.repository import GLib
 
+import logging
 import os
 import shutil
 import re
 import tempfile
 import time
-
-from common.utils import configuration as cfg
-from common.utils import helpers
-from common.utils.dconf import DConfClient
-from common.utils.expectedFailure import expectedFailureJournal
 import unittest as ut
 
+import trackertestutils.dconf
+import trackertestutils.helpers
+
+import configuration as cfg
+from expectedFailure import expectedFailureJournal
+
 
 RDFS_RANGE = "http://www.w3.org/2000/01/rdf-schema#range"
 XSD_DATETIME = "http://www.w3.org/2001/XMLSchema#dateTime"
@@ -49,6 +52,8 @@ TEST_ENV_VARS = {"LC_COLLATE": "en_GB.utf8"}
 
 REASONABLE_TIMEOUT = 5
 
+log = logging.getLogger()
+
 
 class UnableToBootException (Exception):
     pass
@@ -88,12 +93,11 @@ class TrackerSystemAbstraction (object):
             os.environ[var] = directory
 
         if ontodir:
-            helpers.log("export %s=%s" %
-                        ("TRACKER_DB_ONTOLOGIES_DIR", ontodir))
+            log.debug("export %s=%s", "TRACKER_DB_ONTOLOGIES_DIR", ontodir)
             os.environ["TRACKER_DB_ONTOLOGIES_DIR"] = ontodir
 
         for var, value in TEST_ENV_VARS.items():
-            helpers.log("export %s=%s" % (var, value))
+            log.debug("export %s=%s", var, value)
             os.environ[var] = value
 
         # Previous loop should have set DCONF_PROFILE to the test location
@@ -102,7 +106,7 @@ class TrackerSystemAbstraction (object):
 
     def _apply_settings(self, settings):
         for schema_name, contents in settings.items():
-            dconf = DConfClient(schema_name)
+            dconf = trackertestutils.dconf.DConfClient(schema_name)
             dconf.reset()
             for key, value in contents.items():
                 dconf.write(key, value)
@@ -114,7 +118,7 @@ class TrackerSystemAbstraction (object):
         """
         self.set_up_environment(confdir, ontodir)
 
-        self.store = helpers.StoreHelper()
+        self.store = trackertestutils.helpers.StoreHelper(cfg.TRACKER_STORE_PATH)
         self.store.start()
 
     def tracker_store_restart_with_new_ontologies(self, ontodir):
diff --git a/tests/functional-tests/__init__.py b/tests/functional-tests/__init__.py
index a93a4bf16..e69de29bb 100644
--- a/tests/functional-tests/__init__.py
+++ b/tests/functional-tests/__init__.py
@@ -1 +0,0 @@
-#!/usr/bin/python3
diff --git a/tests/functional-tests/common/__init__.py b/tests/functional-tests/common/__init__.py
deleted file mode 100644
index a93a4bf16..000000000
--- a/tests/functional-tests/common/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-#!/usr/bin/python3
diff --git a/tests/functional-tests/common/data/Doc/performance.doc b/tests/functional-tests/common/data/Doc/performance.doc
deleted file mode 100644
index 8450cdd63..000000000
--- a/tests/functional-tests/common/data/Doc/performance.doc
+++ /dev/null
diff --git a/tests/functional-tests/common/data/Images/test-image-2.png b/tests/functional-tests/common/data/Images/test-image-2.png
deleted file mode 100644
index 7ff9788a0..000000000
--- a/tests/functional-tests/common/data/Images/test-image-2.png
+++ /dev/null
diff --git a/tests/functional-tests/common/data/Images/test-image-3.tif b/tests/functional-tests/common/data/Images/test-image-3.tif
deleted file mode 100644
index 8d91556a7..000000000
--- a/tests/functional-tests/common/data/Images/test-image-3.tif
+++ /dev/null
diff --git a/tests/functional-tests/common/data/Pdf/office-tools-test-document.pdf b/tests/functional-tests/common/data/Pdf/office-tools-test-document.pdf
deleted file mode 100644
index 064645c39..000000000
--- a/tests/functional-tests/common/data/Pdf/office-tools-test-document.pdf
+++ /dev/null
diff --git a/tests/functional-tests/common/data/Ppt/al-cont.ppt b/tests/functional-tests/common/data/Ppt/al-cont.ppt
deleted file mode 100644
index 2fc0a733f..000000000
--- a/tests/functional-tests/common/data/Ppt/al-cont.ppt
+++ /dev/null
diff --git a/tests/functional-tests/common/data/Video/.mediaartlocal/video-fec11a5d1e731ccf459f459a9c86cc51-7215ee9c7d9dc229d2921a40e899ec5f.jpeg b/tests/functional-tests/common/data/Video/.mediaartlocal/video-fec11a5d1e731ccf459f459a9c86cc51-7215ee9c7d9dc229d2921a40e899ec5f.jpeg
deleted file mode 100644
index f1f917bb5..000000000
--- a/tests/functional-tests/common/data/Video/.mediaartlocal/video-fec11a5d1e731ccf459f459a9c86cc51-7215ee9c7d9dc229d2921a40e899ec5f.jpeg
+++ /dev/null
diff --git a/tests/functional-tests/common/data/pickled_Images b/tests/functional-tests/common/data/pickled_Images
deleted file mode 100644
index 330cf46c2..000000000
--- a/tests/functional-tests/common/data/pickled_Images
+++ /dev/null
diff --git a/tests/functional-tests/common/data/pickled_Music b/tests/functional-tests/common/data/pickled_Music
deleted file mode 100644
index 1913fc4bf..000000000
--- a/tests/functional-tests/common/data/pickled_Music
+++ /dev/null
diff --git a/tests/functional-tests/common/utils/__init__.py b/tests/functional-tests/common/utils/__init__.py
deleted file mode 100644
index a93a4bf16..000000000
--- a/tests/functional-tests/common/utils/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-#!/usr/bin/python3
diff --git a/tests/functional-tests/common/utils/configuration.py b/tests/functional-tests/common/utils/configuration.py
deleted file mode 100644
index 1a327ab88..000000000
--- a/tests/functional-tests/common/utils/configuration.py
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/usr/bin/python3
-#
-# Copyright (C) 2010, Nokia <jean-luc.lamadon@nokia.com>
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-# 02110-1301, USA.
-#
-
-"Constants describing Tracker D-Bus services"
-
-import json
-import os
-
-if 'TRACKER_FUNCTIONAL_TEST_CONFIG' not in os.environ:
-    raise RuntimeError("The TRACKER_FUNCTIONAL_TEST_CONFIG environment "
-                       "variable must be set to point to the location of "
-                       "the generated configuration.json file.")
-
-with open(os.environ['TRACKER_FUNCTIONAL_TEST_CONFIG']) as f:
-    config = json.load(f)
-
-TRACKER_BUSNAME = 'org.freedesktop.Tracker1'
-TRACKER_OBJ_PATH = '/org/freedesktop/Tracker1/Resources'
-RESOURCES_IFACE = "org.freedesktop.Tracker1.Resources"
-
-MINERFS_BUSNAME = "org.freedesktop.Tracker1.Miner.Files"
-MINERFS_OBJ_PATH = "/org/freedesktop/Tracker1/Miner/Files"
-MINER_IFACE = "org.freedesktop.Tracker1.Miner"
-MINERFS_INDEX_OBJ_PATH = "/org/freedesktop/Tracker1/Miner/Files/Index"
-MINER_INDEX_IFACE = "org.freedesktop.Tracker1.Miner.Files.Index"
-
-TRACKER_BACKUP_OBJ_PATH = "/org/freedesktop/Tracker1/Backup"
-BACKUP_IFACE = "org.freedesktop.Tracker1.Backup"
-
-TRACKER_STATS_OBJ_PATH = "/org/freedesktop/Tracker1/Statistics"
-STATS_IFACE = "org.freedesktop.Tracker1.Statistics"
-
-TRACKER_STATUS_OBJ_PATH = "/org/freedesktop/Tracker1/Status"
-STATUS_IFACE = "org.freedesktop.Tracker1.Status"
-
-TRACKER_EXTRACT_BUSNAME = "org.freedesktop.Tracker1.Miner.Extract"
-TRACKER_EXTRACT_OBJ_PATH = "/org/freedesktop/Tracker1/Miner/Extract"
-
-WRITEBACK_BUSNAME = "org.freedesktop.Tracker1.Writeback"
-
-
-DCONF_MINER_SCHEMA = "org.freedesktop.Tracker.Miner.Files"
-
-# Autoconf substitutes paths in the configuration.json file without
-# expanding variables, so we need to manually insert these.
-
-
-def expandvars(variable):
-    # Note: the order matters!
-    result = variable
-    for var, value in [("${datarootdir}", RAW_DATAROOT_DIR),
-                       ("${exec_prefix}", RAW_EXEC_PREFIX),
-                       ("${prefix}", PREFIX),
-                       ("@top_srcdir@", TOP_SRCDIR),
-                       ("@top_builddir@", TOP_BUILDDIR)]:
-        result = result.replace(var, value)
-
-    return result
-
-
-PREFIX = config['PREFIX']
-RAW_EXEC_PREFIX = config['RAW_EXEC_PREFIX']
-RAW_DATAROOT_DIR = config['RAW_DATAROOT_DIR']
-
-TOP_SRCDIR = os.path.dirname(os.path.dirname(
-    os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
-TOP_BUILDDIR = os.environ['TRACKER_FUNCTIONAL_TEST_BUILD_DIR']
-
-TEST_ONTOLOGIES_DIR = os.path.normpath(
-    expandvars(config['TEST_ONTOLOGIES_DIR']))
-
-TRACKER_STORE_PATH = os.path.normpath(expandvars(config['TRACKER_STORE_PATH']))
-
-disableJournal = (len(config['disableJournal']) == 0)
diff --git a/tests/functional-tests/common/utils/helpers.py b/tests/functional-tests/common/utils/helpers.py
deleted file mode 100644
index 6e5c26c40..000000000
--- a/tests/functional-tests/common/utils/helpers.py
+++ /dev/null
@@ -1,337 +0,0 @@
-#!/usr/bin/python3
-#
-# Copyright (C) 2010, Nokia <jean-luc.lamadon@nokia.com>
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-# 02110-1301, USA.
-#
-from gi.repository import Gio
-from gi.repository import GLib
-import os
-import sys
-import subprocess
-import time
-import re
-
-from common.utils import configuration as cfg
-from common.utils import options as options
-
-
-class NoMetadataException (Exception):
-    pass
-
-REASONABLE_TIMEOUT = 30
-
-
-def log(message):
-    if options.is_verbose():
-        print(message)
-
-
-class Helper:
-    """
-    Abstract helper for Tracker processes. Launches the process manually
-    and waits for it to appear on the session bus.
-
-    The helper will fail if the process is already running. Use
-    test-runner.sh to ensure the processes run inside a separate DBus
-    session bus.
-
-    The process is watched using a timed GLib main loop source. If the process
-    exits with an error code, the test will abort the next time the main loop
-    is entered (or straight away if currently running the main loop).
-    """
-
-    BUS_NAME = None
-    PROCESS_NAME = None
-
-    def __init__(self):
-        self.process = None
-        self.available = False
-
-        self.loop = GLib.MainLoop()
-        self.install_glib_excepthook(self.loop)
-
-        self.bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
-
-    def install_glib_excepthook(self, loop):
-        """
-        Handler to abort test if an exception occurs inside the GLib main loop.
-        """
-        old_hook = sys.excepthook
-
-        def new_hook(etype, evalue, etb):
-            old_hook(etype, evalue, etb)
-            GLib.MainLoop.quit(loop)
-            sys.exit(1)
-        sys.excepthook = new_hook
-
-    def _start_process(self, env=None):
-        path = self.PROCESS_PATH
-        flags = getattr(self,
-                        "FLAGS",
-                        [])
-
-        kws = {}
-
-        if not options.is_verbose():
-            FNULL = open('/dev/null', 'w')
-            kws.update({'stdout': FNULL, 'stderr': subprocess.PIPE})
-
-        if env:
-            kws['env'] = env
-
-        command = [path] + flags
-        log("Starting %s" % ' '.join(command))
-        try:
-            return subprocess.Popen([path] + flags, **kws)
-        except OSError as e:
-            raise RuntimeError("Error starting %s: %s" % (path, e))
-
-    def _bus_name_appeared(self, name, owner, data):
-        log("[%s] appeared in the bus as %s" % (self.PROCESS_NAME, owner))
-        self.available = True
-        self.loop.quit()
-
-    def _bus_name_vanished(self, name, data):
-        log("[%s] disappeared from the bus" % self.PROCESS_NAME)
-        self.available = False
-        self.loop.quit()
-
-    def _process_watch_cb(self):
-        if self.process_watch_timeout == 0:
-            # The GLib seems to call the timeout after we've removed it
-            # sometimes, which causes errors unless we detect it.
-            return False
-
-        status = self.process.poll()
-
-        if status is None:
-            return True    # continue
-        elif status == 0 and not self.abort_if_process_exits_with_status_0:
-            return True    # continue
-        else:
-            self.process_watch_timeout = 0
-            if options.is_verbose():
-                error = ""
-            else:
-                error = self.process.stderr.read()
-            raise RuntimeError("%s exited with status: %i\n%s" %
-                               (self.PROCESS_NAME, status, error))
-
-    def _timeout_on_idle_cb(self):
-        log("[%s] Timeout waiting... asumming idle." % self.PROCESS_NAME)
-        self.loop.quit()
-        self.timeout_id = None
-        return False
-
-    def start(self, env=None):
-        """
-        Start an instance of process and wait for it to appear on the bus.
-        """
-        if self.process is not None:
-            raise RuntimeError(
-                "%s process already started" % self.PROCESS_NAME)
-
-        self._bus_name_watch_id = Gio.bus_watch_name_on_connection(
-            self.bus, self.BUS_NAME, Gio.BusNameWatcherFlags.NONE,
-            self._bus_name_appeared, self._bus_name_vanished)
-        self.loop.run()
-
-        if options.is_manual_start():
-            print("Start %s manually" % self.PROCESS_NAME)
-        else:
-            if self.available:
-                # It's running, but we didn't start it...
-                raise Exception("Unable to start test instance of %s: "
-                                "already running " % self.PROCESS_NAME)
-
-            self.process = self._start_process(env=env)
-            log('[%s] Started process %i' %
-                (self.PROCESS_NAME, self.process.pid))
-            self.process_watch_timeout = GLib.timeout_add(
-                200, self._process_watch_cb)
-
-        self.abort_if_process_exits_with_status_0 = True
-
-        # Run the loop until the bus name appears, or the process dies.
-        self.loop.run()
-
-        self.abort_if_process_exits_with_status_0 = False
-
-    def stop(self):
-        if self.process is None:
-            # Seems that it didn't even start...
-            return
-
-        start = time.time()
-        if self.process.poll() == None:
-            GLib.source_remove(self.process_watch_timeout)
-            self.process_watch_timeout = 0
-
-            self.process.terminate()
-
-            while self.process.poll() == None:
-                time.sleep(0.1)
-
-                if time.time() > (start + REASONABLE_TIMEOUT):
-                    log("[%s] Failed to terminate, sending kill!" %
-                        self.PROCESS_NAME)
-                    self.process.kill()
-                    self.process.wait()
-
-        log("[%s] stopped." % self.PROCESS_NAME)
-
-        # Run the loop until the bus name appears, or the process dies.
-        self.loop.run()
-        Gio.bus_unwatch_name(self._bus_name_watch_id)
-
-        self.process = None
-
-    def kill(self):
-        if options.is_manual_start():
-            log("kill(): ignoring, because process was started manually.")
-            return
-
-        if self.process_watch_timeout != 0:
-            GLib.source_remove(self.process_watch_timeout)
-            self.process_watch_timeout = 0
-
-        self.process.kill()
-
-        # Name owner changed callback should take us out from this loop
-        self.loop.run()
-        Gio.bus_unwatch_name(self._bus_name_watch_id)
-
-        self.process = None
-
-        log("[%s] killed." % self.PROCESS_NAME)
-
-
-class StoreHelper (Helper):
-    """
-    Wrapper for the Store API
-
-    Every method tries to reconnect once if there is a dbus exception
-    (some tests kill the daemon and make the connection useless)
-    """
-
-    PROCESS_NAME = "tracker-store"
-    PROCESS_PATH = cfg.TRACKER_STORE_PATH
-    BUS_NAME = cfg.TRACKER_BUSNAME
-
-    def start(self, env=None):
-        Helper.start(self, env=env)
-
-        self.resources = Gio.DBusProxy.new_sync(
-            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None,
-            cfg.TRACKER_BUSNAME, cfg.TRACKER_OBJ_PATH, cfg.RESOURCES_IFACE)
-
-        self.backup_iface = Gio.DBusProxy.new_sync(
-            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None,
-            cfg.TRACKER_BUSNAME, cfg.TRACKER_BACKUP_OBJ_PATH, cfg.BACKUP_IFACE)
-
-        self.stats_iface = Gio.DBusProxy.new_sync(
-            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None,
-            cfg.TRACKER_BUSNAME, cfg.TRACKER_STATS_OBJ_PATH, cfg.STATS_IFACE)
-
-        self.status_iface = Gio.DBusProxy.new_sync(
-            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None,
-            cfg.TRACKER_BUSNAME, cfg.TRACKER_STATUS_OBJ_PATH, cfg.STATUS_IFACE)
-
-        log("[%s] booting..." % self.PROCESS_NAME)
-        self.status_iface.Wait()
-        log("[%s] ready." % self.PROCESS_NAME)
-
-    def stop(self):
-        Helper.stop(self)
-
-    def query(self, query, timeout=5000, **kwargs):
-        return self.resources.SparqlQuery('(s)', query, timeout=timeout, **kwargs)
-
-    def update(self, update_sparql, timeout=5000, **kwargs):
-        return self.resources.SparqlUpdate('(s)', update_sparql, timeout=timeout, **kwargs)
-
-    def load(self, ttl_uri, timeout=5000, **kwargs):
-        return self.resources.Load('(s)', ttl_uri, timeout=timeout, **kwargs)
-
-    def batch_update(self, update_sparql, **kwargs):
-        return self.resources.BatchSparqlUpdate('(s)', update_sparql, **kwargs)
-
-    def batch_commit(self, **kwargs):
-        return self.resources.BatchCommit(**kwargs)
-
-    def backup(self, backup_file, **kwargs):
-        return self.backup_iface.Save('(s)', backup_file, **kwargs)
-
-    def restore(self, backup_file, **kwargs):
-        return self.backup_iface.Restore('(s)', backup_file, **kwargs)
-
-    def get_stats(self, **kwargs):
-        return self.stats_iface.Get(**kwargs)
-
-    def get_tracker_iface(self):
-        return self.resources
-
-    def count_instances(self, ontology_class):
-        QUERY = """
-        SELECT COUNT(?u) WHERE {
-            ?u a %s .
-        }
-        """
-        result = self.resources.SparqlQuery('(s)', QUERY % (ontology_class))
-
-        if (len(result) == 1):
-            return int(result[0][0])
-        else:
-            return -1
-
-    def get_resource_id_by_uri(self, uri):
-        """
-        Get the internal ID for a given resource, identified by URI.
-        """
-        result = self.query(
-            'SELECT tracker:id(%s) WHERE { }' % uri)
-        if len(result) == 1:
-            return int(result[0][0])
-        elif len(result) == 0:
-            raise Exception("No entry for resource %s" % uri)
-        else:
-            raise Exception("Multiple entries for resource %s" % uri)
-
-    # FIXME: rename to get_resource_id_by_nepomuk_url !!
-    def get_resource_id(self, url):
-        """
-        Get the internal ID for a given resource, identified by URL.
-        """
-        result = self.query(
-            'SELECT tracker:id(?r) WHERE { ?r nie:url "%s" }' % url)
-        if len(result) == 1:
-            return int(result[0][0])
-        elif len(result) == 0:
-            raise Exception("No entry for resource %s" % url)
-        else:
-            raise Exception("Multiple entries for resource %s" % url)
-
-    def ask(self, ask_query):
-        assert ask_query.strip().startswith("ASK")
-        result = self.query(ask_query)
-        assert len(result) == 1
-        if result[0][0] == "true":
-            return True
-        elif result[0][0] == "false":
-            return False
-        else:
-            raise Exception("Something fishy is going on")
diff --git a/tests/functional-tests/common/utils/options.py b/tests/functional-tests/common/utils/options.py
deleted file mode 100644
index d11d5a12f..000000000
--- a/tests/functional-tests/common/utils/options.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import os
-
-
-def get_environment_boolean(variable):
-    '''Parse a yes/no boolean passed through the environment.'''
-
-    value = os.environ.get(variable, 'no').lower()
-    if value in ['no', '0', 'false']:
-        return False
-    elif value in ['yes', '1', 'true']:
-        return True
-    else:
-        raise RuntimeError('Unexpected value for %s: %s' %
-                           (variable, value))
-
-
-def is_verbose():
-    """
-    True to log process status information to stdout
-    """
-    return get_environment_boolean('TRACKER_TESTS_VERBOSE')
-
-
-def is_manual_start():
-    """
-    False to start the processes automatically
-    """
-    return get_environment_boolean('TRACKER_TESTS_MANUAL_START')
diff --git a/tests/functional-tests/common/utils/system.py b/tests/functional-tests/common/utils/system.py
deleted file mode 100644
index a93a4bf16..000000000
--- a/tests/functional-tests/common/utils/system.py
+++ /dev/null
@@ -1 +0,0 @@
-#!/usr/bin/python3
diff --git a/tests/functional-tests/configuration.json.in b/tests/functional-tests/configuration.json.in
index 06a5be195..46c5126f7 100644
--- a/tests/functional-tests/configuration.json.in
+++ b/tests/functional-tests/configuration.json.in
@@ -1,7 +1,4 @@
 {
-    "PREFIX": "@prefix@",
-    "RAW_EXEC_PREFIX": "@exec_prefix@",
-    "RAW_DATAROOT_DIR": "@datarootdir@",
     "TEST_ONTOLOGIES_DIR": "@FUNCTIONAL_TESTS_ONTOLOGIES_DIR@",
     "TRACKER_STORE_PATH": "@FUNCTIONAL_TESTS_TRACKER_STORE_PATH@",
     "disableJournal": "@DISABLE_JOURNAL_TRUE@"
diff --git a/tests/functional-tests/configuration.py b/tests/functional-tests/configuration.py
new file mode 100644
index 000000000..938ad0f19
--- /dev/null
+++ b/tests/functional-tests/configuration.py
@@ -0,0 +1,60 @@
+#
+# Copyright (C) 2010, Nokia <jean-luc.lamadon@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+
+
+import json
+import logging
+import os
+import sys
+
+
+if 'TRACKER_FUNCTIONAL_TEST_CONFIG' not in os.environ:
+    raise RuntimeError("The TRACKER_FUNCTIONAL_TEST_CONFIG environment "
+                       "variable must be set to point to the location of "
+                       "the generated configuration.json file.")
+
+with open(os.environ['TRACKER_FUNCTIONAL_TEST_CONFIG']) as f:
+    config = json.load(f)
+
+
+TOP_SRCDIR = os.path.dirname(os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
+TOP_BUILDDIR = os.environ['TRACKER_FUNCTIONAL_TEST_BUILD_DIR']
+
+TEST_ONTOLOGIES_DIR = config['TEST_ONTOLOGIES_DIR']
+TRACKER_STORE_PATH = config['TRACKER_STORE_PATH']
+disableJournal = (len(config['disableJournal']) == 0)
+
+
+def get_environment_boolean(variable):
+    '''Parse a yes/no boolean passed through the environment.'''
+
+    value = os.environ.get(variable, 'no').lower()
+    if value in ['no', '0', 'false']:
+        return False
+    elif value in ['yes', '1', 'true']:
+        return True
+    else:
+        raise RuntimeError('Unexpected value for %s: %s' %
+                           (variable, value))
+
+
+if get_environment_boolean('TRACKER_TESTS_VERBOSE'):
+    logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
diff --git a/tests/functional-tests/common/utils/expectedFailure.py b/tests/functional-tests/expectedFailure.py
index e7f0eb68c..289e0a9df 100644
--- a/tests/functional-tests/common/utils/expectedFailure.py
+++ b/tests/functional-tests/expectedFailure.py
@@ -1,10 +1,9 @@
-#!/usr/bin/python3
-
 # Code taken and modified from unittest2 framework (case.py)
 
 # Copyright (c) 1999-2003 Steve Purcell
 # Copyright (c) 2003-2010 Python Software Foundation
 # Copyright (c) 2010, Nokia (ivan.frade@nokia.com)
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
 
 # This module is free software, and you may redistribute it and/or modify
 # it under the same terms as Python itself, so long as this copyright message
@@ -25,9 +24,11 @@
 Write values in tracker and check the actual values are written
 on the files. Note that these tests are highly platform dependant.
 """
-import sys
+
 from functools import wraps
-import common.utils.configuration as cfg
+import sys
+
+import configuration as cfg
 
 
 def expectedFailureJournal():
diff --git a/tests/functional-tests/meson.build b/tests/functional-tests/meson.build
index c482fc3df..be3fc2a4a 100644
--- a/tests/functional-tests/meson.build
+++ b/tests/functional-tests/meson.build
@@ -33,6 +33,9 @@ test_env = environment()
 test_env.set('DCONF_PROFILE', dconf_profile_full_path)
 test_env.set('GSETTINGS_SCHEMA_DIR', tracker_uninstalled_gsettings_schema_dir)
 
+tracker_uninstalled_testutils_dir = join_paths(meson.current_source_dir(), '..', '..', 'utils')
+test_env.prepend('PYTHONPATH', tracker_uninstalled_testutils_dir)
+
 test_env.set('TRACKER_DB_ONTOLOGIES_DIR', tracker_uninstalled_nepomuk_ontologies_dir)
 test_env.set('TRACKER_FUNCTIONAL_TEST_BUILD_DIR', build_root)
 test_env.set('TRACKER_FUNCTIONAL_TEST_CONFIG', config_json_full_path)
diff --git a/tests/functional-tests/common/utils/storetest.py b/tests/functional-tests/storetest.py
index dad9a0ac6..ed7aa82c5 100644
--- a/tests/functional-tests/common/utils/storetest.py
+++ b/tests/functional-tests/storetest.py
@@ -1,7 +1,6 @@
-#!/usr/bin/env python
 #
 # Copyright (C) 2010, Nokia <ivan.frade@nokia.com>
-# Copyright (C) 2018, Sam Thursfield <sam@afuera.me.uk>
+# Copyright (C) 2018, 2019, Sam Thursfield <sam@afuera.me.uk>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -19,26 +18,27 @@
 # 02110-1301, USA.
 #
 
-import unittest as ut
-
 import os
 import time
+import unittest as ut
 
-from common.utils.helpers import StoreHelper
-from common.utils import configuration as cfg
+import trackertestutils.helpers
+import configuration as cfg
 
 
 class CommonTrackerStoreTest (ut.TestCase):
     """
     Common superclass for tests that just require a fresh store running
+
+    Note that the store is started per test-suite, not per test.
     """
+
     @classmethod
     def setUpClass(self):
-        env = os.environ
-        env['LC_COLLATE'] = 'en_GB.utf8'
+        extra_env = {'LC_COLLATE': 'en_GB.utf8'}
 
-        self.tracker = StoreHelper()
-        self.tracker.start(env=env)
+        self.tracker = trackertestutils.helpers.StoreHelper(cfg.TRACKER_STORE_PATH)
+        self.tracker.start(extra_env=extra_env)
 
     @classmethod
     def tearDownClass(self):
diff --git a/utils/meson.build b/utils/meson.build
index c624b4914..3b7847501 100644
--- a/utils/meson.build
+++ b/utils/meson.build
@@ -1,3 +1,4 @@
 subdir('mtp')
 subdir('ontology')
 subdir('tracker-resdump')
+subdir('trackertestutils')
diff --git a/utils/trackertestutils/README.md b/utils/trackertestutils/README.md
new file mode 100644
index 000000000..98840ba0a
--- /dev/null
+++ b/utils/trackertestutils/README.md
@@ -0,0 +1,2 @@
+This Python package contains utility functions which are useful when testing
+Tracker.
diff --git a/utils/trackertestutils/__init__.py b/utils/trackertestutils/__init__.py
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/utils/trackertestutils/__init__.py
@@ -0,0 +1 @@
+
diff --git a/tests/functional-tests/common/utils/dconf.py b/utils/trackertestutils/dconf.py
index d4b42dfe9..4ad0e88e9 100644
--- a/tests/functional-tests/common/utils/dconf.py
+++ b/utils/trackertestutils/dconf.py
@@ -1,9 +1,30 @@
+#
+# Copyright (C) 2010, Nokia <jean-luc.lamadon@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+
 from gi.repository import GLib
 from gi.repository import Gio
 
+import logging
 import os
 
-from common.utils.helpers import log
+log = logging.getLogger(__name__)
 
 
 class DConfClient(object):
@@ -75,5 +96,5 @@ class DConfClient(object):
                                 "dconf",
                                 "trackertest")
         if os.path.exists(dconf_db):
-            log("[Conf] Removing dconf database: " + dconf_db)
+            log.debug("[Conf] Removing dconf database: %s", dconf_db)
             os.remove(dconf_db)
diff --git a/utils/trackertestutils/helpers.py b/utils/trackertestutils/helpers.py
new file mode 100644
index 000000000..2b218e5d0
--- /dev/null
+++ b/utils/trackertestutils/helpers.py
@@ -0,0 +1,583 @@
+#
+# Copyright (C) 2010, Nokia <jean-luc.lamadon@nokia.com>
+# Copyright (C) 2019, Sam Thursfield <sam@afuera.me.uk>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+
+from gi.repository import Gio
+from gi.repository import GLib
+
+import atexit
+import logging
+import os
+import subprocess
+
+from . import mainloop
+
+log = logging.getLogger(__name__)
+
+
+class GraphUpdateTimeoutException(RuntimeError):
+    pass
+
+
+class NoMetadataException (Exception):
+    pass
+
+
+REASONABLE_TIMEOUT = 30
+
+
+_process_list = []
+
+
+def _cleanup_processes():
+    for process in _process_list:
+        log.debug("helpers._cleanup_processes: stopping %s", process)
+        process.stop()
+
+
+atexit.register(_cleanup_processes)
+
+
+class Helper:
+    """
+    Abstract helper for Tracker processes. Launches the process
+    and waits for it to appear on the session bus.
+
+    The helper will fail if the process is already running. Use
+    test-runner.sh to ensure the processes run inside a separate DBus
+    session bus.
+
+    The process is watched using a timed GLib main loop source. If the process
+    exits with an error code, the test will abort the next time the main loop
+    is entered (or straight away if currently running the main loop).
+    """
+
+    STARTUP_TIMEOUT = 200   # milliseconds
+    SHUTDOWN_TIMEOUT = 200  #
+
+    def __init__(self, helper_name, bus_name, process_path):
+        self.name = helper_name
+        self.bus_name = bus_name
+        self.process_path = process_path
+
+        self.log = logging.getLogger(f'{__name__}.{self.name}')
+
+        self.process = None
+        self.available = False
+
+        self.loop = mainloop.MainLoop()
+
+        self.bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
+
+    def _start_process(self, command_args=None, extra_env=None):
+        global _process_list
+        _process_list.append(self)
+
+        command = [self.process_path] + (command_args or [])
+        self.log.debug("Starting %s.", ' '.join(command))
+
+        env = os.environ
+        if extra_env:
+            self.log.debug("  starting with extra environment: %s", extra_env)
+            env.update(extra_env)
+
+        try:
+            return subprocess.Popen(command, env=env)
+        except OSError as e:
+            raise RuntimeError("Error starting %s: %s" % (self.process_path, e))
+
+    def _bus_name_appeared(self, connection, name, owner):
+        self.log.debug("%s appeared on the message bus, owned by %s", name, owner)
+        self.available = True
+        self.loop.quit()
+
+    def _bus_name_vanished(self, connection, name):
+        self.log.debug("%s vanished from the message bus", name)
+        self.available = False
+        self.loop.quit()
+
+    def _process_watch_cb(self):
+        if self.process_watch_timeout == 0:
+            # GLib seems to call the timeout after we've removed it
+            # sometimes, which causes errors unless we detect it.
+            return False
+
+        status = self.process.poll()
+
+        if status is None:
+            return True    # continue
+        elif status == 0 and not self.abort_if_process_exits_with_status_0:
+            return True    # continue
+        else:
+            self.process_watch_timeout = 0
+            raise RuntimeError(f"{self.name} exited with status: {self.status}")
+
+    def _process_startup_timeout_cb(self):
+        self.log.debug(f"Process timeout of {self.STARTUP_TIMEOUT}ms was called")
+        self.loop.quit()
+        self.timeout_id = None
+        return False
+
+    def start(self, command_args=None, extra_env=None):
+        """
+        Start an instance of process and wait for it to appear on the bus.
+        """
+        if self.process is not None:
+            raise RuntimeError("%s: already started" % self.name)
+
+        self._bus_name_watch_id = Gio.bus_watch_name_on_connection(
+            self.bus, self.bus_name, Gio.BusNameWatcherFlags.NONE,
+            self._bus_name_appeared, self._bus_name_vanished)
+
+        # We expect the _bus_name_vanished callback to be called here,
+        # causing the loop to exit again.
+        self.loop.run_checked()
+
+        if self.available:
+            # It's running, but we didn't start it...
+            raise RuntimeError("Unable to start test instance of %s: "
+                               "already running" % self.name)
+
+        self.process = self._start_process(command_args=command_args,
+                                           extra_env=extra_env)
+        self.log.debug('Started with PID %i', self.process.pid)
+
+        self.process_startup_timeout = GLib.timeout_add(
+            self.STARTUP_TIMEOUT, self._process_startup_timeout_cb)
+
+        self.abort_if_process_exits_with_status_0 = True
+
+        # Run the loop until the bus name appears, or the process dies.
+        self.loop.run_checked()
+
+        self.abort_if_process_exits_with_status_0 = False
+
+    def stop(self):
+        global _process_list
+
+        if self.process is None:
+            # Seems that it didn't even start...
+            return
+
+        if self.process.poll() == None:
+            GLib.source_remove(self.process_startup_timeout)
+            self.process_startup_timeout = 0
+
+            self.process.terminate()
+            returncode = self.process.wait(timeout=self.SHUTDOWN_TIMEOUT * 1000)
+            if returncode is None:
+                self.log.debug("Process failed to terminate in time, sending kill!")
+                self.process.kill()
+                self.process.wait()
+            elif returncode > 0:
+                self.log.warn("Process returned error code %s", returncode)
+
+        self.log.debug("Process stopped.")
+
+        # Run the loop to handle the expected name_vanished signal.
+        self.loop.run_checked()
+        Gio.bus_unwatch_name(self._bus_name_watch_id)
+
+        self.process = None
+        _process_list.remove(self)
+
+    def kill(self):
+        global _process_list
+
+        if self.process_watch_timeout != 0:
+            GLib.source_remove(self.process_watch_timeout)
+            self.process_watch_timeout = 0
+
+        self.process.kill()
+
+        # Name owner changed callback should take us out from this loop
+        self.loop.run_checked()
+        Gio.bus_unwatch_name(self._bus_name_watch_id)
+
+        self.process = None
+        _process_list.remove(self)
+
+        self.log.debug("Process killed.")
+
+
+class StoreHelper (Helper):
+    """
+    Helper for starting and testing the tracker-store daemon.
+    """
+
+    TRACKER_BUSNAME = 'org.freedesktop.Tracker1'
+    TRACKER_OBJ_PATH = '/org/freedesktop/Tracker1/Resources'
+    RESOURCES_IFACE = "org.freedesktop.Tracker1.Resources"
+
+    TRACKER_BACKUP_OBJ_PATH = "/org/freedesktop/Tracker1/Backup"
+    BACKUP_IFACE = "org.freedesktop.Tracker1.Backup"
+
+    TRACKER_STATS_OBJ_PATH = "/org/freedesktop/Tracker1/Statistics"
+    STATS_IFACE = "org.freedesktop.Tracker1.Statistics"
+
+    TRACKER_STATUS_OBJ_PATH = "/org/freedesktop/Tracker1/Status"
+    STATUS_IFACE = "org.freedesktop.Tracker1.Status"
+
+    def __init__(self, process_path):
+        Helper.__init__(self, "tracker-store", self.TRACKER_BUSNAME, process_path)
+
+    def start(self, command_args=None, extra_env=None):
+        Helper.start(self, command_args, extra_env)
+
+        self.resources = Gio.DBusProxy.new_sync(
+            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None,
+            self.TRACKER_BUSNAME, self.TRACKER_OBJ_PATH, self.RESOURCES_IFACE)
+
+        self.backup_iface = Gio.DBusProxy.new_sync(
+            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None,
+            self.TRACKER_BUSNAME, self.TRACKER_BACKUP_OBJ_PATH, self.BACKUP_IFACE)
+
+        self.stats_iface = Gio.DBusProxy.new_sync(
+            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None,
+            self.TRACKER_BUSNAME, self.TRACKER_STATS_OBJ_PATH, self.STATS_IFACE)
+
+        self.status_iface = Gio.DBusProxy.new_sync(
+            self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None,
+            self.TRACKER_BUSNAME, self.TRACKER_STATUS_OBJ_PATH, self.STATUS_IFACE)
+
+        self.log.debug("Calling %s.Wait() method", self.STATUS_IFACE)
+        self.status_iface.Wait()
+        self.log.debug("Ready")
+
+        self.reset_graph_updates_tracking()
+
+        def signal_handler(proxy, sender_name, signal_name, parameters):
+            if signal_name == 'GraphUpdated':
+                self._graph_updated_cb(*parameters.unpack())
+
+        self.graph_updated_handler_id = self.resources.connect(
+            'g-signal', signal_handler)
+
+    def stop(self):
+        Helper.stop(self)
+
+        if self.graph_updated_handler_id != 0:
+            self.resources.disconnect(self.graph_updated_handler_id)
+
+    # A system to follow GraphUpdated and make sure all changes are tracked.
+    # This code saves every change notification received, and exposes methods
+    # to await insertion or deletion of a certain resource which first check
+    # the list of events already received and wait for more if the event has
+    # not yet happened.
+
+    def reset_graph_updates_tracking(self):
+        self.class_to_track = None
+        self.inserts_list = []
+        self.deletes_list = []
+        self.inserts_match_function = None
+        self.deletes_match_function = None
+
+    def _graph_updated_timeout_cb(self):
+        raise GraphUpdateTimeoutException()
+
+    def _graph_updated_cb(self, class_name, deletes_list, inserts_list):
+        """
+        Process notifications from tracker-store on resource changes.
+        """
+        exit_loop = False
+
+        if class_name == self.class_to_track:
+            self.log.debug("GraphUpdated for %s: %i deletes, %i inserts", class_name, len(deletes_list), len(inserts_list))
+
+            if inserts_list is not None:
+                if self.inserts_match_function is not None:
+                    # The match function will remove matched entries from the list
+                    (exit_loop, inserts_list) = self.inserts_match_function(inserts_list)
+                self.inserts_list += inserts_list
+
+            if not exit_loop and deletes_list is not None:
+                if self.deletes_match_function is not None:
+                    (exit_loop, deletes_list) = self.deletes_match_function(deletes_list)
+                self.deletes_list += deletes_list
+
+            if exit_loop:
+                GLib.source_remove(self.graph_updated_timeout_id)
+                self.graph_updated_timeout_id = 0
+                self.loop.quit()
+        else:
+            self.log.debug("Ignoring GraphUpdated for class %s, currently tracking %s", class_name, self.class_to_track)
+
+    def _enable_await_timeout(self):
+        self.graph_updated_timeout_id = GLib.timeout_add_seconds(REASONABLE_TIMEOUT,
+                                                                 self._graph_updated_timeout_cb)
+
+    def await_resource_inserted(self, rdf_class, url=None, title=None, required_property=None):
+        """
+        Block until a resource matching the parameters becomes available
+        """
+        assert (self.inserts_match_function == None)
+        assert (self.class_to_track == None), "Already waiting for resource of type %s" % self.class_to_track
+
+        self.class_to_track = rdf_class
+
+        self.matched_resource_urn = None
+        self.matched_resource_id = None
+
+        self.log.debug("Await new %s (%i existing inserts)", rdf_class, len(self.inserts_list))
+
+        if required_property is not None:
+            required_property_id = self.get_resource_id_by_uri(required_property)
+            self.log.debug("Required property %s id %i", required_property, required_property_id)
+
+        def find_resource_insertion(inserts_list):
+            matched_creation = (self.matched_resource_id is not None)
+            matched_required_property = False
+            remaining_events = []
+
+            # FIXME: this could be done in an easier way: build one query that filters
+            # based on every subject id in inserts_list, and returns the id of the one
+            # that matched :)
+            for insert in inserts_list:
+                id = insert[1]
+
+                if not matched_creation:
+                    where = "  ?urn a <%s> " % rdf_class
+
+                    if url is not None:
+                        where += "; nie:url \"%s\"" % url
+
+                    if title is not None:
+                        where += "; nie:title \"%s\"" % title
+
+                    query = "SELECT ?urn WHERE { %s FILTER (tracker:id(?urn) = %s)}" % (where, insert[1])
+                    result_set = self.query(query)
+
+                    if len(result_set) > 0:
+                        matched_creation = True
+                        self.matched_resource_urn = result_set[0][0]
+                        self.matched_resource_id = insert[1]
+                        self.log.debug("Matched creation of resource %s (%i)",
+                            self.matched_resource_urn,
+                             self.matched_resource_id)
+                        if required_property is not None:
+                            self.log.debug("Waiting for property %s (%i) to be set",
+                                required_property, required_property_id)
+
+                if required_property is not None and matched_creation and not matched_required_property:
+                    if id == self.matched_resource_id and insert[2] == required_property_id:
+                        matched_required_property = True
+                        self.log.debug("Matched %s %s", self.matched_resource_urn, required_property)
+
+                if not matched_creation or id != self.matched_resource_id:
+                    remaining_events += [insert]
+
+            matched = matched_creation if required_property is None else matched_required_property
+            return matched, remaining_events
+
+        def match_cb(inserts_list):
+            matched, remaining_events = find_resource_insertion(inserts_list)
+            exit_loop = matched
+            return exit_loop, remaining_events
+
+        # Check the list of previously received events for matches
+        (existing_match, self.inserts_list) = find_resource_insertion(self.inserts_list)
+
+        if not existing_match:
+            self._enable_await_timeout()
+            self.inserts_match_function = match_cb
+            # Run the event loop until the correct notification arrives
+            try:
+                self.loop.run_checked()
+            except GraphUpdateTimeoutException:
+                raise GraphUpdateTimeoutException("Timeout waiting for resource: class %s, URL %s, title %s" % (rdf_class, url, title)) from None
+            self.inserts_match_function = None
+
+        self.class_to_track = None
+        return (self.matched_resource_id, self.matched_resource_urn)
+
+    def await_resource_deleted(self, rdf_class, id):
+        """
+        Block until we are notified of a resources deletion
+        """
+        assert (self.deletes_match_function == None)
+        assert (self.class_to_track == None)
+
+        def find_resource_deletion(deletes_list):
+            self.log.debug("find_resource_deletion: looking for %i in %s", id, deletes_list)
+
+            matched = False
+            remaining_events = []
+
+            for delete in deletes_list:
+                if delete[1] == id:
+                    matched = True
+                else:
+                    remaining_events += [delete]
+
+            return matched, remaining_events
+
+        def match_cb(deletes_list):
+            matched, remaining_events = find_resource_deletion(deletes_list)
+            exit_loop = matched
+            return exit_loop, remaining_events
+
+        self.log.debug("Await deletion of %i (%i existing)", id, len(self.deletes_list))
+
+        (existing_match, self.deletes_list) = find_resource_deletion(self.deletes_list)
+
+        if not existing_match:
+            self._enable_await_timeout()
+            self.class_to_track = rdf_class
+            self.deletes_match_function = match_cb
+            # Run the event loop until the correct notification arrives
+            try:
+                self.loop.run_checked()
+            except GraphUpdateTimeoutException:
+                raise GraphUpdateTimeoutException("Resource %i has not been deleted." % id)
+            self.deletes_match_function = None
+            self.class_to_track = None
+
+        return
+
+    def await_property_changed(self, rdf_class, subject_id, property_uri):
+        """
+        Block until a property of a resource is updated or inserted.
+        """
+        assert (self.inserts_match_function == None)
+        assert (self.deletes_match_function == None)
+        assert (self.class_to_track == None)
+
+        self.log.debug("Await change to %i %s (%i, %i existing)", subject_id, property_uri, len(self.inserts_list), len(self.deletes_list))
+
+        self.class_to_track = rdf_class
+
+        property_id = self.get_resource_id_by_uri(property_uri)
+
+        def find_property_change(event_list):
+            matched = False
+            remaining_events = []
+
+            for event in event_list:
+                if event[1] == subject_id and event[2] == property_id:
+                    self.log.debug("Matched property change: %s", str(event))
+                    matched = True
+                else:
+                    remaining_events += [event]
+
+            return matched, remaining_events
+
+        def match_cb(event_list):
+            matched, remaining_events = find_property_change(event_list)
+            exit_loop = matched
+            return exit_loop, remaining_events
+
+        # Check the list of previously received events for matches
+        (existing_match, self.inserts_list) = find_property_change(self.inserts_list)
+        (existing_match, self.deletes_list) = find_property_change(self.deletes_list)
+
+        if not existing_match:
+            self._enable_await_timeout()
+            self.inserts_match_function = match_cb
+            self.deletes_match_function = match_cb
+            # Run the event loop until the correct notification arrives
+            try:
+                self.loop.run_checked()
+            except GraphUpdateTimeoutException:
+                raise GraphUpdateTimeoutException(
+                    "Timeout waiting for property change, subject %i property %s (%i)" % (subject_id, property_uri, property_id))
+            self.inserts_match_function = None
+            self.deletes_match_function = None
+            self.class_to_track = None
+
+    # Note: The methods below call the tracker-store D-Bus API directly. This
+    # is useful for testing this API surface, but we recommand that all regular
+    # applications use libtracker-sparql library to talk to the database.
+
+    def query(self, query, timeout=5000, **kwargs):
+        return self.resources.SparqlQuery('(s)', query, timeout=timeout, **kwargs)
+
+    def update(self, update_sparql, timeout=5000, **kwargs):
+        return self.resources.SparqlUpdate('(s)', update_sparql, timeout=timeout, **kwargs)
+
+    def load(self, ttl_uri, timeout=5000, **kwargs):
+        return self.resources.Load('(s)', ttl_uri, timeout=timeout, **kwargs)
+
+    def batch_update(self, update_sparql, **kwargs):
+        return self.resources.BatchSparqlUpdate('(s)', update_sparql, **kwargs)
+
+    def batch_commit(self, **kwargs):
+        return self.resources.BatchCommit(**kwargs)
+
+    def backup(self, backup_file, **kwargs):
+        return self.backup_iface.Save('(s)', backup_file, **kwargs)
+
+    def restore(self, backup_file, **kwargs):
+        return self.backup_iface.Restore('(s)', backup_file, **kwargs)
+
+    def get_stats(self, **kwargs):
+        return self.stats_iface.Get(**kwargs)
+
+    def get_tracker_iface(self):
+        return self.resources
+
+    def count_instances(self, ontology_class):
+        QUERY = """
+        SELECT COUNT(?u) WHERE {
+            ?u a %s .
+        }
+        """
+        result = self.resources.SparqlQuery('(s)', QUERY % (ontology_class))
+
+        if (len(result) == 1):
+            return int(result[0][0])
+        else:
+            return -1
+
+    def get_resource_id_by_uri(self, uri):
+        """
+        Get the internal ID for a given resource, identified by URI.
+        """
+        result = self.query(
+            'SELECT tracker:id(%s) WHERE { }' % uri)
+        if len(result) == 1:
+            return int(result[0][0])
+        elif len(result) == 0:
+            raise Exception("No entry for resource %s" % uri)
+        else:
+            raise Exception("Multiple entries for resource %s" % uri)
+
+    # FIXME: rename to get_resource_id_by_nepomuk_url !!
+    def get_resource_id(self, url):
+        """
+        Get the internal ID for a given resource, identified by URL.
+        """
+        result = self.query(
+            'SELECT tracker:id(?r) WHERE { ?r nie:url "%s" }' % url)
+        if len(result) == 1:
+            return int(result[0][0])
+        elif len(result) == 0:
+            raise Exception("No entry for resource %s" % url)
+        else:
+            raise Exception("Multiple entries for resource %s" % url)
+
+    def ask(self, ask_query):
+        assert ask_query.strip().startswith("ASK")
+        result = self.query(ask_query)
+        assert len(result) == 1
+        if result[0][0] == "true":
+            return True
+        elif result[0][0] == "false":
+            return False
+        else:
+            raise Exception("Something fishy is going on")
diff --git a/utils/trackertestutils/mainloop.py b/utils/trackertestutils/mainloop.py
new file mode 100644
index 000000000..1e7a46c87
--- /dev/null
+++ b/utils/trackertestutils/mainloop.py
@@ -0,0 +1,58 @@
+# Copyright (C) 2018, Sam Thursfield <sam@afuera.me.uk>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+from gi.repository import GLib
+
+import sys
+
+
+class MainLoop():
+    '''Wrapper for GLib.MainLoop that propagates any unhandled exceptions.
+
+    PyGObject doesn't seem to provide any help with propagating exceptions from
+    the GLib main loop to the main Python execution context. The default
+    behaviour is to print a message and continue, which is useless for tests as
+    it means tests appear to pass when in fact they are broken.
+
+    '''
+
+    def __init__(self):
+        self._loop = GLib.MainLoop.new(None, 0)
+
+    def quit(self):
+        self._loop.quit()
+
+    def run_checked(self):
+        '''Run the loop until quit(), then raise any unhandled exception.'''
+        self._exception = None
+
+        old_hook = sys.excepthook
+
+        def new_hook(etype, evalue, etb):
+            self._loop.quit()
+            self._exception = evalue
+            old_hook(etype, evalue, etb)
+
+        try:
+            sys.excepthook = new_hook
+            self._loop.run()
+        finally:
+            sys.excepthook = old_hook
+
+        if self._exception:
+            raise self._exception
diff --git a/utils/trackertestutils/meson.build b/utils/trackertestutils/meson.build
new file mode 100644
index 000000000..7806ee210
--- /dev/null
+++ b/utils/trackertestutils/meson.build
@@ -0,0 +1,8 @@
+sources = [
+  '__init__.py',
+  'dconf.py',
+  'helpers.py',
+]
+
+install_data(sources,
+  install_dir: join_paths(tracker_internal_libs_dir, 'trackertestutils'))