From fb65a8b99118c519afff63409f428de1d49a345d Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Sat, 31 Aug 2019 04:29:02 +0300 Subject: functional-tests/ipc: Fix invalid INSERT in testcase The test passed, but it was actually hitting a g_critical(). Improved logging meant that the error became visible in the logs! --- tests/functional-tests/ipc/test-insert-or-replace.vala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional-tests/ipc/test-insert-or-replace.vala b/tests/functional-tests/ipc/test-insert-or-replace.vala index 718dfc8f4..ce64eecbe 100644 --- a/tests/functional-tests/ipc/test-insert-or-replace.vala +++ b/tests/functional-tests/ipc/test-insert-or-replace.vala @@ -3,11 +3,11 @@ using Tracker; using Tracker.Sparql; const string insert_query_replace = " -DELETE { ?r nao:hasProperty ?property . } +DELETE { ?r nao:hasTag ?tag . } WHERE { ?r a nco:PhoneNumber; nco:phoneNumber \"02141730585%d\"; - nao:hasProperty ?property . + nao:hasTag ?tag . } DELETE { @@ -57,10 +57,10 @@ INSERT OR REPLACE { }"; const string insert_query_original = " -DELETE { ?r nao:hasProperty ?property . } +DELETE { ?r nao:tag ?tag . } WHERE { ?r a nco:PhoneNumber; nco:phoneNumber \"2141730585%d\"; - nao:hasProperty ?property . + nao:hasTag ?tag . } DELETE { -- cgit v1.2.1 From 2d9c4449ed908214268e23aa5bf7c77cc2cf8601 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 29 Aug 2019 18:01:14 +0300 Subject: functional-tests: Fix brokeness of @expectedFailureJournal decorator --- tests/functional-tests/expectedFailure.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/functional-tests/expectedFailure.py b/tests/functional-tests/expectedFailure.py index fe6769569..92d77ce57 100644 --- a/tests/functional-tests/expectedFailure.py +++ b/tests/functional-tests/expectedFailure.py @@ -27,6 +27,7 @@ on the files. Note that these tests are highly platform dependant. from functools import wraps import sys +import unittest as ut import configuration as cfg @@ -38,11 +39,6 @@ def expectedFailureJournal(): def decorator(func): @wraps(func) def wrapper(*args, **kwargs): - try: - func(*args, **kwargs) - except Exception: - raise ut.case._ExpectedFailure(sys.exc_info()) - raise Exception( - "Unexpected success. This should fail because journal is disabled") + ut.expectedFailure(func) return wrapper return decorator -- cgit v1.2.1 From dae63fea40d9b045bca0e39e855ee61c456e944d Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 27 Aug 2019 00:04:11 +0300 Subject: functional-tests: Start daemons through D-Bus autolaunch Instead of manually running and managing Tracker daemon processes manually in the test, we now rely on our private D-Bus daemon to do so. This makes the test environment more like a real Tracker deployment. Log output from the D-Bus daemon is now captured and output through the Python logging system. This allows for finer-grained filtering of output from the tests themselves and from the Tracker daemons. Some test code is changed to support the new model. --- meson.build | 4 - tests/functional-tests/01-insertion.py | 2 - tests/functional-tests/02-sparql-bugs.py | 5 - tests/functional-tests/03-fts-functions.py | 2 - tests/functional-tests/04-group-concat.py | 2 - tests/functional-tests/05-coalesce.py | 2 - tests/functional-tests/06-distance.py | 2 - tests/functional-tests/07-graph.py | 2 - tests/functional-tests/08-unique-insertions.py | 2 - tests/functional-tests/09-concurrent-query.py | 2 - tests/functional-tests/14-signals.py | 5 +- tests/functional-tests/15-statistics.py | 2 - tests/functional-tests/16-collation.py | 1 - tests/functional-tests/17-ontology-changes.py | 172 +++++------------ tests/functional-tests/configuration.json.in | 8 +- tests/functional-tests/configuration.py | 40 +++- tests/functional-tests/meson.build | 38 ++-- tests/functional-tests/storetest.py | 27 ++- tests/test-bus.conf.in | 1 + utils/sandbox/tracker-sandbox.py | 117 +----------- utils/trackertestutils/dbusdaemon.py | 198 ++++++++++++++++++++ utils/trackertestutils/dconf.py | 69 ++----- utils/trackertestutils/helpers.py | 246 +++++++------------------ utils/trackertestutils/meson.build | 1 + 24 files changed, 415 insertions(+), 535 deletions(-) create mode 100644 utils/trackertestutils/dbusdaemon.py diff --git a/meson.build b/meson.build index 5b30c1982..d6c2fa4a6 100644 --- a/meson.build +++ b/meson.build @@ -283,10 +283,6 @@ conf.set('tracker_store', join_paths ('${libexecdir}', 'tracker-store')) conf.set('ontologies_dir', join_paths ('${datadir}', 'tracker', 'ontologies')) conf.set('domain_ontologies_dir', join_paths('${datadir}', 'tracker', 'domain-ontologies')) -# Configure functional tests to run completely from source tree. -conf.set('FUNCTIONAL_TESTS_ONTOLOGIES_DIR', join_paths(meson.current_source_dir(), 'tests', 'functional-tests', 'test-ontologies')) -conf.set('FUNCTIONAL_TESTS_TRACKER_STORE_PATH', join_paths(meson.current_build_dir(), 'src', 'tracker-store', 'tracker-store')) - configure_file(input: 'config.h.meson.in', output: 'config.h', configuration: conf) diff --git a/tests/functional-tests/01-insertion.py b/tests/functional-tests/01-insertion.py index 00cc4e017..8c8092dd1 100755 --- a/tests/functional-tests/01-insertion.py +++ b/tests/functional-tests/01-insertion.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 -# # Copyright (C) 2010, Nokia # Copyright (C) 2019, Sam Thursfield # diff --git a/tests/functional-tests/02-sparql-bugs.py b/tests/functional-tests/02-sparql-bugs.py index 4305ea0a9..bf0d8ecbb 100755 --- a/tests/functional-tests/02-sparql-bugs.py +++ b/tests/functional-tests/02-sparql-bugs.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 -# # Copyright (C) 2010, Nokia # Copyright (C) 2019, Sam Thursfield # @@ -111,21 +109,18 @@ class TrackerStoreSparqlBugsTests (CommonTrackerStoreTest): """ results1 = self.tracker.query(query1) - print("1", results1) self.assertEqual(len(results1), 1) self.assertEqual(len(results1[0]), 2) self.assertEqual(results1[0][0], "contact:test") self.assertEqual(results1[0][1], "98653") results2 = self.tracker.query(query2) - print("2", results2) self.assertEqual(len(results2), 1) self.assertEqual(len(results2[0]), 2) self.assertEqual(results2[0][0], "contact:test") self.assertEqual(results2[0][1], "98653") results3 = self.tracker.query(query3) - print("3", results3) self.assertEqual(len(results3), 1) self.assertEqual(len(results3[0]), 2) self.assertEqual(results3[0][0], "contact:test") diff --git a/tests/functional-tests/03-fts-functions.py b/tests/functional-tests/03-fts-functions.py index 46c43f368..0938a7e52 100755 --- a/tests/functional-tests/03-fts-functions.py +++ b/tests/functional-tests/03-fts-functions.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 -# # Copyright (C) 2010, Nokia # Copyright (C) 2019, Sam Thursfield # diff --git a/tests/functional-tests/04-group-concat.py b/tests/functional-tests/04-group-concat.py index a8064a828..06e06265b 100755 --- a/tests/functional-tests/04-group-concat.py +++ b/tests/functional-tests/04-group-concat.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 -# # Copyright (C) 2010, Nokia # Copyright (C) 2019, Sam Thursfield # diff --git a/tests/functional-tests/05-coalesce.py b/tests/functional-tests/05-coalesce.py index 48d8e6eb6..eddf3148e 100755 --- a/tests/functional-tests/05-coalesce.py +++ b/tests/functional-tests/05-coalesce.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 -# # Copyright (C) 2010, Nokia # Copyright (C) 2019, Sam Thursfield # diff --git a/tests/functional-tests/06-distance.py b/tests/functional-tests/06-distance.py index 80d35dfb9..b2a22d255 100755 --- a/tests/functional-tests/06-distance.py +++ b/tests/functional-tests/06-distance.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 -# # Copyright (C) 2010, Nokia # Copyright (C) 2019, Sam Thursfield # diff --git a/tests/functional-tests/07-graph.py b/tests/functional-tests/07-graph.py index d9d173bcd..cec6a60d5 100755 --- a/tests/functional-tests/07-graph.py +++ b/tests/functional-tests/07-graph.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 -# # Copyright (C) 2010, Nokia # Copyright (C) 2019, Sam Thursfield # diff --git a/tests/functional-tests/08-unique-insertions.py b/tests/functional-tests/08-unique-insertions.py index 25ea6a13f..a71d2118e 100755 --- a/tests/functional-tests/08-unique-insertions.py +++ b/tests/functional-tests/08-unique-insertions.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 -# # Copyright (C) 2010, Nokia # Copyright (C) 2019, Sam Thursfield # diff --git a/tests/functional-tests/09-concurrent-query.py b/tests/functional-tests/09-concurrent-query.py index b24fdcc40..dacd396aa 100755 --- a/tests/functional-tests/09-concurrent-query.py +++ b/tests/functional-tests/09-concurrent-query.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 -# # Copyright (C) 2010, Nokia # Copyright (C) 2019, Sam Thursfield # diff --git a/tests/functional-tests/14-signals.py b/tests/functional-tests/14-signals.py index 242ae8480..cae97d46d 100755 --- a/tests/functional-tests/14-signals.py +++ b/tests/functional-tests/14-signals.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 -# # Copyright (C) 2010, Nokia # Copyright (C) 2019, Sam Thursfield # @@ -52,7 +50,7 @@ class TrackerStoreSignalsTests (CommonTrackerStoreTest): self.loop = GLib.MainLoop() self.timeout_id = 0 - self.bus = Gio.bus_get_sync(Gio.BusType.SESSION, None) + self.bus = self.sandbox.get_connection() self.results_classname = None self.results_deletes = None @@ -125,7 +123,6 @@ class TrackerStoreSignalsTests (CommonTrackerStoreTest): """ self.__connect_signal() self.tracker.update(CONTACT) - time.sleep(1) self.__wait_for_signal() # validate results diff --git a/tests/functional-tests/15-statistics.py b/tests/functional-tests/15-statistics.py index 6f6ca3014..56fa38eb7 100755 --- a/tests/functional-tests/15-statistics.py +++ b/tests/functional-tests/15-statistics.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 -# # Copyright (C) 2010, Nokia # Copyright (C) 2019, Sam Thursfield # diff --git a/tests/functional-tests/16-collation.py b/tests/functional-tests/16-collation.py index 40a993d82..62765a77a 100755 --- a/tests/functional-tests/16-collation.py +++ b/tests/functional-tests/16-collation.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 # -*- coding: utf-8 -*- # # Copyright (C) 2010, Nokia diff --git a/tests/functional-tests/17-ontology-changes.py b/tests/functional-tests/17-ontology-changes.py index b7e00d9d0..0571defb9 100755 --- a/tests/functional-tests/17-ontology-changes.py +++ b/tests/functional-tests/17-ontology-changes.py @@ -1,5 +1,3 @@ -#!/usr/bin/python3 -# # Copyright (C) 2010, Nokia # Copyright (C) 2019, Sam Thursfield # @@ -26,8 +24,8 @@ changes and checking if the data is still there. from gi.repository import GLib -import logging import os +import pathlib import shutil import re import tempfile @@ -48,101 +46,6 @@ XSD_INTEGER = "http://www.w3.org/2001/XMLSchema#integer" TEST_PREFIX = "http://example.org/ns#" -TEST_ENV_VARS = {"LC_COLLATE": "en_GB.utf8"} - -REASONABLE_TIMEOUT = 5 - -log = logging.getLogger() - - -class UnableToBootException (Exception): - pass - - -class TrackerSystemAbstraction (object): - - def __init__(self, settings=None): - self.store = None - self._dirs = {} - - def xdg_data_home(self): - return os.path.join(self._basedir, 'data') - - def xdg_cache_home(self): - return os.path.join(self._basedir, 'cache') - - def set_up_environment(self, settings=None, ontodir=None): - """ - Sets up the XDG_*_HOME variables and make sure the directories exist - - Settings should be a dict mapping schema names to dicts that hold the - settings that should be changed in those schemas. The contents dicts - should map key->value, where key is a key name and value is a suitable - GLib.Variant instance. - """ - self._basedir = tempfile.mkdtemp() - - self._dirs = { - "XDG_DATA_HOME": self.xdg_data_home(), - "XDG_CACHE_HOME": self.xdg_cache_home() - } - - for var, directory in list(self._dirs.items()): - os.makedirs(directory) - os.makedirs(os.path.join(directory, 'tracker')) - os.environ[var] = directory - - if ontodir: - log.debug("export %s=%s", "TRACKER_DB_ONTOLOGIES_DIR", ontodir) - os.environ["TRACKER_DB_ONTOLOGIES_DIR"] = ontodir - - for var, value in TEST_ENV_VARS.items(): - log.debug("export %s=%s", var, value) - os.environ[var] = value - - # Previous loop should have set DCONF_PROFILE to the test location - if settings is not None: - self._apply_settings(settings) - - def _apply_settings(self, settings): - for schema_name, contents in settings.items(): - dconf = trackertestutils.dconf.DConfClient(schema_name) - dconf.reset() - for key, value in contents.items(): - dconf.write(key, value) - - def tracker_store_testing_start(self, confdir=None, ontodir=None): - """ - Stops any previous instance of the store, calls set_up_environment, - and starts a new instances of the store - """ - self.set_up_environment(confdir, ontodir) - - self.store = trackertestutils.helpers.StoreHelper(cfg.TRACKER_STORE_PATH) - self.store.start() - - def tracker_store_restart_with_new_ontologies(self, ontodir): - self.store.stop() - if ontodir: - os.environ["TRACKER_DB_ONTOLOGIES_DIR"] = ontodir - try: - self.store.start() - except GLib.Error: - raise UnableToBootException( - "Unable to boot the store \n(" + str(e) + ")") - - def finish(self): - """ - Stop all running processes and remove all test data. - """ - - if self.store: - self.store.stop() - - for path in list(self._dirs.values()): - shutil.rmtree(path) - os.rmdir(self._basedir) - class OntologyChangeTestTemplate (ut.TestCase): """ @@ -158,35 +61,43 @@ class OntologyChangeTestTemplate (ut.TestCase): Check doc in those methods for the specific details. """ - def get_ontology_dir(self, param): - return os.path.join(cfg.TEST_ONTOLOGIES_DIR, param) - def setUp(self): - self.system = TrackerSystemAbstraction() + self.tmpdir = tempfile.mkdtemp(prefix='tracker-test-') def tearDown(self): - self.system.finish() + shutil.rmtree(self.tmpdir, ignore_errors=True) - def template_test_ontology_change(self): + def get_ontology_dir(self, param): + return str(pathlib.Path(__file__).parent.joinpath('test-ontologies', param)) + def template_test_ontology_change(self): self.set_ontology_dirs() - basic_ontologies = self.get_ontology_dir(self.FIRST_ONTOLOGY_DIR) - modified_ontologies = self.get_ontology_dir(self.SECOND_ONTOLOGY_DIR) + self.__assert_ontology_dates(self.FIRST_ONTOLOGY_DIR, self.SECOND_ONTOLOGY_DIR) - self.__assert_ontology_dates(basic_ontologies, modified_ontologies) + extra_env = cfg.test_environment(self.tmpdir) + extra_env['LC_COLLATE'] = 'en_GB.utf8' + extra_env['TRACKER_DB_ONTOLOGIES_DIR'] = self.get_ontology_dir(self.FIRST_ONTOLOGY_DIR) - self.system.tracker_store_testing_start(ontodir=basic_ontologies) - self.tracker = self.system.store + sandbox1 = trackertestutils.helpers.TrackerDBusSandbox( + cfg.TEST_DBUS_DAEMON_CONFIG_FILE, extra_env=extra_env) + sandbox1.start() + + self.tracker = trackertestutils.helpers.StoreHelper(sandbox1.get_connection()) + self.tracker.start_and_wait_for_ready() self.insert_data() - try: - # Boot the second set of ontologies - self.system.tracker_store_restart_with_new_ontologies( - modified_ontologies) - except UnableToBootException as e: - self.fail(str(self.__class__) + " " + str(e)) + sandbox1.stop() + + # Boot the second set of ontologies + extra_env['TRACKER_DB_ONTOLOGIES_DIR'] = self.get_ontology_dir(self.SECOND_ONTOLOGY_DIR) + sandbox2 = trackertestutils.helpers.TrackerDBusSandbox( + cfg.TEST_DBUS_DAEMON_CONFIG_FILE, extra_env=extra_env) + sandbox2.start() + + self.tracker = trackertestutils.helpers.StoreHelper(sandbox2.get_connection()) + self.tracker.start_and_wait_for_ready() self.validate_status() @@ -233,7 +144,7 @@ class OntologyChangeTestTemplate (ut.TestCase): (member, dbus_result)) return - def __assert_ontology_dates(self, first_dir, second_dir): + def __assert_ontology_dates(self, first, second): """ Asserts that 91-test.ontology in second_dir has a more recent modification time than in first_dir @@ -241,23 +152,24 @@ class OntologyChangeTestTemplate (ut.TestCase): ISO9601_REGEX = "(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)" def get_ontology_date(ontology): - for line in open(ontology, 'r'): - if "nao:lastModified" in line: - getmodtime = re.compile( - 'nao:lastModified\ \"' + ISO9601_REGEX + '\"') - modtime_match = getmodtime.search(line) - - if (modtime_match): - nao_date = modtime_match.group(1) - return time.strptime(nao_date, "%Y-%m-%dT%H:%M:%SZ") - else: - print("something funky in", line) - break + with open(ontology, 'r') as f: + for line in f: + if "nao:lastModified" in line: + getmodtime = re.compile( + 'nao:lastModified\ \"' + ISO9601_REGEX + '\"') + modtime_match = getmodtime.search(line) + + if (modtime_match): + nao_date = modtime_match.group(1) + return time.strptime(nao_date, "%Y-%m-%dT%H:%M:%SZ") + else: + print("something funky in", line) + break first_date = get_ontology_date( - os.path.join(first_dir, "91-test.ontology")) + os.path.join(self.get_ontology_dir(first), "91-test.ontology")) second_date = get_ontology_date( - os.path.join(second_dir, "91-test.ontology")) + os.path.join(self.get_ontology_dir(second), "91-test.ontology")) if first_date >= second_date: self.fail("nao:modifiedTime in '%s' is not more recent in the second ontology" % ( "91-test.ontology")) diff --git a/tests/functional-tests/configuration.json.in b/tests/functional-tests/configuration.json.in index 686dceae2..c3b71b42d 100644 --- a/tests/functional-tests/configuration.json.in +++ b/tests/functional-tests/configuration.json.in @@ -1,4 +1,8 @@ { - "TEST_ONTOLOGIES_DIR": "@FUNCTIONAL_TESTS_ONTOLOGIES_DIR@", - "TRACKER_STORE_PATH": "@FUNCTIONAL_TESTS_TRACKER_STORE_PATH@" + "TEST_DBUS_DAEMON_CONFIG_FILE": "@TEST_DBUS_DAEMON_CONFIG_FILE@", + "TEST_DCONF_PROFILE": "@TEST_DCONF_PROFILE@", + "TEST_GSETTINGS_SCHEMA_DIR": "@TEST_GSETTINGS_SCHEMA_DIR@", + "TEST_LANGUAGE_STOP_WORDS_DIR": "@TEST_LANGUAGE_STOP_WORDS_DIR@", + "TEST_ONTOLOGIES_DIR": "@TEST_ONTOLOGIES_DIR@", + "TEST_DOMAIN_ONTOLOGY_RULE": "@TEST_DOMAIN_ONTOLOGY_RULE@" } diff --git a/tests/functional-tests/configuration.py b/tests/functional-tests/configuration.py index 7da696c48..cd5cb0cfb 100644 --- a/tests/functional-tests/configuration.py +++ b/tests/functional-tests/configuration.py @@ -18,10 +18,10 @@ # 02110-1301, USA. # - import json import logging import os +import pathlib import sys @@ -34,12 +34,21 @@ with open(os.environ['TRACKER_FUNCTIONAL_TEST_CONFIG']) as f: config = json.load(f) -TOP_SRCDIR = os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) -TOP_BUILDDIR = os.environ['TRACKER_FUNCTIONAL_TEST_BUILD_DIR'] +TEST_DBUS_DAEMON_CONFIG_FILE = config['TEST_DBUS_DAEMON_CONFIG_FILE'] + -TEST_ONTOLOGIES_DIR = config['TEST_ONTOLOGIES_DIR'] -TRACKER_STORE_PATH = config['TRACKER_STORE_PATH'] +def test_environment(tmpdir): + return { + 'DCONF_PROFILE': config['TEST_DCONF_PROFILE'], + 'GSETTINGS_SCHEMA_DIR': config['TEST_GSETTINGS_SCHEMA_DIR'], + 'TRACKER_DB_ONTOLOGIES_DIR': config['TEST_ONTOLOGIES_DIR'], + 'TRACKER_LANGUAGE_STOP_WORDS_DIR': config['TEST_LANGUAGE_STOP_WORDS_DIR'], + 'TRACKER_TEST_DOMAIN_ONTOLOGY_RULE': config['TEST_DOMAIN_ONTOLOGY_RULE'], + 'XDG_CACHE_HOME': os.path.join(tmpdir, 'cache'), + 'XDG_CONFIG_HOME': os.path.join(tmpdir, 'config'), + 'XDG_DATA_HOME': os.path.join(tmpdir, 'data'), + 'XDG_RUNTIME_DIR': os.path.join(tmpdir, 'run'), + } def get_environment_boolean(variable): @@ -55,5 +64,24 @@ def get_environment_boolean(variable): (variable, value)) +def get_environment_int(variable, default=0): + try: + return int(os.environ.get(variable)) + except (TypeError, ValueError): + return default + + if get_environment_boolean('TRACKER_TESTS_VERBOSE'): + # Output all logs to stderr logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) +else: + # Output some messages from D-Bus daemon to stderr by default. In practice, + # only errors and warnings should be output here unless the environment + # contains G_MESSAGES_DEBUG= and/or TRACKER_VERBOSITY=1 or more. + handler_stderr = logging.StreamHandler(stream=sys.stderr) + handler_stderr.addFilter(logging.Filter('trackertestutils.dbusdaemon.stderr')) + handler_stdout = logging.StreamHandler(stream=sys.stderr) + handler_stdout.addFilter(logging.Filter('trackertestutils.dbusdaemon.stdout')) + logging.basicConfig(level=logging.INFO, + handlers=[handler_stderr, handler_stdout], + format='%(message)s') diff --git a/tests/functional-tests/meson.build b/tests/functional-tests/meson.build index be3fc2a4a..21813e410 100644 --- a/tests/functional-tests/meson.build +++ b/tests/functional-tests/meson.build @@ -1,13 +1,22 @@ -test_runner = configure_file( - input: 'test-runner.sh.in', - output: 'test-runner.sh', - configuration: conf) -test_runner = find_program(test_runner) +python = find_program('python3') + +# Configure functional tests to run completely from source tree. +testconf = configuration_data() + +config_json_full_path = join_paths(meson.current_build_dir(), 'configuration.json') +dconf_profile_full_path = join_paths(meson.current_source_dir(), 'trackertest') + +testconf.set('TEST_DBUS_DAEMON_CONFIG_FILE', join_paths(build_root, 'tests', 'test-bus.conf')) +testconf.set('TEST_DCONF_PROFILE', dconf_profile_full_path) +testconf.set('TEST_DOMAIN_ONTOLOGY_RULE', tracker_uninstalled_domain_rule) +testconf.set('TEST_GSETTINGS_SCHEMA_DIR', tracker_uninstalled_gsettings_schema_dir) +testconf.set('TEST_ONTOLOGIES_DIR', tracker_uninstalled_nepomuk_ontologies_dir) +testconf.set('TEST_LANGUAGE_STOP_WORDS_DIR', tracker_uninstalled_stop_words_dir) config_json = configure_file( input: 'configuration.json.in', output: 'configuration.json', - configuration: conf + configuration: testconf ) functional_tests = [ @@ -26,28 +35,21 @@ functional_tests = [ '17-ontology-changes', ] -config_json_full_path = join_paths(meson.current_build_dir(), 'configuration.json') -dconf_profile_full_path = join_paths(meson.current_source_dir(), 'trackertest') - test_env = environment() -test_env.set('DCONF_PROFILE', dconf_profile_full_path) -test_env.set('GSETTINGS_SCHEMA_DIR', tracker_uninstalled_gsettings_schema_dir) tracker_uninstalled_testutils_dir = join_paths(meson.current_source_dir(), '..', '..', 'utils') test_env.prepend('PYTHONPATH', tracker_uninstalled_testutils_dir) -test_env.set('TRACKER_DB_ONTOLOGIES_DIR', tracker_uninstalled_nepomuk_ontologies_dir) -test_env.set('TRACKER_FUNCTIONAL_TEST_BUILD_DIR', build_root) test_env.set('TRACKER_FUNCTIONAL_TEST_CONFIG', config_json_full_path) -test_env.set('TRACKER_LANGUAGE_STOP_WORDS_DIR', tracker_uninstalled_stop_words_dir) -test_env.set('TRACKER_TEST_DOMAIN_ONTOLOGY_RULE', tracker_uninstalled_domain_rule) foreach t: functional_tests - test('functional-' + t, test_runner, - args: './' + t + '.py', + file = '@0@.py'.format(t) + test('functional-' + t, python, + args: [file], env: test_env, workdir: meson.current_source_dir(), timeout: 60) endforeach -subdir('ipc') +# FIXME +#subdir('ipc') diff --git a/tests/functional-tests/storetest.py b/tests/functional-tests/storetest.py index ed7aa82c5..d96294cc4 100644 --- a/tests/functional-tests/storetest.py +++ b/tests/functional-tests/storetest.py @@ -19,6 +19,8 @@ # import os +import shutil +import tempfile import time import unittest as ut @@ -35,11 +37,28 @@ class CommonTrackerStoreTest (ut.TestCase): @classmethod def setUpClass(self): - extra_env = {'LC_COLLATE': 'en_GB.utf8'} + self.tmpdir = tempfile.mkdtemp(prefix='tracker-test-') - self.tracker = trackertestutils.helpers.StoreHelper(cfg.TRACKER_STORE_PATH) - self.tracker.start(extra_env=extra_env) + try: + extra_env = cfg.test_environment(self.tmpdir) + extra_env['LANG'] = 'en_GB.utf8' + extra_env['LC_COLLATE'] = 'en_GB.utf8' + + self.sandbox = trackertestutils.helpers.TrackerDBusSandbox( + dbus_daemon_config_file=cfg.TEST_DBUS_DAEMON_CONFIG_FILE, extra_env=extra_env) + self.sandbox.start() + + self.tracker = trackertestutils.helpers.StoreHelper( + self.sandbox.get_connection()) + self.tracker.start_and_wait_for_ready() + self.tracker.start_watching_updates() + except Exception as e: + shutil.rmtree(self.tmpdir, ignore_errors=True) + raise @classmethod def tearDownClass(self): - self.tracker.stop() + self.tracker.stop_watching_updates() + self.sandbox.stop() + + shutil.rmtree(self.tmpdir, ignore_errors=True) diff --git a/tests/test-bus.conf.in b/tests/test-bus.conf.in index 5b4f51ff9..2f4b2ef1b 100644 --- a/tests/test-bus.conf.in +++ b/tests/test-bus.conf.in @@ -7,6 +7,7 @@ unix:tmpdir=./ @abs_top_builddir@/tests/services/ + diff --git a/utils/sandbox/tracker-sandbox.py b/utils/sandbox/tracker-sandbox.py index cc8ebd786..34ad09ed5 100755 --- a/utils/sandbox/tracker-sandbox.py +++ b/utils/sandbox/tracker-sandbox.py @@ -24,6 +24,7 @@ # import argparse +import configparser import locale import logging import os @@ -33,10 +34,10 @@ import subprocess import sys import threading -import configparser - from gi.repository import GLib +import trackertestutils.dbusdaemon + # Script script_name = 'tracker-sandbox' script_version = '1.0' @@ -84,116 +85,6 @@ log = logging.getLogger('sandbox') dbuslog = logging.getLogger('dbus') -# Private DBus daemon - -class DBusDaemon: - """The private D-Bus instance that provides the sandbox's session bus. - - We support reading and writing the session information to a file. This - means that if the user runs two sandbox instances on the same data - directory at the same time, they will share the same message bus. - """ - - def __init__(self, session_file=None): - self.session_file = session_file - self.existing_session = False - self.process = None - - try: - self.address, self.pid = self.read_session_file(session_file) - self.existing_session = True - except FileNotFoundError: - log.debug("No existing D-Bus session file was found.") - - self.address = None - self.pid = None - - def get_session_file(self): - """Returns the path to the session file if we created it, or None.""" - if self.existing_session: - return None - return self.session_file - - def get_address(self): - return self.address - - @staticmethod - def read_session_file(session_file): - with open(session_file, 'r') as f: - content = f.read() - - try: - address = content.splitlines()[0] - pid = int(content.splitlines()[1]) - except ValueError: - raise RuntimeError(f"D-Bus session file {session_file} is not valid. " - "Remove this file to start a new session.") - - return address, pid - - @staticmethod - def write_session_file(session_file, address, pid): - os.makedirs(os.path.dirname(session_file), exist_ok=True) - - content = '%s\n%s' % (address, pid) - with open(session_file, 'w') as f: - f.write(content) - - def start_if_needed(self): - if self.existing_session: - log.debug('Using existing D-Bus session from file "%s" with address "%s"' - ' with PID %d' % (self.session_file, self.address, self.pid)) - else: - dbus_command = ['dbus-daemon', '--session', '--print-address=1', '--print-pid=1'] - self.process = subprocess.Popen(dbus_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - try: - self.address = self.process.stdout.readline().strip().decode('ascii') - self.pid = int(self.process.stdout.readline().strip().decode('ascii')) - except ValueError: - error = self.process.stderr.read().strip().decode('unicode-escape') - raise RuntimeError(f"Failed to start D-Bus daemon.\n{error}") - - log.debug("Using new D-Bus session with address '%s' with PID %d", - self.address, self.pid) - - self.write_session_file(self.session_file, self.address, self.pid) - log.debug("Wrote D-Bus session file at %s", self.session_file) - - # We must read from the pipes continuously, otherwise the daemon - # process will block. - self._threads=[threading.Thread(target=self.pipe_to_log, args=(self.process.stdout, 'stdout'), daemon=True), - threading.Thread(target=self.pipe_to_log, args=(self.process.stderr, 'stderr'), daemon=True)] - self._threads[0].start() - self._threads[1].start() - - def stop(self): - if self.process: - log.debug(" Stopping DBus daemon") - self.process.terminate() - self.process.wait() - - def pipe_to_log(self, pipe, source): - """This function processes the output from our dbus-daemon instance.""" - while True: - line_raw = pipe.readline() - - if len(line_raw) == 0: - break - - line = line_raw.decode('utf-8').rstrip() - - if line.startswith('(tracker-'): - # We set G_MESSAGES_PREFIXED=all, meaning that all log messages - # output by Tracker processes have a prefix. Note that - # g_print() will NOT be captured here. - dbuslog.info(line) - else: - # Log messages from other daemons, including the dbus-daemon - # itself, go here. Any g_print() messages also end up here. - dbuslog.debug(line) - - # Environment / Clean up def environment_unset(dbus): @@ -260,7 +151,7 @@ def environment_set(index_location, prefix, verbosity=0): dbus_session_file = os.path.join( os.environ['XDG_RUNTIME_DIR'], 'dbus-session') - dbus = DBusDaemon(dbus_session_file) + dbus = trackertestutils.dbusdaemon.DBusDaemon(dbus_session_file) dbus.start_if_needed() # Important, other subprocesses must use our new bus diff --git a/utils/trackertestutils/dbusdaemon.py b/utils/trackertestutils/dbusdaemon.py new file mode 100644 index 000000000..43fe8f146 --- /dev/null +++ b/utils/trackertestutils/dbusdaemon.py @@ -0,0 +1,198 @@ +# Copyright (C) 2018,2019, Sam Thursfield +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + + +from gi.repository import Gio + +import logging +import os +import signal +import subprocess +import threading + +log = logging.getLogger(__name__) +dbus_stderr_log = logging.getLogger(__name__ + '.stderr') +dbus_stdout_log = logging.getLogger(__name__ + '.stdout') + + +class DaemonNotStartedError(Exception): + pass + + +class DBusDaemon: + """The private D-Bus instance that provides the sandbox's session bus. + + We support reading and writing the session information to a file. This + means that if the user runs two sandbox instances on the same data + directory at the same time, they will share the same message bus. + + """ + + def __init__(self, session_file=None): + self.session_file = session_file + self.existing_session = False + self.process = None + + self.address = None + self.pid = None + + self._gdbus_connection = None + self._previous_sigterm_handler = None + + self._threads = [] + + if session_file: + try: + self.address, self.pid = self.read_session_file(session_file) + self.existing_session = True + except FileNotFoundError: + log.debug("No existing D-Bus session file was found.") + + def get_session_file(self): + """Returns the path to the session file if we created it, or None.""" + if self.existing_session: + return None + return self.session_file + + def get_address(self): + if self.address is None: + raise DaemonNotStartedError() + return self.address + + def get_connection(self): + if self._gdbus_connection is None: + raise DaemonNotStartedError() + return self._gdbus_connection + + @staticmethod + def read_session_file(session_file): + with open(session_file, 'r') as f: + content = f.read() + + try: + address = content.splitlines()[0] + pid = int(content.splitlines()[1]) + except ValueError: + raise RuntimeError(f"D-Bus session file {session_file} is not valid. " + "Remove this file to start a new session.") + + return address, pid + + @staticmethod + def write_session_file(session_file, address, pid): + os.makedirs(os.path.dirname(session_file), exist_ok=True) + + content = '%s\n%s' % (address, pid) + with open(session_file, 'w') as f: + f.write(content) + + def start_if_needed(self, config_file=None, env=None): + if self.existing_session: + log.debug('Using existing D-Bus session from file "%s" with address "%s"' + ' with PID %d' % (self.session_file, self.address, self.pid)) + else: + dbus_command = ['dbus-daemon', '--print-address=1', '--print-pid=1'] + if config_file: + dbus_command += ['--config-file=' + config_file] + else: + dbus_command += ['--session'] + log.debug("Running: %s", dbus_command) + self.process = subprocess.Popen( + dbus_command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + self._previous_sigterm_handler = signal.signal( + signal.SIGTERM, self._sigterm_handler) + + try: + self.address = self.process.stdout.readline().strip().decode('ascii') + self.pid = int(self.process.stdout.readline().strip().decode('ascii')) + except ValueError: + error = self.process.stderr.read().strip().decode('unicode-escape') + raise RuntimeError(f"Failed to start D-Bus daemon.\n{error}") + + log.debug("Using new D-Bus session with address '%s' with PID %d", + self.address, self.pid) + + if self.session_file: + self.write_session_file(self.session_file, self.address, self.pid) + log.debug("Wrote D-Bus session file at %s", self.session_file) + + # We must read from the pipes continuously, otherwise the daemon + # process will block. + self._threads=[threading.Thread(target=self.pipe_to_log, args=(self.process.stdout, dbus_stdout_log), daemon=True), + threading.Thread(target=self.pipe_to_log, args=(self.process.stderr, dbus_stdout_log), daemon=True)] + self._threads[0].start() + self._threads[1].start() + + self._gdbus_connection = Gio.DBusConnection.new_for_address_sync( + self.address, + Gio.DBusConnectionFlags.AUTHENTICATION_CLIENT | + Gio.DBusConnectionFlags.MESSAGE_BUS_CONNECTION, None, None) + + log.debug("Pinging the new D-Bus daemon...") + self.ping_sync() + + def stop(self): + if self.process: + log.debug(" Stopping DBus daemon") + self.process.terminate() + self.process.wait() + self.process = None + if len(self._threads) > 0: + log.debug(" Stopping %i pipe reader threads", len(self._threads)) + for thread in self._threads: + thread.join() + self.threads = [] + if self._previous_sigterm_handler: + signal.signal(signal.SIGTERM, self._previous_sigterm_handler) + self._previous_sigterm_handler = None + + def pipe_to_log(self, pipe, dbuslog): + """This function processes the output from our dbus-daemon instance.""" + while True: + line_raw = pipe.readline() + + if len(line_raw) == 0: + break + + line = line_raw.decode('utf-8').rstrip() + + if line.startswith('(tracker-'): + # We set G_MESSAGES_PREFIXED=all, meaning that all log messages + # output by Tracker processes have a prefix. Note that + # g_print() will NOT be captured here. + dbuslog.info(line) + else: + # Log messages from other daemons, including the dbus-daemon + # itself, go here. Any g_print() messages also end up here. + dbuslog.debug(line) + log.debug("Thread stopped") + + # I'm not sure why this is needed, or if it's correct, but without it + # we see warnings like this: + # + # ResourceWarning: unclosed file <_io.BufferedReader name=3> + pipe.close() + + def _sigterm_handler(self, signal, frame): + log.info("Received signal %s", signal) + self.stop() + + def ping_sync(self): + self._gdbus_connection.call_sync( + 'org.freedesktop.DBus', '/', 'org.freedesktop.DBus', 'GetId', + None, None, Gio.DBusCallFlags.NONE, 10000, None) diff --git a/utils/trackertestutils/dconf.py b/utils/trackertestutils/dconf.py index 4ad0e88e9..fe6d981fb 100644 --- a/utils/trackertestutils/dconf.py +++ b/utils/trackertestutils/dconf.py @@ -18,11 +18,9 @@ # 02110-1301, USA. # -from gi.repository import GLib -from gi.repository import Gio - import logging import os +import subprocess log = logging.getLogger(__name__) @@ -36,28 +34,23 @@ class DConfClient(object): this reason, and the constructor will fail if this isn't the profile in use, to avoid any risk of modifying or removing your real configuration. - The constructor will fail if DConf is not the default backend, because this - probably indicates that the memory backend is in use. Without DConf the - required configuration changes will not take effect, causing many tests to - break. + We use the `gsettings` binary rather than using the Gio.Settings API. + This is to avoid the need to set DCONF_PROFILE in our own process + environment. """ - def __init__(self, schema): - self._settings = Gio.Settings.new(schema) - - backend = self._settings.get_property('backend') - self._check_settings_backend_is_dconf(backend) - self._check_using_correct_dconf_profile() - - def _check_settings_backend_is_dconf(self, backend): - typename = type(backend).__name__.split('.')[-1] - if typename != 'DConfSettingsBackend': - raise Exception( - "The functional tests require DConf to be the default " - "GSettings backend. Got %s instead." % typename) + def __init__(self, sandbox): + self.env = os.environ + self.env.update(sandbox.extra_env) + self.env['DBUS_SESSION_BUS_ADDRESS'] = sandbox.daemon.get_address() def _check_using_correct_dconf_profile(self): - profile = os.environ["DCONF_PROFILE"] + profile = self.env.get("DCONF_PROFILE") + if not profile: + raise Exception( + "DCONF_PROFILE is not set in the environment. This class must " + "be created inside a TrackerDBussandbox to avoid risk of " + "interfering with real settings.") if not os.path.exists(profile): raise Exception( "Unable to find DConf profile '%s'. Check that Tracker and " @@ -66,35 +59,11 @@ class DConfClient(object): assert os.path.basename(profile) == "trackertest" - def write(self, key, value): + def write(self, schema, key, value): """ Write a settings value. """ - self._settings.set_value(key, value) - - def read(self, schema, key): - """ - Read a settings value. - """ - return self._settings.get_value(key) - - def reset(self): - """ - Remove all stored values, resetting configuration to the default. - - This can be done by removing the entire 'trackertest' configuration - database. - """ - - self._check_using_correct_dconf_profile() - - # XDG_CONFIG_HOME is useless, so we use HOME. This code should not be - # needed unless for some reason the test is not being run via the - # 'test-runner.sh' script. - dconf_db = os.path.join(os.environ["HOME"], - ".config", - "dconf", - "trackertest") - if os.path.exists(dconf_db): - log.debug("[Conf] Removing dconf database: %s", dconf_db) - os.remove(dconf_db) + subprocess.run(['gsettings', 'set', schema, key, value.print_(False)], + env=self.env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) diff --git a/utils/trackertestutils/helpers.py b/utils/trackertestutils/helpers.py index e6219be3c..45fa67242 100644 --- a/utils/trackertestutils/helpers.py +++ b/utils/trackertestutils/helpers.py @@ -24,8 +24,8 @@ from gi.repository import GLib import atexit import logging import os -import subprocess +from . import dbusdaemon from . import mainloop log = logging.getLogger(__name__) @@ -54,172 +54,9 @@ def _cleanup_processes(): atexit.register(_cleanup_processes) -class Helper: +class StoreHelper(): """ - Abstract helper for Tracker processes. Launches the process - and waits for it to appear on the session bus. - - The helper will fail if the process is already running. Use - test-runner.sh to ensure the processes run inside a separate DBus - session bus. - - The process is watched using a timed GLib main loop source. If the process - exits with an error code, the test will abort the next time the main loop - is entered (or straight away if currently running the main loop). - """ - - STARTUP_TIMEOUT = 200 # milliseconds - SHUTDOWN_TIMEOUT = 200 # - - def __init__(self, helper_name, bus_name, process_path): - self.name = helper_name - self.bus_name = bus_name - self.process_path = process_path - - self.log = logging.getLogger(f'{__name__}.{self.name}') - - self.process = None - self.available = False - - self.loop = mainloop.MainLoop() - - self.bus = Gio.bus_get_sync(Gio.BusType.SESSION, None) - - def _start_process(self, command_args=None, extra_env=None): - global _process_list - _process_list.append(self) - - command = [self.process_path] + (command_args or []) - self.log.debug("Starting %s.", ' '.join(command)) - - env = os.environ - if extra_env: - self.log.debug(" starting with extra environment: %s", extra_env) - env.update(extra_env) - - try: - return subprocess.Popen(command, env=env) - except OSError as e: - raise RuntimeError("Error starting %s: %s" % (self.process_path, e)) - - def _bus_name_appeared(self, connection, name, owner): - self.log.debug("%s appeared on the message bus, owned by %s", name, owner) - self.available = True - self.loop.quit() - - def _bus_name_vanished(self, connection, name): - self.log.debug("%s vanished from the message bus", name) - self.available = False - self.loop.quit() - - def _process_watch_cb(self): - if self.process_watch_timeout == 0: - # GLib seems to call the timeout after we've removed it - # sometimes, which causes errors unless we detect it. - return False - - status = self.process.poll() - - if status is None: - return True # continue - elif status == 0 and not self.abort_if_process_exits_with_status_0: - return True # continue - else: - self.process_watch_timeout = 0 - raise RuntimeError(f"{self.name} exited with status: {self.status}") - - def _process_startup_timeout_cb(self): - self.log.debug(f"Process timeout of {self.STARTUP_TIMEOUT}ms was called") - self.loop.quit() - self.timeout_id = None - return False - - def start(self, command_args=None, extra_env=None): - """ - Start an instance of process and wait for it to appear on the bus. - """ - if self.process is not None: - raise RuntimeError("%s: already started" % self.name) - - self._bus_name_watch_id = Gio.bus_watch_name_on_connection( - self.bus, self.bus_name, Gio.BusNameWatcherFlags.NONE, - self._bus_name_appeared, self._bus_name_vanished) - - # We expect the _bus_name_vanished callback to be called here, - # causing the loop to exit again. - self.loop.run_checked() - - if self.available: - # It's running, but we didn't start it... - raise RuntimeError("Unable to start test instance of %s: " - "already running" % self.name) - - self.process = self._start_process(command_args=command_args, - extra_env=extra_env) - self.log.debug('Started with PID %i', self.process.pid) - - self.process_startup_timeout = GLib.timeout_add( - self.STARTUP_TIMEOUT, self._process_startup_timeout_cb) - - self.abort_if_process_exits_with_status_0 = True - - # Run the loop until the bus name appears, or the process dies. - self.loop.run_checked() - - self.abort_if_process_exits_with_status_0 = False - - def stop(self): - global _process_list - - if self.process is None: - # Seems that it didn't even start... - return - - if self.process_startup_timeout != 0: - GLib.source_remove(self.process_startup_timeout) - self.process_startup_timeout = 0 - - if self.process.poll() == None: - self.process.terminate() - returncode = self.process.wait(timeout=self.SHUTDOWN_TIMEOUT * 1000) - if returncode is None: - self.log.debug("Process failed to terminate in time, sending kill!") - self.process.kill() - self.process.wait() - elif returncode > 0: - self.log.warn("Process returned error code %s", returncode) - - self.log.debug("Process stopped.") - - # Run the loop to handle the expected name_vanished signal. - self.loop.run_checked() - Gio.bus_unwatch_name(self._bus_name_watch_id) - - self.process = None - _process_list.remove(self) - - def kill(self): - global _process_list - - if self.process_watch_timeout != 0: - GLib.source_remove(self.process_watch_timeout) - self.process_watch_timeout = 0 - - self.process.kill() - - # Name owner changed callback should take us out from this loop - self.loop.run_checked() - Gio.bus_unwatch_name(self._bus_name_watch_id) - - self.process = None - _process_list.remove(self) - - self.log.debug("Process killed.") - - -class StoreHelper (Helper): - """ - Helper for starting and testing the tracker-store daemon. + Helper for testing the tracker-store daemon. """ TRACKER_BUSNAME = 'org.freedesktop.Tracker1' @@ -235,32 +72,41 @@ class StoreHelper (Helper): TRACKER_STATUS_OBJ_PATH = "/org/freedesktop/Tracker1/Status" STATUS_IFACE = "org.freedesktop.Tracker1.Status" - def __init__(self, process_path): - Helper.__init__(self, "tracker-store", self.TRACKER_BUSNAME, process_path) + def __init__(self, dbus_connection): + self.log = logging.getLogger(__name__) + self.loop = mainloop.MainLoop() - def start(self, command_args=None, extra_env=None): - Helper.start(self, command_args, extra_env) + self.bus = dbus_connection + self.graph_updated_handler_id = 0 self.resources = Gio.DBusProxy.new_sync( - self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None, + self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None, self.TRACKER_BUSNAME, self.TRACKER_OBJ_PATH, self.RESOURCES_IFACE) self.backup_iface = Gio.DBusProxy.new_sync( - self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None, + self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None, self.TRACKER_BUSNAME, self.TRACKER_BACKUP_OBJ_PATH, self.BACKUP_IFACE) self.stats_iface = Gio.DBusProxy.new_sync( - self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None, + self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None, self.TRACKER_BUSNAME, self.TRACKER_STATS_OBJ_PATH, self.STATS_IFACE) self.status_iface = Gio.DBusProxy.new_sync( - self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START, None, + self.bus, Gio.DBusProxyFlags.DO_NOT_AUTO_START_AT_CONSTRUCTION, None, self.TRACKER_BUSNAME, self.TRACKER_STATUS_OBJ_PATH, self.STATUS_IFACE) + def start_and_wait_for_ready(self): + # The daemon is autostarted as soon as a method is called. + # + # We set a big timeout to avoid interfering when a daemon is being + # interactively debugged. self.log.debug("Calling %s.Wait() method", self.STATUS_IFACE) - self.status_iface.Wait() + self.status_iface.call_sync('Wait', None, Gio.DBusCallFlags.NONE, 1000000, None) self.log.debug("Ready") + def start_watching_updates(self): + assert self.graph_updated_handler_id == 0 + self.reset_graph_updates_tracking() def signal_handler(proxy, sender_name, signal_name, parameters): @@ -269,12 +115,13 @@ class StoreHelper (Helper): self.graph_updated_handler_id = self.resources.connect( 'g-signal', signal_handler) + self.log.debug("Watching for updates from Resources interface") - def stop(self): - Helper.stop(self) - + def stop_watching_updates(self): if self.graph_updated_handler_id != 0: + self.log.debug("No longer watching for updates from Resources interface") self.resources.disconnect(self.graph_updated_handler_id) + self.graph_updated_handler_id = 0 # A system to follow GraphUpdated and make sure all changes are tracked. # This code saves every change notification received, and exposes methods @@ -329,6 +176,7 @@ class StoreHelper (Helper): """ assert (self.inserts_match_function == None) assert (self.class_to_track == None), "Already waiting for resource of type %s" % self.class_to_track + assert (self.graph_updated_handler_id != 0), "You must call start_watching_updates() first." self.class_to_track = rdf_class @@ -413,6 +261,7 @@ class StoreHelper (Helper): """ assert (self.deletes_match_function == None) assert (self.class_to_track == None) + assert (self.graph_updated_handler_id != 0), "You must call start_watching_updates() first." def find_resource_deletion(deletes_list): self.log.debug("find_resource_deletion: looking for %i in %s", id, deletes_list) @@ -444,8 +293,8 @@ class StoreHelper (Helper): # Run the event loop until the correct notification arrives try: self.loop.run_checked() - except GraphUpdateTimeoutException: - raise GraphUpdateTimeoutException("Resource %i has not been deleted." % id) + except GraphUpdateTimeoutException as e: + raise GraphUpdateTimeoutException("Resource %i has not been deleted." % id) from e self.deletes_match_function = None self.class_to_track = None @@ -458,6 +307,7 @@ class StoreHelper (Helper): assert (self.inserts_match_function == None) assert (self.deletes_match_function == None) assert (self.class_to_track == None) + assert (self.graph_updated_handler_id != 0), "You must call start_watching_updates() first." self.log.debug("Await change to %i %s (%i, %i existing)", subject_id, property_uri, len(self.inserts_list), len(self.deletes_list)) @@ -582,3 +432,39 @@ class StoreHelper (Helper): return False else: raise Exception("Something fishy is going on") + + +class TrackerDBusSandbox: + """ + Private D-Bus session bus which executes a sandboxed Tracker instance. + + """ + def __init__(self, dbus_daemon_config_file, extra_env=None): + self.dbus_daemon_config_file = dbus_daemon_config_file + self.extra_env = extra_env or {} + + self.daemon = dbusdaemon.DBusDaemon() + + def start(self): + env = os.environ + env.update(self.extra_env) + env['G_MESSAGES_PREFIXED'] = 'all' + + # Precreate runtime dir, to avoid this warning from dbus-daemon: + # + # Unable to set up transient service directory: XDG_RUNTIME_DIR "/home/sam/tracker-tests/tmp_59i3ev1/run" not available: No such file or directory + # + xdg_runtime_dir = env.get('XDG_RUNTIME_DIR') + if xdg_runtime_dir: + os.makedirs(xdg_runtime_dir, exist_ok=True) + + log.info("Starting D-Bus daemon for sandbox.") + log.debug("Added environment variables: %s", self.extra_env) + self.daemon.start_if_needed(self.dbus_daemon_config_file, env=env) + + def stop(self): + log.info("Stopping D-Bus daemon for sandbox.") + self.daemon.stop() + + def get_connection(self): + return self.daemon.get_connection() diff --git a/utils/trackertestutils/meson.build b/utils/trackertestutils/meson.build index 99573e323..a144794d5 100644 --- a/utils/trackertestutils/meson.build +++ b/utils/trackertestutils/meson.build @@ -1,5 +1,6 @@ sources = [ '__init__.py', + 'dbusdaemon.py', 'dconf.py', 'helpers.py', 'mainloop.py' -- cgit v1.2.1 From 5c60dc4499965ea3fcb0f764ef1bae82acb94e18 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 10 Sep 2019 11:38:21 +0200 Subject: functional-tests: Remove +x bit from functional-tests We expect them to be run from `meson test` now. If you do want to run them manually, run the test with `meson test --verbose` and it will show you the commandline it uses, which you can then use to run the test from your shell. --- tests/functional-tests/01-insertion.py | 0 tests/functional-tests/02-sparql-bugs.py | 0 tests/functional-tests/03-fts-functions.py | 0 tests/functional-tests/04-group-concat.py | 0 tests/functional-tests/05-coalesce.py | 0 tests/functional-tests/06-distance.py | 0 tests/functional-tests/07-graph.py | 0 tests/functional-tests/08-unique-insertions.py | 0 tests/functional-tests/09-concurrent-query.py | 0 tests/functional-tests/14-signals.py | 0 tests/functional-tests/15-statistics.py | 0 tests/functional-tests/16-collation.py | 0 tests/functional-tests/17-ontology-changes.py | 0 13 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 tests/functional-tests/01-insertion.py mode change 100755 => 100644 tests/functional-tests/02-sparql-bugs.py mode change 100755 => 100644 tests/functional-tests/03-fts-functions.py mode change 100755 => 100644 tests/functional-tests/04-group-concat.py mode change 100755 => 100644 tests/functional-tests/05-coalesce.py mode change 100755 => 100644 tests/functional-tests/06-distance.py mode change 100755 => 100644 tests/functional-tests/07-graph.py mode change 100755 => 100644 tests/functional-tests/08-unique-insertions.py mode change 100755 => 100644 tests/functional-tests/09-concurrent-query.py mode change 100755 => 100644 tests/functional-tests/14-signals.py mode change 100755 => 100644 tests/functional-tests/15-statistics.py mode change 100755 => 100644 tests/functional-tests/16-collation.py mode change 100755 => 100644 tests/functional-tests/17-ontology-changes.py diff --git a/tests/functional-tests/01-insertion.py b/tests/functional-tests/01-insertion.py old mode 100755 new mode 100644 diff --git a/tests/functional-tests/02-sparql-bugs.py b/tests/functional-tests/02-sparql-bugs.py old mode 100755 new mode 100644 diff --git a/tests/functional-tests/03-fts-functions.py b/tests/functional-tests/03-fts-functions.py old mode 100755 new mode 100644 diff --git a/tests/functional-tests/04-group-concat.py b/tests/functional-tests/04-group-concat.py old mode 100755 new mode 100644 diff --git a/tests/functional-tests/05-coalesce.py b/tests/functional-tests/05-coalesce.py old mode 100755 new mode 100644 diff --git a/tests/functional-tests/06-distance.py b/tests/functional-tests/06-distance.py old mode 100755 new mode 100644 diff --git a/tests/functional-tests/07-graph.py b/tests/functional-tests/07-graph.py old mode 100755 new mode 100644 diff --git a/tests/functional-tests/08-unique-insertions.py b/tests/functional-tests/08-unique-insertions.py old mode 100755 new mode 100644 diff --git a/tests/functional-tests/09-concurrent-query.py b/tests/functional-tests/09-concurrent-query.py old mode 100755 new mode 100644 diff --git a/tests/functional-tests/14-signals.py b/tests/functional-tests/14-signals.py old mode 100755 new mode 100644 diff --git a/tests/functional-tests/15-statistics.py b/tests/functional-tests/15-statistics.py old mode 100755 new mode 100644 diff --git a/tests/functional-tests/16-collation.py b/tests/functional-tests/16-collation.py old mode 100755 new mode 100644 diff --git a/tests/functional-tests/17-ontology-changes.py b/tests/functional-tests/17-ontology-changes.py old mode 100755 new mode 100644 -- cgit v1.2.1 From 072ad286680b10a63babd4e69e8d2e85eb6de2c3 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Sat, 31 Aug 2019 04:12:44 +0300 Subject: utils/sandbox: Add --dbus-config option This allows tracker-sandbox to be used as a test runner for when the daemons are not yet installed. --- utils/sandbox/tracker-sandbox.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utils/sandbox/tracker-sandbox.py b/utils/sandbox/tracker-sandbox.py index 34ad09ed5..687e2c48c 100755 --- a/utils/sandbox/tracker-sandbox.py +++ b/utils/sandbox/tracker-sandbox.py @@ -117,7 +117,7 @@ def environment_set_and_add_path(env, prefix, suffix): os.environ[env] = full -def environment_set(index_location, prefix, verbosity=0): +def environment_set(index_location, prefix, verbosity=0, dbus_config=None): # Environment index_location = os.path.abspath(index_location) prefix = os.path.abspath(os.path.expanduser(prefix)) @@ -152,7 +152,7 @@ def environment_set(index_location, prefix, verbosity=0): os.environ['XDG_RUNTIME_DIR'], 'dbus-session') dbus = trackertestutils.dbusdaemon.DBusDaemon(dbus_session_file) - dbus.start_if_needed() + dbus.start_if_needed(config_file=dbus_config) # Important, other subprocesses must use our new bus os.environ['DBUS_SESSION_BUS_ADDRESS'] = dbus.get_address() @@ -238,6 +238,8 @@ def argument_parser(): "only show messages logged by Tracker daemons.") parser.add_argument('--debug-sandbox', action='store_true', help="show debugging info from tracker-sandbox") + parser.add_argument('--dbus-config', metavar='FILE', + help="use a custom config file for the private D-Bus daemon") parser.add_argument('-v', '--verbosity', default='0', choices=['0', '1', '2', '3', 'errors', 'minimal', 'detailed', 'debug'], help="show debugging info from Tracker processes") @@ -297,7 +299,7 @@ if __name__ == "__main__": verbosity = verbosity_as_int(args.verbosity) # Set up environment variables and foo needed to get started. - dbus = environment_set(args.index_location, args.prefix, verbosity) + dbus = environment_set(args.index_location, args.prefix, verbosity, dbus_config=args.dbus_config) config_set() link_to_mime_data() -- cgit v1.2.1 From 7b64753d6c5982642afc5378b32945252f8da9fe Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Sat, 31 Aug 2019 04:28:17 +0300 Subject: utils/sandbox: Add --index-tmpdir option This makes the tracker-sandbox utility more useful as a test harness. --- utils/sandbox/tracker-sandbox.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/utils/sandbox/tracker-sandbox.py b/utils/sandbox/tracker-sandbox.py index 687e2c48c..db24679f3 100755 --- a/utils/sandbox/tracker-sandbox.py +++ b/utils/sandbox/tracker-sandbox.py @@ -29,10 +29,11 @@ import locale import logging import os import shlex +import shutil import signal import subprocess import sys -import threading +import tempfile from gi.repository import GLib @@ -248,6 +249,9 @@ def argument_parser(): parser.add_argument('-i', '--index', metavar='DIR', type=str, default=default_index_location, dest='index_location', help=f"directory to the index (default={default_index_location})") + parser.add_argument('--index-tmpdir', action='store_true', + help="create index in a temporary directory and " + "delete it on exit (useful for automated testing)") parser.add_argument('command', type=str, nargs='*', help="Command to run inside the shell") return parser @@ -298,8 +302,18 @@ if __name__ == "__main__": verbosity = verbosity_as_int(args.verbosity) + index_location = None + index_tmpdir = None + + if args.index_location != default_index_location and args.index_tmpdir: + raise RuntimeError("The --index-tmpdir flag is enabled, but --index= was also passed.") + if args.index_tmpdir: + index_location = index_tmpdir = tempfile.mkdtemp(prefix='tracker-sandbox') + else: + index_location = args.index_location + # Set up environment variables and foo needed to get started. - dbus = environment_set(args.index_location, args.prefix, verbosity, dbus_config=args.dbus_config) + dbus = environment_set(index_location, args.prefix, verbosity, dbus_config=args.dbus_config) config_set() link_to_mime_data() @@ -316,3 +330,5 @@ if __name__ == "__main__": os.system(shell) finally: environment_unset(dbus) + if index_tmpdir: + shutil.rmtree(index_tmpdir, ignore_errors=True) -- cgit v1.2.1 From 14e557aad2675c8a07f680a2d584d676cbc85a28 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Sat, 31 Aug 2019 04:30:03 +0300 Subject: functional-tests: Run the IPC tests through tracker-sandbox We can't use trackertestutils directly in these tests, because they are not written in Python. Now we use tracker-sandbox and a few environment variables set in meson.build to achieve the same thing. --- tests/functional-tests/ipc/meson.build | 18 ++++++++++++++---- tests/functional-tests/meson.build | 7 ++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/functional-tests/ipc/meson.build b/tests/functional-tests/ipc/meson.build index 8ee7175fe..1c558c514 100644 --- a/tests/functional-tests/ipc/meson.build +++ b/tests/functional-tests/ipc/meson.build @@ -3,10 +3,21 @@ functional_ipc_test_c_args = [ '-DTEST_ONTOLOGIES_DIR="@0@"'.format(tracker_uninstalled_nepomuk_ontologies_dir), ] +tracker_sandbox = find_program(join_paths(source_root, 'utils', 'sandbox', 'tracker-sandbox.py')) +sandbox_args = ['--dbus-config', test_dbus_config, '--debug-sandbox', '--index-tmpdir', '--'] + +sandbox_env = environment() + +test_env.prepend('PYTHONPATH', tracker_uninstalled_testutils_dir) +test_env.set('TRACKER_DB_ONTOLOGIES_DIR', tracker_uninstalled_nepomuk_ontologies_dir) +test_env.set('TRACKER_LANGUAGE_STOP_WORDS_DIR', tracker_uninstalled_stop_words_dir) +test_env.set('TRACKER_TEST_DOMAIN_ONTOLOGY_RULE', tracker_uninstalled_domain_rule) + insert_or_replace_test = executable('test-insert-or-replace', 'test-insert-or-replace.vala', tracker_sparql_vapi, dependencies: [tracker_common_dep, tracker_sparql_dep]) -test('functional-ipc-insert-or-replace', insert_or_replace_test, +test('functional-ipc-insert-or-replace', tracker_sandbox, + args: sandbox_args + [insert_or_replace_test], env: test_env) bus_query_cancellation_test = executable('test-bus-query-cancellation', @@ -14,7 +25,6 @@ bus_query_cancellation_test = executable('test-bus-query-cancellation', c_args: functional_ipc_test_c_args, dependencies: [tracker_common_dep, tracker_sparql_dep]) -test('functional-ipc-bus-query-cancellation', - test_runner, - args: bus_query_cancellation_test, +test('functional-ipc-bus-query-cancellation', tracker_sandbox, + args: sandbox_args + [bus_query_cancellation_test], env: test_env) diff --git a/tests/functional-tests/meson.build b/tests/functional-tests/meson.build index 21813e410..8bc70c446 100644 --- a/tests/functional-tests/meson.build +++ b/tests/functional-tests/meson.build @@ -6,7 +6,9 @@ testconf = configuration_data() config_json_full_path = join_paths(meson.current_build_dir(), 'configuration.json') dconf_profile_full_path = join_paths(meson.current_source_dir(), 'trackertest') -testconf.set('TEST_DBUS_DAEMON_CONFIG_FILE', join_paths(build_root, 'tests', 'test-bus.conf')) +test_dbus_config = join_paths(build_root, 'tests', 'test-bus.conf') + +testconf.set('TEST_DBUS_DAEMON_CONFIG_FILE', test_dbus_config) testconf.set('TEST_DCONF_PROFILE', dconf_profile_full_path) testconf.set('TEST_DOMAIN_ONTOLOGY_RULE', tracker_uninstalled_domain_rule) testconf.set('TEST_GSETTINGS_SCHEMA_DIR', tracker_uninstalled_gsettings_schema_dir) @@ -51,5 +53,4 @@ foreach t: functional_tests timeout: 60) endforeach -# FIXME -#subdir('ipc') +subdir('ipc') -- cgit v1.2.1 From d286930db174ee5e0576c5c965d261a6b92ab052 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 10 Sep 2019 11:39:38 +0200 Subject: Remove test-runner.sh.in --- tests/functional-tests/test-runner.sh.in | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100755 tests/functional-tests/test-runner.sh.in diff --git a/tests/functional-tests/test-runner.sh.in b/tests/functional-tests/test-runner.sh.in deleted file mode 100755 index 6e295aec6..000000000 --- a/tests/functional-tests/test-runner.sh.in +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# -# Test runner script for Tracker's functional tests - -set -e - -export TEMP_DIR=`mktemp --tmpdir -d tracker-test-XXXX` - -# We need to use the actual home directory for some tests because -# Tracker will explicitly ignore files in /tmp ... -export REAL_HOME=`echo ~` - -# ... but /tmp is preferred for test data, to avoid leaving debris -# in the filesystem -HOME=$TEMP_DIR - -echo "Running $@" -dbus-run-session --config-file=@abs_top_builddir@/tests/test-bus.conf -- "$@" - -rm -R $TEMP_DIR -- cgit v1.2.1 From b1b130ac93ad6b6875a7e5a264388d76c5c39b53 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 29 Aug 2019 17:30:23 +0300 Subject: Add HACKING.md This talks about how to enable different kinds of logging output from the functional tests, now that the previous commit has introduced some sanity into the logging. Attaching debugger is also discussed. --- HACKING.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 +- 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 HACKING.md diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 000000000..389b9c2a1 --- /dev/null +++ b/HACKING.md @@ -0,0 +1,58 @@ +# Logging + +The following environment variables control logging from Tracker daemons: + + * `TRACKER_VERBOSITY`: takes a value of 1, 2 or 3 and causes increasing + amounts of log output from Tracker code to be written to stdout. + * `G_MESSAGES_DEBUG`: controls log output from GLib-based libraries that + are used in the Tracker process. Use `G_MESSAGES_DEBUG=all` for maximal + log output. + +Internally, Tracker will set `G_MESSAGES_DEBUG=Tracker` if `TRACKER_VERBOSITY` +is set and `G_MESSAGES_DEBUG` is not set, to enable printing of its own log +messages to stdout. + +You can set these variables when using `tracker-sandbox`, and when running the +Tracker test suite. Note that Meson will not print log output from tests by +default, use `meson test --verbose` or `meson test --print-errorlogs` to +enable. + +The functional tests understand an additional variable, `TRACKER_TESTS_VERBOSE` +with can be set to `1` or `yes` to see detailed logging from the test harness +itself, and full log output from the internal D-Bus daemon. By default, these +tests filter output from the D-Bus daemon to only show log messages from +Tracker processes. Anything written directly to stdout, for example by +`g_print()` or by the dbus-daemon itself, will not be displayed unless +`TRACKER_TESTS_VERBOSE` is set. + +# Attaching a debugger to Tracker daemons + +Tracker daemons are not started directly. Instead they are started by the D-Bus +daemon by request. When using tracker-sandbox or the functional-tests, it's +difficult to start the daemon manually under `gdb`. + +Instead, we recommend adding a 10 second timeout at the top of the daemon's +main() function. In Vala code, try this: + + print("Pausing to attach debugger. Run: gdb attach %i\n", Posix.getpid()); + Posix.usleep(10 * 1000 * 1000); + print("Waking up again\n"); + +Run the test, using the `meson build --timeout-multiplier=10000` +option to avoid your process being killed by the test runner. When you see +the 'Pausing' message, run the `gdb attach``command in another terminal within +10 seconds. + +# Running Tracker daemons under Valgrind + +The Tracker daemons are launched using D-Bus autolaunch. When running them from +the source tree using tracker-sandbox or the functional tests, the commandline +is controlled by the D-Bus .service.in files stored in `./tests/services`. Just +change the `Exec=` line to add Valgrind, like this: + + Exec=/usr/bin/valgrind @abs_top_builddir@/src/tracker-store/tracker-store + +By default the tracker-sandbox utility and the functional-tests will only +show output from Tracker code. For the functional-tests, set +TRACKER_TESTS_VERBOSE=1 to see output from Valgrind. For tracker-sandbox use +the `--debug-dbus` option. diff --git a/README.md b/README.md index f1ec0f855..c94639b87 100644 --- a/README.md +++ b/README.md @@ -131,4 +131,4 @@ interactive shell inside the sandbox. From here you can use debugging tools such as GDB. For more information about developing Tracker, look at -https://wiki.gnome.org/Projects/Tracker. +https://wiki.gnome.org/Projects/Tracker and HACKING.md. -- cgit v1.2.1 From d5c4f7200f7f1b772163694731e1c2d0468e9df5 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 29 Aug 2019 17:32:35 +0300 Subject: Disable D-Bus service startup timeout This timeout interferes with debugging. If you attach GDB to a daemon as it starts up and step through the code, the timeout can cause dbus-daemon to kill the process that you're debugging. We can rely on `meson test` to kill tests that actually don't finish, and the `meson test` timeout can be conveniently increased with the `--timeout-multiplier` commandline option. --- tests/test-bus.conf.in | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test-bus.conf.in b/tests/test-bus.conf.in index 2f4b2ef1b..f2df619e3 100644 --- a/tests/test-bus.conf.in +++ b/tests/test-bus.conf.in @@ -9,6 +9,12 @@ @abs_top_builddir@/tests/services/ + + 1000000 + @@ -17,5 +23,4 @@ - -- cgit v1.2.1 From 240268b928bec3e3c6709757ac959f89e43ac43a Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 29 Aug 2019 19:03:37 +0300 Subject: Enable G_MESSAGES_DEBUG=Tracker rather than G_MESSAGES_DEBUG=all In order for our g_info()/g_message()/g_debug() log messages to be handled, we need to have G_MESSAGES_DEBUG=Tracker enabled in the environment. Previously we would enable G_MESSAGES_DEBUG=all, which has the effect of also enabling log output from GLib/GIO and other dependencies. In the interests of keeping logs clean, we now require the user to explicitly set G_MESSAGES_DEBUG=all if they want maximum debug output. --- src/libtracker-common/tracker-log.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libtracker-common/tracker-log.c b/src/libtracker-common/tracker-log.c index d300ee795..f3228516e 100644 --- a/src/libtracker-common/tracker-log.c +++ b/src/libtracker-common/tracker-log.c @@ -151,6 +151,18 @@ hide_log_handler (const gchar *domain, /* do nothing */ } +static void +ensure_g_messages_debug_set () +{ + const gchar *value; + + value = g_getenv ("G_MESSAGES_DEBUG"); + + if (value == NULL) { + g_setenv ("G_MESSAGES_DEBUG", "Tracker", TRUE); + } +} + gboolean tracker_log_init (gint this_verbosity, gchar **used_filename) @@ -192,8 +204,8 @@ tracker_log_init (gint this_verbosity, /* If we have debug enabled, we imply G_MESSAGES_DEBUG or we * see nothing, this came in since GLib 2.32. */ - if (this_verbosity > 1) { - g_setenv ("G_MESSAGES_DEBUG", "all", TRUE); + if (this_verbosity > 0) { + ensure_g_messages_debug_set (); } if (use_log_files) { -- cgit v1.2.1 From 12f095cc07d921c06a666aef05ab3f37b58804ad Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 7 Aug 2019 17:10:24 +0200 Subject: Use g_message() instead of g_print() in daemons All logs should go through the GLib logging system so they can be properly enabled and disabled with the TRACKER_VERBOSITY variable. --- src/tracker-store/tracker-main.vala | 6 ++---- src/tracker/tracker-daemon.c | 7 +++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/tracker-store/tracker-main.vala b/src/tracker-store/tracker-main.vala index 1248709b7..e1b0cc86b 100644 --- a/src/tracker-store/tracker-main.vala +++ b/src/tracker-store/tracker-main.vala @@ -118,14 +118,12 @@ License which can be viewed at: do_shutdown (); if (strsignal (signo) != null) { - print ("\n"); - print ("Received signal:%d->'%s'", signo, strsignal (signo)); + message ("Received signal:%d->'%s'", signo, strsignal (signo)); } break; default: if (strsignal (signo) != null) { - print ("\n"); - print ("Received signal:%d->'%s'", signo, strsignal (signo)); + message ("Received signal:%d->'%s'", signo, strsignal (signo)); } break; } diff --git a/src/tracker/tracker-daemon.c b/src/tracker/tracker-daemon.c index 2c25b2c07..c73658477 100644 --- a/src/tracker/tracker-daemon.c +++ b/src/tracker/tracker-daemon.c @@ -228,10 +228,9 @@ signal_handler (gpointer user_data) /* Fall through */ default: if (g_strsignal (signo)) { - g_print ("\n"); - g_print ("Received signal:%d->'%s'\n", - signo, - g_strsignal (signo)); + g_message ("Received signal:%d->'%s'", + signo, + g_strsignal (signo)); } break; } -- cgit v1.2.1 From e13d9087af7ce4c601c4c2a4bda4083960eaf0c0 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Sat, 31 Aug 2019 03:20:05 +0300 Subject: Expose dbus_services_dir to dependent projects The tracker-miners tests now depend on the tests/services/org.freedesktop.Tracker1.service file. We need to expose it. The pkg-config variable is for situations where we build against an installed Tracker core (so we use the normal service file, and honour any custom location that the user specified for it). The Meson variable is for when Tracker core is built as a subproject of tracker-miners. --- meson.build | 1 + src/libtracker-sparql/tracker-sparql.pc.in | 1 + tests/services/meson.build | 2 ++ 3 files changed, 4 insertions(+) diff --git a/meson.build b/meson.build index d6c2fa4a6..af7ad6bc7 100644 --- a/meson.build +++ b/meson.build @@ -273,6 +273,7 @@ conf.set('exec_prefix', get_option('prefix')) conf.set('bindir', join_paths(get_option('prefix'), get_option('bindir'))) conf.set('datadir', datadir) conf.set('datarootdir', join_paths(get_option('prefix'), get_option('datadir'))) +conf.set('dbus_services_dir', dbus_services_dir) conf.set('includedir', join_paths(get_option('prefix'), get_option('includedir'))) conf.set('libdir', libdir) conf.set('libexecdir', join_paths(get_option('prefix'), get_option('libexecdir'))) diff --git a/src/libtracker-sparql/tracker-sparql.pc.in b/src/libtracker-sparql/tracker-sparql.pc.in index dcdf0b7df..7b3aa3718 100644 --- a/src/libtracker-sparql/tracker-sparql.pc.in +++ b/src/libtracker-sparql/tracker-sparql.pc.in @@ -6,6 +6,7 @@ libexecdir=@libexecdir@ datadir=@datadir@ tracker_store=@tracker_store@ ontologies_dir=@ontologies_dir@ +dbus_services_dir=@dbus_services_dir@ domain_ontologies_dir=@domain_ontologies_dir@ Name: tracker-sparql diff --git a/tests/services/meson.build b/tests/services/meson.build index 182ba1e74..0223867e9 100644 --- a/tests/services/meson.build +++ b/tests/services/meson.build @@ -2,3 +2,5 @@ test_dbus_service_file = configure_file( input: 'org.freedesktop.Tracker1.service.in', output: 'org.freedesktop.Tracker1.service', configuration: conf) + +tracker_test_dbus_services_dir = meson.current_build_dir() -- cgit v1.2.1 From b7e45448c34ca06095b5ceb51ae8a944cc1d2059 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 10 Sep 2019 11:13:36 +0200 Subject: ci: Allow controlling verbosity of tests in CI There is also a MESON_TEST_EXTRA_ARGS variable which can be set to cause only one test to execute. This is to help with debugging test failures that can only be reproduced on the CI runners. --- .gitlab-ci.yml | 21 ++++++++++++++++++++- HACKING.md | 5 +++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1922acfb5..29a459acd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,12 @@ +variables: + # These can be used to see verbose log output from the functional-tests. + # See HACKING.md for more information. + TRACKER_VERBOSITY: "0" + TRACKER_TESTS_VERBOSE: "no" + + # This can be used when debugging test failures that only occur within GitLab CI. + MESON_TEST_EXTRA_ARGS: "" + stages: - test @@ -15,7 +24,17 @@ test-fedora-latest: # screenful of junk each time unless we strip these. unset $(env|grep -o '^CI_[^=]*') - su tracker -c 'cd build; meson test --print-errorlogs' + su tracker -c 'cd build; meson test --print-errorlogs ${MESON_TEST_EXTRA_ARGS}' + + after_script: + - | + echo "Test suite settings:" + echo + echo " TRACKER_VERBOSITY: ${TRACKER_VERBOSITY}" + echo " TRACKER_TESTS_VERBOSE: ${TRACKER_TESTS_VERBOSE}" + echo " MESON_TEST_EXTRA_ARGS: ${MESON_TEST_EXTRA_ARGS}" + echo + echo "These values can be set at https://gitlab.gnome.org/GNOME/tracker/pipelines/new" artifacts: when: always diff --git a/HACKING.md b/HACKING.md index 389b9c2a1..b907f3112 100644 --- a/HACKING.md +++ b/HACKING.md @@ -25,6 +25,11 @@ Tracker processes. Anything written directly to stdout, for example by `g_print()` or by the dbus-daemon itself, will not be displayed unless `TRACKER_TESTS_VERBOSE` is set. +When working with GitLab CI, you can use the +[Run Pipeline dialog](https://gitlab.gnome.org/GNOME/tracker/pipelines/new) +to set the values of these variables and increase the verbosity of the tests in +CI. + # Attaching a debugger to Tracker daemons Tracker daemons are not started directly. Instead they are started by the D-Bus -- cgit v1.2.1 From 2473e123d68113bbe3a1a0bd6b3c5b517a1727c7 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 10 Sep 2019 12:14:35 +0200 Subject: functional-tests: Terminate Tracker processes before the message bus This reduces the volume of criticals like this that we see in the log output: (tracker-miner-fs:14955): GLib-GIO-CRITICAL **: 11:38:40.386: Error while sending AddMatch() message: The connection is closed I did still see one test that output a bunch of similar criticals from tracker-extract. --- utils/trackertestutils/dbusdaemon.py | 24 +++++++++ utils/trackertestutils/helpers.py | 25 +++++++++ utils/trackertestutils/meson.build | 3 +- utils/trackertestutils/psutil_mini.py | 98 +++++++++++++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 utils/trackertestutils/psutil_mini.py diff --git a/utils/trackertestutils/dbusdaemon.py b/utils/trackertestutils/dbusdaemon.py index 43fe8f146..c7e4707f3 100644 --- a/utils/trackertestutils/dbusdaemon.py +++ b/utils/trackertestutils/dbusdaemon.py @@ -17,6 +17,7 @@ from gi.repository import Gio +from gi.repository import GLib import logging import os @@ -193,6 +194,29 @@ class DBusDaemon: self.stop() def ping_sync(self): + """Call the daemon Ping() method to check that it is alive.""" self._gdbus_connection.call_sync( 'org.freedesktop.DBus', '/', 'org.freedesktop.DBus', 'GetId', None, None, Gio.DBusCallFlags.NONE, 10000, None) + + def list_names_sync(self): + """Get the name of every client connected to the bus.""" + conn = self.get_connection() + result = conn.call_sync('org.freedesktop.DBus', + '/org/freedesktop/DBus', + 'org.freedesktop.DBus', 'ListNames', None, + GLib.VariantType('(as)'), + Gio.DBusCallFlags.NONE, -1, None) + return result[0] + + def get_connection_unix_process_id_sync(self, name): + """Get the process ID for one of the names connected to the bus.""" + conn = self.get_connection() + result = conn.call_sync('org.freedesktop.DBus', + '/org/freedesktop/DBus', + 'org.freedesktop.DBus', + 'GetConnectionUnixProcessID', + GLib.Variant('(s)', [name]), + GLib.VariantType('(u)'), + Gio.DBusCallFlags.NONE, -1, None) + return result[0] diff --git a/utils/trackertestutils/helpers.py b/utils/trackertestutils/helpers.py index 45fa67242..fc81b52e6 100644 --- a/utils/trackertestutils/helpers.py +++ b/utils/trackertestutils/helpers.py @@ -24,9 +24,12 @@ from gi.repository import GLib import atexit import logging import os +import signal from . import dbusdaemon from . import mainloop +from . import psutil_mini as psutil + log = logging.getLogger(__name__) @@ -463,6 +466,28 @@ class TrackerDBusSandbox: self.daemon.start_if_needed(self.dbus_daemon_config_file, env=env) def stop(self): + tracker_processes = [] + + log.info("Looking for active Tracker processes on the bus") + for busname in self.daemon.list_names_sync(): + if busname.startswith('org.freedesktop.Tracker1'): + pid = self.daemon.get_connection_unix_process_id_sync(busname) + tracker_processes.append(pid) + + log.info("Terminating %i Tracker processes", len(tracker_processes)) + for pid in tracker_processes: + os.kill(pid, signal.SIGTERM) + + log.info("Waiting for %i Tracker processes", len(tracker_processes)) + for pid in tracker_processes: + psutil.wait_pid(pid) + + # We need to wait until Tracker processes have stopped before we + # terminate the D-Bus daemon, otherwise lots of criticals like this + # appear in the log output: + # + # (tracker-miner-fs:14955): GLib-GIO-CRITICAL **: 11:38:40.386: Error while sending AddMatch() message: The connection is closed + log.info("Stopping D-Bus daemon for sandbox.") self.daemon.stop() diff --git a/utils/trackertestutils/meson.build b/utils/trackertestutils/meson.build index a144794d5..e8ab94c72 100644 --- a/utils/trackertestutils/meson.build +++ b/utils/trackertestutils/meson.build @@ -3,7 +3,8 @@ sources = [ 'dbusdaemon.py', 'dconf.py', 'helpers.py', - 'mainloop.py' + 'mainloop.py', + 'psutil_mini.py', ] install_data(sources, diff --git a/utils/trackertestutils/psutil_mini.py b/utils/trackertestutils/psutil_mini.py new file mode 100644 index 000000000..d0c93565d --- /dev/null +++ b/utils/trackertestutils/psutil_mini.py @@ -0,0 +1,98 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# +# Use of this source code is governed by a BSD-style license that can be +# found at https://github.com/giampaolo/psutil/blob/master/LICENSE +# +# Taken from https://github.com/giampaolo/psutil/blob/master/psutil/_psposix.py +# by Sam Thursfield to avoid adding a dependency between the Tracker testsuite +# and the 'psutil' module. + + +import os +import time + + +class TimeoutExpired(Exception): + pass + + +def pid_exists(pid): + """Check whether pid exists in the current process table.""" + if pid == 0: + # According to "man 2 kill" PID 0 has a special meaning: + # it refers to <> so we don't want to go any further. + # If we get here it means this UNIX platform *does* have + # a process with id 0. + return True + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + # EPERM clearly means there's a process to deny access to + return True + # According to "man 2 kill" possible error values are + # (EINVAL, EPERM, ESRCH) + else: + return True + + +def wait_pid(pid, timeout=None, proc_name=None): + """Wait for process with pid 'pid' to terminate and return its + exit status code as an integer. + If pid is not a children of os.getpid() (current process) just + waits until the process disappears and return None. + If pid does not exist at all return None immediately. + Raise TimeoutExpired on timeout expired. + """ + def check_timeout(delay): + if timeout is not None: + if timer() >= stop_at: + raise TimeoutExpired(timeout, pid=pid, name=proc_name) + time.sleep(delay) + return min(delay * 2, 0.04) + + timer = getattr(time, 'monotonic', time.time) + if timeout is not None: + def waitcall(): + return os.waitpid(pid, os.WNOHANG) + stop_at = timer() + timeout + else: + def waitcall(): + return os.waitpid(pid, 0) + + delay = 0.0001 + while True: + try: + retpid, status = waitcall() + except InterruptedError: + delay = check_timeout(delay) + except ChildProcessError: + # This has two meanings: + # - pid is not a child of os.getpid() in which case + # we keep polling until it's gone + # - pid never existed in the first place + # In both cases we'll eventually return None as we + # can't determine its exit status code. + while True: + if pid_exists(pid): + delay = check_timeout(delay) + else: + return + else: + if retpid == 0: + # WNOHANG was used, pid is still running + delay = check_timeout(delay) + continue + # process exited due to a signal; return the integer of + # that signal + if os.WIFSIGNALED(status): + return -os.WTERMSIG(status) + # process exited using exit(2) system call; return the + # integer exit(2) system call has been called with + elif os.WIFEXITED(status): + return os.WEXITSTATUS(status) + else: + # should never happen + raise ValueError("unknown process exit status %r" % status) -- cgit v1.2.1 From eb72f5b7210d63a3ba5e668bb6931a385f05c2a9 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Sat, 14 Sep 2019 10:59:06 +0200 Subject: functional-tests: Set test runner verbosity to 2 Tests would previously output progress dots and a summary, like this: .(tracker-store:5918): GLib-GObject-CRITICAL **: 10:53:10.135: g_object_unref: assertion 'G_IS_OBJECT (object)' failed ..(tracker-store:5952): GLib-GObject-CRITICAL **: 10:53:10.857: g_object_unref: assertion 'G_IS_OBJECT (object)' failed ..................(tracker-store:5986): GLib-GObject-CRITICAL **: 10:53:13.275: g_object_unref: assertion 'G_IS_OBJECT (object)' failed ..(tracker-store:6042): GLib-GObject-CRITICAL **: 10:53:13.985: g_object_unref: assertion 'G_IS_OBJECT (object)' failed ---------------------------------------------------------------------- Ran 23 tests in 4.555s OK This not very useful when we see error output during the test, as you can see in the example. The test log now contains the name of each test. Errors from Tracker daemons now appear like this: test_batch_insert_01 (__main__.TrackerStoreBatchUpdateTest) ... ok (tracker-store:6341): GLib-GObject-CRITICAL **: 10:56:18.859: g_object_unref: assertion 'G_IS_OBJECT (object)' failed test_delete_01 (__main__.TrackerStoreDeleteTests) ... ok test_delete_02 (__main__.TrackerStoreDeleteTests) ... ok (tracker-store:6374): GLib-GObject-CRITICAL **: 10:56:19.581: g_object_unref: assertion 'G_IS_OBJECT (object)' failed test_insert_01 (__main__.TrackerStoreInsertionTests) ... ok test_insert_02 (__main__.TrackerStoreInsertionTests) ... ok --- tests/functional-tests/01-insertion.py | 2 +- tests/functional-tests/02-sparql-bugs.py | 2 +- tests/functional-tests/03-fts-functions.py | 2 +- tests/functional-tests/04-group-concat.py | 2 +- tests/functional-tests/05-coalesce.py | 2 +- tests/functional-tests/06-distance.py | 2 +- tests/functional-tests/07-graph.py | 2 +- tests/functional-tests/08-unique-insertions.py | 2 +- tests/functional-tests/09-concurrent-query.py | 2 +- tests/functional-tests/14-signals.py | 2 +- tests/functional-tests/15-statistics.py | 2 +- tests/functional-tests/16-collation.py | 2 +- tests/functional-tests/17-ontology-changes.py | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/functional-tests/01-insertion.py b/tests/functional-tests/01-insertion.py index 8c8092dd1..8b5b40c80 100644 --- a/tests/functional-tests/01-insertion.py +++ b/tests/functional-tests/01-insertion.py @@ -869,4 +869,4 @@ class TrackerStorePhoneNumberTest (CommonTrackerStoreTest): if __name__ == "__main__": - ut.main() + ut.main(verbosity=2) diff --git a/tests/functional-tests/02-sparql-bugs.py b/tests/functional-tests/02-sparql-bugs.py index bf0d8ecbb..e312e9109 100644 --- a/tests/functional-tests/02-sparql-bugs.py +++ b/tests/functional-tests/02-sparql-bugs.py @@ -238,4 +238,4 @@ class TrackerStoreSparqlBugsTests (CommonTrackerStoreTest): if __name__ == "__main__": - ut.main() + ut.main(verbosity=2) diff --git a/tests/functional-tests/03-fts-functions.py b/tests/functional-tests/03-fts-functions.py index 0938a7e52..ec23a3ed6 100644 --- a/tests/functional-tests/03-fts-functions.py +++ b/tests/functional-tests/03-fts-functions.py @@ -127,4 +127,4 @@ class TestFTSFunctions (CommonTrackerStoreTest): if __name__ == '__main__': - ut.main() + ut.main(verbosity=2) diff --git a/tests/functional-tests/04-group-concat.py b/tests/functional-tests/04-group-concat.py index 06e06265b..d36523004 100644 --- a/tests/functional-tests/04-group-concat.py +++ b/tests/functional-tests/04-group-concat.py @@ -87,4 +87,4 @@ class TestGroupConcat (CommonTrackerStoreTest): if __name__ == '__main__': - ut.main() + ut.main(verbosity=2) diff --git a/tests/functional-tests/05-coalesce.py b/tests/functional-tests/05-coalesce.py index eddf3148e..176ae6b66 100644 --- a/tests/functional-tests/05-coalesce.py +++ b/tests/functional-tests/05-coalesce.py @@ -114,4 +114,4 @@ class TestCoalesce (CommonTrackerStoreTest): if __name__ == '__main__': - ut.main() + ut.main(verbosity=2) diff --git a/tests/functional-tests/06-distance.py b/tests/functional-tests/06-distance.py index b2a22d255..42989a946 100644 --- a/tests/functional-tests/06-distance.py +++ b/tests/functional-tests/06-distance.py @@ -124,4 +124,4 @@ class TestDistanceFunctions (CommonTrackerStoreTest): if __name__ == '__main__': - ut.main() + ut.main(verbosity=2) diff --git a/tests/functional-tests/07-graph.py b/tests/functional-tests/07-graph.py index cec6a60d5..c1a6e8697 100644 --- a/tests/functional-tests/07-graph.py +++ b/tests/functional-tests/07-graph.py @@ -145,4 +145,4 @@ class TestGraphs (CommonTrackerStoreTest): if __name__ == '__main__': - ut.main() + ut.main(verbosity=2) diff --git a/tests/functional-tests/08-unique-insertions.py b/tests/functional-tests/08-unique-insertions.py index a71d2118e..69e16573b 100644 --- a/tests/functional-tests/08-unique-insertions.py +++ b/tests/functional-tests/08-unique-insertions.py @@ -75,4 +75,4 @@ class TestMinerInsertBehaviour (CommonTrackerStoreTest): if __name__ == '__main__': - ut.main() + ut.main(verbosity=2) diff --git a/tests/functional-tests/09-concurrent-query.py b/tests/functional-tests/09-concurrent-query.py index dacd396aa..7164babb6 100644 --- a/tests/functional-tests/09-concurrent-query.py +++ b/tests/functional-tests/09-concurrent-query.py @@ -97,4 +97,4 @@ class TestConcurrentQuery (CommonTrackerStoreTest): return False if __name__ == "__main__": - ut.main() + ut.main(verbosity=2) diff --git a/tests/functional-tests/14-signals.py b/tests/functional-tests/14-signals.py index cae97d46d..8cf349edd 100644 --- a/tests/functional-tests/14-signals.py +++ b/tests/functional-tests/14-signals.py @@ -189,4 +189,4 @@ class TrackerStoreSignalsTests (CommonTrackerStoreTest): if __name__ == "__main__": - ut.main() + ut.main(verbosity=2) diff --git a/tests/functional-tests/15-statistics.py b/tests/functional-tests/15-statistics.py index 56fa38eb7..a883b1708 100644 --- a/tests/functional-tests/15-statistics.py +++ b/tests/functional-tests/15-statistics.py @@ -123,4 +123,4 @@ class TrackerStoreStatisticsTests (CommonTrackerStoreTest): self.assertEqual(old_stats[k], new_stats[k]) if __name__ == "__main__": - ut.main() + ut.main(verbosity=2) diff --git a/tests/functional-tests/16-collation.py b/tests/functional-tests/16-collation.py index 62765a77a..962db9640 100644 --- a/tests/functional-tests/16-collation.py +++ b/tests/functional-tests/16-collation.py @@ -131,4 +131,4 @@ if __name__ == "__main__": * Check what happens in non-english encoding * Dynamic change of collation (not implemented yet in tracker) """) - ut.main() + ut.main(verbosity=2) diff --git a/tests/functional-tests/17-ontology-changes.py b/tests/functional-tests/17-ontology-changes.py index 0571defb9..e2cb132f1 100644 --- a/tests/functional-tests/17-ontology-changes.py +++ b/tests/functional-tests/17-ontology-changes.py @@ -917,4 +917,4 @@ class PropertyRelegationTest (OntologyChangeTestTemplate): if __name__ == "__main__": - ut.main() + ut.main(verbosity=2) -- cgit v1.2.1 From 60b0f7fb6abb4cbd03707260e14419b885e30635 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Sat, 14 Sep 2019 12:27:02 +0200 Subject: trackertestutils: Use default timeout when calling tracker-store D-Bus methods The default timeout of a GDBusProxy object is 25 seconds, but we were reducing that to 5 seconds. This resulted in occasional test failures (especially in functional-07-graph). (The test failures would occur on systems with many CPUs where the tests ran in parallel, which in practice meant they only failed in GitLab CI. I found that I could reproduce the failure locally by running `meson test --num-processes=100` though.) --- utils/trackertestutils/helpers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/trackertestutils/helpers.py b/utils/trackertestutils/helpers.py index fc81b52e6..ef450ee05 100644 --- a/utils/trackertestutils/helpers.py +++ b/utils/trackertestutils/helpers.py @@ -358,14 +358,14 @@ class StoreHelper(): # is useful for testing this API surface, but we recommand that all regular # applications use libtracker-sparql library to talk to the database. - def query(self, query, timeout=5000, **kwargs): - return self.resources.SparqlQuery('(s)', query, timeout=timeout, **kwargs) + def query(self, query, **kwargs): + return self.resources.SparqlQuery('(s)', query, **kwargs) - def update(self, update_sparql, timeout=5000, **kwargs): - return self.resources.SparqlUpdate('(s)', update_sparql, timeout=timeout, **kwargs) + def update(self, update_sparql, **kwargs): + return self.resources.SparqlUpdate('(s)', update_sparql, **kwargs) - def load(self, ttl_uri, timeout=5000, **kwargs): - return self.resources.Load('(s)', ttl_uri, timeout=timeout, **kwargs) + def load(self, ttl_uri, **kwargs): + return self.resources.Load('(s)', ttl_uri, **kwargs) def batch_update(self, update_sparql, **kwargs): return self.resources.BatchSparqlUpdate('(s)', update_sparql, **kwargs) -- cgit v1.2.1