From 7712c2c7e10b3c28b2cc2ec8dfeeb8a89bd57945 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Sat, 2 May 2020 16:46:28 +0200 Subject: Port to Tracker 3 Mostly the port is straightforward, we connect to tracker-miner-fs explicitly over D-Bus instead of the centralized tracker-store daemon we connected to previously. The search-engine-tracker test is now isolated from the user's real Tracker index using the `tracker-sandbox` script provided by Tracker, and it lets tracker-miner-fs index the test file rather than trying to synthesize the expected database contents. There are more changes in nautilus-tag-manager.c. Until now, starred file information was stored in the tracker-miner-fs database. This has some downsides, firstly the data is deleted if someone runs `tracker reset --hard`, secondly it isn't possible to do this from inside a Flatpak sandbox with Tracker 3.0. because the This commit changes the NautilusTagManager to set up a private database inside XDG_DATA_HOME/nautilus/tags. This stores the starred file information. The database is managed with Tracker, which allows us to continue using the rename-tracking that tracker-miner-fs provides. The same limitations apply as before that only files in indexed locations can be starred. --- data/meson.build | 2 + data/ontology/meson.build | 8 + data/ontology/nautilus.description | 9 + data/ontology/nautilus.ontology | 19 + meson.build | 2 +- src/nautilus-batch-rename-utilities.c | 36 +- src/nautilus-global-preferences.c | 2 +- src/nautilus-search-engine-tracker.c | 38 +- src/nautilus-tag-manager.c | 567 ++++++++++++--------- test/automated/displayless/meson.build | 32 +- .../test-nautilus-search-engine-tracker.c | 157 ++++-- 11 files changed, 559 insertions(+), 313 deletions(-) create mode 100644 data/ontology/meson.build create mode 100644 data/ontology/nautilus.description create mode 100644 data/ontology/nautilus.ontology diff --git a/data/meson.build b/data/meson.build index 7218b84e0..e51ed50b1 100644 --- a/data/meson.build +++ b/data/meson.build @@ -137,3 +137,5 @@ if appstream_util.found() ] ) endif + +subdir('ontology') diff --git a/data/ontology/meson.build b/data/ontology/meson.build new file mode 100644 index 000000000..cb80e86a1 --- /dev/null +++ b/data/ontology/meson.build @@ -0,0 +1,8 @@ +ontology_data = files( + 'nautilus.description', + 'nautilus.ontology', +) + +install_data(ontology_data, + install_dir: join_paths(datadir, 'nautilus', 'ontology') +) diff --git a/data/ontology/nautilus.description b/data/ontology/nautilus.description new file mode 100644 index 000000000..42f4102c3 --- /dev/null +++ b/data/ontology/nautilus.description @@ -0,0 +1,9 @@ +@prefix dsc: . + + a dsc:Ontology ; + dsc:title "Nautilus ontology" ; + dsc:description "Tracker database schema for Nautilus private data." ; + + dsc:localPrefix "nautilus" ; + dsc:baseUrl "https://gitlab.gnome.org/GNOME/nautilus#" ; + dsc:relativePath "./nautilus.ontology" ; diff --git a/data/ontology/nautilus.ontology b/data/ontology/nautilus.ontology new file mode 100644 index 000000000..ed56df447 --- /dev/null +++ b/data/ontology/nautilus.ontology @@ -0,0 +1,19 @@ +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . +@prefix nrl: . +@prefix nautilus: . + +nautilus: a nrl:Namespace, nrl:Ontology ; + nrl:prefix "nautilus" ; + nrl:lastModified "2020-05-01T10:00:00Z" . + +nautilus:FileReference a rdfs:Class ; + rdfs:comment "A unique ID for a file, corresponding with the nie:InformationElement created by tracker-miner-fs. We use a unique ID rather than the file URL so the star will follow renames." ; + rdfs:subClassOf rdfs:Resource . + +nautilus:starred a rdf:Property ; + rdfs:comment "Marks resources that are starred in Nautilus." ; + rdfs:domain nautilus:FileReference ; + rdfs:range xsd:boolean ; + nrl:maxCardinality 1 . diff --git a/meson.build b/meson.build index 03026e04c..bb36b8a20 100644 --- a/meson.build +++ b/meson.build @@ -124,7 +124,7 @@ selinux = [] if get_option('selinux') selinux = dependency('libselinux', version: '>= 2.0') endif -tracker_sparql = dependency('tracker-sparql-2.0') +tracker_sparql = dependency('tracker-sparql-3.0') xml = dependency('libxml-2.0', version: '>= 2.7.8') #################### diff --git a/src/nautilus-batch-rename-utilities.c b/src/nautilus-batch-rename-utilities.c index fcc0a32e9..b6cea25fa 100644 --- a/src/nautilus-batch-rename-utilities.c +++ b/src/nautilus-batch-rename-utilities.c @@ -63,6 +63,8 @@ enum ALBUM_NAME_INDEX, } QueryMetadata; +#define TRACKER_MINER_FS_BUSNAME "org.freedesktop.Tracker3.Miner.Files" + static void on_cursor_callback (GObject *object, GAsyncResult *result, gpointer user_data); @@ -1056,21 +1058,21 @@ check_metadata_for_selection (NautilusBatchRenameDialog *dialog, query = g_string_new ("SELECT " "nfo:fileName(?file) " - "nie:contentCreated(?file) " - "year(nie:contentCreated(?file)) " - "month(nie:contentCreated(?file)) " - "day(nie:contentCreated(?file)) " - "hours(nie:contentCreated(?file)) " - "minutes(nie:contentCreated(?file)) " - "seconds(nie:contentCreated(?file)) " - "nfo:model(nfo:equipment(?file)) " - "nmm:season(?file) " - "nmm:episodeNumber(?file) " - "nmm:trackNumber(?file) " - "nmm:artistName(nmm:performer(?file)) " - "nie:title(?file) " - "nmm:albumTitle(nmm:musicAlbum(?file)) " - "WHERE { ?file a nfo:FileDataObject. ?file nie:url ?url. "); + "nie:contentCreated(?content) " + "year(nie:contentCreated(?content)) " + "month(nie:contentCreated(?content)) " + "day(nie:contentCreated(?content)) " + "hours(nie:contentCreated(?content)) " + "minutes(nie:contentCreated(?content)) " + "seconds(nie:contentCreated(?content)) " + "nfo:model(nfo:equipment(?content)) " + "nmm:seasonNumber(?content) " + "nmm:episodeNumber(?content) " + "nmm:trackNumber(?content) " + "nmm:artistName(nmm:performer(?content)) " + "nie:title(?content) " + "nie:title(nmm:musicAlbum(?content)) " + "WHERE { ?file a nfo:FileDataObject. ?file nie:url ?url. ?content nie:isStoredAs ?file. "); parent_uri = nautilus_file_get_parent_uri (NAUTILUS_FILE (selection->data)); @@ -1115,9 +1117,9 @@ check_metadata_for_selection (NautilusBatchRenameDialog *dialog, selection_metadata = g_list_reverse (selection_metadata); - g_string_append (query, "} ORDER BY ASC(nie:contentCreated(?file))"); + g_string_append (query, "} ORDER BY ASC(nie:contentCreated(?content))"); - connection = tracker_sparql_connection_get (NULL, &error); + connection = tracker_sparql_connection_bus_new (TRACKER_MINER_FS_BUSNAME, NULL, NULL, &error); if (!connection) { if (error) diff --git a/src/nautilus-global-preferences.c b/src/nautilus-global-preferences.c index 0e7e011b6..685ced48c 100644 --- a/src/nautilus-global-preferences.c +++ b/src/nautilus-global-preferences.c @@ -66,5 +66,5 @@ nautilus_global_preferences_init (void) gnome_lockdown_preferences = g_settings_new ("org.gnome.desktop.lockdown"); gnome_background_preferences = g_settings_new ("org.gnome.desktop.background"); gnome_interface_preferences = g_settings_new ("org.gnome.desktop.interface"); - tracker_preferences = g_settings_new ("org.freedesktop.Tracker.Miner.Files"); + tracker_preferences = g_settings_new ("org.freedesktop.Tracker3.Miner.Files"); } diff --git a/src/nautilus-search-engine-tracker.c b/src/nautilus-search-engine-tracker.c index cff28eb71..44f4a9cf6 100644 --- a/src/nautilus-search-engine-tracker.c +++ b/src/nautilus-search-engine-tracker.c @@ -55,6 +55,8 @@ enum LAST_PROP }; +#define TRACKER_MINER_FS_BUSNAME "org.freedesktop.Tracker3.Miner.Files" + static void nautilus_search_provider_init (NautilusSearchProviderInterface *iface); G_DEFINE_TYPE_WITH_CODE (NautilusSearchEngineTracker, @@ -334,43 +336,53 @@ nautilus_search_engine_tracker_start (NautilusSearchProvider *provider) mimetypes = nautilus_query_get_mime_types (tracker->query); sparql = g_string_new ("SELECT DISTINCT" - " nie:url(?urn)" + " ?url" " xsd:double(COALESCE(?rank2, ?rank1)) AS ?rank" - " nfo:fileLastModified(?urn)" - " nfo:fileLastAccessed(?urn)"); + " nfo:fileLastModified(?file)" + " nfo:fileLastAccessed(?file)"); + + if (tracker->fts_enabled && *search_text) + { + g_string_append (sparql, " fts:snippet(?content)"); + } + + g_string_append (sparql, "FROM tracker:FileSystem "); if (tracker->fts_enabled) { - g_string_append (sparql, " fts:snippet(?urn)"); + g_string_append (sparql, "FROM tracker:Documents "); } g_string_append (sparql, "\nWHERE {" - " ?urn a nfo:FileDataObject;" + " ?file a nfo:FileDataObject;" " nfo:fileLastModified ?mtime;" " nfo:fileLastAccessed ?atime;" - " tracker:available true;" - " nie:url ?url"); + " nie:dataSource/tracker:available true;" + " nie:url ?url."); if (mimetypes->len > 0) { - g_string_append (sparql, "; nie:mimeType ?mime"); + g_string_append (sparql, + " ?content nie:isStoredAs ?file;" + " nie:mimeType ?mime"); } if (tracker->fts_enabled) { /* Use fts:match only for content search to not lose some filename results due to stop words. */ g_string_append_printf (sparql, - " {" - " ?urn fts:match '\"nie:plainTextContent\" : \"%s\"*' ." - " BIND(fts:rank(?urn) AS ?rank1) ." + " { " + " ?content nie:isStoredAs ?file ." + " ?content fts:match \"%s*\" ." + " BIND(fts:rank(?content) AS ?rank1) ." " } UNION", search_text); } g_string_append_printf (sparql, " {" - " ?urn nfo:fileName ?filename ." + " ?file nfo:fileName ?filename ." " FILTER(fn:contains(fn:lower-case(?filename), '%s')) ." " BIND(" FILENAME_RANK " AS ?rank2) ." " }", @@ -561,7 +573,7 @@ nautilus_search_engine_tracker_init (NautilusSearchEngineTracker *engine) engine->hits_pending = g_queue_new (); - engine->connection = tracker_sparql_connection_get (NULL, &error); + engine->connection = tracker_sparql_connection_bus_new (TRACKER_MINER_FS_BUSNAME, NULL, NULL, &error); if (error) { diff --git a/src/nautilus-tag-manager.c b/src/nautilus-tag-manager.c index 345ddb75b..8009e6ca9 100644 --- a/src/nautilus-tag-manager.c +++ b/src/nautilus-tag-manager.c @@ -1,6 +1,7 @@ /* nautilus-tag-manager.c * * Copyright (C) 2017 Alexandru Pandelea + * Copyright (C) 2020 Sam Thursfield * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,17 +21,30 @@ #include "nautilus-file.h" #include "nautilus-file-undo-operations.h" #include "nautilus-file-undo-manager.h" +#define DEBUG_FLAG NAUTILUS_DEBUG_TAG_MANAGER +#include "nautilus-debug.h" #include +#include "config.h" + struct _NautilusTagManager { GObject object; + gboolean tracker_ok; + TrackerSparqlConnection *local; + TrackerSparqlConnection *miner_fs; TrackerNotifier *notifier; - GError *notifier_error; - GHashTable *starred_files; + TrackerSparqlStatement *query_starred_files; + TrackerSparqlStatement *query_updated_file_url; + + /* Map of URI -> tracker ID */ + GHashTable *starred_file_uris; + /* Map of tracker ID -> URI */ + GHashTable *starred_file_ids; + GCancellable *cancellable; }; @@ -67,29 +81,30 @@ enum LAST_SIGNAL }; -#define STARRED_TAG "" +#define TRACKER_MINER_FS_BUSNAME "org.freedesktop.Tracker3.Miner.Files" + +#define QUERY_STARRED_FILES \ + "SELECT ?file_url ?content_id " \ + "{ " \ + " ?content_urn a nautilus:FileReference ; " \ + " nautilus:starred true . " \ + " SERVICE { " \ + " ?content_urn nie:isStoredAs ?file_url . " \ + " BIND (tracker:id (?content_urn) AS ?content_id) " \ + " } " \ + "}" + +#define QUERY_UPDATED_FILE_URL \ + "SELECT ?file_url EXISTS { ?content_urn nautilus:starred true } AS ?starred" \ + "{ " \ + " SERVICE { " \ + " ?content_urn nie:isStoredAs ?file_url . " \ + " FILTER (tracker:id(?content_urn) = ~id) " \ + " }" \ + "}" -static guint signals[LAST_SIGNAL]; - -static const gchar * -nautilus_tag_manager_file_with_id_changed_url (GHashTable *hash_table, - gint64 id, - const gchar *url) -{ - GHashTableIter iter; - gpointer key, value; - - g_hash_table_iter_init (&iter, hash_table); - while (g_hash_table_iter_next (&iter, &key, &value)) - { - if ((gint64) value == id && g_strcmp0 (url, key) != 0) - { - return key; - } - } - return NULL; -} +static guint signals[LAST_SIGNAL]; static void destroy_insert_task_data (gpointer data) @@ -109,7 +124,7 @@ add_selection_filter (GList *selection, NautilusFile *file; GList *l; - g_string_append (query, " FILTER(?url IN ("); + g_string_append (query, " FILTER(?file_url IN ("); for (l = selection; l != NULL; l = l->next) { @@ -119,8 +134,7 @@ add_selection_filter (GList *selection, file = l->data; uri = nautilus_file_get_uri (file); - escaped_uri = tracker_sparql_escape_string (uri); - g_string_append_printf (query, "'%s'", escaped_uri); + g_string_append_printf (query, "<%s>", uri); if (l->next != NULL) { @@ -134,29 +148,24 @@ add_selection_filter (GList *selection, } static void -start_query_or_update (GString *query, - GAsyncReadyCallback callback, - gpointer user_data, - gboolean is_query, - GCancellable *cancellable) +start_query_or_update (TrackerSparqlConnection *local, + GString *query, + GAsyncReadyCallback callback, + gpointer user_data, + gboolean is_query, + GCancellable *cancellable) { g_autoptr (GError) error = NULL; - TrackerSparqlConnection *connection; - connection = tracker_sparql_connection_get (cancellable, &error); - if (!connection) + if (!local) { - if (error) - { - g_warning ("Error on getting connection: %s", error->message); - } - + g_message ("nautilus-tag-manager: No Tracker connection"); return; } if (is_query) { - tracker_sparql_connection_query_async (connection, + tracker_sparql_connection_query_async (local, query->str, cancellable, callback, @@ -164,65 +173,12 @@ start_query_or_update (GString *query, } else { - tracker_sparql_connection_update_async (connection, + tracker_sparql_connection_update_async (local, query->str, - G_PRIORITY_DEFAULT, cancellable, callback, user_data); } - - g_object_unref (connection); -} - -static void -on_query_callback (GObject *object, - GAsyncResult *result, - gpointer user_data, - GAsyncReadyCallback callback, - OperationType op_type, - GCancellable *cancellable) -{ - TrackerSparqlCursor *cursor; - g_autoptr (GError) error = NULL; - TrackerSparqlConnection *connection; - GTask *task; - - task = user_data; - - connection = TRACKER_SPARQL_CONNECTION (object); - - cursor = tracker_sparql_connection_query_finish (connection, - result, - &error); - - if (error != NULL) - { - if (error->code != G_IO_ERROR_CANCELLED) - { - if (op_type == GET_STARRED_FILES) - { - g_warning ("Error on getting starred files: %s", error->message); - } - else if (op_type == GET_IDS_FOR_URLS) - { - g_warning ("Error on getting id for url: %s", error->message); - g_task_return_pointer (task, g_task_get_task_data (task), NULL); - g_object_unref (task); - } - else - { - g_warning ("Error on getting query callback: %s", error->message); - } - } - } - else - { - tracker_sparql_cursor_next_async (cursor, - cancellable, - callback, - user_data); - } } static void @@ -230,10 +186,10 @@ on_update_callback (GObject *object, GAsyncResult *result, gpointer user_data) { - TrackerSparqlConnection *connection; + TrackerSparqlConnection *local; GError *error; UpdateData *data; - gint64 *id; + gint64 *new_id; GList *l; gchar *uri; @@ -241,9 +197,9 @@ on_update_callback (GObject *object, error = NULL; - connection = TRACKER_SPARQL_CONNECTION (object); + local = TRACKER_SPARQL_CONNECTION (object); - tracker_sparql_connection_update_finish (connection, result, &error); + tracker_sparql_connection_update_finish (local, result, &error); if (error == NULL) { @@ -255,17 +211,29 @@ on_update_callback (GObject *object, { if (g_hash_table_contains (data->ids, uri)) { - id = g_new0 (gint64, 1); + new_id = g_new0 (gint64, 1); - *id = (gint64) g_hash_table_lookup (data->ids, uri); - g_hash_table_insert (data->tag_manager->starred_files, + *new_id = GPOINTER_TO_INT (g_hash_table_lookup (data->ids, uri)); + g_hash_table_insert (data->tag_manager->starred_file_uris, nautilus_file_get_uri (NAUTILUS_FILE (l->data)), - id); + new_id); + g_hash_table_insert (data->tag_manager->starred_file_ids, + new_id, + nautilus_file_get_uri (NAUTILUS_FILE (l->data))); + } + else + { + g_message ("Ignoring star request for %s as we didn't get the resource ID from Tracker.", uri); } } else { - g_hash_table_remove (data->tag_manager->starred_files, uri); + new_id = g_hash_table_lookup (data->tag_manager->starred_file_uris, uri); + if (new_id) + { + g_hash_table_remove (data->tag_manager->starred_file_uris, uri); + g_hash_table_remove (data->tag_manager->starred_file_ids, new_id); + } } g_free (uri); @@ -292,9 +260,9 @@ on_update_callback (GObject *object, } else { + g_warning ("error updating tags: %s", error->message); g_task_return_error (data->task, error); g_object_unref (data->task); - g_warning ("error updating tags: %s", error->message); } if (data->ids) @@ -351,7 +319,7 @@ get_query_status (TrackerSparqlCursor *cursor, GList * nautilus_tag_manager_get_starred_files (NautilusTagManager *self) { - return g_hash_table_get_keys (self->starred_files); + return g_hash_table_get_keys (self->starred_file_uris); } static void @@ -382,9 +350,8 @@ on_get_starred_files_cursor_callback (GObject *object, url = tracker_sparql_cursor_get_string (cursor, 0, NULL); *id = tracker_sparql_cursor_get_integer (cursor, 1); - g_hash_table_insert (self->starred_files, - g_strdup (url), - id); + g_hash_table_insert (self->starred_file_uris, g_strdup (url), id); + g_hash_table_insert (self->starred_file_ids, id, g_strdup (url)); file = nautilus_file_get_by_uri (url); changed_files = g_list_prepend (NULL, file); @@ -404,36 +371,50 @@ on_get_starred_files_query_callback (GObject *object, GAsyncResult *result, gpointer user_data) { + TrackerSparqlCursor *cursor; + g_autoptr (GError) error = NULL; + TrackerSparqlStatement *statement; NautilusTagManager *self; self = NAUTILUS_TAG_MANAGER (user_data); + statement = TRACKER_SPARQL_STATEMENT (object); + + cursor = tracker_sparql_statement_execute_finish (statement, + result, + &error); - on_query_callback (object, - result, - user_data, - on_get_starred_files_cursor_callback, - GET_STARRED_FILES, - self->cancellable); + if (error != NULL) + { + if (error->code != G_IO_ERROR_CANCELLED) + { + g_warning ("Error on getting starred files: %s", error->message); + } + } + else + { + tracker_sparql_cursor_next_async (cursor, + self->cancellable, + on_get_starred_files_cursor_callback, + user_data); + } } static void nautilus_tag_manager_query_starred_files (NautilusTagManager *self, GCancellable *cancellable) { - GString *query; + if (!self->tracker_ok) + { + g_message ("nautilus-tag-manager: No Tracker connection"); + return; + } self->cancellable = cancellable; - query = g_string_new ("SELECT ?url tracker:id(?urn) " - "WHERE { ?urn nie:url ?url ; nao:hasTag " STARRED_TAG "}"); - - start_query_or_update (query, - on_get_starred_files_query_callback, - self, - TRUE, - cancellable); - - g_string_free (query, TRUE); + tracker_sparql_statement_execute_async (self->query_starred_files, + cancellable, + on_get_starred_files_query_callback, + self); } static gpointer @@ -452,12 +433,17 @@ nautilus_tag_manager_delete_tag (NautilusTagManager *self, GString *query) { g_string_append (query, - "DELETE { ?urn nao:hasTag " STARRED_TAG " }" - "WHERE { ?urn a nfo:FileDataObject ; nie:url ?url ."); + "DELETE { " + " ?content_urn a nautilus:FileReference ; " + " nautilus:starred true . " + "} " + "WHERE { " + " SERVICE { " + " ?content_urn nie:isStoredAs ?file_url . "); query = add_selection_filter (selection, query); - g_string_append (query, "}\n"); + g_string_append (query, " } }\n"); return query; } @@ -468,13 +454,16 @@ nautilus_tag_manager_insert_tag (NautilusTagManager *self, GString *query) { g_string_append (query, - "INSERT DATA { " STARRED_TAG " a nao:Tag .}\n" - "INSERT { ?urn nao:hasTag " STARRED_TAG " }" - "WHERE { ?urn a nfo:FileDataObject ; nie:url ?url ."); + "INSERT { " + " ?content_urn a nautilus:FileReference . " + " ?content_urn nautilus:starred true . " + "} WHERE { " + " SERVICE { " + " ?content_urn nie:isStoredAs ?file_url . "); query = add_selection_filter (selection, query); - g_string_append (query, "}\n"); + g_string_append (query, "} }\n"); return query; } @@ -483,7 +472,7 @@ gboolean nautilus_tag_manager_file_is_starred (NautilusTagManager *self, const gchar *file_name) { - return g_hash_table_contains (self->starred_files, file_name); + return g_hash_table_contains (self->starred_file_uris, file_name); } static void @@ -546,32 +535,53 @@ on_get_file_ids_for_urls_query_callback (GObject *object, GAsyncResult *result, gpointer user_data) { + TrackerSparqlCursor *cursor; + g_autoptr (GError) error = NULL; + TrackerSparqlConnection *local; GTask *task; + local = TRACKER_SPARQL_CONNECTION (object); task = user_data; - on_query_callback (object, - result, - user_data, - on_get_file_ids_for_urls_cursor_callback, - GET_IDS_FOR_URLS, - g_task_get_cancellable (task)); + cursor = tracker_sparql_connection_query_finish (local, result, &error); + + if (error != NULL) + { + if (error->code != G_IO_ERROR_CANCELLED) + { + g_warning ("Error on getting id for url: %s", error->message); + g_task_return_pointer (task, g_task_get_task_data (task), NULL); + g_object_unref (task); + } + } + else + { + tracker_sparql_cursor_next_async (cursor, + g_task_get_cancellable (task), + on_get_file_ids_for_urls_cursor_callback, + user_data); + } } static void -nautilus_tag_manager_get_file_ids_for_urls (GObject *object, - GList *selection, - GTask *task) +nautilus_tag_manager_get_file_ids_for_urls (NautilusTagManager *self, + GList *selection, + GTask *task) { GString *query; - query = g_string_new ("SELECT ?url tracker:id(?urn) WHERE { ?urn nie:url ?url; ."); + query = g_string_new ("SELECT ?file_url ?content_id " + "WHERE { " + " SERVICE { " + " ?content_urn nie:isStoredAs ?file_url . " + " BIND (tracker:id (?content_urn) AS ?content_id) "); query = add_selection_filter (selection, query); - g_string_append (query, "}\n"); + g_string_append (query, "} }\n"); - start_query_or_update (query, + start_query_or_update (self->local, + query, on_get_file_ids_for_urls_query_callback, task, TRUE, @@ -616,7 +626,8 @@ on_star_files_callback (GObject *object, */ destroy_insert_task_data (data); - start_query_or_update (query, + start_query_or_update (self->local, + query, on_update_callback, update_data, FALSE, @@ -645,12 +656,14 @@ nautilus_tag_manager_star_files (NautilusTagManager *self, data->object = object; data->cancellable = cancellable; + DEBUG ("Starring %i files", g_list_length (selection)); + task = g_task_new (self, cancellable, on_star_files_callback, NULL); g_task_set_task_data (task, data, NULL); - nautilus_tag_manager_get_file_ids_for_urls (G_OBJECT (self), selection, task); + nautilus_tag_manager_get_file_ids_for_urls (self, selection, task); } void @@ -664,6 +677,8 @@ nautilus_tag_manager_unstar_files (NautilusTagManager *self, GTask *task; UpdateData *update_data; + DEBUG ("Unstarring %i files", g_list_length (selection)); + task = g_task_new (object, cancellable, callback, NULL); query = g_string_new (""); @@ -678,7 +693,8 @@ nautilus_tag_manager_unstar_files (NautilusTagManager *self, update_data->selection = nautilus_file_list_copy (selection); update_data->star = FALSE; - start_query_or_update (query, + start_query_or_update (self->local, + query, on_update_callback, update_data, FALSE, @@ -689,22 +705,23 @@ nautilus_tag_manager_unstar_files (NautilusTagManager *self, static void on_tracker_notifier_events (TrackerNotifier *notifier, + gchar *service, + gchar *graph, GPtrArray *events, gpointer user_data) { TrackerNotifierEvent *event; NautilusTagManager *self; int i; - const gchar *location_uri; - const gchar *new_location_uri; + const gchar *file_url; + const gchar *new_file_url; GError *error = NULL; - TrackerSparqlConnection *connection; TrackerSparqlCursor *cursor; - GString *query; - gboolean query_has_results; - gint64 *id; + gboolean query_has_results = FALSE; + gboolean is_starred; + gint64 id, *new_id; GList *changed_files; - NautilusFile *file; + NautilusFile *changed_file; self = NAUTILUS_TAG_MANAGER (user_data); @@ -712,101 +729,95 @@ on_tracker_notifier_events (TrackerNotifier *notifier, { event = g_ptr_array_index (events, i); - location_uri = tracker_notifier_event_get_location (event); - - query = g_string_new (""); - g_string_append_printf (query, - "SELECT ?url WHERE { ?urn nie:url ?url; nao:hasTag " STARRED_TAG " . FILTER (tracker:id(?urn) = %" G_GINT64_FORMAT ")}", - tracker_notifier_event_get_id (event)); - - /* check if the file changed it's location and update hash table if so */ - new_location_uri = nautilus_tag_manager_file_with_id_changed_url (self->starred_files, - tracker_notifier_event_get_id (event), - location_uri); - if (new_location_uri) - { - id = g_new0 (gint64, 1); - *id = tracker_notifier_event_get_id (event); - - g_hash_table_remove (self->starred_files, new_location_uri); - g_hash_table_insert (self->starred_files, - g_strdup (location_uri), - id); + id = tracker_notifier_event_get_id (event); + file_url = g_hash_table_lookup (self->starred_file_ids, &id); + changed_file = NULL; - file = nautilus_file_get_by_uri (location_uri); - changed_files = g_list_prepend (NULL, file); + DEBUG ("Got event for tracker resource id %" G_GINT64_FORMAT, id); - g_signal_emit_by_name (self, "starred-changed", changed_files); + tracker_sparql_statement_bind_int (self->query_updated_file_url, "id", tracker_notifier_event_get_id (event)); + cursor = tracker_sparql_statement_execute (self->query_updated_file_url, + NULL, + &error); - nautilus_file_list_free (changed_files); + if (cursor) + { + query_has_results = tracker_sparql_cursor_next (cursor, NULL, &error); } - connection = tracker_sparql_connection_get (NULL, &error); - - if (!connection) + if (error || !cursor) { - g_printerr ("Couldn't obtain a direct connection to the Tracker store: %s", - error ? error->message : "unknown error"); + g_warning ("Couldn't query the Tracker Store: '%s'", error ? error->message : "(null error)"); g_clear_error (&error); - return; } - cursor = tracker_sparql_connection_query (connection, - query->str, - NULL, - &error); - - if (error) + if (!query_has_results) { - g_printerr ("Couldn't query the Tracker Store: '%s'", error->message); + if (g_hash_table_contains (self->starred_file_ids, &id)) + { + /* The file was deleted from the filesystem or is no longer indexed by Tracker. */ + file_url = g_hash_table_lookup (self->starred_file_ids, &id); + DEBUG ("Removing %s from starred files list, as id %" G_GINT64_FORMAT " no longer present in Tracker index. ", file_url, id); - g_clear_error (&error); + changed_file = nautilus_file_get_by_uri (file_url); - return; + g_hash_table_remove (self->starred_file_ids, &id); + g_hash_table_remove (self->starred_file_uris, file_url); + } } - - if (cursor) + else { - query_has_results = tracker_sparql_cursor_next (cursor, NULL, &error); + new_file_url = tracker_sparql_cursor_get_string (cursor, 0, NULL); + is_starred = tracker_sparql_cursor_get_boolean (cursor, 1); - /* if no results are found, then the file isn't marked as starred. - * If needed, update the hashtable. - */ - if (!query_has_results && location_uri && g_hash_table_contains (self->starred_files, location_uri)) + if (g_hash_table_contains (self->starred_file_ids, &id)) { - g_hash_table_remove (self->starred_files, location_uri); + if (is_starred && strcmp (file_url, new_file_url) != 0) + { + new_id = g_new0 (gint64, 1); + *new_id = id; - file = nautilus_file_get_by_uri (location_uri); - changed_files = g_list_prepend (NULL, file); + DEBUG ("Starred file changed URI from %s to %s.", file_url, new_file_url); + g_hash_table_remove (self->starred_file_uris, file_url); - g_signal_emit_by_name (self, "starred-changed", changed_files); + g_hash_table_insert (self->starred_file_ids, new_id, g_strdup (new_file_url)); + g_hash_table_insert (self->starred_file_uris, g_strdup (new_file_url), new_id); - nautilus_file_list_free (changed_files); + changed_file = nautilus_file_get_by_uri (new_file_url); + } + else if (!is_starred) + { + DEBUG ("File is no longer starred: %s", file_url); + g_hash_table_remove (self->starred_file_uris, file_url); + g_hash_table_remove (self->starred_file_ids, &id); + + changed_file = nautilus_file_get_by_uri (new_file_url); + } } - else if (query_has_results && location_uri && !g_hash_table_contains (self->starred_files, location_uri)) + else if (is_starred) { - id = g_new0 (gint64, 1); - *id = tracker_notifier_event_get_id (event); + DEBUG ("File is now starred: %s", new_file_url); + new_id = g_new0 (gint64, 1); + *new_id = id; - g_hash_table_insert (self->starred_files, - g_strdup (location_uri), - id); + g_hash_table_insert (self->starred_file_ids, new_id, g_strdup (new_file_url)); + g_hash_table_insert (self->starred_file_uris, g_strdup (new_file_url), new_id); - file = nautilus_file_get_by_uri (location_uri); - changed_files = g_list_prepend (NULL, file); + changed_file = nautilus_file_get_by_uri (new_file_url); + } + } - g_signal_emit_by_name (self, "starred-changed", changed_files); + if (changed_file) + { + changed_files = g_list_prepend (NULL, changed_file); - nautilus_file_list_free (changed_files); - } + g_signal_emit_by_name (self, "starred-changed", changed_files); - g_object_unref (cursor); + nautilus_file_list_free (changed_files); } - g_object_unref (connection); - - g_string_free (query, TRUE); + g_object_unref (cursor); } } @@ -825,8 +836,13 @@ nautilus_tag_manager_finalize (GObject *object) } g_clear_object (&self->notifier); + g_clear_object (&self->local); + g_clear_object (&self->miner_fs); + g_clear_object (&self->query_updated_file_url); + g_clear_object (&self->query_starred_files); - g_hash_table_destroy (self->starred_files); + g_hash_table_destroy (self->starred_file_ids); + g_hash_table_destroy (self->starred_file_uris); G_OBJECT_CLASS (nautilus_tag_manager_parent_class)->finalize (object); } @@ -868,30 +884,109 @@ nautilus_tag_manager_get (void) return tag_manager; } +static gboolean +setup_tracker_connections (NautilusTagManager *self, + GCancellable *cancellable, + GError **error) +{ + const gchar *datadir; + g_autofree gchar *store_path = NULL; + g_autofree gchar *ontology_path = NULL; + g_autoptr (GFile) store = NULL; + g_autoptr (GFile) ontology = NULL; + + /* Connect to private database to store nautilus:starred property. */ + + datadir = NAUTILUS_DATADIR; + + store_path = g_build_filename (g_get_user_data_dir (), "nautilus", "tags", NULL); + ontology_path = g_build_filename (datadir, "ontology", NULL); + + store = g_file_new_for_path (store_path); + ontology = g_file_new_for_path (ontology_path); + + self->local = tracker_sparql_connection_new (TRACKER_SPARQL_CONNECTION_FLAGS_NONE, + store, + ontology, + cancellable, + error); + + if (*error) + { + return FALSE; + } + + /* Connect to Tracker filesystem index to follow file renames. */ + self->miner_fs = tracker_sparql_connection_bus_new (TRACKER_MINER_FS_BUSNAME, + NULL, + NULL, + error); + + if (*error) + { + return FALSE; + } + + /* Prepare reusable queries. */ + self->query_updated_file_url = tracker_sparql_connection_query_statement (self->local, + QUERY_UPDATED_FILE_URL, + cancellable, + error); + + if (*error) + { + return FALSE; + } + + self->query_starred_files = tracker_sparql_connection_query_statement (self->local, + QUERY_STARRED_FILES, + cancellable, + error); + + if (*error) + { + return FALSE; + } + + return TRUE; +} + +/* Initialize the tag mananger. */ void nautilus_tag_manager_set_cancellable (NautilusTagManager *self, GCancellable *cancellable) { - nautilus_tag_manager_query_starred_files (self, cancellable); + g_autoptr (GError) error = NULL; - self->notifier = tracker_notifier_new (NULL, - TRACKER_NOTIFIER_FLAG_QUERY_LOCATION, - cancellable, - &self->notifier_error); - if (self->notifier != NULL) + self->tracker_ok = setup_tracker_connections (self, cancellable, &error); + + if (error) { - g_signal_connect (self->notifier, - "events", - G_CALLBACK (on_tracker_notifier_events), - self); + g_warning ("Unable to initialize tag manager: %s", error->message); + return; } + + self->notifier = tracker_sparql_connection_create_notifier (self->miner_fs); + + nautilus_tag_manager_query_starred_files (self, cancellable); + + g_signal_connect (self->notifier, + "events", + G_CALLBACK (on_tracker_notifier_events), + self); } static void nautilus_tag_manager_init (NautilusTagManager *self) { - self->starred_files = g_hash_table_new_full (g_str_hash, - g_str_equal, - (GDestroyNotify) g_free, - (GDestroyNotify) g_free); + self->starred_file_uris = g_hash_table_new_full (g_str_hash, + g_str_equal, + (GDestroyNotify) g_free, + /* values are keys in the other hash table + * and are freed there */ + NULL); + self->starred_file_ids = g_hash_table_new_full (g_int_hash, + g_int_equal, + (GDestroyNotify) g_free, + (GDestroyNotify) g_free); } diff --git a/test/automated/displayless/meson.build b/test/automated/displayless/meson.build index eecc1f406..64b23bd2a 100644 --- a/test/automated/displayless/meson.build +++ b/test/automated/displayless/meson.build @@ -1,3 +1,7 @@ +trackertestutils = dependency('tracker-testutils-3.0') + +tracker_sandbox = find_program(trackertestutils.get_pkgconfig_variable('command')) + tests = [ ['test-file-utilities-get-common-filename-prefix', [ 'test-file-utilities-get-common-filename-prefix.c' @@ -23,9 +27,6 @@ tests = [ ['test-nautilus-search-engine-model', [ 'test-nautilus-search-engine-model.c' ]], - ['test-nautilus-search-engine-tracker', [ - 'test-nautilus-search-engine-tracker.c' - ]], ['test-file-operations-copy-files', [ 'test-file-operations-copy-files.c' ]], @@ -34,6 +35,12 @@ tests = [ ]] ] +tracker_tests = [ + ['test-nautilus-search-engine-tracker', [ + 'test-nautilus-search-engine-tracker.c', + ]], +] + foreach t: tests test( t[0], @@ -46,3 +53,22 @@ foreach t: tests timeout: 480 ) endforeach + + + +# Tests that read and write from the Tracker index are run using 'tracker-sandbox' +# script to use a temporary instance of tracker-miner-fs instead of the session one. +foreach t: tracker_tests + test_exe = executable(t[0], t[1], files('test-utilities.c'), dependencies: libnautilus_dep) + test( + t[0], + tracker_sandbox, + args: ['--store-tmpdir', '--index-recursive-tmpdir', test_exe], + env: [ + test_env, + 'G_TEST_BUILDDIR=@0@'.format(meson.current_build_dir()), + 'G_TEST_SRCDIR=@0@'.format(meson.current_source_dir()) + ], + timeout: 480 + ) +endforeach diff --git a/test/automated/displayless/test-nautilus-search-engine-tracker.c b/test/automated/displayless/test-nautilus-search-engine-tracker.c index fdbca124f..0c96ef76d 100644 --- a/test/automated/displayless/test-nautilus-search-engine-tracker.c +++ b/test/automated/displayless/test-nautilus-search-engine-tracker.c @@ -1,7 +1,106 @@ #include "test-utilities.h" +/* Time in seconds we allow for Tracker Miners to index the file */ +#define TRACKER_MINERS_AWAIT_TIMEOUT 1000 + static guint total_hits = 0; +typedef struct +{ + GMainLoop *main_loop; + gchar *uri; + gboolean created; +} TrackerAwaitFileData; + +static TrackerAwaitFileData * +tracker_await_file_data_new (const char *uri, + GMainLoop *main_loop) +{ + TrackerAwaitFileData *data; + + data = g_slice_new0 (TrackerAwaitFileData); + data->uri = g_strdup (uri); + data->main_loop = g_main_loop_ref (main_loop); + + return data; +} + +static void +tracker_await_file_data_free (TrackerAwaitFileData *data) +{ + g_free (data->uri); + g_main_loop_unref (data->main_loop); + g_slice_free (TrackerAwaitFileData, data); +} + +static gboolean timeout_cb (gpointer user_data) +{ + TrackerAwaitFileData *data = user_data; + g_error ("Timeout waiting for %s to be indexed by Tracker.", data->uri); + return G_SOURCE_REMOVE; +} + +static void +tracker_events_cb (TrackerNotifier *self, + gchar *service, + gchar *graph, + GPtrArray *events, + gpointer user_data) +{ + TrackerAwaitFileData *data = user_data; + int i; + + for (i = 0; i < events->len; i++) + { + TrackerNotifierEvent *event = g_ptr_array_index (events, i); + + if (tracker_notifier_event_get_event_type (event) == TRACKER_NOTIFIER_EVENT_CREATE) + { + const gchar *urn = tracker_notifier_event_get_urn (event); + g_debug ("Got CREATED event for %s", urn); + if (strcmp (urn, data->uri) == 0) + { + data->created = TRUE; + g_main_loop_quit (data->main_loop); + } + } + } +} + +/* Create data that the Tracker indexer will find, and wait for the database to be updated. */ +static void +create_test_data (TrackerSparqlConnection *connection, + const gchar *indexed_tmpdir) +{ + g_autoptr (GFile) test_file = NULL; + g_autoptr (GMainLoop) main_loop = NULL; + g_autoptr (GError) error = NULL; + g_autoptr (TrackerNotifier) notifier = NULL; + TrackerAwaitFileData *await_data; + gulong signal_id, timeout_id; + + test_file = g_file_new_build_filename (indexed_tmpdir, "target_file.txt", NULL); + + main_loop = g_main_loop_new (NULL, 0); + await_data = tracker_await_file_data_new (g_file_get_uri (test_file), main_loop); + + notifier = tracker_sparql_connection_create_notifier (connection); + + signal_id = g_signal_connect (notifier, "events", G_CALLBACK (tracker_events_cb), await_data); + timeout_id = g_timeout_add_seconds (TRACKER_MINERS_AWAIT_TIMEOUT, timeout_cb, await_data); + + g_file_set_contents (g_file_peek_path (test_file), "Please show me in the search results", -1, &error); + g_assert_no_error (error); + + g_main_loop_run (main_loop); + + g_assert (await_data->created); + g_source_remove (timeout_id); + g_clear_signal_handler (&signal_id, notifier); + + tracker_await_file_data_free (await_data); +} + static void hits_added_cb (NautilusSearchEngine *engine, GSList *hits) @@ -19,21 +118,10 @@ finished_cb (NautilusSearchEngine *engine, NautilusSearchProviderStatus status, gpointer user_data) { - TrackerSparqlConnection *connection; - g_autofree gchar *sparql_query = NULL; - nautilus_search_provider_stop (NAUTILUS_SEARCH_PROVIDER (engine)); g_print ("\nNautilus search engine tracker finished!\n"); - connection = tracker_sparql_connection_get (NULL, NULL); - sparql_query = g_strdup_printf ("DELETE WHERE { ?p ?o }"); - tracker_sparql_connection_update (connection, - sparql_query, - 0, - NULL, - NULL); - g_main_loop_quit (user_data); } @@ -42,14 +130,24 @@ main (int argc, char *argv[]) { g_autoptr (GMainLoop) loop = NULL; + g_autoptr (TrackerSparqlConnection) connection = NULL; NautilusSearchEngine *engine; g_autoptr (NautilusDirectory) directory = NULL; g_autoptr (NautilusQuery) query = NULL; g_autoptr (GFile) location = NULL; - TrackerSparqlConnection *connection; - g_autofree gchar *sparql_query = NULL; + g_autoptr (GError) error = NULL; + const gchar *indexed_tmpdir; + + indexed_tmpdir = g_getenv ("TRACKER_INDEXED_TMPDIR"); + if (!indexed_tmpdir) + { + g_error ("This test must be inside the `tracker-sandbox` script " + "to ensure a private Tracker indexer daemon is used."); + } - connection = tracker_sparql_connection_get (NULL, NULL); + connection = tracker_sparql_connection_bus_new ("org.freedesktop.Tracker3.Miner.Files", NULL, NULL, &error); + + g_assert_no_error (error); loop = g_main_loop_new (NULL, FALSE); @@ -60,6 +158,8 @@ main (int argc, */ nautilus_global_preferences_init (); + create_test_data (connection, indexed_tmpdir); + engine = nautilus_search_engine_new (); g_signal_connect (engine, "hits-added", G_CALLBACK (hits_added_cb), NULL); @@ -70,37 +170,10 @@ main (int argc, nautilus_query_set_text (query, "target"); nautilus_search_provider_set_query (NAUTILUS_SEARCH_PROVIDER (engine), query); - location = g_file_new_for_path (g_get_tmp_dir ()); + location = g_file_new_for_path (indexed_tmpdir); directory = nautilus_directory_get (location); nautilus_query_set_location (query, location); - /* This sparql query with the update call create a virtual file - * in tracker, so it sees a file named "target_file" in /tmp. - * The file's MIME type is text/plain and the name tracker is - * using for search is "target". For the engine tracker to hit, - * we also need to set the last time the file was accessed and modified, - * which we set to 2001-01-01, at 00:00:01 (the date needs to be a full - * ISO 8601 date string) and tracker:available be set to true (in order - * for the file to be accessible). - */ - - sparql_query = g_strdup_printf ("INSERT DATA {\n "); - sparql_query = g_strconcat (sparql_query, "a nfo:FileDataObject ;", NULL); - sparql_query = g_strconcat (sparql_query, "\na nie:InformationElement ;", NULL); - sparql_query = g_strconcat (sparql_query, "\nnie:url 'file:///tmp/target_file';", NULL); - sparql_query = g_strconcat (sparql_query, "\nnie:mimeType 'text/plain';", NULL); - sparql_query = g_strconcat (sparql_query, "\nnfo:fileName 'target';", NULL); - sparql_query = g_strconcat (sparql_query, "\nnfo:fileLastModified '2001-01-01T00:00:01Z';", NULL); - sparql_query = g_strconcat (sparql_query, "\nnfo:fileLastAccessed '2001-01-01T00:00:01Z';", NULL); - sparql_query = g_strconcat (sparql_query, "\ntracker:available true", NULL); - sparql_query = g_strconcat (sparql_query, ".\n}\n", NULL); - - tracker_sparql_connection_update (connection, - sparql_query, - 0, - NULL, - NULL); - nautilus_search_engine_start_by_target (NAUTILUS_SEARCH_PROVIDER (engine), NAUTILUS_SEARCH_ENGINE_TRACKER_ENGINE); -- cgit v1.2.1