From fd5a83cce02d17e0a99f9ccec2aa0e154f554c78 Mon Sep 17 00:00:00 2001 From: Carlos Garnacho Date: Wed, 12 Feb 2014 23:45:11 +0100 Subject: libtracker-miner: Detach early the processed file in TrackerFileNotifier This is so priv->pending_index_roots can be prepended/appended without confusing ongoing crawling on the current folder. --- src/libtracker-miner/tracker-file-notifier.c | 80 ++++++++++++++-------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/src/libtracker-miner/tracker-file-notifier.c b/src/libtracker-miner/tracker-file-notifier.c index 9ef01aaba..e14657306 100644 --- a/src/libtracker-miner/tracker-file-notifier.c +++ b/src/libtracker-miner/tracker-file-notifier.c @@ -69,6 +69,7 @@ typedef struct { * trees to get data from */ GList *pending_index_roots; + GFile *current_index_root; guint stopped : 1; } TrackerFileNotifierPrivate; @@ -160,9 +161,8 @@ crawler_check_directory_cb (TrackerCrawler *crawler, * currently processed, bypass it, it will be processed * when the time arrives. */ - if (canonical && - root == canonical && - root != priv->pending_index_roots->data) { + if (canonical && root == canonical && + root != priv->current_index_root) { return FALSE; } @@ -254,6 +254,22 @@ file_notifier_traverse_tree_foreach (GFile *file, return FALSE; } +static gboolean +notifier_check_next_root (TrackerFileNotifier *notifier) +{ + TrackerFileNotifierPrivate *priv; + + priv = notifier->priv; + + if (priv->pending_index_roots) { + return crawl_directories_start (notifier); + } else { + priv->current_index_root = NULL; + g_signal_emit (notifier, signals[FINISHED], 0); + return FALSE; + } +} + static void file_notifier_traverse_tree (TrackerFileNotifier *notifier) { @@ -262,7 +278,7 @@ file_notifier_traverse_tree (TrackerFileNotifier *notifier) TrackerDirectoryFlags flags; priv = notifier->priv; - current_root = priv->pending_index_roots->data; + current_root = priv->current_index_root; config_root = tracker_indexing_tree_get_root (priv->indexing_tree, current_root, &flags); @@ -288,16 +304,7 @@ file_notifier_traverse_tree (TrackerFileNotifier *notifier) tracker_info (" Notified files after %2.2f seconds", g_timer_elapsed (priv->timer, NULL)); - /* We've finished crawling/querying on the first element - * of the pending list, continue onto the next */ - priv->pending_index_roots = g_list_delete_link (priv->pending_index_roots, - priv->pending_index_roots); - - if (priv->pending_index_roots) { - crawl_directories_start (notifier); - } else { - g_signal_emit (notifier, signals[FINISHED], 0); - } + notifier_check_next_root (notifier); } static gboolean @@ -408,9 +415,8 @@ sparql_file_query_populate (TrackerFileNotifier *notifier, canonical = tracker_file_system_peek_file (priv->file_system, file); root = tracker_indexing_tree_get_root (priv->indexing_tree, file, NULL); - if (canonical && - root == file && - root != priv->pending_index_roots->data) { + if (canonical && root == file && + root != priv->current_index_root) { g_object_unref (file); continue; } @@ -469,7 +475,7 @@ sparql_query_cb (GObject *object, /* Mark the directory root as queried */ tracker_file_system_set_property (priv->file_system, - priv->pending_index_roots->data, + priv->current_index_root, quark_property_queried, GUINT_TO_POINTER (TRUE)); @@ -478,7 +484,7 @@ sparql_query_cb (GObject *object, /* If it's also been crawled, finish operation */ if (tracker_file_system_get_property (priv->file_system, - priv->pending_index_roots->data, + priv->current_index_root, quark_property_crawled)) { file_notifier_traverse_tree (notifier); } @@ -565,7 +571,9 @@ crawl_directories_start (TrackerFileNotifier *notifier) } while (priv->pending_index_roots) { - directory = priv->pending_index_roots->data; + directory = priv->current_index_root = priv->pending_index_roots->data; + priv->pending_index_roots = g_list_delete_link (priv->pending_index_roots, + priv->pending_index_roots); tracker_indexing_tree_get_root (priv->indexing_tree, directory, @@ -614,10 +622,6 @@ crawl_directories_start (TrackerFileNotifier *notifier) g_signal_emit (notifier, signals[DIRECTORY_FINISHED], 0, directory, 0, 0, 0, 0); } - - /* Remove index root and try the next one */ - priv->pending_index_roots = g_list_delete_link (priv->pending_index_roots, - priv->pending_index_roots); } g_signal_emit (notifier, signals[FINISHED], 0); @@ -640,7 +644,7 @@ crawler_finished_cb (TrackerCrawler *crawler, if (!was_interrupted) { GFile *directory; - directory = priv->pending_index_roots->data; + directory = priv->current_index_root; /* Mark the directory root as crawled */ tracker_file_system_set_property (priv->file_system, directory, @@ -1018,7 +1022,8 @@ indexing_tree_directory_added (TrackerIndexingTree *indexing_tree, if (!g_list_find (priv->pending_index_roots, directory)) { priv->pending_index_roots = g_list_append (priv->pending_index_roots, - directory); + directory); + if (start_crawler) { crawl_directories_start (notifier); } @@ -1080,22 +1085,15 @@ indexing_tree_directory_removed (TrackerIndexingTree *indexing_tree, g_signal_emit (notifier, signals[FILE_DELETED], 0, directory); } - if (priv->pending_index_roots) { - gboolean start_crawler = FALSE; - - if (directory == priv->pending_index_roots->data) { - /* Directory being currently processed */ - tracker_crawler_stop (priv->crawler); - g_cancellable_cancel (priv->cancellable); - start_crawler = TRUE; - } + priv->pending_index_roots = g_list_remove_all (priv->pending_index_roots, + directory); - priv->pending_index_roots = g_list_remove_all (priv->pending_index_roots, - directory); + if (directory == priv->current_index_root) { + /* Directory being currently processed */ + tracker_crawler_stop (priv->crawler); + g_cancellable_cancel (priv->cancellable); - if (start_crawler && priv->pending_index_roots != NULL) { - crawl_directories_start (notifier); - } + notifier_check_next_root (notifier); } /* Remove monitors if any */ @@ -1379,7 +1377,7 @@ tracker_file_notifier_is_active (TrackerFileNotifier *notifier) g_return_val_if_fail (TRACKER_IS_FILE_NOTIFIER (notifier), FALSE); priv = notifier->priv; - return priv->pending_index_roots != NULL; + return priv->pending_index_roots || priv->current_index_root; } const gchar * -- cgit v1.2.1 From 24a4c24e65da41aab7388adf60ce90a8cb0663ab Mon Sep 17 00:00:00 2001 From: Carlos Garnacho Date: Thu, 13 Feb 2014 00:03:09 +0100 Subject: libtracker-miner: Add/handle a "priority" TrackerDirectoryFlag If that flag is set on a file in the TrackerIndexingTree, the TrackerFileNotifier will append these when processing, so these directories will be crawled and inspected as soon as possible --- src/libtracker-miner/tracker-file-notifier.c | 60 ++++++++++++++-------------- src/libtracker-miner/tracker-miner-enums.h | 3 +- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/src/libtracker-miner/tracker-file-notifier.c b/src/libtracker-miner/tracker-file-notifier.c index e14657306..7de13f5de 100644 --- a/src/libtracker-miner/tracker-file-notifier.c +++ b/src/libtracker-miner/tracker-file-notifier.c @@ -660,6 +660,20 @@ crawler_finished_cb (TrackerCrawler *crawler, } } +static void +notifier_queue_file (TrackerFileNotifier *notifier, + GFile *file, + TrackerDirectoryFlags flags) +{ + TrackerFileNotifierPrivate *priv = notifier->priv; + + if (flags & TRACKER_DIRECTORY_FLAG_PRIORITY) { + priv->pending_index_roots = g_list_prepend (priv->pending_index_roots, file); + } else { + priv->pending_index_roots = g_list_append (priv->pending_index_roots, file); + } +} + /* Monitor signal handlers */ static void monitor_item_created_cb (TrackerMonitor *monitor, @@ -719,10 +733,7 @@ monitor_item_created_cb (TrackerMonitor *monitor, file, file_type, NULL); - priv->pending_index_roots = - g_list_append (priv->pending_index_roots, - canonical); - + notifier_queue_file (notifier, canonical, flags); crawl_directories_start (notifier); return; } @@ -820,6 +831,7 @@ monitor_item_deleted_cb (TrackerMonitor *monitor, } if (!is_directory) { + TrackerDirectoryFlags flags; gboolean indexable; GList *children; GFile *parent; @@ -845,9 +857,9 @@ monitor_item_deleted_cb (TrackerMonitor *monitor, file, G_FILE_TYPE_DIRECTORY, NULL); - priv->pending_index_roots = - g_list_append (priv->pending_index_roots, file); - + tracker_indexing_tree_get_root (priv->indexing_tree, + file, &flags); + notifier_queue_file (notifier, file, flags); crawl_directories_start (notifier); return; } @@ -874,9 +886,11 @@ monitor_item_moved_cb (TrackerMonitor *monitor, { TrackerFileNotifier *notifier; TrackerFileNotifierPrivate *priv; + TrackerDirectoryFlags flags; notifier = user_data; priv = notifier->priv; + tracker_indexing_tree_get_root (priv->indexing_tree, other_file, &flags); if (!is_source_monitored) { if (is_directory) { @@ -888,9 +902,7 @@ monitor_item_moved_cb (TrackerMonitor *monitor, other_file, G_FILE_TYPE_DIRECTORY, NULL); - priv->pending_index_roots = - g_list_append (priv->pending_index_roots, file); - + notifier_queue_file (notifier, file, flags); crawl_directories_start (notifier); } /* else, file, do nothing */ @@ -941,10 +953,7 @@ monitor_item_moved_cb (TrackerMonitor *monitor, other_file, G_FILE_TYPE_DIRECTORY, NULL); - priv->pending_index_roots = - g_list_append (priv->pending_index_roots, - other_file); - + notifier_queue_file (notifier, other_file, flags); crawl_directories_start (notifier); } } @@ -961,20 +970,16 @@ monitor_item_moved_cb (TrackerMonitor *monitor, /* Handle move */ if (is_directory) { gboolean dest_is_recursive, source_is_recursive; - TrackerDirectoryFlags flags; + TrackerDirectoryFlags source_flags; tracker_monitor_move (priv->monitor, file, other_file); tracker_indexing_tree_get_root (priv->indexing_tree, - other_file, - &flags); + file, &source_flags); + source_is_recursive = (source_flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0; dest_is_recursive = (flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0; - tracker_indexing_tree_get_root (priv->indexing_tree, - file, &flags); - source_is_recursive = (flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0; - if (source_is_recursive && !dest_is_recursive) { /* A directory is being moved from a * recursive location to a non-recursive @@ -987,10 +992,7 @@ monitor_item_moved_cb (TrackerMonitor *monitor, other_file, G_FILE_TYPE_DIRECTORY, NULL); - priv->pending_index_roots = - g_list_append (priv->pending_index_roots, - file); - + notifier_queue_file (notifier, file, flags); crawl_directories_start (notifier); } } @@ -1021,8 +1023,7 @@ indexing_tree_directory_added (TrackerIndexingTree *indexing_tree, } if (!g_list_find (priv->pending_index_roots, directory)) { - priv->pending_index_roots = g_list_append (priv->pending_index_roots, - directory); + notifier_queue_file (notifier, directory, flags); if (start_crawler) { crawl_directories_start (notifier); @@ -1064,10 +1065,7 @@ indexing_tree_directory_removed (TrackerIndexingTree *indexing_tree, &parent_flags); if (parent_flags & TRACKER_DIRECTORY_FLAG_RECURSE) { - priv->pending_index_roots = - g_list_append (priv->pending_index_roots, - g_object_ref (directory)); - + notifier_queue_file (notifier, directory, flags); crawl_directories_start (notifier); } else if (tracker_indexing_tree_file_is_root (indexing_tree, parent)) { diff --git a/src/libtracker-miner/tracker-miner-enums.h b/src/libtracker-miner/tracker-miner-enums.h index e2b978fbd..ecfaf3527 100644 --- a/src/libtracker-miner/tracker-miner-enums.h +++ b/src/libtracker-miner/tracker-miner-enums.h @@ -50,7 +50,8 @@ typedef enum { TRACKER_DIRECTORY_FLAG_CHECK_MTIME = 1 << 2, TRACKER_DIRECTORY_FLAG_MONITOR = 1 << 3, TRACKER_DIRECTORY_FLAG_IGNORE = 1 << 4, - TRACKER_DIRECTORY_FLAG_PRESERVE = 1 << 5 + TRACKER_DIRECTORY_FLAG_PRESERVE = 1 << 5, + TRACKER_DIRECTORY_FLAG_PRIORITY = 1 << 6 } TrackerDirectoryFlags; /** -- cgit v1.2.1 From f5c764c7cf3ed5f192da0cab903c8ed47561d874 Mon Sep 17 00:00:00 2001 From: Carlos Garnacho Date: Thu, 13 Feb 2014 00:34:40 +0100 Subject: libtracker-miner: Obey the "priority" flag when adding files to processing queues If a file belongs to a root directory with that flag set, it will get G_PRIORITY_HIGH at the time of processing the file in TrackerMinerFS queues. Also, make tracker_miner_fs_check_directory() use G_PRIORITY_HIGH, like check_file() does. --- src/libtracker-miner/tracker-miner-fs.c | 72 +++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c index fc7a0da9e..1bf742e33 100644 --- a/src/libtracker-miner/tracker-miner-fs.c +++ b/src/libtracker-miner/tracker-miner-fs.c @@ -2508,6 +2508,30 @@ cancel_writeback_task (TrackerMinerFS *fs, } } +static gint +miner_fs_get_queue_priority (TrackerMinerFS *fs, + GFile *file) +{ + TrackerDirectoryFlags flags; + + tracker_indexing_tree_get_root (fs->priv->indexing_tree, + file, &flags); + + return (flags & TRACKER_DIRECTORY_FLAG_PRIORITY) ? + G_PRIORITY_HIGH : G_PRIORITY_DEFAULT; +} + +static void +miner_fs_queue_file (TrackerMinerFS *fs, + TrackerPriorityQueue *item_queue, + GFile *file) +{ + gint priority; + + priority = miner_fs_get_queue_priority (fs, file); + tracker_priority_queue_add (item_queue, g_object_ref (file), priority); +} + /* Checks previous created/updated/deleted/moved/writeback queues for * monitor events. Returns TRUE if the item should still * be added to the queue. @@ -2634,9 +2658,7 @@ check_item_queues (TrackerMinerFS *fs, */ g_debug (" Found matching unhandled CREATED event " "for source file, merging both events together"); - tracker_priority_queue_add (fs->priv->items_created, - g_object_ref (other_file), - G_PRIORITY_DEFAULT); + miner_fs_queue_file (fs, fs->priv->items_created, other_file); return FALSE; } @@ -2671,9 +2693,7 @@ file_notifier_file_created (TrackerFileNotifier *notifier, TrackerMinerFS *fs = user_data; if (check_item_queues (fs, QUEUE_CREATED, file, NULL)) { - tracker_priority_queue_add (fs->priv->items_created, - g_object_ref (file), - G_PRIORITY_DEFAULT); + miner_fs_queue_file (fs, fs->priv->items_created, file); item_queue_handlers_set_up (fs); } } @@ -2686,9 +2706,7 @@ file_notifier_file_deleted (TrackerFileNotifier *notifier, TrackerMinerFS *fs = user_data; if (check_item_queues (fs, QUEUE_DELETED, file, NULL)) { - tracker_priority_queue_add (fs->priv->items_deleted, - g_object_ref (file), - G_PRIORITY_DEFAULT); + miner_fs_queue_file (fs, fs->priv->items_deleted, file); item_queue_handlers_set_up (fs); } } @@ -2719,9 +2737,7 @@ file_notifier_file_updated (TrackerFileNotifier *notifier, GINT_TO_POINTER (TRUE)); } - tracker_priority_queue_add (fs->priv->items_updated, - g_object_ref (file), - G_PRIORITY_DEFAULT); + miner_fs_queue_file (fs, fs->priv->items_updated, file); item_queue_handlers_set_up (fs); } } @@ -2735,9 +2751,12 @@ file_notifier_file_moved (TrackerFileNotifier *notifier, TrackerMinerFS *fs = user_data; if (check_item_queues (fs, QUEUE_MOVED, source, dest)) { + gint priority; + + priority = miner_fs_get_queue_priority (fs, dest); tracker_priority_queue_add (fs->priv->items_moved, item_moved_data_new (dest, source), - G_PRIORITY_DEFAULT); + priority); item_queue_handlers_set_up (fs); } } @@ -3041,9 +3060,7 @@ tracker_miner_fs_directory_remove_full (TrackerMinerFS *fs, * to preserve remove_full() semantics. */ trace_eq_push_tail ("DELETED", file, "on remove full"); - tracker_priority_queue_add (fs->priv->items_deleted, - g_object_ref (file), - G_PRIORITY_DEFAULT); + miner_fs_queue_file (fs, fs->priv->items_deleted, file); item_queue_handlers_set_up (fs); } @@ -3087,9 +3104,8 @@ check_file_parents (TrackerMinerFS *fs, for (p = parents; p; p = p->next) { trace_eq_push_tail ("UPDATED", p->data, "checking file parents"); - tracker_priority_queue_add (fs->priv->items_updated, - p->data, - G_PRIORITY_DEFAULT); + miner_fs_queue_file (fs, fs->priv->items_updated, p->data); + g_object_unref (p->data); } g_list_free (parents); @@ -3313,16 +3329,22 @@ tracker_miner_fs_check_directory_with_priority (TrackerMinerFS *fs, path); if (should_process) { + TrackerDirectoryFlags flags; + if (check_parents && !check_file_parents (fs, file)) { return; } - /* FIXME: Apply priority */ + flags = TRACKER_DIRECTORY_FLAG_RECURSE | + TRACKER_DIRECTORY_FLAG_CHECK_MTIME | + TRACKER_DIRECTORY_FLAG_MONITOR; + + /* Priorities run from positive to negative */ + if (priority < G_PRIORITY_DEFAULT) + flags |= TRACKER_DIRECTORY_FLAG_PRIORITY; + tracker_indexing_tree_add (fs->priv->indexing_tree, - file, - TRACKER_DIRECTORY_FLAG_RECURSE | - TRACKER_DIRECTORY_FLAG_CHECK_MTIME | - TRACKER_DIRECTORY_FLAG_MONITOR); + file, flags); } g_free (path); @@ -3346,7 +3368,7 @@ tracker_miner_fs_check_directory (TrackerMinerFS *fs, gboolean check_parents) { tracker_miner_fs_check_directory_with_priority (fs, file, - G_PRIORITY_DEFAULT, + G_PRIORITY_HIGH, check_parents); } -- cgit v1.2.1 From 80551719642f82f859912c80a7adb1cff8fe0b41 Mon Sep 17 00:00:00 2001 From: Carlos Garnacho Date: Thu, 13 Feb 2014 12:50:46 +0100 Subject: miner-fs: Set the priority flag on mounted volumes. This is so their content is inspected as soon as possible, even if tracker is busy crawling over local directories. --- src/miners/fs/tracker-miner-files.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/miners/fs/tracker-miner-files.c b/src/miners/fs/tracker-miner-files.c index ea31eaf35..e103dcb3a 100644 --- a/src/miners/fs/tracker-miner-files.c +++ b/src/miners/fs/tracker-miner-files.c @@ -2764,7 +2764,8 @@ miner_files_add_removable_or_optical_directory (TrackerMinerFiles *mf, indexing_tree = tracker_miner_fs_get_indexing_tree (TRACKER_MINER_FS (mf)); flags = TRACKER_DIRECTORY_FLAG_RECURSE | TRACKER_DIRECTORY_FLAG_CHECK_MTIME | - TRACKER_DIRECTORY_FLAG_PRESERVE; + TRACKER_DIRECTORY_FLAG_PRESERVE | + TRACKER_DIRECTORY_FLAG_PRIORITY; if (tracker_config_get_enable_monitors (mf->private->config)) { flags |= TRACKER_DIRECTORY_FLAG_MONITOR; -- cgit v1.2.1