Merge branch 'prioritized-crawling'

4 files changed, 118 insertions, 98 deletions

diff --git a/src/libtracker-miner/tracker-file-notifier.c b/src/libtracker-miner/tracker-file-notifier.c
index 9ef01aaba..7de13f5de 100644
--- a/src/libtracker-miner/tracker-file-notifier.c
+++ b/src/libtracker-miner/tracker-file-notifier.c
@@ -69,6 +69,7 @@ typedef struct {
 	 * trees to get data from
 	 */
 	GList *pending_index_roots;
+	GFile *current_index_root;
 
 	guint stopped : 1;
 } TrackerFileNotifierPrivate;
@@ -160,9 +161,8 @@ crawler_check_directory_cb (TrackerCrawler *crawler,
 	 * currently processed, bypass it, it will be processed
 	 * when the time arrives.
 	 */
-	if (canonical &&
-	    root == canonical &&
-	    root != priv->pending_index_roots->data) {
+	if (canonical && root == canonical &&
+	    root != priv->current_index_root) {
 		return FALSE;
 	}
 
@@ -254,6 +254,22 @@ file_notifier_traverse_tree_foreach (GFile    *file,
 	return FALSE;
 }
 
+static gboolean
+notifier_check_next_root (TrackerFileNotifier *notifier)
+{
+	TrackerFileNotifierPrivate *priv;
+
+	priv = notifier->priv;
+
+	if (priv->pending_index_roots) {
+		return crawl_directories_start (notifier);
+	} else {
+		priv->current_index_root = NULL;
+		g_signal_emit (notifier, signals[FINISHED], 0);
+		return FALSE;
+	}
+}
+
 static void
 file_notifier_traverse_tree (TrackerFileNotifier *notifier)
 {
@@ -262,7 +278,7 @@ file_notifier_traverse_tree (TrackerFileNotifier *notifier)
 	TrackerDirectoryFlags flags;
 
 	priv = notifier->priv;
-	current_root = priv->pending_index_roots->data;
+	current_root = priv->current_index_root;
 	config_root = tracker_indexing_tree_get_root (priv->indexing_tree,
 						      current_root, &flags);
 
@@ -288,16 +304,7 @@ file_notifier_traverse_tree (TrackerFileNotifier *notifier)
 	tracker_info ("  Notified files after %2.2f seconds",
 	              g_timer_elapsed (priv->timer, NULL));
 
-	/* We've finished crawling/querying on the first element
-	 * of the pending list, continue onto the next */
-	priv->pending_index_roots = g_list_delete_link (priv->pending_index_roots,
-	                                                priv->pending_index_roots);
-
-	if (priv->pending_index_roots) {
-		crawl_directories_start (notifier);
-	} else {
-		g_signal_emit (notifier, signals[FINISHED], 0);
-	}
+	notifier_check_next_root (notifier);
 }
 
 static gboolean
@@ -408,9 +415,8 @@ sparql_file_query_populate (TrackerFileNotifier *notifier,
 			canonical = tracker_file_system_peek_file (priv->file_system, file);
 			root = tracker_indexing_tree_get_root (priv->indexing_tree, file, NULL);
 
-			if (canonical &&
-			    root == file &&
-			    root != priv->pending_index_roots->data) {
+			if (canonical && root == file &&
+			    root != priv->current_index_root) {
 				g_object_unref (file);
 				continue;
 			}
@@ -469,7 +475,7 @@ sparql_query_cb (GObject      *object,
 
 	/* Mark the directory root as queried */
 	tracker_file_system_set_property (priv->file_system,
-	                                  priv->pending_index_roots->data,
+	                                  priv->current_index_root,
 	                                  quark_property_queried,
 	                                  GUINT_TO_POINTER (TRUE));
 
@@ -478,7 +484,7 @@ sparql_query_cb (GObject      *object,
 
 	/* If it's also been crawled, finish operation */
 	if (tracker_file_system_get_property (priv->file_system,
-	                                      priv->pending_index_roots->data,
+	                                      priv->current_index_root,
 	                                      quark_property_crawled)) {
 		file_notifier_traverse_tree (notifier);
 	}
@@ -565,7 +571,9 @@ crawl_directories_start (TrackerFileNotifier *notifier)
 	}
 
 	while (priv->pending_index_roots) {
-		directory = priv->pending_index_roots->data;
+		directory = priv->current_index_root = priv->pending_index_roots->data;
+		priv->pending_index_roots = g_list_delete_link (priv->pending_index_roots,
+		                                                priv->pending_index_roots);
 
 		tracker_indexing_tree_get_root (priv->indexing_tree,
 		                                directory,
@@ -614,10 +622,6 @@ crawl_directories_start (TrackerFileNotifier *notifier)
 			g_signal_emit (notifier, signals[DIRECTORY_FINISHED], 0,
 			               directory, 0, 0, 0, 0);
 		}
-
-		/* Remove index root and try the next one */
-		priv->pending_index_roots = g_list_delete_link (priv->pending_index_roots,
-		                                                priv->pending_index_roots);
 	}
 
 	g_signal_emit (notifier, signals[FINISHED], 0);
@@ -640,7 +644,7 @@ crawler_finished_cb (TrackerCrawler *crawler,
 	if (!was_interrupted) {
 		GFile *directory;
 
-		directory = priv->pending_index_roots->data;
+		directory = priv->current_index_root;
 
 		/* Mark the directory root as crawled */
 		tracker_file_system_set_property (priv->file_system, directory,
@@ -656,6 +660,20 @@ crawler_finished_cb (TrackerCrawler *crawler,
 	}
 }
 
+static void
+notifier_queue_file (TrackerFileNotifier   *notifier,
+                     GFile                 *file,
+                     TrackerDirectoryFlags  flags)
+{
+	TrackerFileNotifierPrivate *priv = notifier->priv;
+
+	if (flags & TRACKER_DIRECTORY_FLAG_PRIORITY) {
+		priv->pending_index_roots = g_list_prepend (priv->pending_index_roots, file);
+	} else {
+		priv->pending_index_roots = g_list_append (priv->pending_index_roots, file);
+	}
+}
+
 /* Monitor signal handlers */
 static void
 monitor_item_created_cb (TrackerMonitor *monitor,
@@ -715,10 +733,7 @@ monitor_item_created_cb (TrackerMonitor *monitor,
 			                                          file,
 			                                          file_type,
 			                                          NULL);
-			priv->pending_index_roots =
-				g_list_append (priv->pending_index_roots,
-				               canonical);
-
+			notifier_queue_file (notifier, canonical, flags);
 			crawl_directories_start (notifier);
 			return;
 		}
@@ -816,6 +831,7 @@ monitor_item_deleted_cb (TrackerMonitor *monitor,
 	}
 
 	if (!is_directory) {
+		TrackerDirectoryFlags flags;
 		gboolean indexable;
 		GList *children;
 		GFile *parent;
@@ -841,9 +857,9 @@ monitor_item_deleted_cb (TrackerMonitor *monitor,
 			                                     file,
 			                                     G_FILE_TYPE_DIRECTORY,
 			                                     NULL);
-			priv->pending_index_roots =
-				g_list_append (priv->pending_index_roots, file);
-
+			tracker_indexing_tree_get_root (priv->indexing_tree,
+							file, &flags);
+			notifier_queue_file (notifier, file, flags);
 			crawl_directories_start (notifier);
 			return;
 		}
@@ -870,9 +886,11 @@ monitor_item_moved_cb (TrackerMonitor *monitor,
 {
 	TrackerFileNotifier *notifier;
 	TrackerFileNotifierPrivate *priv;
+	TrackerDirectoryFlags flags;
 
 	notifier = user_data;
 	priv = notifier->priv;
+	tracker_indexing_tree_get_root (priv->indexing_tree, other_file, &flags);
 
 	if (!is_source_monitored) {
 		if (is_directory) {
@@ -884,9 +902,7 @@ monitor_item_moved_cb (TrackerMonitor *monitor,
 			                                     other_file,
 			                                     G_FILE_TYPE_DIRECTORY,
 			                                     NULL);
-			priv->pending_index_roots =
-				g_list_append (priv->pending_index_roots, file);
-
+			notifier_queue_file (notifier, file, flags);
 			crawl_directories_start (notifier);
 		}
 		/* else, file, do nothing */
@@ -937,10 +953,7 @@ monitor_item_moved_cb (TrackerMonitor *monitor,
 					                                           other_file,
 					                                           G_FILE_TYPE_DIRECTORY,
 					                                           NULL);
-					priv->pending_index_roots =
-						g_list_append (priv->pending_index_roots,
-						               other_file);
-
+					notifier_queue_file (notifier, other_file, flags);
 					crawl_directories_start (notifier);
 				}
 			}
@@ -957,20 +970,16 @@ monitor_item_moved_cb (TrackerMonitor *monitor,
 			/* Handle move */
 			if (is_directory) {
 				gboolean dest_is_recursive, source_is_recursive;
-				TrackerDirectoryFlags flags;
+				TrackerDirectoryFlags source_flags;
 
 				tracker_monitor_move (priv->monitor,
 				                      file, other_file);
 
 				tracker_indexing_tree_get_root (priv->indexing_tree,
-				                                other_file,
-				                                &flags);
+				                                file, &source_flags);
+				source_is_recursive = (source_flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0;
 				dest_is_recursive = (flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0;
 
-				tracker_indexing_tree_get_root (priv->indexing_tree,
-				                                file, &flags);
-				source_is_recursive = (flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0;
-
 				if (source_is_recursive && !dest_is_recursive) {
 					/* A directory is being moved from a
 					 * recursive location to a non-recursive
@@ -983,10 +992,7 @@ monitor_item_moved_cb (TrackerMonitor *monitor,
 					                                     other_file,
 					                                     G_FILE_TYPE_DIRECTORY,
 					                                     NULL);
-					priv->pending_index_roots =
-						g_list_append (priv->pending_index_roots,
-						               file);
-
+					notifier_queue_file (notifier, file, flags);
 					crawl_directories_start (notifier);
 				}
 			}
@@ -1017,8 +1023,8 @@ indexing_tree_directory_added (TrackerIndexingTree *indexing_tree,
 	}
 
 	if (!g_list_find (priv->pending_index_roots, directory)) {
-		priv->pending_index_roots = g_list_append (priv->pending_index_roots,
-		                                           directory);
+		notifier_queue_file (notifier, directory, flags);
+
 		if (start_crawler) {
 			crawl_directories_start (notifier);
 		}
@@ -1059,10 +1065,7 @@ indexing_tree_directory_removed (TrackerIndexingTree *indexing_tree,
 			                                &parent_flags);
 
 			if (parent_flags & TRACKER_DIRECTORY_FLAG_RECURSE) {
-				priv->pending_index_roots =
-					g_list_append (priv->pending_index_roots,
-						       g_object_ref (directory));
-
+				notifier_queue_file (notifier, directory, flags);
 				crawl_directories_start (notifier);
 			} else if (tracker_indexing_tree_file_is_root (indexing_tree,
 			                                               parent)) {
@@ -1080,22 +1083,15 @@ indexing_tree_directory_removed (TrackerIndexingTree *indexing_tree,
 		g_signal_emit (notifier, signals[FILE_DELETED], 0, directory);
 	}
 
-	if (priv->pending_index_roots) {
-		gboolean start_crawler = FALSE;
-
-		if (directory == priv->pending_index_roots->data) {
-			/* Directory being currently processed */
-			tracker_crawler_stop (priv->crawler);
-			g_cancellable_cancel (priv->cancellable);
-			start_crawler = TRUE;
-		}
+	priv->pending_index_roots = g_list_remove_all (priv->pending_index_roots,
+	                                               directory);
 
-		priv->pending_index_roots = g_list_remove_all (priv->pending_index_roots,
-		                                               directory);
+	if (directory == priv->current_index_root) {
+		/* Directory being currently processed */
+		tracker_crawler_stop (priv->crawler);
+		g_cancellable_cancel (priv->cancellable);
 
-		if (start_crawler && priv->pending_index_roots != NULL) {
-			crawl_directories_start (notifier);
-		}
+		notifier_check_next_root (notifier);
 	}
 
 	/* Remove monitors if any */
@@ -1379,7 +1375,7 @@ tracker_file_notifier_is_active (TrackerFileNotifier *notifier)
 	g_return_val_if_fail (TRACKER_IS_FILE_NOTIFIER (notifier), FALSE);
 
 	priv = notifier->priv;
-	return priv->pending_index_roots != NULL;
+	return priv->pending_index_roots || priv->current_index_root;
 }
 
 const gchar *
diff --git a/src/libtracker-miner/tracker-miner-enums.h b/src/libtracker-miner/tracker-miner-enums.h
index e2b978fbd..ecfaf3527 100644
--- a/src/libtracker-miner/tracker-miner-enums.h
+++ b/src/libtracker-miner/tracker-miner-enums.h
@@ -50,7 +50,8 @@ typedef enum {
 	TRACKER_DIRECTORY_FLAG_CHECK_MTIME = 1 << 2,
 	TRACKER_DIRECTORY_FLAG_MONITOR     = 1 << 3,
 	TRACKER_DIRECTORY_FLAG_IGNORE      = 1 << 4,
-	TRACKER_DIRECTORY_FLAG_PRESERVE    = 1 << 5
+	TRACKER_DIRECTORY_FLAG_PRESERVE    = 1 << 5,
+	TRACKER_DIRECTORY_FLAG_PRIORITY    = 1 << 6
 } TrackerDirectoryFlags;
 
 /**
diff --git a/src/libtracker-miner/tracker-miner-fs.c b/src/libtracker-miner/tracker-miner-fs.c
index fc7a0da9e..1bf742e33 100644
--- a/src/libtracker-miner/tracker-miner-fs.c
+++ b/src/libtracker-miner/tracker-miner-fs.c
@@ -2508,6 +2508,30 @@ cancel_writeback_task (TrackerMinerFS *fs,
 	}
 }
 
+static gint
+miner_fs_get_queue_priority (TrackerMinerFS *fs,
+                             GFile          *file)
+{
+	TrackerDirectoryFlags flags;
+
+	tracker_indexing_tree_get_root (fs->priv->indexing_tree,
+	                                file, &flags);
+
+	return (flags & TRACKER_DIRECTORY_FLAG_PRIORITY) ?
+	        G_PRIORITY_HIGH : G_PRIORITY_DEFAULT;
+}
+
+static void
+miner_fs_queue_file (TrackerMinerFS       *fs,
+		     TrackerPriorityQueue *item_queue,
+		     GFile                *file)
+{
+	gint priority;
+
+	priority = miner_fs_get_queue_priority (fs, file);
+	tracker_priority_queue_add (item_queue, g_object_ref (file), priority);
+}
+
 /* Checks previous created/updated/deleted/moved/writeback queues for
  * monitor events. Returns TRUE if the item should still
  * be added to the queue.
@@ -2634,9 +2658,7 @@ check_item_queues (TrackerMinerFS *fs,
 			 */
 			g_debug ("  Found matching unhandled CREATED event "
 			         "for source file, merging both events together");
-			tracker_priority_queue_add (fs->priv->items_created,
-			                            g_object_ref (other_file),
-			                            G_PRIORITY_DEFAULT);
+			miner_fs_queue_file (fs, fs->priv->items_created, other_file);
 
 			return FALSE;
 		}
@@ -2671,9 +2693,7 @@ file_notifier_file_created (TrackerFileNotifier  *notifier,
 	TrackerMinerFS *fs = user_data;
 
 	if (check_item_queues (fs, QUEUE_CREATED, file, NULL)) {
-		tracker_priority_queue_add (fs->priv->items_created,
-		                            g_object_ref (file),
-		                            G_PRIORITY_DEFAULT);
+		miner_fs_queue_file (fs, fs->priv->items_created, file);
 		item_queue_handlers_set_up (fs);
 	}
 }
@@ -2686,9 +2706,7 @@ file_notifier_file_deleted (TrackerFileNotifier  *notifier,
 	TrackerMinerFS *fs = user_data;
 
 	if (check_item_queues (fs, QUEUE_DELETED, file, NULL)) {
-		tracker_priority_queue_add (fs->priv->items_deleted,
-		                            g_object_ref (file),
-		                            G_PRIORITY_DEFAULT);
+		miner_fs_queue_file (fs, fs->priv->items_deleted, file);
 		item_queue_handlers_set_up (fs);
 	}
 }
@@ -2719,9 +2737,7 @@ file_notifier_file_updated (TrackerFileNotifier  *notifier,
 			                    GINT_TO_POINTER (TRUE));
 		}
 
-		tracker_priority_queue_add (fs->priv->items_updated,
-		                            g_object_ref (file),
-		                            G_PRIORITY_DEFAULT);
+		miner_fs_queue_file (fs, fs->priv->items_updated, file);
 		item_queue_handlers_set_up (fs);
 	}
 }
@@ -2735,9 +2751,12 @@ file_notifier_file_moved (TrackerFileNotifier *notifier,
 	TrackerMinerFS *fs = user_data;
 
 	if (check_item_queues (fs, QUEUE_MOVED, source, dest)) {
+		gint priority;
+
+		priority = miner_fs_get_queue_priority (fs, dest);
 		tracker_priority_queue_add (fs->priv->items_moved,
 		                            item_moved_data_new (dest, source),
-		                            G_PRIORITY_DEFAULT);
+					    priority);
 		item_queue_handlers_set_up (fs);
 	}
 }
@@ -3041,9 +3060,7 @@ tracker_miner_fs_directory_remove_full (TrackerMinerFS *fs,
 			 * to preserve remove_full() semantics.
 			 */
 			trace_eq_push_tail ("DELETED", file, "on remove full");
-			tracker_priority_queue_add (fs->priv->items_deleted,
-			                            g_object_ref (file),
-			                            G_PRIORITY_DEFAULT);
+			miner_fs_queue_file (fs, fs->priv->items_deleted, file);
 			item_queue_handlers_set_up (fs);
 		}
 
@@ -3087,9 +3104,8 @@ check_file_parents (TrackerMinerFS *fs,
 
 	for (p = parents; p; p = p->next) {
 		trace_eq_push_tail ("UPDATED", p->data, "checking file parents");
-		tracker_priority_queue_add (fs->priv->items_updated,
-		                            p->data,
-		                            G_PRIORITY_DEFAULT);
+		miner_fs_queue_file (fs, fs->priv->items_updated, p->data);
+		g_object_unref (p->data);
 	}
 
 	g_list_free (parents);
@@ -3313,16 +3329,22 @@ tracker_miner_fs_check_directory_with_priority (TrackerMinerFS *fs,
 	         path);
 
 	if (should_process) {
+		TrackerDirectoryFlags flags;
+
 		if (check_parents && !check_file_parents (fs, file)) {
 			return;
 		}
 
-		/* FIXME: Apply priority */
+		flags = TRACKER_DIRECTORY_FLAG_RECURSE |
+			TRACKER_DIRECTORY_FLAG_CHECK_MTIME |
+			TRACKER_DIRECTORY_FLAG_MONITOR;
+
+		/* Priorities run from positive to negative */
+		if (priority < G_PRIORITY_DEFAULT)
+			flags |= TRACKER_DIRECTORY_FLAG_PRIORITY;
+
 		tracker_indexing_tree_add (fs->priv->indexing_tree,
-		                           file,
-		                           TRACKER_DIRECTORY_FLAG_RECURSE |
-		                           TRACKER_DIRECTORY_FLAG_CHECK_MTIME |
-		                           TRACKER_DIRECTORY_FLAG_MONITOR);
+		                           file, flags);
 	}
 
 	g_free (path);
@@ -3346,7 +3368,7 @@ tracker_miner_fs_check_directory (TrackerMinerFS *fs,
                                   gboolean        check_parents)
 {
 	tracker_miner_fs_check_directory_with_priority (fs, file,
-	                                                G_PRIORITY_DEFAULT,
+	                                                G_PRIORITY_HIGH,
 	                                                check_parents);
 }
 
diff --git a/src/miners/fs/tracker-miner-files.c b/src/miners/fs/tracker-miner-files.c
index ea31eaf35..e103dcb3a 100644
--- a/src/miners/fs/tracker-miner-files.c
+++ b/src/miners/fs/tracker-miner-files.c
@@ -2764,7 +2764,8 @@ miner_files_add_removable_or_optical_directory (TrackerMinerFiles *mf,
 	indexing_tree = tracker_miner_fs_get_indexing_tree (TRACKER_MINER_FS (mf));
 	flags = TRACKER_DIRECTORY_FLAG_RECURSE |
 		TRACKER_DIRECTORY_FLAG_CHECK_MTIME |
-		TRACKER_DIRECTORY_FLAG_PRESERVE;
+		TRACKER_DIRECTORY_FLAG_PRESERVE |
+		TRACKER_DIRECTORY_FLAG_PRIORITY;
 
 	if (tracker_config_get_enable_monitors (mf->private->config)) {
 		flags |= TRACKER_DIRECTORY_FLAG_MONITOR;