Don't use single find_events call during reindex, instead do find_event_ids + get_events

4 files changed, 63 insertions, 16 deletions

diff --git a/extensions/fts++/controller.cpp b/extensions/fts++/controller.cpp
index d945a327..b849d819 100644
--- a/extensions/fts++/controller.cpp
+++ b/extensions/fts++/controller.cpp
@@ -37,19 +37,21 @@ void Controller::Run ()
 void Controller::RebuildIndex ()
 {
   GError *error = NULL;
-  GPtrArray *events;
+  guint32 *event_ids;
+  gint event_ids_size;
   GPtrArray *templates = g_ptr_array_new ();
   ZeitgeistTimeRange *time_range = zeitgeist_time_range_new_anytime ();
 
   g_debug ("asking reader for all events");
-  events = zeitgeist_db_reader_find_events (zg_reader,
-                                            time_range,
-                                            templates,
-                                            ZEITGEIST_STORAGE_STATE_ANY,
-                                            0,
-                                            ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
-                                            NULL,
-                                            &error);
+  event_ids = zeitgeist_db_reader_find_event_ids (zg_reader,
+                                                  time_range,
+                                                  templates,
+                                                  ZEITGEIST_STORAGE_STATE_ANY,
+                                                  0,
+                                                  ZEITGEIST_RESULT_TYPE_MOST_RECENT_EVENTS,
+                                                  NULL,
+                                                  &event_ids_size,
+                                                  &error);
 
   if (error)
   {
@@ -58,10 +60,10 @@ void Controller::RebuildIndex ()
   }
   else
   {
-    g_debug ("reader returned %u events", events->len);
+    g_debug ("reader returned %d events", event_ids_size);
 
-    IndexEvents (events);
-    g_ptr_array_unref (events);
+    IndexEvents (event_ids, event_ids_size);
+    g_free (event_ids);
 
     // Set the db metadata key only once we're done
     PushTask (new MetadataTask ("fts_index_version", INDEX_VERSION));
@@ -81,6 +83,19 @@ void Controller::IndexEvents (GPtrArray *events)
   }
 }
 
+void Controller::IndexEvents (guint *event_ids, int event_ids_size)
+{
+  const int CHUNK_SIZE = 64;
+  // Break down index tasks into suitable chunks
+  for (int i = 0; i < event_ids_size; i += CHUNK_SIZE)
+  {
+    int num_ids = i + CHUNK_SIZE > event_ids_size ? 
+      event_ids_size - i : CHUNK_SIZE;
+    PushTask (new IndexEventsTask (zg_reader,
+        std::vector<guint> (&event_ids[i], &event_ids[i + num_ids])));
+  }
+}
+
 void Controller::DeleteEvents (guint *event_ids, int event_ids_size)
 {
   // FIXME: Should we break the task here as well?
diff --git a/extensions/fts++/controller.h b/extensions/fts++/controller.h
index b62c2918..e6dd0603 100644
--- a/extensions/fts++/controller.h
+++ b/extensions/fts++/controller.h
@@ -49,6 +49,7 @@ public:
   void RebuildIndex ();
 
   void IndexEvents (GPtrArray *events);
+  void IndexEvents (guint *event_ids, int event_ids_size);
   void DeleteEvents (guint *event_ids, int event_ids_size);
 
   void PushTask (Task* task);
diff --git a/extensions/fts++/task.cpp b/extensions/fts++/task.cpp
index 67346ac6..caa17ced 100644
--- a/extensions/fts++/task.cpp
+++ b/extensions/fts++/task.cpp
@@ -23,10 +23,36 @@ namespace ZeitgeistFTS {
 
 void IndexEventsTask::Process (Indexer *indexer)
 {
-  unsigned end_index = MIN (start_index + event_count, events->len);
-  for (unsigned i = start_index; i < end_index; i++)
+  if (events)
   {
-    indexer->IndexEvent ((ZeitgeistEvent*) g_ptr_array_index (events, i));
+    unsigned end_index = MIN (start_index + event_count, events->len);
+    for (unsigned i = start_index; i < end_index; i++)
+    {
+      indexer->IndexEvent ((ZeitgeistEvent*) g_ptr_array_index (events, i));
+    }
+  }
+  else if (!event_ids.empty ())
+  {
+    GError *error = NULL;
+    GPtrArray *results = zeitgeist_db_reader_get_events (zg_reader,
+                                                         &event_ids[0],
+                                                         event_ids.size (),
+                                                         NULL,
+                                                         &error);
+    if (error)
+    {
+      g_warning ("Unable to get events: %s", error->message);
+      return;
+    }
+    else
+    {
+      for (unsigned i = 0; i < results->len; i++)
+      {
+        indexer->IndexEvent ((ZeitgeistEvent*) g_ptr_array_index (results, i));
+      }
+    }
+
+    g_ptr_array_unref (results);
   }
 }
 
diff --git a/extensions/fts++/task.h b/extensions/fts++/task.h
index 3d4b3dd9..630503a6 100644
--- a/extensions/fts++/task.h
+++ b/extensions/fts++/task.h
@@ -49,16 +49,21 @@ public:
 
   IndexEventsTask (GPtrArray *event_arr, unsigned index, unsigned count)
     : events (event_arr), start_index (index), event_count (count) {}
+  
+  IndexEventsTask (ZeitgeistDbReader *reader, std::vector<unsigned> const &ids)
+    : events (NULL), zg_reader (reader), event_ids (ids) {}
 
   virtual ~IndexEventsTask ()
   {
-    g_ptr_array_unref (events);
+    if (events) g_ptr_array_unref (events);
   }
 
 private:
   GPtrArray *events;
   unsigned start_index;
   unsigned event_count;
+  ZeitgeistDbReader *zg_reader;
+  std::vector<unsigned> event_ids;
 };
 
 class DeleteEventsTask : public Task