completed optimisation of indexing

svn path=/trunk/; revision=407

11 files changed, 734 insertions, 813 deletions

diff --git a/src/trackerd/tracker-db-email.c b/src/trackerd/tracker-db-email.c
index 00137fe27..405288931 100644
--- a/src/trackerd/tracker-db-email.c
+++ b/src/trackerd/tracker-db-email.c
@@ -190,6 +190,7 @@ tracker_db_email_save_email (DBConnection *db_con, MailMessage *mm)
 	const GSList	*tmp;
 	char		/* *to_print, */ *name, *path;
 	int		mbox_id, id;
+	GHashTable	*index_table;
 
 	g_return_if_fail (db_con);
 	g_return_if_fail (mm);
@@ -207,6 +208,8 @@ tracker_db_email_save_email (DBConnection *db_con, MailMessage *mm)
 
 	name = tracker_get_vfs_name (mm->uri);
 	path = tracker_get_vfs_path (mm->uri);
+	
+	tracker_db_start_transaction (db_con);
 
 	tracker_db_create_service (db_con, path, name, "Emails", "email", 0, FALSE, FALSE, mm->offset, 0, mbox_id);
 
@@ -217,13 +220,14 @@ tracker_db_email_save_email (DBConnection *db_con, MailMessage *mm)
 
 		tracker_log ("saving email with uri \"%s\" and subject \"%s\" from \"%s\"", mm->uri, mm->subject, mm->from);
 
+		index_table = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
 		str_id = tracker_int_to_str (id);
 		str_date = tracker_int_to_str (mm->date);
 
-		tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:Body", mm->body);
-		tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:Date", str_date);
-		tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:Sender", mm->from);
-		tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:Subject", mm->subject);
+		tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:Body", mm->body, index_table);
+		tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:Date", str_date, index_table);
+		tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:Sender", mm->from, index_table);
+		tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:Subject", mm->subject, index_table);
 
 		g_free (str_date);
 
@@ -234,7 +238,7 @@ tracker_db_email_save_email (DBConnection *db_con, MailMessage *mm)
 			mp = tmp->data;
 
 			str = g_strconcat (mp->name, ":", mp->addr, NULL);
-			tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:SentTo", str);
+			tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:SentTo", str, index_table);
 			g_free (str);
 		}
 
@@ -245,7 +249,7 @@ tracker_db_email_save_email (DBConnection *db_con, MailMessage *mm)
 			mp = tmp->data;
 
 			str = g_strconcat (mp->name, ":", mp->addr, NULL);
-			tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:CC", str);
+			tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:CC", str, index_table);
 			g_free (str);
 		}
 
@@ -254,7 +258,7 @@ tracker_db_email_save_email (DBConnection *db_con, MailMessage *mm)
 
 			ma = tmp->data;
 
-			tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:Attachments", ma->attachment_name);
+			tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:Attachments", ma->attachment_name, index_table);
 
 			/* delimit attachment names so hyphens and underscores are removed so that they can be indexed separately */
 			if (strchr (ma->attachment_name, '_') || strchr (ma->attachment_name, '-')) {
@@ -262,19 +266,23 @@ tracker_db_email_save_email (DBConnection *db_con, MailMessage *mm)
 
 				delimited = g_strdup (ma->attachment_name);
 				delimited =  g_strdelimit (delimited, "-_" , ' ');
-				tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:AttachmentsDelimted", delimited);
+				tracker_db_insert_embedded_metadata (db_con, "Emails", str_id, "Email:AttachmentsDelimted", delimited, index_table);
 				g_free (delimited);
 			}
 		}
 
-		tracker_db_update_indexes_for_new_service (db_con, db_con->user_data, id, tracker_get_id_for_service ("Emails"), NULL);
-
 		tracker_db_refresh_all_display_metadata (db_con, str_id);
 
+		tracker_db_end_transaction (db_con);
+
+		tracker_db_update_indexes_for_new_service (id, tracker_get_id_for_service ("Emails"), index_table);
+
 		g_free (str_id);
-	}
 
+		return;
+	}
 
+	tracker_db_end_transaction (db_con);
 
 	/* sometimes we will create a new record, sometimes we will update previous entries */
 
diff --git a/src/trackerd/tracker-db-sqlite.c b/src/trackerd/tracker-db-sqlite.c
index 578061b73..5a7f3f5dc 100644
--- a/src/trackerd/tracker-db-sqlite.c
+++ b/src/trackerd/tracker-db-sqlite.c
@@ -45,6 +45,34 @@ static GMutex *cache_mutex;
 gboolean use_nfs_safe_locking = FALSE;
 
 
+/* slqite utf-8 user defined collation sequence */
+
+static int 
+sqlite3_utf8_collation (void *NotUsed,  int len1, const void *str1,  int len2, const void *str2)
+{
+	char *s, *word1, *word2;
+	int result;
+
+	/* normalize words */
+	s = g_utf8_casefold (str1, len1);
+	word1 = g_utf8_normalize (s, len1, G_NORMALIZE_NFD);
+	g_free (s);
+	
+	s = g_utf8_casefold (str2, len2);
+	word2 = g_utf8_normalize (s, len2, G_NORMALIZE_NFD);
+	g_free (s);
+	
+	result = strcmp (word1, word2);
+	
+	g_free (word1);
+	g_free (word2);
+
+	return result;
+}
+ 	
+
+
+
 /* sqlite user defined functions for use in sql */
 
 /* converts date/time in UTC format to ISO 8160 standardised format for display */
@@ -588,6 +616,12 @@ tracker_db_connect (void)
 
 	tracker_db_exec_no_reply (db_con, "PRAGMA encoding = \"UTF-8\"");
 
+	/* create user defined utf-8 collation sequence */
+	if (SQLITE_OK != sqlite3_create_collation (db_con->db, "UTF8", SQLITE_UTF8, 0, &sqlite3_utf8_collation)) {
+		tracker_log ("Collation sequence failed due to %s", sqlite3_errmsg (db_con->db));
+	}
+	
+
 	/* create user defined functions that can be used in sql */
 	if (SQLITE_OK != sqlite3_create_function (db_con->db, "FormatDate", 1, SQLITE_ANY, NULL, &sqlite3_date_to_str, NULL, NULL)) {
 		tracker_log ("Function FormatDate failed due to %s", sqlite3_errmsg (db_con->db));
@@ -1374,10 +1408,12 @@ tracker_update_db (DBConnection *db_con)
 		return TRUE;
 	}
 
-	if (i < 14) {
-		tracker_exec_sql (db_con, "insert Into MetaDataTypes (MetaName, DatatypeID, MultipleValues, Weight) values  ('File:Contents', 6, 0, 1)");
-		tracker_exec_sql (db_con, "insert Into MetaDataTypes (MetaName, DatatypeID, MultipleValues, Weight) values  ('Email:Body', 6, 0, 1)");
-		tracker_exec_sql (db_con, "update Options set OptionValue = '14' where OptionKey = 'DBVersion'");
+	if (i == 14) {
+		tracker_db_exec_no_reply (db_con, "delete from MetaDataTypes where MetaName = 'Email:Body'");
+		tracker_db_exec_no_reply (db_con, "delete from MetaDataTypes where MetaName = 'File:Contents'");
+		tracker_db_exec_no_reply (db_con, "insert Into MetaDataTypes (MetaName, DatatypeID, MultipleValues, Weight) values  ('Email:Body', 0, 0, 1)");
+		tracker_exec_sql (db_con, "update Options set OptionValue = '15' where OptionKey = 'DBVersion'");
+	} else {
 		tracker_exec_sql (db_con, "ANALYZE");
 	}
 
@@ -1473,7 +1509,7 @@ get_file_contents_words (DBConnection *db_con, guint32 id)
 
 			if (st) {
 
-				old_table = tracker_parse_text (old_table, st, 1);
+				old_table = tracker_parse_text (old_table, st, 1, TRUE);
 			}
 
 			continue;
@@ -1509,7 +1545,7 @@ get_indexable_content_words (DBConnection *db_con, guint32 id, GHashTable *table
 		for (k = 0; (row = tracker_db_get_row (res, k)); k++) {
 
 			if (row[0] && row[1]) {
-				table = tracker_parse_text (table, row[0], atoi (row[1]));
+				table = tracker_parse_text (table, row[0], atoi (row[1]), TRUE);
 			}
 		}
 
@@ -1526,7 +1562,7 @@ get_indexable_content_words (DBConnection *db_con, guint32 id, GHashTable *table
 		for (k = 0; (row = tracker_db_get_row (res, k)); k++) {
 
 			if (row[0] && row[1]) {
-				table = tracker_parse_text (table, row[0], atoi (row[1]));
+				table = tracker_parse_text (table, row[0], atoi (row[1]), TRUE);
 			}
 		}
 
@@ -1625,15 +1661,13 @@ save_full_text (DBConnection *blob_db_con, const char *str_file_id, const char *
 
 
 void
-tracker_db_save_file_contents (DBConnection *db_con, DBConnection *blob_db_con, DBConnection *cache_db_con, const char *file_name, FileInfo *info)
+tracker_db_save_file_contents (DBConnection *db_con, DBConnection *blob_db_con, GHashTable *index_table, const char *file_name, FileInfo *info)
 {
 	FILE 		*file;
 	char 		buffer[65565];
 	int  		bytes_read, throttle_count;
 	char		*str_file_id, *value;
 	GString 	*str;
-	GHashTable 	*new_table;
-
 
 	file = g_fopen (file_name, "r");
 
@@ -1648,12 +1682,14 @@ tracker_db_save_file_contents (DBConnection *db_con, DBConnection *blob_db_con,
 
 	value = NULL;
 
-	new_table = NULL;
-
 	bytes_read = 0;
 
 	throttle_count = 0;
 
+	if (!index_table) {
+		index_table = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+	}
+
 	while (fgets (buffer, 65565, file)) {
 		unsigned int buffer_length;
 
@@ -1678,21 +1714,25 @@ tracker_db_save_file_contents (DBConnection *db_con, DBConnection *blob_db_con,
 
 			str = g_string_append (str, value);
 
-			new_table = tracker_parse_text (new_table, value, 1);
+			index_table = tracker_parse_text (index_table, value, 1, TRUE);
 
 			bytes_read += strlen (value);
 			g_free (value);
 
 		} else {
 			str = g_string_append (str, buffer);
-			new_table = tracker_parse_text (new_table, buffer, 1);
+	
+			index_table = tracker_parse_text (index_table, buffer, 1, TRUE);
+			
 			bytes_read += buffer_length;
 		}
 
-		throttle_count++;
-		if (throttle_count > (10 + (20 - tracker->throttle))) {
-			tracker_throttle (10);
-			throttle_count= 0;
+		if (!tracker->throttle == 0) {
+			throttle_count++;
+			if (throttle_count > (10 + (20 - tracker->throttle))) {
+				tracker_throttle (10);
+				throttle_count= 0;
+			}
 		}
 
 		/* set upper limit on text we read in to approx 1MB */
@@ -1705,29 +1745,20 @@ tracker_db_save_file_contents (DBConnection *db_con, DBConnection *blob_db_con,
 	
 	value = g_string_free (str, FALSE);
 
-	//tracker_log ("text is %s", value);
-
 	fclose (file);
 
 
-	if (info->is_new) {
-
-		tracker_db_update_indexes_for_new_service (db_con, cache_db_con, info->file_id, info->service_type_id, new_table);
-
-		if (new_table) {
-			g_hash_table_destroy (new_table);
-		}
-
-	} else {
+	if (!info->is_new) {
 		GHashTable *old_table;
 
 		/* get old data and compare with new */
 		old_table = get_file_contents_words (blob_db_con, info->file_id);
 
-		tracker_db_update_differential_index (cache_db_con, old_table, new_table, str_file_id, info->service_type_id);
+		tracker_db_update_differential_index (old_table, index_table, str_file_id, info->service_type_id);
 
-		if (new_table) {
-			g_hash_table_destroy (new_table);
+		if (index_table) {
+			g_hash_table_destroy (index_table);
+			index_table = NULL;
 		}
 
 		if (old_table) {
@@ -1735,15 +1766,7 @@ tracker_db_save_file_contents (DBConnection *db_con, DBConnection *blob_db_con,
 		}
 	}
 
-
-	if (!lock_db ()) {
-		if (value) {
-			g_free (str_file_id);
-			g_free (value);
-		}
-		return;
-	}
-
+	//tracker_log ("saving full text with size %d", bytes_read);
 	save_full_text (blob_db_con, str_file_id, value, bytes_read);
 
 	g_free (str_file_id);
@@ -1751,6 +1774,7 @@ tracker_db_save_file_contents (DBConnection *db_con, DBConnection *blob_db_con,
 	if (value) {
 		g_free (value);
 	}
+
 }
 
 
@@ -2063,7 +2087,7 @@ update_metadata_index (DBConnection *db_con, const char *id, const char *service
 	int	   weight;
 	char	   ***res;
 	GHashTable *old_table, *new_table;
-
+	gboolean filter_words = FALSE;
 	weight = -1;
 
 	/* get meta info for metadata type */
@@ -2096,21 +2120,21 @@ update_metadata_index (DBConnection *db_con, const char *id, const char *service
 	old_table = NULL;
 	new_table = NULL;
 
-	old_table = tracker_parse_text (old_table, old_value, weight);
+	filter_words = ((strcmp (meta_name, "File:Delimited") != 0) && (strcmp (meta_name, "File:Name") != 0));
+
+	old_table = tracker_parse_text (old_table, old_value, weight, filter_words);
 
 	/* parse new metadata value */
-	new_table = tracker_parse_text (new_table, new_value, weight);
+	new_table = tracker_parse_text (new_table, new_value, weight, filter_words);
 
 	/* we only do differential updates so only changed words scores are updated */
 	if (new_table) {
 		int sid;
 
 		sid = tracker_get_id_for_service (service);
-		if (!db_con->user_data) {
-			tracker_log ("WARNING: Cache Database not found");
-		} else {
-			tracker_db_update_differential_index (db_con->user_data, old_table, new_table, id, sid);
-		}
+		g_debug ("updating differential metadata for %s", meta_name);
+		tracker_db_update_differential_index (old_table, new_table, id, sid);
+		
 		g_hash_table_destroy (new_table);
 	}
 
@@ -2258,8 +2282,6 @@ tracker_db_refresh_all_display_metadata (DBConnection *db_con, const char *id)
 {
 	char ***res;
 
-	tracker_db_start_transaction (db_con);
-
 	tracker_exec_proc (db_con, "DeleteAllDisplayMetadata", 1, id);
 
 	res = tracker_exec_proc (db_con, "GetAllDisplayMetadataTypes", 3, id, id, id);
@@ -2286,7 +2308,6 @@ tracker_db_refresh_all_display_metadata (DBConnection *db_con, const char *id)
 		
 	}
 
-	tracker_db_end_transaction (db_con);
 }
 
 
@@ -2316,53 +2337,144 @@ tracker_get_metadata_table (DataTypes type)
 
 
 
-void 
-tracker_db_delete_metadata_value (DBConnection *db_con, const char *service, const char *id, const char *key, const char *value, gboolean embedded) 
+/* fast insert of embedded metadata for new values only (no checks for overwriting, multiple values). Table parameter is used to build up a unique word list of indexable contents */ 
+void
+tracker_db_insert_embedded_metadata (DBConnection *db_con, const char *service, const char *id, const char *key, const char *value, GHashTable *table)
 {
+	FieldDef   *def;
 
-	char 		*old_value = NULL, *new_value = NULL;
-	FieldDef	*def;
-	gboolean 	update_index = FALSE;
+	g_return_if_fail (id);
 
-	g_return_if_fail (id && key && service && db_con);
+	def = tracker_db_get_field_def (db_con, key);
+
+	if (!def) {
+		tracker_log ("metadata type %s not found", key);
+		return;
+	}
+
+	switch (def->type) {
+
+		case DATA_INDEX:
+			if (table) {
+				gboolean filter_words =  ((strcmp (key,  "File:Delimited") != 0) && (strcmp (key, "File:Name") != 0));
+				table = tracker_parse_text (table, value, def->weight, filter_words);
+			}
+
+		case DATA_STRING:
+
+			tracker_exec_proc (db_con, "SetMetadataString", 4, id, def->id, value, "1"); 
+			break;
+
+		case DATA_NUMERIC:
+		case DATA_DATE:
+
+			tracker_exec_proc (db_con, "SetMetadataNumeric", 4, id, def->id, value, "1"); 
+
+			break;
+
+		case DATA_BLOB :
+			
+			tracker_log ("Error: metadata could not be set as type %d for metadata %s is not supported", def->type, key);
+			break;
+
+		case DATA_KEYWORD:
+			if (table) {
+				table = tracker_parse_text (table, value, def->weight, TRUE);
+			}
+
+			tracker_exec_proc (db_con, "SetMetadataKeyword", 4, id, def->id, value, "1");
+			break;
+
+		case DATA_FULLTEXT:
+			if (table) {
+				table = tracker_parse_text  (table, value, def->weight, TRUE);
+			}
+
+			if (value) {
+				save_full_text (db_con->user_data2, id, value, strlen (value));
+			}
+			break;
+	}
+
+
+	tracker_db_free_field_def (def);
+
+	
+}
+
+
+
+void
+tracker_db_set_metadata (DBConnection *db_con, const char *service, const char *id, const char *key, const char *value, gboolean generate_display_metadata, gboolean index, gboolean embedded)
+{
+	FieldDef   *def;
+	char *old_value = NULL, *new_value = NULL;
+	const char *str_embedded;
+	gboolean update_index = FALSE;
+
+	g_return_if_fail (id);
 
-	/* get type details */
 	def = tracker_db_get_field_def (db_con, key);
 
 	if (!def) {
+		tracker_log ("metadata type %s not found", key);
 		return;
 	}
 
 
-	/* get current value */	
-	char ***res = tracker_db_get_metadata (db_con, service, id, key);
+	if (def->type != DATA_INDEX && def->type != DATA_KEYWORD && def->type !=  DATA_FULLTEXT) { 
+		index = FALSE;
+	}
 
-	if (res) {
-		char **row;
 
-		row = tracker_db_get_row (res, 0);
+	/* get old value for comparison if indexing  */
 
-		if (row && row[0]) {
-			old_value = g_strdup (row[0]);
+	if (index) {
+
+		char ***res = tracker_db_get_metadata (db_con, service, id, key);
+
+		if (res) {
+			char **row;
+
+			row = tracker_db_get_row (res, 0);
+
+			if (row && row[0]) {
+				old_value = g_strdup (row[0]);
+			}
+
+			tracker_db_free_result (res);
 		}
+	}
 
-		tracker_db_free_result (res);
+
+	/* delete old value if metadata does not support multiple values */
+	if (!def->multiple_values) {
+		tracker_db_delete_metadata (db_con, service, id, key, FALSE);
 	}
 
 
-	
-	
-	/* perform deletion */
+
+	if (embedded) {
+		str_embedded = "1";
+	} else {
+		str_embedded = "0";
+	}
+
+
+
+
 	switch (def->type) {
 
 		case DATA_INDEX:
-			update_index = TRUE;
+			if (index) {
+				update_index = TRUE;
+			}
 
 		case DATA_STRING:
 
-			tracker_exec_proc (db_con, "DeleteMetadataStringValue", 3, id, def->id, value); 
+			tracker_exec_proc (db_con, "SetMetadataString", 4, id, def->id, value, str_embedded); 
 			
-			if (def->multiple_values) {
+			if (generate_display_metadata && def->multiple_values) {
 				new_value = tracker_db_refresh_display_metadata (db_con, id, def->id, def->type, key);
 			} else {
 				new_value = g_strdup (value);
@@ -2374,36 +2486,45 @@ tracker_db_delete_metadata_value (DBConnection *db_con, const char *service, con
 		case DATA_NUMERIC:
 		case DATA_DATE:
 
-			tracker_exec_proc (db_con, "DeleteMetadataNumericValue", 3, id, def->id, value);  
+			tracker_exec_proc (db_con, "SetMetadataNumeric", 4, id, def->id, value, str_embedded); 
 
-			if (def->multiple_values) {
+			if (generate_display_metadata && def->multiple_values) {
 				new_value = tracker_db_refresh_display_metadata (db_con, id, def->id, def->type, key);
 			} else {
-				new_value = g_strdup (value);			
+				new_value = g_strdup (value);
 			}
 
 			break;
 
 		case DATA_BLOB :
-		case DATA_FULLTEXT:
 			
 			tracker_log ("Error: metadata could not be set as type %d for metadata %s is not supported", def->type, key);
 			break;
 
 		case DATA_KEYWORD:
 
-			update_index = TRUE;
+			if (index) {
+				update_index = TRUE;
+			}
 
-			tracker_exec_proc (db_con, "DeleteMetadataKeywordValue", 3, id, def->id, value); 
+			tracker_exec_proc (db_con, "SetMetadataKeyword", 4, id, def->id, value, str_embedded);
 
-			if (def->multiple_values) {
+			if (generate_display_metadata && def->multiple_values) {
 				new_value = tracker_db_refresh_display_metadata (db_con, id, def->id, def->type, key);
 			} else {
-				new_value = g_strdup (value);			
+				new_value = g_strdup (value);
 			}
 
 			break;
 
+		case DATA_FULLTEXT:
+
+			if (index) {
+				update_index = TRUE;
+			}
+
+			new_value = g_strdup (value);
+			save_full_text (db_con->user_data2, id, value, strlen (value));
 
 	}
 
@@ -2416,20 +2537,21 @@ tracker_db_delete_metadata_value (DBConnection *db_con, const char *service, con
 
 	g_free (new_value);
 	g_free (old_value);
-	tracker_db_free_field_def (def);	
+	tracker_db_free_field_def (def);
 }
 
 
+
 void 
-tracker_db_delete_metadata (DBConnection *db_con, const char *service, const char *id, const char *key) 
+tracker_db_delete_metadata_value (DBConnection *db_con, const char *service, const char *id, const char *key, const char *value, gboolean embedded) 
 {
-	char 		*old_value = NULL;
+
+	char 		*old_value = NULL, *new_value = NULL;
 	FieldDef	*def;
 	gboolean 	update_index = FALSE;
 
 	g_return_if_fail (id && key && service && db_con);
 
-
 	/* get type details */
 	def = tracker_db_get_field_def (db_con, key);
 
@@ -2453,9 +2575,7 @@ tracker_db_delete_metadata (DBConnection *db_con, const char *service, const cha
 		tracker_db_free_result (res);
 	}
 
-	if (def->multiple_values) {
-		tracker_exec_proc (db_con, "DeleteMetadataDisplay", 2, id, def->id);
-	}
+
 	
 	
 	/* perform deletion */
@@ -2465,239 +2585,155 @@ tracker_db_delete_metadata (DBConnection *db_con, const char *service, const cha
 			update_index = TRUE;
 
 		case DATA_STRING:
-			tracker_exec_proc (db_con, "DeleteMetadataString", 2, id, def->id); 
+
+			tracker_exec_proc (db_con, "DeleteMetadataStringValue", 3, id, def->id, value); 
+			
+			if (def->multiple_values) {
+				new_value = tracker_db_refresh_display_metadata (db_con, id, def->id, def->type, key);
+			} else {
+				new_value = g_strdup (value);
+			}
+	 	
 			break;
 
+		
 		case DATA_NUMERIC:
 		case DATA_DATE:
-			tracker_exec_proc (db_con, "DeleteMetadataNumeric", 2, id, def->id);  
+
+			tracker_exec_proc (db_con, "DeleteMetadataNumericValue", 3, id, def->id, value);  
+
+			if (def->multiple_values) {
+				new_value = tracker_db_refresh_display_metadata (db_con, id, def->id, def->type, key);
+			} else {
+				new_value = g_strdup (value);			
+			}
+
 			break;
 
 		case DATA_BLOB :
+		case DATA_FULLTEXT:
+			
 			tracker_log ("Error: metadata could not be set as type %d for metadata %s is not supported", def->type, key);
 			break;
 
 		case DATA_KEYWORD:
-			update_index = TRUE;
 
-			tracker_exec_proc (db_con, "DeleteMetadataKeyword", 2, id, def->id); 
-			break;
+			update_index = TRUE;
 
-		case DATA_FULLTEXT:
+			tracker_exec_proc (db_con, "DeleteMetadataKeywordValue", 3, id, def->id, value); 
 
-			update_index = TRUE;
+			if (def->multiple_values) {
+				new_value = tracker_db_refresh_display_metadata (db_con, id, def->id, def->type, key);
+			} else {
+				new_value = g_strdup (value);			
+			}
 
-			tracker_exec_proc (db_con, "DeleteFileContents", 1, id); 
-						
+			break;
 
 
 	}
 
+//	tracker_log ("replacing old value %s with new value %s for key %s", old_value, new_value, key);
 	
-	/* update fulltext index differentially with old values and NULL */
+	/* update fulltext index differentially with current and new values */
 	if (update_index) {
-		update_metadata_index (db_con, id, service, key, old_value, " ");
+		update_metadata_index (db_con, id, service, key, old_value, new_value);
 	}
 
-	
+	g_free (new_value);
 	g_free (old_value);
-
-
+	tracker_db_free_field_def (def);	
 }
 
 
-/* fast insert of embedded metadata for new values only (no checks for overwriting, multiple values or support for indexing) */ 
-void
-tracker_db_insert_embedded_metadata (DBConnection *db_con, const char *service, const char *id, const char *key, const char *value)
+void 
+tracker_db_delete_metadata (DBConnection *db_con, const char *service, const char *id, const char *key, gboolean update_indexes) 
 {
-	FieldDef   *def;
-
-	g_return_if_fail (id);
-
-	def = tracker_db_get_field_def (db_con, key);
-
-	if (!def) {
-		tracker_log ("metadata type %s not found", key);
-		return;
-	}
-
-	switch (def->type) {
-
-		case DATA_INDEX:
-		case DATA_STRING:
-
-			tracker_exec_proc (db_con, "SetMetadataString", 4, id, def->id, value, "1"); 
-			break;
-
-		case DATA_NUMERIC:
-		case DATA_DATE:
-
-			tracker_exec_proc (db_con, "SetMetadataNumeric", 4, id, def->id, value, "1"); 
-
-			break;
-
-		case DATA_BLOB :
-			
-			tracker_log ("Error: metadata could not be set as type %d for metadata %s is not supported", def->type, key);
-			break;
-
-		case DATA_KEYWORD:
-
-			tracker_exec_proc (db_con, "SetMetadataKeyword", 4, id, def->id, value, "1");
-			break;
-
-		case DATA_FULLTEXT:
-
-			if (value) {
-				save_full_text (db_con->user_data2, id, value, strlen (value));
-			}
-			break;
-	}
-
-
-	tracker_db_free_field_def (def);
-
-	
-}
+	char 		*old_value = NULL;
+	FieldDef	*def;
+	gboolean 	update_index = FALSE;
 
+	g_return_if_fail (id && key && service && db_con);
 
-void
-tracker_db_set_metadata (DBConnection *db_con, const char *service, const char *id, const char *key, const char *value, gboolean generate_display_metadata, gboolean index, gboolean embedded)
-{
-	FieldDef   *def;
-	char *old_value = NULL, *new_value = NULL;
-	const char *str_embedded;
-	gboolean update_index = FALSE;
-
-	g_return_if_fail (id);
 
+	/* get type details */
 	def = tracker_db_get_field_def (db_con, key);
 
 	if (!def) {
-		tracker_log ("metadata type %s not found", key);
 		return;
 	}
 
 
-	if (def->type != DATA_INDEX && def->type != DATA_KEYWORD && def->type !=  DATA_FULLTEXT) { 
-		index = FALSE;
-	}
-
-
-	/* get old value for comparison if indexing  */
-
-	if (index) {
-
-		char ***res = tracker_db_get_metadata (db_con, service, id, key);
-
-		if (res) {
-			char **row;
+	/* get current value */	
+	char ***res = tracker_db_get_metadata (db_con, service, id, key);
 
-			row = tracker_db_get_row (res, 0);
+	if (res) {
+		char **row;
 
-			if (row && row[0]) {
-				old_value = g_strdup (row[0]);
-			}
+		row = tracker_db_get_row (res, 0);
 
-			tracker_db_free_result (res);
+		if (row && row[0]) {
+			old_value = g_strdup (row[0]);
 		}
-	}
 
-
-	/* delete old value if metadata does not support multiple values */
-	if (!def->multiple_values) {
-		tracker_db_delete_metadata (db_con, service, id, key);
+		tracker_db_free_result (res);
 	}
 
-
-
-	if (embedded) {
-		str_embedded = "1";
-	} else {
-		str_embedded = "0";
+	if (def->multiple_values) {
+		tracker_exec_proc (db_con, "DeleteMetadataDisplay", 2, id, def->id);
 	}
-
-
-
-
+	
+	
+	/* perform deletion */
 	switch (def->type) {
 
 		case DATA_INDEX:
-			if (index) {
-				update_index = TRUE;
-			}
+			update_index = TRUE;
 
 		case DATA_STRING:
-
-			tracker_exec_proc (db_con, "SetMetadataString", 4, id, def->id, value, str_embedded); 
-			
-			if (generate_display_metadata && def->multiple_values) {
-				new_value = tracker_db_refresh_display_metadata (db_con, id, def->id, def->type, key);
-			} else {
-				new_value = g_strdup (value);
-			}
-	 	
+			tracker_exec_proc (db_con, "DeleteMetadataString", 2, id, def->id); 
 			break;
 
-		
 		case DATA_NUMERIC:
 		case DATA_DATE:
-
-			tracker_exec_proc (db_con, "SetMetadataNumeric", 4, id, def->id, value, str_embedded); 
-
-			if (generate_display_metadata && def->multiple_values) {
-				new_value = tracker_db_refresh_display_metadata (db_con, id, def->id, def->type, key);
-			} else {
-				new_value = g_strdup (value);
-			}
-
+			tracker_exec_proc (db_con, "DeleteMetadataNumeric", 2, id, def->id);  
 			break;
 
 		case DATA_BLOB :
-			
 			tracker_log ("Error: metadata could not be set as type %d for metadata %s is not supported", def->type, key);
 			break;
 
 		case DATA_KEYWORD:
+			update_index = TRUE;
 
-			if (index) {
-				update_index = TRUE;
-			}
-
-			tracker_exec_proc (db_con, "SetMetadataKeyword", 4, id, def->id, value, str_embedded);
-
-			if (generate_display_metadata && def->multiple_values) {
-				new_value = tracker_db_refresh_display_metadata (db_con, id, def->id, def->type, key);
-			} else {
-				new_value = g_strdup (value);
-			}
-
+			tracker_exec_proc (db_con, "DeleteMetadataKeyword", 2, id, def->id); 
 			break;
 
 		case DATA_FULLTEXT:
 
-			if (index) {
-				update_index = TRUE;
-			}
+			update_index = TRUE;
+
+			tracker_exec_proc (db_con, "DeleteFileContents", 1, id); 
+						
 
-			new_value = g_strdup (value);
-			save_full_text (db_con->user_data2, id, value, strlen (value));
 
 	}
 
-//	tracker_log ("replacing old value %s with new value %s for key %s", old_value, new_value, key);
 	
-	/* update fulltext index differentially with current and new values */
-	if (update_index) {
-		update_metadata_index (db_con, id, service, key, old_value, new_value);
+	/* update fulltext index differentially with old values and NULL */
+	if (update_index && update_indexes) {
+		update_metadata_index (db_con, id, service, key, old_value, " ");
 	}
 
-	g_free (new_value);
+	
 	g_free (old_value);
-	tracker_db_free_field_def (def);
+
+
 }
 
 
+
+
 guint32
 tracker_db_create_service (DBConnection *db_con, const char *path, const char *name, const char *service, const char *mime, guint32 filesize, gboolean is_dir, gboolean is_link, int offset, guint32 mtime, guint aux_id)
 {
@@ -2818,6 +2854,10 @@ delete_id_from_list (gpointer         key,
 	guint32 file_id = GPOINTER_TO_UINT (data);
 
 	list = value;
+
+	if (!list) {
+		return 1;
+	}
 	
 	for (l=list;l;l=l->next) {
 		wd = l->data;
@@ -2843,7 +2883,7 @@ delete_id_from_list (gpointer         key,
 static void
 delete_cache_words (guint32 file_id)
 {
-  	g_hash_table_foreach (tracker->cached_table, (GHFunc) delete_id_from_list, GUINT_TO_POINTER (file_id));	
+  	//g_hash_table_foreach (tracker->cached_table, (GHFunc) delete_id_from_list, GUINT_TO_POINTER (file_id));	
 }
 
 
@@ -3348,7 +3388,7 @@ tracker_db_search_text_location (DBConnection *db_con, const char *text, const c
 
 	location_prefix = g_strconcat (location, G_DIR_SEPARATOR_S, NULL);
 
-	array = tracker_parse_text_into_array ( text);
+	array = tracker_parse_text_into_array (text);
 
 	hit_list = tracker_indexer_get_hits (tracker->file_indexer, array, 0, 9, 0, 999999, FALSE, &count);
 
@@ -3997,6 +4037,10 @@ update_index_data (gpointer key,
 	score = GPOINTER_TO_INT (value);
 	info = user_data;
 
+	if (score == 0) return;
+
+	g_debug ("updating index for word %s with score %d", word, score);
+
 	if (!update_cache_word (info->db_con, word, info->service_id, score)) {
 		tracker_indexer_update_word (tracker->file_indexer, word, info->service_id, info->service_type_id, score, FALSE);
 	}
@@ -4031,10 +4075,9 @@ update_index_data (gpointer key,
 
 
 void
-tracker_db_update_indexes_for_new_service (DBConnection *db_con, DBConnection *cache_db_con, guint32 service_id, int service_type_id, GHashTable *table)
+tracker_db_update_indexes_for_new_service (guint32 service_id, int service_type_id, GHashTable *table)
 {
-	table = get_indexable_content_words (db_con, service_id, table);
-
+	
 	if (table) {
 		ServiceTypeInfo *info;
 
@@ -4042,11 +4085,8 @@ tracker_db_update_indexes_for_new_service (DBConnection *db_con, DBConnection *c
 
 		info->service_id = service_id;
 		info->service_type_id = service_type_id;
-		info->db_con = cache_db_con;
-
-		tracker_db_start_transaction (db_con);
+	
 		g_hash_table_foreach (table, append_index_data, info);
-		tracker_db_end_transaction (db_con);
 		g_free (info);
 	}
 }
@@ -4068,8 +4108,6 @@ cmp_data (gpointer key,
 
 	lookup_score = GPOINTER_TO_INT (g_hash_table_lookup (new_table, word));
 
-	g_debug ("word %s has old score %d and new score %d so updating with total score %d", word, score, lookup_score, lookup_score-score);
-
 	/* subtract scores so only words with score != 0 are updated (when score is zero, old word score is same as new word so no updating necessary)
 	   negative scores mean either word exists in old but no new data or has a lower score in new than old */
 	g_hash_table_insert (new_table, g_strdup (word), GINT_TO_POINTER (lookup_score - score));
@@ -4077,7 +4115,7 @@ cmp_data (gpointer key,
 
 
 void
-tracker_db_update_differential_index (DBConnection *db_con, GHashTable *old_table, GHashTable *new_table, const char *id, int service_type_id)
+tracker_db_update_differential_index (GHashTable *old_table, GHashTable *new_table, const char *id, int service_type_id)
 {
 	ServiceTypeInfo *info;
 
@@ -4096,7 +4134,6 @@ tracker_db_update_differential_index (DBConnection *db_con, GHashTable *old_tabl
 
 	info->service_id = strtoul (id, NULL, 10);
 	info->service_type_id = service_type_id;
-	info->db_con = db_con;
 
 	g_hash_table_foreach (new_table, update_index_data, info);
 
@@ -4129,3 +4166,4 @@ tracker_db_get_keyword_list (DBConnection *db_con, const char *service)
 
 	return res;
 }
+
diff --git a/src/trackerd/tracker-db-sqlite.h b/src/trackerd/tracker-db-sqlite.h
index 66772c921..210d8aa63 100644
--- a/src/trackerd/tracker-db-sqlite.h
+++ b/src/trackerd/tracker-db-sqlite.h
@@ -45,7 +45,12 @@ typedef enum {
 	DB_CACHE
 } DBTypes;
 
-
+typedef enum {
+	METADATA_INDEX,
+	METADATA_REINDEX,
+	METADATA_USER,
+	METADATA_USER_REPLACE
+} MetadataAction;
 
 typedef struct {
 	char		*id;
@@ -96,14 +101,15 @@ void		tracker_db_exec_no_reply 	(DBConnection *db_con, const char *query);
 void		tracker_log_sql			(DBConnection *db_con, const char *query);
 void		tracker_create_db		(void);
 void		tracker_db_load_stored_procs	(DBConnection *db_con);
-void		tracker_db_save_file_contents	(DBConnection *db_con, DBConnection *blob_db_con, DBConnection *cache_db_con, const char *file_name, FileInfo *info);
+void		tracker_db_save_file_contents	(DBConnection *db_con, DBConnection *blob_db_con, GHashTable *index_table, const char *file_name, FileInfo *info);
 void		tracker_db_clear_temp		(DBConnection *db_con);
 void		tracker_db_check_tables		(DBConnection *db_con);
 void		tracker_db_start_transaction	(DBConnection *db_con);
 void		tracker_db_end_transaction	(DBConnection *db_con);
 
-void		tracker_db_update_indexes_for_new_service	(DBConnection *db_con, DBConnection *cache_db_con, guint32 service_id, int service_type_id, GHashTable *table);
-void		tracker_db_update_differential_index		(DBConnection *db_con, GHashTable *old_table, GHashTable *new_table, const char *id, int service_type_id);
+void		tracker_db_update_indexes_for_new_service	(guint32 service_id, int service_type_id, GHashTable *table);
+void		tracker_db_update_differential_index		(GHashTable *old_table, GHashTable *new_table, const char *id, int service_type_id);
+void		tracker_db_update_index_file_contents 		(DBConnection *blob_db_con, GHashTable *index_table);
 int		tracker_db_flush_words_to_qdbm 			(DBConnection *db_con, int limit);
 
 void		tracker_db_release_memory 	();
@@ -122,10 +128,10 @@ char ***	tracker_db_get_metadata		(DBConnection *db_con, const char *service, co
 
 /* gets metadata using a separate row for each value it has */
 char ***	tracker_db_get_metadata_values 	(DBConnection *db_con, const char *service, const char *id, const char *key);
-void		tracker_db_insert_embedded_metadata (DBConnection *db_con, const char *service, const char *id, const char *key, const char *value);
+void		tracker_db_insert_embedded_metadata (DBConnection *db_con, const char *service, const char *id, const char *key, const char *value, GHashTable *table);
 void		tracker_db_set_metadata		 (DBConnection *db_con, const char *service, const char *id, const char *key, const char *value, gboolean generate_display_metadata, gboolean index, gboolean embedded);
 void		tracker_db_delete_metadata_value (DBConnection *db_con, const char *service, const char *id, const char *key, const char *value, gboolean embedded);
-void 		tracker_db_delete_metadata 	 (DBConnection *db_con, const char *service, const char *id, const char *key);
+void 		tracker_db_delete_metadata 	 (DBConnection *db_con, const char *service, const char *id, const char *key, gboolean update_indexes); 
 
 char *		tracker_db_refresh_display_metadata 	(DBConnection *db_con, const char *id,  const char *metadata_id, int data_type, const char *key);
 void		tracker_db_refresh_all_display_metadata (DBConnection *db_con, const char *id);
@@ -163,4 +169,6 @@ char ***	tracker_db_delete_sub_watches	(DBConnection *db_con, const char *dir);
 
 char ***	tracker_db_get_keyword_list	(DBConnection *db_con, const char *service);
 
+
+
 #endif
diff --git a/src/trackerd/tracker-db.c b/src/trackerd/tracker-db.c
index 9570d8176..3998872a9 100644
--- a/src/trackerd/tracker-db.c
+++ b/src/trackerd/tracker-db.c
@@ -19,10 +19,11 @@
 
 #include <stdlib.h>
 #include <string.h>
-#include <time.h>
-
+#include <time.h> 
+#include <glib/gstdio.h>
 #include "tracker-db.h"
-
+#include "tracker-email.h"
+#include "tracker-metadata.h"
 
 extern Tracker *tracker;
 
@@ -30,6 +31,7 @@ extern Tracker *tracker;
 typedef struct {
 	DBConnection 	*db_con;
 	char 		*file_id;
+	GHashTable	*table;
 } DatabaseAction;
 
 
@@ -211,7 +213,7 @@ tracker_db_get_file_info (DBConnection *db_con, FileInfo *info)
 
 
 static void
-tracker_db_add_embedded_keywords (DBConnection *db_con, const char *file_id, const char *keyword_type, const char *keywords, gboolean index)
+tracker_db_add_embedded_keywords (DBConnection *db_con, const char *file_id, const char *keyword_type, const char *keywords, gboolean is_new, GHashTable *table)
 {
 	char **array, **tags;
 	char *tag;
@@ -224,9 +226,15 @@ tracker_db_add_embedded_keywords (DBConnection *db_con, const char *file_id, con
 		tag = g_strstrip (tag);
 
 		if (strlen (tag) > 0) {
-			tracker_log ("Auto-tagging file with %s", tag);
-			tracker_db_set_metadata (db_con, "Files", file_id, keyword_type, tag, FALSE, index, TRUE);
-//			tracker_exec_proc (db_con, "AddEmbeddedKeyword", 3, file_id, keyword_type, tag);
+			if (tracker->verbosity > 0) {
+				tracker_log ("Auto-tagging file with %s", tag);
+			}
+		}
+
+		if (is_new) {
+			tracker_db_insert_embedded_metadata (db_con, "Files", file_id, keyword_type, tag, table);
+		} else {
+			tracker_db_set_metadata (db_con, "Files", file_id, keyword_type, tag, FALSE, TRUE, TRUE);
 		}
 	}
 
@@ -294,7 +302,7 @@ get_meta_table_data (gpointer key,
 
 	/* auto-tag keyword related metadata */	
 	if ( (strcasecmp (mtype, "Doc:Keywords") == 0) || (strcasecmp (mtype, "Image:Keywords") == 0) ) {
-		tracker_db_add_embedded_keywords (db_action->db_con, db_action->file_id, mtype, mvalue, TRUE);
+		tracker_db_add_embedded_keywords (db_action->db_con, db_action->file_id, mtype, mvalue, FALSE, NULL);
 	} else {
 		tracker_db_set_metadata (db_action->db_con, "Files", db_action->file_id, mtype, mvalue, FALSE, TRUE, TRUE);
 	}
@@ -364,9 +372,9 @@ get_meta_table_data_new (gpointer key,
 
 	/* auto-tag keyword related metadata */	
 	if ( (strcasecmp (mtype, "Doc:Keywords") == 0) || (strcasecmp (mtype, "Image:Keywords") == 0) ) {
-		tracker_db_add_embedded_keywords (db_action->db_con, db_action->file_id, mtype, mvalue, FALSE);
+		tracker_db_add_embedded_keywords (db_action->db_con, db_action->file_id, mtype, mvalue, TRUE, db_action->table);
 	} else {
-		tracker_db_insert_embedded_metadata (db_action->db_con, "Files", db_action->file_id, mtype, mvalue);
+		tracker_db_insert_embedded_metadata (db_action->db_con, "Files", db_action->file_id, mtype, mvalue, db_action->table);
 	}
 
 	if (mvalue) {
@@ -375,28 +383,39 @@ get_meta_table_data_new (gpointer key,
 }
 
 
-void
-tracker_db_save_metadata (DBConnection *db_con, GHashTable *table, guint32 file_id, gboolean new_file)
+GHashTable *
+tracker_db_save_metadata (DBConnection *db_con, GHashTable *table, GHashTable *index_table, guint32 file_id, gboolean new_file)
 {
 	DatabaseAction db_action;
-
-	g_return_if_fail (file_id != 0 || table || db_con);
+	
+	if ((file_id == 0 || !table || !db_con)) {
+		return NULL;
+	}
 
 	db_action.db_con = db_con;
 
-	db_action.file_id = tracker_uint_to_str ( file_id);
+	if (new_file && index_table) {
+		db_action.table = index_table;
+	} else {
+		db_action.table = NULL;
+	}
+
+	db_action.file_id = tracker_uint_to_str (file_id);
 
 	if (table) {
-		tracker_db_start_transaction (db_con);
+		
 		if (new_file) {
 			g_hash_table_foreach (table, get_meta_table_data_new, &db_action);
 		} else {
 			g_hash_table_foreach (table, get_meta_table_data, &db_action);
 		}
-		tracker_db_end_transaction (db_con);
+		
 	}
 
 	g_free (db_action.file_id);
+
+	return db_action.table;
+
 }
 
 
@@ -745,3 +764,332 @@ tracker_is_valid_service (DBConnection *db_con, const char *service)
 }
 
 
+
+void
+tracker_db_index_service (DBConnection *db_con, FileInfo *info, const char *service, GHashTable *meta_table, gboolean is_attachment, gboolean get_embedded, gboolean get_full_text, gboolean get_thumbs)
+{
+	char	*name, *path, *str_file_id;
+	GHashTable *index_table;
+
+	if (!service) {
+		/* its an external service - TODO get external service name */
+		tracker_log ("External service %s not supported yet", service);
+		return;
+	}
+
+
+	info->service_type_id = tracker_get_id_for_service (service);
+	if (info->service_type_id == -1) {
+		tracker_log ("Service %s not supported yet", service);
+		return;
+	}
+
+
+	if (tracker->verbosity > 0) {
+		if (info->is_new) {
+			tracker_log ("Indexing %s with service %s and mime %s (new)", info->uri, service, info->mime);
+		} else {
+			tracker_log ("Indexing %s with service %s and mime %s (existing)", info->uri, service, info->mime);
+		}
+	}
+
+	str_file_id = tracker_uint_to_str (info->file_id);
+
+	if (info->is_new) {
+		index_table = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+	} else {
+		index_table = NULL;
+	}
+
+
+
+	/* get embedded metadata filter */
+	if (get_embedded && meta_table) {	
+
+		tracker_metadata_get_embedded (info->uri, info->mime, meta_table);
+		/* to do - emit dbus signal here for EmbeddedMetadataChanged */
+		
+	} 
+
+	if (info->is_new) {
+
+		
+		name = g_path_get_basename (info->uri);
+		path = g_path_get_dirname (info->uri);
+
+		if (is_attachment) {
+			info->file_id = tracker_db_create_service (db_con, path, name, "EmailAttachments", info->mime, info->file_size, info->is_directory, info->is_link, info->offset, info->mtime, -1);
+		} else {
+			info->file_id = tracker_db_create_service (db_con, path, name, service, info->mime, info->file_size, info->is_directory, info->is_link, info->offset, info->mtime, -1);
+		}
+
+		g_free (name);
+		g_free (path);
+
+		if (info->file_id == 0) {
+			tracker_log ("Error: Could not get file id for %s - unable to continue indexing this file", info->uri);
+			return;
+		}
+
+		if (info->service_type_id == -1) {
+			tracker_log ("******ERROR****** Unknown service type for %s with service %s and mime %s", info->uri, service, info->mime);
+		}
+	}
+
+	if (get_thumbs && tracker->enable_thumbnails) {
+		char *small_thumb_file = NULL, *large_thumb_file = NULL;
+
+		small_thumb_file = tracker_metadata_get_thumbnail (info->uri, info->mime, "normal");
+
+		if (small_thumb_file) {
+			tracker_db_save_thumbs (db_con, small_thumb_file, large_thumb_file, info->file_id);
+		}
+
+		/* to do - emit dbus signal ThumbNailChanged */
+		if (small_thumb_file) {
+			g_free (small_thumb_file);
+		}
+
+	}
+
+	if (get_full_text && tracker->enable_content_indexing) {
+		char *file_as_text = tracker_metadata_get_text_file (info->uri, info->mime);
+
+		if (file_as_text) {
+			const char *tmp_dir;
+			char	   *dir;
+
+			tracker_db_save_file_contents (db_con, db_con->user_data2, index_table, file_as_text, info);
+			
+			tmp_dir = g_get_tmp_dir ();
+
+			dir = g_strconcat (tmp_dir, G_DIR_SEPARATOR_S, NULL);
+
+			/* clear up if text contents are in a temp file */
+			if (g_str_has_prefix (file_as_text, dir)) {
+				g_unlink (file_as_text);
+			}
+
+			g_free (dir);
+
+			g_free (file_as_text);
+
+		} else {
+			get_full_text = FALSE;
+		}
+
+	}
+
+
+
+	/* save stuff to Db */
+	tracker_db_start_transaction (db_con);
+
+	
+	if (!info->is_new) {
+
+		/* update existing file entry */
+		tracker_db_update_file (db_con, info);
+
+		/* mark metadata that needs to be deleted (IE all derived metadata in DB for an updated file) */
+		tracker_exec_proc (db_con, "MarkEmbeddedServiceMetadata1", 1, str_file_id);
+		tracker_exec_proc (db_con, "MarkEmbeddedServiceMetadata2", 1, str_file_id);
+		tracker_exec_proc (db_con, "MarkEmbeddedServiceMetadata3", 1, str_file_id);
+		tracker_exec_proc (db_con, "MarkEmbeddedServiceMetadata4", 1, str_file_id);
+	}
+
+	if (meta_table && (g_hash_table_size (meta_table) > 0)) {
+		tracker_db_save_metadata (db_con, meta_table, index_table, info->file_id, info->is_new);
+	}
+
+	if (!info->is_new) {
+		/* delete any old metadata that was not updated  */
+		tracker_exec_proc (db_con, "DeleteEmbeddedServiceMetadata1", 1, str_file_id);
+		tracker_exec_proc (db_con, "DeleteEmbeddedServiceMetadata2", 1, str_file_id);
+		tracker_exec_proc (db_con, "DeleteEmbeddedServiceMetadata3", 1, str_file_id);
+		tracker_exec_proc (db_con, "DeleteEmbeddedServiceMetadata4", 1, str_file_id);
+	}
+
+	/* update metadata display table */
+	tracker_db_refresh_all_display_metadata (db_con, str_file_id);
+
+	tracker_db_end_transaction (db_con);
+
+
+	/* update full text indexes */
+	if (info->is_new) {
+		tracker_db_update_indexes_for_new_service (info->file_id, info->service_type_id, index_table);
+	}
+
+	if (index_table) {
+		g_hash_table_destroy (index_table);
+	}
+
+	g_free (str_file_id);
+
+}
+
+void
+tracker_db_index_file (DBConnection *db_con, FileInfo *info, gboolean is_attachment)
+{
+	GHashTable	*meta_table;
+	const char 	*ext;
+	char 		*str_link_uri, *service_name;
+
+	char *services_with_metadata[] = {"Documents", "Music", "Videos", "Images", NULL};
+	char *services_with_text[] = {"Documents", "Text Files", "Development Files", NULL};
+	char *services_with_thumbs[] = {"Documents", "Images", "Videos", NULL};
+
+	meta_table = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
+
+	if (info->mime) {
+		g_free (info->mime);
+	}
+
+	if (!info->is_directory) {
+		info->mime = tracker_get_mime_type (info->uri);
+
+		if (!info->mime) {
+			info->mime = g_strdup ("unknown");
+		}
+	} else {
+		info->mime = g_strdup ("Folder");
+		info->file_size = 0;
+	}
+
+	if (info->is_link) {
+		str_link_uri = g_build_filename (info->link_path, info->link_name, NULL);
+	} else {
+		str_link_uri = g_strdup (" ");
+	}
+
+	meta_table = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
+
+	/* delimit file uri so hyphens and underscores are removed so that they can be indexed separately */
+	if (strchr (info->uri, '_') || strchr (info->uri, '-')) {
+		char	*delimited;
+
+		delimited = g_strdup (info->uri);
+		delimited =  g_strdelimit (delimited, "-_" , ' ');
+		g_debug ("delimited file name is %s", delimited);
+		g_hash_table_insert (meta_table, g_strdup ("File:NameDelimited"), g_strdup (delimited));
+		g_free (delimited);
+	}
+
+	ext = strrchr (info->uri, '.');
+	if (ext) {
+		ext++;
+		g_debug ("file extension is %s", ext);
+		g_hash_table_insert (meta_table, g_strdup ("File:Ext"), g_strdup (ext));
+	}
+
+	g_hash_table_insert (meta_table, g_strdup ("File:Path"), g_path_get_dirname (info->uri));
+	g_hash_table_insert (meta_table, g_strdup ("File:Name"), g_path_get_basename (info->uri));
+	g_hash_table_insert (meta_table, g_strdup ("File:Link"), g_strdup (str_link_uri));
+	g_hash_table_insert (meta_table, g_strdup ("File:Mime"), g_strdup (info->mime));
+	g_hash_table_insert (meta_table, g_strdup ("File:Size"), tracker_uint_to_str (info->file_size));
+	g_hash_table_insert (meta_table, g_strdup ("File:Permissions"), g_strdup (info->permissions));
+	g_hash_table_insert (meta_table, g_strdup ("File:Modified"), tracker_date_to_str (info->mtime));
+	g_hash_table_insert (meta_table, g_strdup ("File:Accessed"), tracker_date_to_str (info->atime));
+
+	g_free (str_link_uri);
+
+	if (info->is_directory) {
+		service_name = g_strdup ("Folders");
+	} else {
+		service_name = tracker_get_service_type_for_mime (info->mime);
+	}
+			
+	if (!info->mime) {
+		info->mime = g_strdup ("unknown");
+	}
+
+	gboolean  is_file_indexable, service_has_metadata, is_external_service, service_has_fulltext, service_has_thumbs;
+
+	is_external_service = g_str_has_prefix (info->mime, "service/");
+	is_file_indexable = !info->is_directory && (strcmp (info->mime, "unknown") != 0) && (strcmp (info->mime, "symlink") != 0);
+	
+	service_has_metadata = is_external_service || (is_file_indexable && (tracker_str_in_array (service_name, services_with_metadata) != -1));
+	service_has_fulltext = is_external_service || (is_file_indexable && (tracker_str_in_array (service_name, services_with_text) != -1));
+	service_has_thumbs = is_external_service || (is_file_indexable && (tracker_str_in_array (service_name, services_with_thumbs) != -1));
+
+	
+//	tracker_log ("file %s has fulltext %d, %d, %d", info->uri, (tracker_str_in_array (service_name, services_with_text) != -1), service_has_fulltext, is_file_known);
+	tracker_db_index_service (db_con, info, service_name, meta_table, is_attachment, service_has_metadata, service_has_fulltext, service_has_thumbs);
+
+	/* add extra delay to throttle back indexing of larger files */
+	if (service_has_fulltext && (info->file_size > 10000)) {
+		tracker_throttle (5000);
+	}
+
+	if (is_attachment) {
+		tracker_email_unlink_email_attachment (info->uri);
+	}
+
+	tracker_dec_info_ref (info);
+	g_hash_table_destroy (meta_table);
+	g_free (service_name);
+
+	
+	
+}
+
+
+
+static void 
+index_conversation (DBConnection *db_con, FileInfo *info)
+{
+
+/* todo */
+}
+
+
+static void
+index_application (DBConnection *db_con, FileInfo *info)
+{
+
+/* todo */
+}
+
+
+void
+tracker_db_index_entity (DBConnection *db_con, FileInfo *info)
+{
+
+	g_return_if_fail (info->uri && (info->uri[0] == '/'));
+
+	if (!info->is_directory) {
+		/* sleep  to throttle back indexing */
+		tracker_throttle (10000);
+	}
+
+	if (!tracker_file_is_valid (info->uri)) {
+		tracker_log ("Warning - file %s no longer exists - abandoning index on this file", info->uri);
+		return;
+	}
+
+	if (tracker_email_file_is_interesting (db_con, info)) {
+		tracker_email_index_file (db_con, info);
+
+	} else if (tracker_email_is_an_attachment (info)) {
+		tracker_db_index_file (db_con, info, TRUE);
+		
+	} else if (!info->mime || strcmp (info->mime, "unknown") == 0) {
+		g_free (info->mime);
+		info->mime = tracker_get_service_for_uri (info->uri);
+	}	
+
+	if (strcmp (info->mime, "Files") == 0) {
+		tracker_db_index_file (db_con, info, FALSE);
+		
+	} else if (strcmp (info->mime, "Conversations") == 0) {
+		index_conversation (db_con, info);
+
+	} else if (strcmp (info->mime, "Applications") == 0) {
+		index_application (db_con, info);
+
+	} else if (g_str_has_prefix (info->mime, "service/")) {
+		tracker_db_index_service (db_con, info, NULL, NULL, FALSE, TRUE, TRUE, TRUE);
+	}
+	
+}
diff --git a/src/trackerd/tracker-db.h b/src/trackerd/tracker-db.h
index b1783772c..645ec0c86 100644
--- a/src/trackerd/tracker-db.h
+++ b/src/trackerd/tracker-db.h
@@ -35,7 +35,7 @@ FileInfo *	tracker_db_get_file_info	(DBConnection *db_con, FileInfo *info);
 guint32		tracker_db_get_file_id		(DBConnection *db_con, const char *uri);
 gboolean	tracker_is_valid_service	(DBConnection *db_con, const char *service);
 char *		tracker_db_get_id		(DBConnection *db_con, const char *service, const char *uri);
-void		tracker_db_save_metadata	(DBConnection *db_con, GHashTable *table, guint32 file_id, gboolean new_file);
+GHashTable *	tracker_db_save_metadata	(DBConnection *db_con, GHashTable *table, GHashTable *index_table, guint32 file_id, gboolean new_file);
 void		tracker_db_save_thumbs		(DBConnection *db_con, const char *small_thumb, const char *large_thumb, guint32 file_id);
 char **		tracker_db_get_files_in_folder	(DBConnection *db_con, const char *folder_uri);
 FieldDef *	tracker_db_get_field_def	(DBConnection *db_con, const char *field_name);
@@ -48,4 +48,7 @@ void		tracker_db_add_to_extract_queue	(DBConnection *db_con, FileInfo *info);
 gboolean	tracker_db_has_pending_files	(DBConnection *db_con);
 gboolean	tracker_db_has_pending_metadata	(DBConnection *db_con);
 
+void		tracker_db_index_service 	(DBConnection *db_con, FileInfo *info, const char *service, GHashTable *meta_table, gboolean is_attachment, gboolean get_embedded, gboolean get_full_text, gboolean get_thumbs);
+void		tracker_db_index_file 		(DBConnection *db_con, FileInfo *info, gboolean is_attachment);
+void		tracker_db_index_entity 	(DBConnection *db_con, FileInfo *info);
 #endif
diff --git a/src/trackerd/tracker-dbus-keywords.c b/src/trackerd/tracker-dbus-keywords.c
index fb469649c..0d948a008 100644
--- a/src/trackerd/tracker-dbus-keywords.c
+++ b/src/trackerd/tracker-dbus-keywords.c
@@ -401,7 +401,7 @@ tracker_dbus_method_keywords_remove_all (DBusRec *rec)
 		return;
 	}
 
-	tracker_db_delete_metadata (db_con, service, id, "DC:Keywords");
+	tracker_db_delete_metadata (db_con, service, id, "DC:Keywords", TRUE);
 	
 	g_free (id);
 	
diff --git a/src/trackerd/tracker-parser.c b/src/trackerd/tracker-parser.c
index 20df34675..fe5a43a93 100644
--- a/src/trackerd/tracker-parser.c
+++ b/src/trackerd/tracker-parser.c
@@ -148,7 +148,7 @@ tracker_parse_text_into_array (const char *text)
 	int count;
 	GString *str;
 
-	table = tracker_parse_text (NULL, text, 1);
+	table = tracker_parse_text (NULL, text, 1, TRUE);
 
 	if (!table || g_hash_table_size (table) == 0) {
 		return NULL;
@@ -280,7 +280,7 @@ process_word (const char *index_word)
 
 
 GHashTable *
-tracker_parse_text (GHashTable *word_table, const char *text, int weight)
+tracker_parse_text (GHashTable *word_table, const char *text, int weight, gboolean filter_words)
 {
 	char *delimit_text;
 	int  i, j, total_words;
@@ -333,11 +333,12 @@ tracker_parse_text (GHashTable *word_table, const char *text, int weight)
 
 				word = g_strndup (start, i-j);
 
-				index_word = process_word (word);
-
-				//tracker_log ("word is %s and indexed word is %s", word, index_word);
-				
-				g_free (word);
+				if (filter_words) {
+					index_word = process_word (word);
+					g_free (word);
+				} else {
+					index_word = word;
+				}
 
 				if (!index_word) {
 					start = NULL;
diff --git a/src/trackerd/tracker-parser.h b/src/trackerd/tracker-parser.h
index 3469d2990..fdc45d879 100644
--- a/src/trackerd/tracker-parser.h
+++ b/src/trackerd/tracker-parser.h
@@ -36,7 +36,7 @@
  * 
  * returns the word_table.
  */
-GHashTable *	tracker_parse_text (GHashTable *word_table, const char *text,  int weight);
+GHashTable *	tracker_parse_text (GHashTable *word_table, const char *text, int weight, gboolean filter_words);
 
 char **		tracker_parse_text_into_array (const char *text);
 
diff --git a/src/trackerd/tracker-rdf-query.c b/src/trackerd/tracker-rdf-query.c
index 4a07e566e..616a25934 100644
--- a/src/trackerd/tracker-rdf-query.c
+++ b/src/trackerd/tracker-rdf-query.c
@@ -849,9 +849,9 @@ build_sql (ParserData *data)
 			sub = strchr (data->current_value, '*');
 
 			if (sub) {
-				g_string_append_printf (str, " (%s glob '*%s') ", field_data->meta_field, data->current_value);
+				g_string_append_printf (str, " (%s like '%s%s%s') ", field_data->meta_field, "%", data->current_value, "%");
 			} else {
-				g_string_append_printf (str, " (%s glob  '*%s*') ", field_data->meta_field, data->current_value);
+				g_string_append_printf (str, " (%s like '%s%s%s') ", field_data->meta_field, "%", data->current_value, "%");
 			}
 
 			break;
@@ -861,9 +861,9 @@ build_sql (ParserData *data)
 			sub = strchr (data->current_value, '*');
 
 			if (sub) {
-				g_string_append_printf (str, " (%s glob '%s') ", field_data->meta_field, data->current_value);
+				g_string_append_printf (str, " (%s like '%s') ", field_data->meta_field, data->current_value);
 			} else {
-				g_string_append_printf (str, " (%s glob '%s*') ", field_data->meta_field, data->current_value);
+				g_string_append_printf (str, " (%s like '%s%s') ", field_data->meta_field, data->current_value, "%");
 			}
 			
 			break;
diff --git a/src/trackerd/tracker-utils.h b/src/trackerd/tracker-utils.h
index 88931f02f..8b9942b57 100644
--- a/src/trackerd/tracker-utils.h
+++ b/src/trackerd/tracker-utils.h
@@ -99,6 +99,8 @@ typedef struct {
 	guint32		watch_limit;
 	guint32		poll_interval;
 
+	int		verbosity;
+
 	/* data directories */
 	char 		*data_dir;
 	char		*backup_dir;
diff --git a/src/trackerd/trackerd.c b/src/trackerd/trackerd.c
index 228d984cb..8d956c838 100644
--- a/src/trackerd/trackerd.c
+++ b/src/trackerd/trackerd.c
@@ -54,15 +54,12 @@
 #endif
 
 #include "tracker-db.h"
-#include "tracker-email.h"
-#include "tracker-metadata.h"
-
 #include "tracker-dbus-methods.h"
 #include "tracker-dbus-metadata.h"
 #include "tracker-dbus-keywords.h"
 #include "tracker-dbus-search.h"
 #include "tracker-dbus-files.h"
-
+#include "tracker-email.h"
 #include  "tracker-indexer.h"
 
 Tracker		       *tracker;
@@ -134,17 +131,16 @@ static char **no_watch_dirs = NULL;
 static char **watch_dirs = NULL;
 static char *language = NULL;
 static gboolean disable_indexing = FALSE;
-static gboolean low_memory, turbo, enable_debug, enable_evolution, enable_thunderbird, enable_kmail;
-static int throttle = 0, throttle_battery = 10;
+static gboolean low_memory, enable_evolution, enable_thunderbird, enable_kmail;
+static int throttle = 0;
+static int verbosity = 0;
 
 static GOptionEntry entries[] = {
 	{"exclude-dir", 'e', 0, G_OPTION_ARG_STRING_ARRAY, &no_watch_dirs, N_("Directory to exclude from indexing"), N_("/PATH/DIR")},
 	{"include-dir", 'i', 0, G_OPTION_ARG_STRING_ARRAY, &watch_dirs, N_("Directory to include in indexing"), N_("/PATH/DIR")},
 	{"no-indexing", 0, 0, G_OPTION_ARG_NONE, &disable_indexing, N_("Disable any indexing or watching taking place"), NULL },
-	{"debug", 0, 0, G_OPTION_ARG_NONE, &enable_debug, N_("Enables more verbose debug messages"), NULL },
-	{"throttle", 0, 0, G_OPTION_ARG_INT, &throttle, N_("Value to use for throttling indexing. Value must be in range 0-20 (default 5) with lower values increasing indexing speed"), N_("value") },
-	{"battery-throttle", 0, 0, G_OPTION_ARG_INT, &throttle_battery, N_("Value to use for throttling indexing and on battery power . Value must be in range 0-20 (default 15) with lower values increasing indexing speed"), N_("value") },
-	{"turbo", 't', 0, G_OPTION_ARG_NONE, &turbo, N_("Faster indexing, use more memory and CPU. Equivalent to --throttle=0"), NULL },
+	{"verbosity", 0, 0, G_OPTION_ARG_INT, &verbosity, N_("Value that controls the level of logging. Valid values are 0 (default minimal), 1 (detailed) and 2 (debug)"), N_("value") },
+	{"throttle", 0, 0, G_OPTION_ARG_INT, &throttle, N_("Value to use for throttling indexing. Value must be in range 0-20 (default 0) with lower values increasing indexing speed"), N_("value") },
 	{"low-memory", 'm', 0, G_OPTION_ARG_NONE, &low_memory, N_("Minimizes the use of memory but may slow indexing down"), NULL },
 	{"language", 'l', 0, G_OPTION_ARG_STRING, &language, N_("Language to use for stemmer and stop words list (ISO 639-1 2 characters code)"), N_("LANG")},
 	{NULL}
@@ -821,318 +817,8 @@ delete_directory (DBConnection *db_con, DBConnection *blob_db_con, FileInfo *inf
 
 
 
-static void
-index_service (DBConnection *db_con, FileInfo *info, const char *service, GHashTable *meta_table, gboolean is_attachment, gboolean get_embedded, gboolean get_full_text, gboolean get_thumbs)
-{
-	char	*name, *path, *str_file_id;
-
-	if (!service) {
-		/* its an external service - TODO get external service name */
-		tracker_log ("External service %s not supported yet", service);
-		return;
-	}
-
-
-	info->service_type_id = tracker_get_id_for_service (service);
-	if (info->service_type_id == -1) {
-		tracker_log ("Service %s not supported yet", service);
-		return;
-	}
-
-
-	if (info->is_new) {
-		tracker_log ("Indexing %s with service %s and mime %s (new)", info->uri, service, info->mime);
-	} else {
-		tracker_log ("Indexing %s with service %s and mime %s (existing)", info->uri, service, info->mime);
-	}
-
-	str_file_id = tracker_uint_to_str (info->file_id);
-
-	if (info->is_new) {
-	
-		name = g_path_get_basename (info->uri);
-		path = g_path_get_dirname (info->uri);
-
-		if (is_attachment) {
-			info->file_id = tracker_db_create_service (db_con, path, name, "EmailAttachments", info->mime, info->file_size, info->is_directory, info->is_link, info->offset, info->mtime, -1);
-		} else {
-			info->file_id = tracker_db_create_service (db_con, path, name, service, info->mime, info->file_size, info->is_directory, info->is_link, info->offset, info->mtime, -1);
-		}
-
-		g_free (name);
-		g_free (path);
-
-		if (info->file_id == 0) {
-			tracker_log ("Error: Could not get file id for %s - unable to continue indexing this file", info->uri);
-			return;
-		}
-
-		if (info->service_type_id == -1) {
-			tracker_log ("******ERROR****** Unknown service type for %s with service %s and mime %s", info->uri, service, info->mime);
-		}
-	} else {
-
-		/* update existing file entry */
-		tracker_db_update_file (db_con, info);
-
-		/* mark metadata that needs to be deleted (IE all derived metadata in DB for an updated file) */
-		
-		tracker_db_start_transaction (db_con);
-		tracker_exec_proc (db_con, "MarkEmbeddedServiceMetadata1", 1, str_file_id);
-		tracker_exec_proc (db_con, "MarkEmbeddedServiceMetadata2", 1, str_file_id);
-		tracker_exec_proc (db_con, "MarkEmbeddedServiceMetadata3", 1, str_file_id);
-		tracker_exec_proc (db_con, "MarkEmbeddedServiceMetadata4", 1, str_file_id);
-		tracker_db_end_transaction (db_con);
-		
-	}
-
-	
-
-	/* get embedded metadata filter */
-	if (get_embedded) {	
-
-		if (!meta_table) {
-			meta_table = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
-		}
-
-		tracker_metadata_get_embedded (info->uri, info->mime, meta_table);
-
-		/* to do - emit dbus signal here for EmbeddedMetadataChanged */
-		
-	} 
-
-	if (meta_table && (g_hash_table_size (meta_table) > 0)) {
-		tracker_db_save_metadata (db_con, meta_table, info->file_id, info->is_new);
-	}
-
-	if (get_thumbs && tracker->enable_thumbnails) {
-		char *small_thumb_file = NULL, *large_thumb_file = NULL;
-
-		small_thumb_file = tracker_metadata_get_thumbnail (info->uri, info->mime, "normal");
-
-		if (small_thumb_file) {
-			tracker_db_save_thumbs (db_con, small_thumb_file, large_thumb_file, info->file_id);
-		}
-
-		/* to do - emit dbus signal ThumbNailChanged */
-		if (small_thumb_file) {
-			g_free (small_thumb_file);
-		}
-
-	}
-
-	if (get_full_text && tracker->enable_content_indexing) {
-		char *file_as_text = tracker_metadata_get_text_file (info->uri, info->mime);
-
-		if (file_as_text) {
-			const char *tmp_dir;
-			char	   *dir;
-
-			tracker_db_save_file_contents (db_con, db_con->user_data2, db_con->user_data, file_as_text, info);
-
-			tmp_dir = g_get_tmp_dir ();
-
-			dir = g_strconcat (tmp_dir, G_DIR_SEPARATOR_S, NULL);
-
-			/* clear up if text contents are in a temp file */
-			if (g_str_has_prefix (file_as_text, dir)) {
-				g_unlink (file_as_text);
-			}
-
-			g_free (dir);
-
-			g_free (file_as_text);
-
-		} else {
-			get_full_text = FALSE;
-		}
-
-	}
-
-	if (!(get_full_text && tracker->enable_content_indexing) && info->is_new) {
-		tracker_db_update_indexes_for_new_service (db_con, db_con->user_data, info->file_id, info->service_type_id, NULL);
-	} else {
-
-		/* delete any old metadata that was not updated  */
-		tracker_exec_proc (db_con, "DeleteEmbeddedServiceMetadata1", 1, str_file_id);
-		tracker_exec_proc (db_con, "DeleteEmbeddedServiceMetadata2", 1, str_file_id);
-		tracker_exec_proc (db_con, "DeleteEmbeddedServiceMetadata3", 1, str_file_id);
-		tracker_exec_proc (db_con, "DeleteEmbeddedServiceMetadata4", 1, str_file_id);
-		
-	}
-
-	/* update metadata display table */
-	tracker_db_refresh_all_display_metadata (db_con, str_file_id);
-	g_free (str_file_id);
-
-
-
-}
-
-static void
-index_file (DBConnection *db_con, FileInfo *info, gboolean is_attachment)
-{
-	GHashTable	*meta_table;
-	const char 	*ext;
-	char 		*str_link_uri, *service_name;
-
-	char *services_with_metadata[] = {"Documents", "Music", "Videos", "Images", NULL};
-	char *services_with_text[] = {"Documents", "Text Files", "Development Files", NULL};
-	char *services_with_thumbs[] = {"Documents", "Images", "Videos", NULL};
-
-	meta_table = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
-
-	if (info->mime) {
-		g_free (info->mime);
-	}
-
-	if (!info->is_directory) {
-		info->mime = tracker_get_mime_type (info->uri);
-
-		if (!info->mime) {
-			info->mime = g_strdup ("unknown");
-		}
-	} else {
-		info->mime = g_strdup ("Folder");
-		info->file_size = 0;
-	}
-
-	if (info->is_link) {
-		str_link_uri = g_build_filename (info->link_path, info->link_name, NULL);
-	} else {
-		str_link_uri = g_strdup (" ");
-	}
-
-	meta_table = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
-
-	/* delimit file uri so hyphens and underscores are removed so that they can be indexed separately */
-	if (strchr (info->uri, '_') || strchr (info->uri, '-')) {
-		char	*delimited;
-
-		delimited = g_strdup (info->uri);
-		delimited =  g_strdelimit (delimited, "-_" , ' ');
-		g_debug ("delimited file name is %s", delimited);
-		g_hash_table_insert (meta_table, g_strdup ("File:NameDelimited"), g_strdup (delimited));
-		g_free (delimited);
-	}
-
-	ext = strrchr (info->uri, '.');
-	if (ext) {
-		ext++;
-		g_debug ("file extension is %s", ext);
-		g_hash_table_insert (meta_table, g_strdup ("File:Ext"), g_strdup (ext));
-	}
-
-	g_hash_table_insert (meta_table, g_strdup ("File:Path"), g_path_get_dirname (info->uri));
-	g_hash_table_insert (meta_table, g_strdup ("File:Name"), g_path_get_basename (info->uri));
-	g_hash_table_insert (meta_table, g_strdup ("File:Link"), g_strdup (str_link_uri));
-	g_hash_table_insert (meta_table, g_strdup ("File:Mime"), g_strdup (info->mime));
-	g_hash_table_insert (meta_table, g_strdup ("File:Size"), tracker_uint_to_str (info->file_size));
-	g_hash_table_insert (meta_table, g_strdup ("File:Permissions"), g_strdup (info->permissions));
-	g_hash_table_insert (meta_table, g_strdup ("File:Modified"), tracker_date_to_str (info->mtime));
-	g_hash_table_insert (meta_table, g_strdup ("File:Accessed"), tracker_date_to_str (info->atime));
-
-	g_free (str_link_uri);
-
-	if (info->is_directory) {
-		service_name = g_strdup ("Folders");
-	} else {
-		service_name = tracker_get_service_type_for_mime (info->mime);
-	}
-			
-	if (!info->mime) {
-		info->mime = g_strdup ("unknown");
-	}
-
-	gboolean  is_file_indexable, service_has_metadata, is_external_service, service_has_fulltext, service_has_thumbs;
-
-	is_external_service = g_str_has_prefix (info->mime, "service/");
-	is_file_indexable = !info->is_directory && (strcmp (info->mime, "unknown") != 0) && (strcmp (info->mime, "symlink") != 0);
-	
-	service_has_metadata = is_external_service || (is_file_indexable && (tracker_str_in_array (service_name, services_with_metadata) != -1));
-	service_has_fulltext = is_external_service || (is_file_indexable && (tracker_str_in_array (service_name, services_with_text) != -1));
-	service_has_thumbs = is_external_service || (is_file_indexable && (tracker_str_in_array (service_name, services_with_thumbs) != -1));
-
-	
-//	tracker_log ("file %s has fulltext %d, %d, %d", info->uri, (tracker_str_in_array (service_name, services_with_text) != -1), service_has_fulltext, is_file_known);
-	index_service (db_con, info, service_name, meta_table, is_attachment, service_has_metadata, service_has_fulltext, service_has_thumbs);
-
-	/* add extra delay to throttle back indexing of larger files */
-	if (service_has_fulltext && (info->file_size > 10000)) {
-		tracker_throttle (5000);
-	}
-
-	if (is_attachment) {
-		tracker_email_unlink_email_attachment (info->uri);
-	}
-
-	tracker_dec_info_ref (info);
-	g_hash_table_destroy (meta_table);
-	g_free (service_name);
-
-	
-	
-}
-
-
-
-static void 
-index_conversation (DBConnection *db_con, FileInfo *info)
-{
-
-/* todo */
-}
-
-
-static void
-index_application (DBConnection *db_con, FileInfo *info)
-{
-
-/* todo */
-}
-
-
-static void
-index_entity (DBConnection *db_con, FileInfo *info)
-{
-
-	g_return_if_fail (info->uri && (info->uri[0] == '/'));
-
-	if (!info->is_directory) {
-		/* sleep  to throttle back indexing */
-		tracker_throttle (10000);
-	}
-
-	if (!tracker_file_is_valid (info->uri)) {
-		tracker_log ("Warning - file %s no longer exists - abandoning index on this file", info->uri);
-		return;
-	}
 
-	if (tracker_email_file_is_interesting (db_con, info)) {
-		tracker_email_index_file (db_con, info);
 
-	} else if (tracker_email_is_an_attachment (info)) {
-		index_file (db_con, info, TRUE);
-		
-	} else if (!info->mime || strcmp (info->mime, "unknown") == 0) {
-		g_free (info->mime);
-		info->mime = tracker_get_service_for_uri (info->uri);
-	}	
-
-	if (strcmp (info->mime, "Files") == 0) {
-		index_file (db_con, info, FALSE);
-		
-	} else if (strcmp (info->mime, "Conversations") == 0) {
-		index_conversation (db_con, info);
-
-	} else if (strcmp (info->mime, "Applications") == 0) {
-		index_application (db_con, info);
-
-	} else if (g_str_has_prefix (info->mime, "service/")) {
-		index_service (db_con, info, NULL, NULL, FALSE, TRUE, TRUE, TRUE);
-	}
-	
-}
 
 static void
 schedule_file_check (const char *uri, DBConnection *db_con)
@@ -1403,127 +1089,7 @@ extract_metadata_thread (void)
 			continue;
 		}
 
-		if (info) {
-			if (info->uri && (info->uri[0] == '/')) {
-				char *file_as_text;
-
-				GHashTable *meta_table;
-
-				if (!tracker->is_indexing) {
-					tracker->is_indexing = TRUE;
-					//g_timeout_add (3000, (GSourceFunc) flush_when_indexing_finished, NULL);
-				}
-
-				if (info->service_type_id == -1) {
-
-					char *service_name = tracker_get_service_type_for_mime (info->mime);
-
-					info->service_type_id = tracker_get_id_for_service (service_name);
-					
-					if (info->service_type_id == -1) {
-						tracker_log ("******ERROR****** Unknown service type for %s with service %s and mime %s", info->uri, service_name, info->mime);
-					}
-				
-					g_free (service_name);
-				}
-
-
-				if (info->is_new) {
-					tracker_log ("Extracting Metadata for *new* file %s with mime %s and service type %d", info->uri, info->mime, info->service_type_id);
-				} else {
-					tracker_log ("Extracting Metadata for *existing* file %s with mime %s and service type %d", info->uri, info->mime, info->service_type_id);
-				}
-
-				/* refresh stat data in case its changed */
-				info = tracker_get_file_info (info);
-
-
-				if (info->file_id == 0) {
-					info->file_id = tracker_db_get_file_id (db_con, info->uri);
-				}
-
-				meta_table = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
-
-				tracker_metadata_get_embedded (info->uri, info->mime, meta_table);
-
-				if (g_hash_table_size (meta_table) > 0) {
-					tracker_db_save_metadata (db_con, meta_table, info->file_id, info->is_new);
-
-					/* to do - emit dbus signal here for EmbeddedMetadataChanged */
-				}
-
-				g_hash_table_destroy (meta_table);
-
-
-
-
-				if (tracker->enable_thumbnails) {
-					char *small_thumb_file = NULL, *large_thumb_file = NULL;
-
-					small_thumb_file = tracker_metadata_get_thumbnail (info->uri, info->mime, "normal");
-
-					tracker_db_save_thumbs (db_con, small_thumb_file, large_thumb_file, info->file_id);
-
-					/* to do - emit dbus signal ThumbNailChanged */
-					if (small_thumb_file) {
-						g_free (small_thumb_file);
-					}
-
-
-				}
-
-				file_as_text = tracker_metadata_get_text_file (info->uri, info->mime);
-
-				if (tracker->enable_content_indexing && file_as_text) {
-					const char *tmp_dir;
-					char	   *dir;
-
-					tracker_db_save_file_contents (db_con, blob_db_con, cache_db_con, file_as_text, info);
-
-					tmp_dir = g_get_tmp_dir ();
-
-					dir = g_strconcat (tmp_dir, G_DIR_SEPARATOR_S, NULL);
-
-					/* clear up if text contents are in a temp file */
-					if (g_str_has_prefix (file_as_text, dir)) {
-						g_unlink (file_as_text);
-					}
-
-					g_free (dir);
-
-					g_free (file_as_text);
-
-				} else {
-					if (info->is_new) {
-						tracker_db_update_indexes_for_new_service (db_con, cache_db_con, info->file_id, info->service_type_id, NULL);
-					}
-				}
-
-				//tracker_indexer_sync (tracker->file_indexer);
-
-				char *str_id;
-				str_id = tracker_uint_to_str (info->file_id);
-
-				/* delete any old metadata that was not updated  */
-				if (!info->is_new) {
-					tracker_exec_proc (db_con, "DeleteEmbeddedServiceMetadata1", 1, str_id);
-					tracker_exec_proc (db_con, "DeleteEmbeddedServiceMetadata2", 1, str_id);
-					tracker_exec_proc (db_con, "DeleteEmbeddedServiceMetadata3", 1, str_id);
-					tracker_exec_proc (db_con, "DeleteEmbeddedServiceMetadata4", 1, str_id);
-				}
-
-				/* update metadata display table */
-				tracker_db_refresh_all_display_metadata (db_con, str_id);
-
-				g_free (str_id);
-
-				if (tracker_email_is_an_attachment (info)) {
-					tracker_email_unlink_email_attachment (info->uri);
-				}
-			}
-
-			tracker_dec_info_ref (info);
-		}
+		
 	}
 
 	tracker_db_close (db_con);
@@ -1604,7 +1170,6 @@ process_files_thread (void)
 
 	DBConnection *db_con;
 	DBConnection *blob_db_con = NULL;
-	DBConnection *cache_db_con;
 
 	GSList	     *moved_from_list; /* list to hold moved_from events whilst waiting for a matching moved_to event */
 	gboolean pushed_events, first_run;
@@ -1621,10 +1186,8 @@ process_files_thread (void)
 
 	db_con = tracker_db_connect ();
 	blob_db_con = tracker_db_connect_full_text ();
-	cache_db_con = tracker_db_connect_cache ();
 
 	db_con->thread = "files";
-	db_con->user_data = cache_db_con;
 	db_con->user_data2 = blob_db_con;
 
 	pushed_events = FALSE;
@@ -1659,45 +1222,6 @@ process_files_thread (void)
 			tracker_flush_rare_words ();
 		}
 
-	/*
-		if (!tracker->in_flush && (tracker->number_of_cached_words > tracker->cache_word_limit)) {
-			int words_left;
-			tracker->in_flush = TRUE;
-			tracker_log ("Please wait while data is being flushed to the inverted word index...");
-			tracker->in_flush = TRUE;
-
-			words_left = tracker_db_flush_words_to_qdbm (cache_db_con, 1000);
-
-			while (tracker->number_of_cached_words > tracker->cache_word_min) {
-				
-
-				if (words_left > 5000) {
-					words_left = tracker_db_flush_words_to_qdbm (cache_db_con, 3000);
-				} else if (words_left > 4000) {
-					words_left = tracker_db_flush_words_to_qdbm (cache_db_con, 2000);
-				} else if (words_left > 2500) {
-					words_left = tracker_db_flush_words_to_qdbm (cache_db_con, 1000);
-				} else if (words_left > 1000) {
-					words_left = tracker_db_flush_words_to_qdbm (cache_db_con, 500);
-				} else if (words_left > 500) {
-					words_left = tracker_db_flush_words_to_qdbm (cache_db_con, 200);
-				} else {
-					words_left = tracker_db_flush_words_to_qdbm (cache_db_con, 50);
-				}
-
-				tracker->number_of_cached_words = words_left;
-				tracker_log ("flushing data (%d words left) to inverted word index - please wait", words_left);
-			}	
-			if (tracker->first_time_index && tracker->first_flush) {
-				tracker->first_flush = FALSE;
-				tracker_db_exec_no_reply (cache_db_con, "ANALYZE");
-			}
-			tracker->in_flush = FALSE;
-			
-		}
-
-*/
-
 		info = g_async_queue_try_pop (tracker->file_process_queue);
 
 		/* check pending table if we haven't got anything */
@@ -1973,10 +1497,7 @@ process_files_thread (void)
 		}
 
 		if (need_index) {
-
-			
-
-			index_entity (db_con, info);
+			tracker_db_index_entity (db_con, info);
 		}
 
 		tracker_dec_info_ref (info);
@@ -1988,7 +1509,6 @@ process_files_thread (void)
 		tracker_db_close (blob_db_con);
 	}
 
-	tracker_db_close (cache_db_con);
 
 	tracker_db_thread_end ();
 	g_debug ("files thread has exited successfully");
@@ -2409,7 +1929,7 @@ log_handler (const gchar *domain, GLogLevelFlags levels, const char* message, gp
 	GTimeVal	start;
 
 
-	if ((levels & G_LOG_LEVEL_DEBUG) && (!tracker->enable_debug)) {
+	if ((levels & G_LOG_LEVEL_DEBUG) && (tracker->verbosity < 2)) {
 		return;
 	}
 
@@ -2575,9 +2095,14 @@ sanity_check_option_values ()
 		tracker->watch_directory_roots_list = g_slist_prepend (tracker->watch_directory_roots_list, g_strdup (g_get_home_dir()));
 	}
 
+	if (tracker->throttle > 20) {
+		tracker->throttle = 20;
+	} else 	if (tracker->throttle < 0) {
+		tracker->throttle = 0;
+	}
 
-	if (tracker->slow) {
-		tracker->turbo = FALSE;
+	if (tracker->throttle == 0) {
+		tracker->turbo = TRUE;
 	}
 
 	char *bools[] = {"no", "yes"};
@@ -2641,17 +2166,12 @@ sanity_check_option_values ()
 		tracker_log ("\t");
 	}
 
-	if (tracker->throttle > 20) {
-		tracker->throttle = 20;
-	} else 	if (tracker->throttle < 0) {
-		tracker->throttle = 0;
-	}
 
-	if (tracker->throttle == 0) {
-		tracker->turbo = TRUE;
+	if (tracker->verbosity < 0) {
+		tracker->verbosity = 0;
+	} else if (tracker->verbosity > 2) {
+		tracker->verbosity = 2;
 	}
-
-
 }
 
 	
@@ -2864,11 +2384,6 @@ main (int argc, char **argv)
 		tracker->language = language;
 	}
 
-
-	if (enable_debug) {
-		tracker->enable_debug = TRUE;
-	}
-
 	if (enable_evolution) {
 		tracker->index_evolution_emails = TRUE;
 	}
@@ -2883,14 +2398,12 @@ main (int argc, char **argv)
 
 	tracker->throttle = throttle;
 
-	if (turbo) {
-		tracker->turbo = TRUE;
-	}
-
 	if (low_memory) {
 		tracker->use_extra_memory = FALSE;
 	}
 
+	tracker->verbosity = verbosity;
+
 	sanity_check_option_values ();
 
 	tracker_data_dir = g_build_filename (g_get_home_dir (), ".Tracker", "data", NULL);