diff options
author | Carlos Garnacho <carlosg@gnome.org> | 2022-08-30 16:50:52 +0000 |
---|---|---|
committer | Carlos Garnacho <carlosg@gnome.org> | 2022-08-30 16:50:52 +0000 |
commit | 3cdc00edc66e3101386a7effe185ea060909bd9a (patch) | |
tree | 6ae5df004c162236bf8c8e02a1fe51aab78598e0 | |
parent | a100df349c55e859d44b04c4fb0a1bea350b1a1c (diff) | |
parent | 3487ebf90f01f4f97df6cac208cbe0b3d63c26be (diff) | |
download | tracker-3cdc00edc66e3101386a7effe185ea060909bd9a.tar.gz |
Merge branch 'wip/carlosg/update-perf' into 'master'
Improve performance of database updates
See merge request GNOME/tracker!532
23 files changed, 1935 insertions, 1511 deletions
diff --git a/src/libtracker-sparql/core/tracker-data-manager.c b/src/libtracker-sparql/core/tracker-data-manager.c index 05ebf1c66..3bbdc0d0d 100644 --- a/src/libtracker-sparql/core/tracker-data-manager.c +++ b/src/libtracker-sparql/core/tracker-data-manager.c @@ -4194,6 +4194,25 @@ tracker_data_manager_update_from_version (TrackerDataManager *manager, goto error; } + if (version < TRACKER_DB_VERSION_3_4) { + GHashTableIter iter; + const gchar *graph; + + if (!tracker_db_interface_sqlite_fts_delete_table (iface, "main", &internal_error)) + goto error; + if (!tracker_data_manager_update_fts (manager, iface, "main", &internal_error)) + goto error; + + g_hash_table_iter_init (&iter, manager->graphs); + + while (g_hash_table_iter_next (&iter, (gpointer *) &graph, NULL)) { + if (!tracker_db_interface_sqlite_fts_delete_table (iface, graph, &internal_error)) + goto error; + if (!tracker_data_manager_update_fts (manager, iface, graph, &internal_error)) + goto error; + } + } + tracker_db_manager_update_version (manager->db_manager); return TRUE; @@ -4948,6 +4967,8 @@ tracker_data_manager_dispose (GObject *object) GError *error = NULL; gboolean readonly = TRUE; + g_clear_object (&manager->data_update); + if (manager->db_manager) { readonly = (tracker_db_manager_get_flags (manager->db_manager, NULL, NULL) & TRACKER_DB_MANAGER_READONLY) != 0; @@ -4980,7 +5001,6 @@ tracker_data_manager_finalize (GObject *object) TrackerDataManager *manager = TRACKER_DATA_MANAGER (object); g_clear_object (&manager->ontologies); - g_clear_object (&manager->data_update); g_clear_object (&manager->ontology_location); g_clear_object (&manager->cache_location); g_clear_pointer (&manager->graphs, g_hash_table_unref); @@ -5452,7 +5472,7 @@ tracker_data_manager_expand_prefix (TrackerDataManager *manager, if (expanded) { if (sep) { - *expanded = g_strdup_printf ("%s%s", expanded_ns, sep); + *expanded = g_strconcat (expanded_ns, sep, NULL); } else { *expanded = g_strdup (expanded_ns); } diff --git a/src/libtracker-sparql/core/tracker-data-query.c b/src/libtracker-sparql/core/tracker-data-query.c index 09f41961e..7aa563338 100644 --- a/src/libtracker-sparql/core/tracker-data-query.c +++ b/src/libtracker-sparql/core/tracker-data-query.c @@ -33,75 +33,14 @@ #include "tracker-ontologies.h" #include "tracker-sparql.h" -GPtrArray* -tracker_data_query_rdf_type (TrackerDataManager *manager, - const gchar *graph, - TrackerRowid id, - GError **error) -{ - TrackerDBCursor *cursor = NULL; - TrackerDBInterface *iface; - TrackerDBStatement *stmt; - GPtrArray *ret = NULL; - GError *inner_error = NULL; - TrackerOntologies *ontologies; - - iface = tracker_data_manager_get_writable_db_interface (manager); - ontologies = tracker_data_manager_get_ontologies (manager); - - stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_SELECT, &inner_error, - "SELECT (SELECT Uri FROM Resource WHERE ID = \"rdf:type\") " - "FROM \"%s\".\"rdfs:Resource_rdf:type\" " - "WHERE ID = ?", - graph ? graph : "main"); - - if (stmt) { - tracker_db_statement_bind_int (stmt, 0, id); - cursor = tracker_db_statement_start_cursor (stmt, &inner_error); - g_object_unref (stmt); - } - - if (cursor) { - - /* Query is usually a rather small result, but let's try to - * avoid reallocs in gptrarray.c as much as possible (this - * function is called fairly often) */ - - ret = g_ptr_array_sized_new (20); - while (tracker_db_cursor_iter_next (cursor, NULL, &inner_error)) { - const gchar *class_uri; - TrackerClass *cl; - - class_uri = tracker_db_cursor_get_string (cursor, 0, NULL); - cl = tracker_ontologies_get_class_by_uri (ontologies, class_uri); - if (!cl) { - g_critical ("Unknown class %s", class_uri); - continue; - } - g_ptr_array_add (ret, cl); - } - g_object_unref (cursor); - } - - if (G_UNLIKELY (inner_error)) { - g_propagate_prefixed_error (error, - inner_error, - "Querying RDF type:"); - g_clear_pointer (&ret, g_ptr_array_unref); - return NULL; - } - - return ret; -} - gchar * tracker_data_query_resource_urn (TrackerDataManager *manager, TrackerDBInterface *iface, TrackerRowid id) { - TrackerDBCursor *cursor = NULL; TrackerDBStatement *stmt; gchar *uri = NULL; + GArray *res = NULL; g_return_val_if_fail (id != 0, NULL); @@ -111,16 +50,15 @@ tracker_data_query_resource_urn (TrackerDataManager *manager, return NULL; tracker_db_statement_bind_int (stmt, 0, id); - cursor = tracker_db_statement_start_cursor (stmt, NULL); + res = tracker_db_statement_get_values (stmt, + TRACKER_PROPERTY_TYPE_STRING, + NULL); g_object_unref (stmt); - if (!cursor) - return NULL; + if (res && res->len == 1) + uri = g_value_dup_string (&g_array_index (res, GValue, 0)); - if (tracker_db_cursor_iter_next (cursor, NULL, NULL)) - uri = g_strdup (tracker_db_cursor_get_string (cursor, 0, NULL)); - - g_object_unref (cursor); + g_clear_pointer (&res, g_array_unref); return uri; } @@ -131,10 +69,10 @@ tracker_data_query_resource_id (TrackerDataManager *manager, const gchar *uri, GError **error) { - TrackerDBCursor *cursor = NULL; TrackerDBStatement *stmt; GError *inner_error = NULL; TrackerRowid id = 0; + GArray *res = NULL; g_return_val_if_fail (uri != NULL, 0); @@ -143,17 +81,16 @@ tracker_data_query_resource_id (TrackerDataManager *manager, if (stmt) { tracker_db_statement_bind_text (stmt, 0, uri); - cursor = tracker_db_statement_start_cursor (stmt, &inner_error); + res = tracker_db_statement_get_values (stmt, + TRACKER_PROPERTY_TYPE_INTEGER, + &inner_error); g_object_unref (stmt); } - if (cursor) { - if (tracker_db_cursor_iter_next (cursor, NULL, &inner_error)) { - id = tracker_db_cursor_get_int (cursor, 0); - } + if (res && res->len == 1) + id = g_value_get_int64 (&g_array_index (res, GValue, 0)); - g_object_unref (cursor); - } + g_clear_pointer (&res, g_array_unref); if (G_UNLIKELY (inner_error)) { g_propagate_prefixed_error (error, diff --git a/src/libtracker-sparql/core/tracker-data-query.h b/src/libtracker-sparql/core/tracker-data-query.h index 94d84400d..11f7c6762 100644 --- a/src/libtracker-sparql/core/tracker-data-query.h +++ b/src/libtracker-sparql/core/tracker-data-query.h @@ -41,11 +41,6 @@ TrackerDBCursor *tracker_data_query_sparql_cursor (TrackerDataManager *mana const gchar *query, GError **error); -GPtrArray* tracker_data_query_rdf_type (TrackerDataManager *manager, - const gchar *graph, - TrackerRowid id, - GError **error); - gboolean tracker_data_query_string_to_value (TrackerDataManager *manager, const gchar *value, const gchar *langtag, diff --git a/src/libtracker-sparql/core/tracker-data-update.c b/src/libtracker-sparql/core/tracker-data-update.c index f4c857cd3..0347877b9 100644 --- a/src/libtracker-sparql/core/tracker-data-update.c +++ b/src/libtracker-sparql/core/tracker-data-update.c @@ -28,6 +28,8 @@ #include <libtracker-common/tracker-common.h> #include <libtracker-sparql/tracker-deserializer-rdf.h> +#include <libtracker-sparql/tracker-private.h> +#include <libtracker-sparql/tracker-uri.h> #include "tracker-class.h" #include "tracker-data-manager.h" @@ -43,60 +45,99 @@ typedef struct _TrackerDataUpdateBuffer TrackerDataUpdateBuffer; typedef struct _TrackerDataUpdateBufferGraph TrackerDataUpdateBufferGraph; typedef struct _TrackerDataUpdateBufferResource TrackerDataUpdateBufferResource; -typedef struct _TrackerDataUpdateBufferProperty TrackerDataUpdateBufferProperty; -typedef struct _TrackerDataUpdateBufferTable TrackerDataUpdateBufferTable; typedef struct _TrackerDataBlankBuffer TrackerDataBlankBuffer; typedef struct _TrackerStatementDelegate TrackerStatementDelegate; typedef struct _TrackerCommitDelegate TrackerCommitDelegate; +#define UPDATE_LOG_SIZE 64 + +typedef enum { + TRACKER_LOG_CLASS_INSERT, + TRACKER_LOG_CLASS_UPDATE, + TRACKER_LOG_CLASS_DELETE, + TRACKER_LOG_MULTIVALUED_PROPERTY_INSERT, + TRACKER_LOG_MULTIVALUED_PROPERTY_DELETE, + TRACKER_LOG_MULTIVALUED_PROPERTY_CLEAR, +} TrackerDataLogEntryType; + +typedef struct { + gint prev; + TrackerProperty *property; + GValue value; +} TrackerDataPropertyEntry; + +typedef struct { + TrackerDataLogEntryType type; + const TrackerDataUpdateBufferGraph *graph; + TrackerRowid id; + union { + struct { + TrackerClass *class; + gint last_property_idx; /* Index in properties_ptr array */ + } class; + struct { + TrackerProperty *property; + gint change_idx; /* Index in properties_ptr array */ + } multivalued; + GObject *any; + } table; + GArray *properties_ptr; +} TrackerDataLogEntry; + struct _TrackerDataUpdateBuffer { /* string -> ID */ GHashTable *resource_cache; /* set of IDs, key is same pointer than resource_cache, and owned there */ GHashTable *new_resources; - /* string -> TrackerDataUpdateBufferGraph */ + /* TrackerDataUpdateBufferGraph */ GPtrArray *graphs; + /* Statement to insert in Resource table */ + TrackerDBStatement *insert_resource; + TrackerDBStatement *query_resource; + + /* Array of TrackerDataPropertyEntry */ + GArray *properties; + /* Array of TrackerDataLogEntry */ + GArray *update_log; + /* Set of TrackerDataLogEntry. Used for class events lookups in order to + * coalesce single-valued property changes. + */ + GHashTable *class_updates; + + TrackerDBStatementMru stmt_mru; }; +typedef struct { + TrackerRowid rowid; + gint refcount_change; +} RefcountEntry; + struct _TrackerDataUpdateBufferGraph { gchar *graph; - TrackerRowid id; - /* string -> TrackerDataUpdateBufferResource */ + /* id -> TrackerDataUpdateBufferResource */ GHashTable *resources; /* id -> integer */ - GHashTable *refcounts; + GArray *refcounts; + + TrackerDBStatement *insert_ref; + TrackerDBStatement *update_ref; + TrackerDBStatement *delete_ref; + TrackerDBStatement *query_rdf_types; + TrackerDBStatementMru values_mru; }; struct _TrackerDataUpdateBufferResource { - const TrackerDataUpdateBufferGraph *graph; + TrackerDataUpdateBufferGraph *graph; TrackerRowid id; gboolean create; gboolean modified; - /* TrackerProperty -> GArray */ + /* TrackerProperty -> GArray of GValue */ GHashTable *predicates; - /* string -> TrackerDataUpdateBufferTable */ - GHashTable *tables; /* TrackerClass */ GPtrArray *types; - gboolean fts_updated; -}; - -struct _TrackerDataUpdateBufferProperty { - const gchar *name; - GValue value; - guint delete_all_values : 1; - guint delete_value : 1; -}; - -struct _TrackerDataUpdateBufferTable { - gboolean insert; - gboolean delete_row; - gboolean multiple_values; - TrackerClass *class; - /* TrackerDataUpdateBufferProperty */ - GArray *properties; + GList *fts_properties; }; struct _TrackerStatementDelegate { @@ -142,14 +183,9 @@ enum { G_DEFINE_TYPE (TrackerData, tracker_data, G_TYPE_OBJECT) -static void cache_insert_value (TrackerData *data, - const gchar *table_name, - const gchar *field_name, - const GValue *value, - gboolean multiple_values); -static GArray *get_old_property_values (TrackerData *data, - TrackerProperty *property, - GError **error); +static GArray *get_property_values (TrackerData *data, + TrackerProperty *property, + GError **error); static gboolean delete_metadata_decomposed (TrackerData *data, TrackerProperty *property, const GValue *object, @@ -162,19 +198,154 @@ static gboolean update_resource_single (TrackerData *data, TrackerRowid *id, GError **error); -void tracker_data_insert_statement_with_uri (TrackerData *data, - const gchar *graph, - TrackerRowid subject, - TrackerProperty *predicate, - const GValue *object, - GError **error); -void tracker_data_insert_statement_with_string (TrackerData *data, - const gchar *graph, - TrackerRowid subject, - TrackerProperty *predicate, - const GValue *object, - GError **error); +static void tracker_data_insert_statement_with_uri (TrackerData *data, + const gchar *graph, + TrackerRowid subject, + TrackerProperty *predicate, + const GValue *object, + GError **error); +static void tracker_data_insert_statement_with_string (TrackerData *data, + const gchar *graph, + TrackerRowid subject, + TrackerProperty *predicate, + const GValue *object, + GError **error); +static guint +tracker_data_log_entry_hash (gconstpointer value) +{ + const TrackerDataLogEntry *entry = value; + + return (g_direct_hash (entry->graph) ^ + tracker_rowid_hash (&entry->id) ^ + g_direct_hash (entry->table.any)); +} + +static gboolean +tracker_data_log_entry_equal (gconstpointer value1, + gconstpointer value2) +{ + const TrackerDataLogEntry *entry1 = value1, *entry2 = value2; + + return (entry1->graph == entry2->graph && + entry1->id == entry2->id && + entry1->table.any == entry2->table.any); +} + +static guint +tracker_data_log_entry_schema_hash (gconstpointer value) +{ + const TrackerDataLogEntry *entry = value; + guint hash = 0; + + hash = (entry->type ^ + g_direct_hash (entry->graph) ^ + g_direct_hash (entry->table.any)); + + if (entry->type == TRACKER_LOG_CLASS_INSERT || + entry->type == TRACKER_LOG_CLASS_UPDATE) { + TrackerDataPropertyEntry *prop; + gint idx; + + /* Unite with hash of properties */ + idx = entry->table.class.last_property_idx; + + while (idx >= 0) { + prop = &g_array_index (entry->properties_ptr, + TrackerDataPropertyEntry, idx); + hash ^= g_direct_hash (prop->property); + idx = prop->prev; + } + } + + return hash; +} + +static gboolean +tracker_data_log_entry_schema_equal (gconstpointer value1, + gconstpointer value2) +{ + const TrackerDataLogEntry *entry1 = value1, *entry2 = value2; + + if (value1 == value2) + return TRUE; + + if (entry1->type != entry2->type || + entry1->graph != entry2->graph || + entry1->table.any != entry2->table.any) + return FALSE; + + if (entry1->type == TRACKER_LOG_CLASS_INSERT || + entry1->type == TRACKER_LOG_CLASS_UPDATE) { + TrackerDataPropertyEntry *prop1, *prop2; + gint idx1, idx2; + + /* Compare properties */ + idx1 = entry1->table.class.last_property_idx; + idx2 = entry2->table.class.last_property_idx; + + while (idx1 >= 0 && idx2 >= 0) { + prop1 = &g_array_index (entry1->properties_ptr, + TrackerDataPropertyEntry, idx1); + prop2 = &g_array_index (entry2->properties_ptr, + TrackerDataPropertyEntry, idx2); + + if (prop1->property != prop2->property) + return FALSE; + + idx1 = prop1->prev; + idx2 = prop2->prev; + } + + if (idx1 >= 0 || idx2 >= 0) + return FALSE; + } + + return TRUE; +} + +static TrackerDataLogEntry * +tracker_data_log_entry_copy (TrackerDataLogEntry *entry) +{ + TrackerDataLogEntry *copy; + + copy = g_slice_copy (sizeof (TrackerDataLogEntry), entry); + + if (entry->type == TRACKER_LOG_CLASS_INSERT || + entry->type == TRACKER_LOG_CLASS_UPDATE) { + TrackerDataPropertyEntry prop; + GArray *properties_copy; + gint idx; + + properties_copy = g_array_new (FALSE, TRUE, sizeof (TrackerDataPropertyEntry)); + idx = entry->table.class.last_property_idx; + + while (idx >= 0) { + prop = g_array_index (entry->properties_ptr, TrackerDataPropertyEntry, idx); + idx = prop.prev; + if (idx >= 0) + prop.prev = properties_copy->len + 1; + g_array_append_val (properties_copy, prop); + } + + copy->properties_ptr = properties_copy; + + if (properties_copy->len > 0) + copy->table.class.last_property_idx = 0; + } + + return copy; +} + +static void +tracker_data_log_entry_free (TrackerDataLogEntry *entry) +{ + if (entry->type == TRACKER_LOG_CLASS_INSERT || + entry->type == TRACKER_LOG_CLASS_UPDATE) + g_array_unref (entry->properties_ptr); + + g_slice_free (TrackerDataLogEntry, entry); +} void tracker_data_add_commit_statement_callback (TrackerData *data, @@ -329,8 +500,7 @@ tracker_data_dispatch_insert_statement_callbacks (TrackerData *data, TrackerStatementDelegate *delegate; delegate = g_ptr_array_index (data->insert_callbacks, n); - delegate->callback (data->resource_buffer->graph->id, - data->resource_buffer->graph->graph, + delegate->callback (data->resource_buffer->graph->graph, data->resource_buffer->id, predicate_id, class_id, @@ -390,8 +560,7 @@ tracker_data_dispatch_delete_statement_callbacks (TrackerData *data, TrackerStatementDelegate *delegate; delegate = g_ptr_array_index (data->delete_callbacks, n); - delegate->callback (data->resource_buffer->graph->id, - data->resource_buffer->graph->graph, + delegate->callback (data->resource_buffer->graph->graph, data->resource_buffer->id, predicate_id, class_id, @@ -405,13 +574,13 @@ static gboolean tracker_data_update_initialize_modseq (TrackerData *data, GError **error) { - TrackerDBCursor *cursor = NULL; TrackerDBInterface *temp_iface; TrackerDBStatement *stmt; - TrackerOntologies *ontologies; - TrackerProperty *property; - GError *inner_error = NULL; - gint max_modseq = 0; + TrackerOntologies *ontologies; + TrackerProperty *property; + GArray *res = NULL; + GError *inner_error = NULL; + gint max_modseq = 0; /* Is it already initialized? */ if (data->transaction_modseq != 0) @@ -419,25 +588,24 @@ tracker_data_update_initialize_modseq (TrackerData *data, temp_iface = tracker_data_manager_get_writable_db_interface (data->manager); ontologies = tracker_data_manager_get_ontologies (data->manager); - property = tracker_ontologies_get_property_by_uri (ontologies, TRACKER_PREFIX_NRL "modified"); + property = tracker_ontologies_get_nrl_modified (ontologies); - stmt = tracker_db_interface_create_vstatement (temp_iface, TRACKER_DB_STATEMENT_CACHE_TYPE_SELECT, &inner_error, + stmt = tracker_db_interface_create_vstatement (temp_iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, &inner_error, "SELECT MAX(object) FROM tracker_triples " "WHERE predicate = %" G_GINT64_FORMAT, tracker_property_get_id (property)); if (stmt) { - cursor = tracker_db_statement_start_cursor (stmt, &inner_error); + res = tracker_db_statement_get_values (stmt, + TRACKER_PROPERTY_TYPE_INTEGER, + &inner_error); g_object_unref (stmt); } - if (cursor) { - if (tracker_db_cursor_iter_next (cursor, NULL, &inner_error)) { - max_modseq = tracker_db_cursor_get_int (cursor, 0); - data->transaction_modseq = max_modseq + 1; - } - - g_object_unref (cursor); + if (res) { + max_modseq = g_value_get_int64 (&g_array_index (res, GValue, 0)); + data->transaction_modseq = max_modseq + 1; + g_array_unref (res); } if (G_UNLIKELY (inner_error)) { @@ -490,12 +658,36 @@ tracker_data_get_property (GObject *object, } static void +tracker_data_finalize (GObject *object) +{ + TrackerData *data = TRACKER_DATA (object); + + g_clear_pointer (&data->update_buffer.graphs, g_ptr_array_unref); + g_clear_pointer (&data->update_buffer.new_resources, g_hash_table_unref); + g_clear_pointer (&data->update_buffer.resource_cache, g_hash_table_unref); + g_clear_pointer (&data->update_buffer.properties, g_array_unref); + g_clear_pointer (&data->update_buffer.update_log, g_array_unref); + g_clear_pointer (&data->update_buffer.class_updates, g_hash_table_unref); + g_clear_object (&data->update_buffer.insert_resource); + g_clear_object (&data->update_buffer.query_resource); + tracker_db_statement_mru_finish (&data->update_buffer.stmt_mru); + + g_clear_pointer (&data->insert_callbacks, g_ptr_array_unref); + g_clear_pointer (&data->delete_callbacks, g_ptr_array_unref); + g_clear_pointer (&data->commit_callbacks, g_ptr_array_unref); + g_clear_pointer (&data->rollback_callbacks, g_ptr_array_unref); + + G_OBJECT_CLASS (tracker_data_parent_class)->finalize (object); +} + +static void tracker_data_class_init (TrackerDataClass *klass) { GObjectClass *object_class = G_OBJECT_CLASS (klass); object_class->set_property = tracker_data_set_property; object_class->get_property = tracker_data_get_property; + object_class->finalize = tracker_data_finalize; g_object_class_install_property (object_class, PROP_MANAGER, @@ -526,140 +718,169 @@ get_transaction_modseq (TrackerData *data) return data->transaction_modseq; } -static TrackerDataUpdateBufferTable * -cache_table_new (gboolean multiple_values) -{ - TrackerDataUpdateBufferTable *table; - - table = g_slice_new0 (TrackerDataUpdateBufferTable); - table->multiple_values = multiple_values; - table->properties = g_array_sized_new (FALSE, FALSE, sizeof (TrackerDataUpdateBufferProperty), 4); - - return table; -} - static void -cache_table_free (TrackerDataUpdateBufferTable *table) +log_entry_for_multi_value_property (TrackerData *data, + TrackerDataLogEntryType type, + TrackerProperty *property, + const GValue *value) { - TrackerDataUpdateBufferProperty *property; - guint i; + TrackerDataLogEntry entry = { 0, }; + TrackerDataPropertyEntry prop = { 0, }; + guint prop_idx; - for (i = 0; i < table->properties->len; i++) { - property = &g_array_index (table->properties, TrackerDataUpdateBufferProperty, i); - g_value_unset (&property->value); + prop.property = property; + prop.prev = -1; + + if (type != TRACKER_LOG_MULTIVALUED_PROPERTY_CLEAR) { + g_value_init (&prop.value, G_VALUE_TYPE (value)); + g_value_copy (value, &prop.value); } - g_array_free (table->properties, TRUE); - g_slice_free (TrackerDataUpdateBufferTable, table); + g_array_append_val (data->update_buffer.properties, prop); + prop_idx = data->update_buffer.properties->len - 1; + + entry.type = type; + entry.graph = data->resource_buffer->graph; + entry.id = data->resource_buffer->id; + entry.table.multivalued.property = property; + entry.table.multivalued.change_idx = prop_idx; + entry.properties_ptr = data->update_buffer.properties; + g_array_append_val (data->update_buffer.update_log, entry); } -static TrackerDataUpdateBufferTable * -cache_ensure_table (TrackerData *data, - const gchar *table_name, - gboolean multiple_values) +static void +log_entry_for_single_value_property (TrackerData *data, + TrackerClass *class, + TrackerProperty *property, + const GValue *value) { - TrackerDataUpdateBufferTable *table; - - if (!data->resource_buffer->modified) { - /* first modification of this particular resource, update nrl:modified */ + TrackerDataLogEntry entry = { 0, }, *entry_ptr; + TrackerDataPropertyEntry prop = { 0, }; + guint prop_idx; - GValue gvalue = { 0 }; + entry.type = TRACKER_LOG_CLASS_UPDATE; + entry.graph = data->resource_buffer->graph; + entry.id = data->resource_buffer->id; + entry.table.class.class = class; + entry.table.class.last_property_idx = -1; + entry.properties_ptr = data->update_buffer.properties; - data->resource_buffer->modified = TRUE; + entry_ptr = g_hash_table_lookup (data->update_buffer.class_updates, &entry); - g_value_init (&gvalue, G_TYPE_INT64); - g_value_set_int64 (&gvalue, get_transaction_modseq (data)); - cache_insert_value (data, "rdfs:Resource", "nrl:modified", - &gvalue, FALSE); + if (!entry_ptr) { + g_array_append_val (data->update_buffer.update_log, entry); + entry_ptr = &g_array_index (data->update_buffer.update_log, + TrackerDataLogEntry, + data->update_buffer.update_log->len - 1); + g_hash_table_add (data->update_buffer.class_updates, entry_ptr); } - table = g_hash_table_lookup (data->resource_buffer->tables, table_name); - if (table == NULL) { - table = cache_table_new (multiple_values); - g_hash_table_insert (data->resource_buffer->tables, g_strdup (table_name), table); - table->insert = multiple_values; + prop.property = property; + prop.prev = entry_ptr->table.class.last_property_idx; + if (value) { + g_value_init (&prop.value, G_VALUE_TYPE (value)); + g_value_copy (value, &prop.value); } + g_array_append_val (data->update_buffer.properties, prop); + prop_idx = data->update_buffer.properties->len - 1; - return table; + entry_ptr->table.class.last_property_idx = prop_idx; } static void -cache_insert_row (TrackerData *data, - TrackerClass *class) +log_entry_for_class (TrackerData *data, + TrackerDataLogEntryType type, + TrackerClass *class) { - TrackerDataUpdateBufferTable *table; + TrackerDataLogEntry entry = { 0, }, *entry_ptr; - table = cache_ensure_table (data, tracker_class_get_name (class), FALSE); - table->class = class; - table->insert = TRUE; -} + entry.type = type; + entry.graph = data->resource_buffer->graph; + entry.id = data->resource_buffer->id; + entry.table.class.class = class; + entry.table.class.last_property_idx = -1; + entry.properties_ptr = data->update_buffer.properties; -static void -cache_insert_value (TrackerData *data, - const gchar *table_name, - const gchar *field_name, - const GValue *value, - gboolean multiple_values) -{ - TrackerDataUpdateBufferTable *table; - TrackerDataUpdateBufferProperty property = { 0 }; + entry_ptr = g_hash_table_lookup (data->update_buffer.class_updates, &entry); + + if (entry_ptr && entry_ptr->type == type) + return; - /* No need to strdup here, the incoming string is either always static, or - * long-standing as tracker_property_get_name return value. */ - property.name = field_name; + entry.properties_ptr = data->update_buffer.properties; + g_array_append_val (data->update_buffer.update_log, entry); - g_value_init (&property.value, G_VALUE_TYPE (value)); - g_value_copy (value, &property.value); + entry_ptr = &g_array_index (data->update_buffer.update_log, + TrackerDataLogEntry, + data->update_buffer.update_log->len - 1); - table = cache_ensure_table (data, table_name, multiple_values); - g_array_append_val (table->properties, property); + if (type == TRACKER_LOG_CLASS_DELETE) + g_hash_table_remove (data->update_buffer.class_updates, entry_ptr); + else + g_hash_table_add (data->update_buffer.class_updates, entry_ptr); } -static void -cache_delete_row (TrackerData *data, - TrackerClass *class) +static GPtrArray* +tracker_data_query_rdf_type (TrackerData *data, + TrackerDataUpdateBufferGraph *graph, + TrackerRowid id, + GError **error) { - TrackerDataUpdateBufferTable *table; + TrackerDBInterface *iface; + TrackerDBStatement *stmt; + GArray *classes = NULL; + GPtrArray *ret = NULL; + GError *inner_error = NULL; + TrackerOntologies *ontologies; + const gchar *class_uri; + guint i; - table = cache_ensure_table (data, tracker_class_get_name (class), FALSE); - table->class = class; - table->delete_row = TRUE; -} + iface = tracker_data_manager_get_writable_db_interface (data->manager); + ontologies = tracker_data_manager_get_ontologies (data->manager); -/* Use only for multi-valued properties */ -static void -cache_delete_all_values (TrackerData *data, - const gchar *table_name, - const gchar *field_name) -{ - TrackerDataUpdateBufferTable *table; - TrackerDataUpdateBufferProperty property = { 0 }; + stmt = graph->query_rdf_types; - property.name = field_name; - property.delete_all_values = TRUE; + if (!stmt) { + stmt = graph->query_rdf_types = + tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, &inner_error, + "SELECT (SELECT Uri FROM Resource WHERE ID = \"rdf:type\") " + "FROM \"%s\".\"rdfs:Resource_rdf:type\" " + "WHERE ID = ?", + graph->graph ? graph->graph : "main"); + } - table = cache_ensure_table (data, table_name, TRUE); - g_array_append_val (table->properties, property); -} + if (stmt) { + tracker_db_statement_bind_int (stmt, 0, id); + classes = tracker_db_statement_get_values (stmt, + TRACKER_PROPERTY_TYPE_STRING, + &inner_error); + } -static void -cache_delete_value (TrackerData *data, - const gchar *table_name, - const gchar *field_name, - const GValue *value, - gboolean multiple_values) -{ - TrackerDataUpdateBufferTable *table; - TrackerDataUpdateBufferProperty property = { 0 }; + if (G_UNLIKELY (inner_error)) { + g_propagate_prefixed_error (error, + inner_error, + "Querying RDF type:"); + return NULL; + } + + if (classes) { + ret = g_ptr_array_sized_new (classes->len); - property.name = field_name; - property.delete_value = TRUE; + for (i = 0; i < classes->len; i++) { + TrackerClass *cl; + + class_uri = g_value_get_string (&g_array_index (classes, GValue, i)); + cl = tracker_ontologies_get_class_by_uri (ontologies, class_uri); + if (!cl) { + g_critical ("Unknown class %s", class_uri); + continue; + } + g_ptr_array_add (ret, cl); + } - g_value_init (&property.value, G_VALUE_TYPE (value)); - g_value_copy (value, &property.value); + g_array_unref (classes); + } - table = cache_ensure_table (data, table_name, multiple_values); - g_array_append_val (table->properties, property); + return ret; } static TrackerRowid @@ -668,23 +889,66 @@ query_resource_id (TrackerData *data, GError **error) { TrackerDBInterface *iface; - TrackerRowid *value, id; + TrackerDBStatement *stmt; + TrackerRowid *value, id = 0; + GError *inner_error = NULL; + GArray *res = NULL; value = g_hash_table_lookup (data->update_buffer.resource_cache, uri); + if (value) + return *value; - if (value == NULL) { + stmt = data->update_buffer.query_resource; + if (!stmt) { iface = tracker_data_manager_get_writable_db_interface (data->manager); - id = tracker_data_query_resource_id (data->manager, iface, uri, error); + stmt = data->update_buffer.query_resource = + tracker_db_interface_create_statement (iface, + TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, + &inner_error, + "SELECT ID FROM Resource WHERE Uri = ?"); + } - if (id != 0) { - g_hash_table_insert (data->update_buffer.resource_cache, g_strdup (uri), - tracker_rowid_copy (&id)); - } + if (stmt) { + tracker_db_statement_bind_text (stmt, 0, uri); + res = tracker_db_statement_get_values (stmt, + TRACKER_PROPERTY_TYPE_INTEGER, + &inner_error); + } - return id; + if (G_UNLIKELY (inner_error)) { + g_propagate_prefixed_error (error, + inner_error, + "Querying resource ID:"); + return 0; + } + + if (res && res->len == 1) { + id = g_value_get_int64 (&g_array_index (res, GValue, 0)); + g_hash_table_insert (data->update_buffer.resource_cache, g_strdup (uri), + tracker_rowid_copy (&id)); } - return *value; + g_clear_pointer (&res, g_array_unref); + + return id; +} + +static gboolean +tracker_data_ensure_insert_resource_stmt (TrackerData *data, + GError **error) +{ + TrackerDBInterface *iface; + + if (G_LIKELY (data->update_buffer.insert_resource)) + return TRUE; + + iface = tracker_data_manager_get_writable_db_interface (data->manager); + + data->update_buffer.insert_resource = + tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, error, + "INSERT INTO Resource (Uri, BlankNode) VALUES (?, ?)"); + + return data->update_buffer.insert_resource != NULL; } TrackerRowid @@ -696,9 +960,10 @@ tracker_data_update_ensure_resource (TrackerData *data, TrackerDBManagerFlags db_flags; TrackerDBInterface *iface; TrackerDBStatement *stmt = NULL; - GError *inner_error = NULL; - gchar *key; - TrackerRowid *value, id; + gboolean inserted; + TrackerRowid *value, id = 0; + TrackerOntologies *ontologies; + TrackerClass *class; value = g_hash_table_lookup (data->update_buffer.resource_cache, uri); @@ -706,6 +971,27 @@ tracker_data_update_ensure_resource (TrackerData *data, return *value; } + ontologies = tracker_data_manager_get_ontologies (data->manager); + class = tracker_ontologies_get_class_by_uri (ontologies, uri); + + /* Fast path, look up classes/properties directly */ + if (class) { + id = tracker_class_get_id (class); + } else { + TrackerProperty *property; + + property = tracker_ontologies_get_property_by_uri (ontologies, uri); + if (property) + id = tracker_property_get_id (property); + } + + if (id != 0) { + g_hash_table_insert (data->update_buffer.resource_cache, + g_strdup (uri), + tracker_rowid_copy (&id)); + return id; + } + db_manager = tracker_data_manager_get_db_manager (data->manager); db_flags = tracker_db_manager_get_flags (db_manager, NULL, NULL); @@ -718,48 +1004,35 @@ tracker_data_update_ensure_resource (TrackerData *data, return id; } - iface = tracker_data_manager_get_writable_db_interface (data->manager); + if (!tracker_data_ensure_insert_resource_stmt (data, error)) + return 0; - stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, &inner_error, - "INSERT INTO Resource (Uri, BlankNode) VALUES (?, ?)"); + stmt = data->update_buffer.insert_resource; + tracker_db_statement_bind_text (stmt, 0, uri); + tracker_db_statement_bind_int (stmt, 1, FALSE); + inserted = tracker_db_statement_execute (stmt, NULL); - if (stmt) { - tracker_db_statement_bind_text (stmt, 0, uri); - tracker_db_statement_bind_int (stmt, 1, FALSE); - tracker_db_statement_execute (stmt, &inner_error); - g_object_unref (stmt); + if (inserted) { + iface = tracker_data_manager_get_writable_db_interface (data->manager); + id = tracker_db_interface_sqlite_get_last_insert_id (iface); + g_hash_table_add (data->update_buffer.new_resources, + tracker_rowid_copy (&id)); + } else { + id = query_resource_id (data, uri, error); } - if (inner_error) { - if (g_error_matches (inner_error, - TRACKER_DB_INTERFACE_ERROR, - TRACKER_DB_CONSTRAINT)) { - g_clear_error (&inner_error); - id = query_resource_id (data, uri, &inner_error); - - if (id != 0) - return id; - } - - g_propagate_error (error, inner_error); - - return 0; + if (id != 0) { + g_hash_table_insert (data->update_buffer.resource_cache, + g_strdup (uri), + tracker_rowid_copy (&id)); } - id = tracker_db_interface_sqlite_get_last_insert_id (iface); - key = g_strdup (uri); - g_hash_table_insert (data->update_buffer.resource_cache, key, - tracker_rowid_copy (&id)); - - g_hash_table_add (data->update_buffer.new_resources, - tracker_rowid_copy (&id)); - return id; } static void statement_bind_gvalue (TrackerDBStatement *stmt, - gint *idx, + gint idx, const GValue *value) { GType type; @@ -767,19 +1040,19 @@ statement_bind_gvalue (TrackerDBStatement *stmt, type = G_VALUE_TYPE (value); switch (type) { case G_TYPE_STRING: - tracker_db_statement_bind_text (stmt, (*idx)++, g_value_get_string (value)); + tracker_db_statement_bind_text (stmt, idx, g_value_get_string (value)); break; case G_TYPE_INT: - tracker_db_statement_bind_int (stmt, (*idx)++, g_value_get_int (value)); + tracker_db_statement_bind_int (stmt, idx, g_value_get_int (value)); break; case G_TYPE_INT64: - tracker_db_statement_bind_int (stmt, (*idx)++, g_value_get_int64 (value)); + tracker_db_statement_bind_int (stmt, idx, g_value_get_int64 (value)); break; case G_TYPE_DOUBLE: - tracker_db_statement_bind_double (stmt, (*idx)++, g_value_get_double (value)); + tracker_db_statement_bind_double (stmt, idx, g_value_get_double (value)); break; case G_TYPE_BOOLEAN: - tracker_db_statement_bind_int (stmt, (*idx)++, g_value_get_boolean (value)); + tracker_db_statement_bind_int (stmt, idx, g_value_get_boolean (value)); break; default: if (type == G_TYPE_DATE_TIME) { @@ -793,10 +1066,10 @@ statement_bind_gvalue (TrackerDBStatement *stmt, gchar *str; str = tracker_date_format_iso8601 (datetime); - tracker_db_statement_bind_text (stmt, (*idx)++, str); + tracker_db_statement_bind_text (stmt, idx, str); g_free (str); } else { - tracker_db_statement_bind_int (stmt, (*idx)++, + tracker_db_statement_bind_int (stmt, idx, g_date_time_to_unix (datetime)); } } else if (type == G_TYPE_BYTES) { @@ -809,14 +1082,13 @@ statement_bind_gvalue (TrackerDBStatement *stmt, if (len == strlen (data) + 1) { /* No ancillary data */ - tracker_db_statement_bind_text (stmt, (*idx)++, data); + tracker_db_statement_bind_text (stmt, idx, data); } else { /* String with langtag */ - tracker_db_statement_bind_bytes (stmt, (*idx)++, bytes); + tracker_db_statement_bind_bytes (stmt, idx, bytes); } - } else if (g_strcmp0 (g_type_name (type), "TrackerUri") == 0) { - /* FIXME: We can't access TrackerUri GType here */ - tracker_db_statement_bind_text (stmt, (*idx)++, g_value_get_string (value)); + } else if (type == TRACKER_TYPE_URI) { + tracker_db_statement_bind_text (stmt, idx, g_value_get_string (value)); } else { g_warning ("Unknown type for binding: %s\n", G_VALUE_TYPE_NAME (value)); } @@ -824,249 +1096,207 @@ statement_bind_gvalue (TrackerDBStatement *stmt, } } -static void -tracker_data_resource_buffer_flush (TrackerData *data, - TrackerDataUpdateBufferResource *resource, - GError **error) -{ - TrackerDBInterface *iface; - TrackerDBStatement *stmt; - TrackerDataUpdateBufferTable *table; - TrackerDataUpdateBufferProperty *property; - GHashTableIter iter; - const gchar *table_name, *database; - guint i; - gint param; - GError *actual_error = NULL; +static TrackerDBStatement * +tracker_data_ensure_update_statement (TrackerData *data, + TrackerDataLogEntry *entry, + GError **error) +{ + TrackerDBStatement *stmt; + TrackerDBInterface *iface; + const gchar *database; + + stmt = tracker_db_statement_mru_lookup (&data->update_buffer.stmt_mru, entry); + if (stmt) { + tracker_db_statement_mru_update (&data->update_buffer.stmt_mru, stmt); + return g_object_ref (stmt); + } iface = tracker_data_manager_get_writable_db_interface (data->manager); - database = resource->graph->graph ? resource->graph->graph : "main"; - - g_hash_table_iter_init (&iter, resource->tables); - while (g_hash_table_iter_next (&iter, (gpointer*) &table_name, (gpointer*) &table)) { - if (table->multiple_values) { - for (i = 0; i < table->properties->len; i++) { - property = &g_array_index (table->properties, TrackerDataUpdateBufferProperty, i); - - if (property->delete_all_values) { - stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, &actual_error, - "DELETE FROM \"%s\".\"%s\" WHERE ID = ?", - database, - table_name); - } else if (property->delete_value) { - /* delete rows for multiple value properties */ - stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, &actual_error, - "DELETE FROM \"%s\".\"%s\" WHERE ID = ? AND \"%s\" = ?", - database, - table_name, - property->name); - } else { - stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, &actual_error, - "INSERT OR IGNORE INTO \"%s\".\"%s\" (ID, \"%s\") VALUES (?, ?)", - database, - table_name, - property->name); - } + database = entry->graph->graph ? entry->graph->graph : "main"; + + if (entry->type == TRACKER_LOG_MULTIVALUED_PROPERTY_CLEAR) { + stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, error, + "DELETE FROM \"%s\".\"%s\" WHERE ID = ?", + database, + tracker_property_get_table_name (entry->table.multivalued.property)); + } else if (entry->type == TRACKER_LOG_MULTIVALUED_PROPERTY_DELETE) { + stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, error, + "DELETE FROM \"%s\".\"%s\" WHERE ID = ? AND \"%s\" = ?", + database, + tracker_property_get_table_name (entry->table.multivalued.property), + tracker_property_get_name (entry->table.multivalued.property)); + } else if (entry->type == TRACKER_LOG_MULTIVALUED_PROPERTY_INSERT) { + stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, error, + "INSERT OR IGNORE INTO \"%s\".\"%s\" (ID, \"%s\") VALUES (?, ?)", + database, + tracker_property_get_table_name (entry->table.multivalued.property), + tracker_property_get_name (entry->table.multivalued.property)); + } else if (entry->type == TRACKER_LOG_CLASS_DELETE) { + stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, error, + "DELETE FROM \"%s\".\"%s\" WHERE ID = ?", + database, + tracker_class_get_name (entry->table.class.class)); + } else { + GHashTable *visited_properties; + TrackerDataPropertyEntry *property_entry; + gint param, property_idx; + GString *sql; - if (actual_error) { - g_propagate_error (error, actual_error); - return; - } + sql = g_string_new (NULL); + visited_properties = g_hash_table_new (NULL, NULL); + param = 2; - param = 0; + if (entry->type == TRACKER_LOG_CLASS_INSERT) { + GString *values_sql; - tracker_db_statement_bind_int (stmt, param++, resource->id); + g_string_append_printf (sql, + "INSERT INTO \"%s\".\"%s\" (ID", + database, + tracker_class_get_name (entry->table.class.class)); + values_sql = g_string_new ("VALUES (?1"); - if (!property->delete_all_values) - statement_bind_gvalue (stmt, ¶m, &property->value); + property_idx = entry->table.class.last_property_idx; - tracker_db_statement_execute (stmt, &actual_error); - g_object_unref (stmt); + while (property_idx >= 0) { + property_entry = &g_array_index (entry->properties_ptr, + TrackerDataPropertyEntry, + property_idx); + property_idx = property_entry->prev; - if (actual_error) { - g_propagate_error (error, actual_error); - return; - } + if (g_hash_table_contains (visited_properties, property_entry->property)) + continue; + + g_string_append_printf (sql, ", \"%s\"", tracker_property_get_name (property_entry->property)); + g_string_append_printf (values_sql, ", ?%d", param++); + g_hash_table_add (visited_properties, property_entry->property); } - } else { - GString *sql, *values_sql; - GHashTable *visited_properties; - gint n; - - if (table->delete_row) { - /* remove row from class table */ - stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, &actual_error, - "DELETE FROM \"%s\".\"%s\" WHERE ID = ?", - database, table_name); - - if (stmt) { - tracker_db_statement_bind_int (stmt, 0, resource->id); - tracker_db_statement_execute (stmt, &actual_error); - g_object_unref (stmt); - } - if (actual_error) { - g_propagate_error (error, actual_error); - return; - } + g_string_append (sql, ")"); + g_string_append (values_sql, ")"); + + stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, error, + "%s %s", sql->str, values_sql->str); + g_string_free (sql, TRUE); + g_string_free (values_sql, TRUE); + } else if (entry->type == TRACKER_LOG_CLASS_UPDATE) { + g_string_append_printf (sql, + "UPDATE \"%s\".\"%s\" SET ", + database, + tracker_class_get_name (entry->table.class.class)); + property_idx = entry->table.class.last_property_idx; + + while (property_idx >= 0) { + TrackerDataPropertyEntry *property_entry; + + property_entry = &g_array_index (entry->properties_ptr, + TrackerDataPropertyEntry, + property_idx); + property_idx = property_entry->prev; + + if (g_hash_table_contains (visited_properties, property_entry->property)) + continue; - continue; - } + if (param > 2) + g_string_append (sql, ", "); - if (table->insert) { - sql = g_string_new ("INSERT INTO "); - values_sql = g_string_new ("VALUES (?"); - } else { - sql = g_string_new ("UPDATE "); - values_sql = NULL; + g_string_append_printf (sql, "\"%s\" = ?%d", + tracker_property_get_name (property_entry->property), + param++); + g_hash_table_add (visited_properties, property_entry->property); } - g_string_append_printf (sql, "\"%s\".\"%s\"", - database, table_name); - - if (table->insert) { - g_string_append (sql, " (ID"); + g_string_append (sql, " WHERE ID = ?1"); - if (strcmp (table_name, "rdfs:Resource") == 0) { - g_string_append (sql, ", \"nrl:added\", \"nrl:modified\""); - g_string_append (values_sql, ", ?, ?"); - } - } else { - g_string_append (sql, " SET "); - } + stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, error, + sql->str); + g_string_free (sql, TRUE); + } - visited_properties = g_hash_table_new (g_str_hash, g_str_equal); + g_hash_table_unref (visited_properties); + } - for (n = table->properties->len - 1; n >= 0; n--) { - property = &g_array_index (table->properties, TrackerDataUpdateBufferProperty, n); - if (g_hash_table_contains (visited_properties, property->name)) - continue; + if (stmt) { + tracker_db_statement_mru_insert (&data->update_buffer.stmt_mru, + tracker_data_log_entry_copy (entry), + stmt); + } - if (table->insert) { - g_string_append_printf (sql, ", \"%s\"", property->name); - g_string_append (values_sql, ", ?"); - } else { - if (n < (int) table->properties->len - 1) { - g_string_append (sql, ", "); - } - g_string_append_printf (sql, "\"%s\" = ?", property->name); - } + return stmt; +} - g_hash_table_add (visited_properties, (gpointer) property->name); - } +static gboolean +tracker_data_flush_log (TrackerData *data, + GError **error) +{ + TrackerDBStatement *stmt = NULL; + TrackerDataPropertyEntry *property_entry; + guint i; + GError *inner_error = NULL; - g_hash_table_unref (visited_properties); + for (i = 0; i < data->update_buffer.update_log->len; i++) { + TrackerDataLogEntry *entry; - if (table->insert) { - g_string_append (sql, ")"); - g_string_append (values_sql, ")"); + entry = &g_array_index (data->update_buffer.update_log, + TrackerDataLogEntry, i); - stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, &actual_error, - "%s %s", sql->str, values_sql->str); - g_string_free (sql, TRUE); - g_string_free (values_sql, TRUE); - } else { - g_string_append (sql, " WHERE ID = ?"); + stmt = tracker_data_ensure_update_statement (data, entry, error); + if (!stmt) + return FALSE; - stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, &actual_error, - sql->str); - g_string_free (sql, TRUE); - } + if (entry->type == TRACKER_LOG_CLASS_DELETE || + entry->type == TRACKER_LOG_MULTIVALUED_PROPERTY_CLEAR) { + tracker_db_statement_bind_int (stmt, 0, entry->id); + } else if (entry->type == TRACKER_LOG_MULTIVALUED_PROPERTY_DELETE || + entry->type == TRACKER_LOG_MULTIVALUED_PROPERTY_INSERT) { + tracker_db_statement_bind_int (stmt, 0, entry->id); + + property_entry = &g_array_index (entry->properties_ptr, + TrackerDataPropertyEntry, + entry->table.multivalued.change_idx); + statement_bind_gvalue (stmt, 1, &property_entry->value); + } else { + GList *visited_properties = NULL; + gint param, property_idx; - if (actual_error) { - g_propagate_error (error, actual_error); - return; - } + tracker_db_statement_bind_int (stmt, 0, entry->id); + param = 1; - if (table->insert) { - tracker_db_statement_bind_int (stmt, 0, resource->id); + property_idx = entry->table.class.last_property_idx; - if (strcmp (table_name, "rdfs:Resource") == 0) { - g_warn_if_fail (data->resource_time != 0); - tracker_db_statement_bind_int (stmt, 1, (gint64) data->resource_time); - tracker_db_statement_bind_int (stmt, 2, get_transaction_modseq (data)); - param = 3; - } else { - param = 1; - } - } else { - param = 0; - } + while (property_idx >= 0) { + TrackerDataPropertyEntry *property_entry; - visited_properties = g_hash_table_new (g_str_hash, g_str_equal); + property_entry = &g_array_index (entry->properties_ptr, + TrackerDataPropertyEntry, + property_idx); + property_idx = property_entry->prev; - for (n = table->properties->len - 1; n >= 0; n--) { - property = &g_array_index (table->properties, TrackerDataUpdateBufferProperty, n); - if (g_hash_table_contains (visited_properties, property->name)) + if (g_list_find (visited_properties, property_entry->property)) continue; - if (property->delete_value) { + if (G_VALUE_TYPE (&property_entry->value) == G_TYPE_INVALID) { /* just set value to NULL for single value properties */ tracker_db_statement_bind_null (stmt, param++); } else { - statement_bind_gvalue (stmt, ¶m, &property->value); + statement_bind_gvalue (stmt, param++, &property_entry->value); } - g_hash_table_add (visited_properties, (gpointer) property->name); - } - - g_hash_table_unref (visited_properties); - - if (!table->insert) { - tracker_db_statement_bind_int (stmt, param++, resource->id); - } - - tracker_db_statement_execute (stmt, &actual_error); - g_object_unref (stmt); - - if (actual_error) { - g_propagate_error (error, actual_error); - return; + visited_properties = g_list_prepend (visited_properties, property_entry->property); } - } - } - - if (resource->fts_updated) { - TrackerProperty *prop; - GArray *values; - GPtrArray *properties, *text; - properties = text = NULL; - g_hash_table_iter_init (&iter, resource->predicates); - while (g_hash_table_iter_next (&iter, (gpointer*) &prop, (gpointer*) &values)) { - if (tracker_property_get_fulltext_indexed (prop)) { - GString *fts; - - fts = g_string_new (""); - for (i = 0; i < values->len; i++) { - GValue *v = &g_array_index (values, GValue, i); - g_string_append (fts, g_value_get_string (v)); - g_string_append_c (fts, ' '); - } - - if (!properties && !text) { - properties = g_ptr_array_new (); - text = g_ptr_array_new_with_free_func ((GDestroyNotify) g_free); - } - - g_ptr_array_add (properties, (gpointer) tracker_property_get_name (prop)); - g_ptr_array_add (text, g_string_free (fts, FALSE)); - } + g_list_free (visited_properties); } - if (properties && text) { - g_ptr_array_add (properties, NULL); - g_ptr_array_add (text, NULL); + tracker_db_statement_execute (stmt, &inner_error); + g_object_unref (stmt); - tracker_db_interface_sqlite_fts_update_text (iface, - database, - resource->id, - (const gchar **) properties->pdata, - (const gchar **) text->pdata); - g_ptr_array_free (properties, TRUE); - g_ptr_array_free (text, TRUE); + if (inner_error) { + g_propagate_error (error, inner_error); + return FALSE; } } + + return TRUE; } static void @@ -1075,15 +1305,25 @@ tracker_data_update_refcount (TrackerData *data, gint refcount) { const TrackerDataUpdateBufferGraph *graph; - gint old_refcount; + RefcountEntry entry; + guint i; g_assert (data->resource_buffer != NULL); graph = data->resource_buffer->graph; - old_refcount = GPOINTER_TO_INT (g_hash_table_lookup (graph->refcounts, &id)); - g_hash_table_insert (graph->refcounts, - tracker_rowid_copy (&id), - GINT_TO_POINTER (old_refcount + refcount)); + for (i = 0; i < graph->refcounts->len; i++) { + RefcountEntry *ptr; + + ptr = &g_array_index (graph->refcounts, RefcountEntry, i); + if (ptr->rowid == id) { + ptr->refcount_change += refcount; + return; + } + } + + entry.rowid = id; + entry.refcount_change = refcount; + g_array_append_val (graph->refcounts, entry); } static void @@ -1120,7 +1360,7 @@ tracker_data_resource_unref_all (TrackerData *data, g_assert (tracker_property_get_multiple_values (property) == TRUE); g_assert (tracker_property_get_data_type (property) == TRACKER_PROPERTY_TYPE_RESOURCE); - old_values = get_old_property_values (data, property, error); + old_values = get_property_values (data, property, error); if (!old_values) return FALSE; @@ -1140,44 +1380,41 @@ tracker_data_flush_graph_refcounts (TrackerData *data, GError **error) { TrackerDBInterface *iface; - TrackerDBStatement *stmt; - GHashTableIter iter; - gpointer key, value; TrackerRowid id; gint refcount; + guint i; GError *inner_error = NULL; const gchar *database; - gchar *insert_query; - gchar *update_query; - gchar *delete_query; + gchar *query; iface = tracker_data_manager_get_writable_db_interface (data->manager); database = graph->graph ? graph->graph : "main"; - insert_query = g_strdup_printf ("INSERT OR IGNORE INTO \"%s\".Refcount (ROWID, Refcount) VALUES (?1, 0)", - database); - update_query = g_strdup_printf ("UPDATE \"%s\".Refcount SET Refcount = Refcount + ?2 WHERE Refcount.ROWID = ?1", - database); - delete_query = g_strdup_printf ("DELETE FROM \"%s\".Refcount WHERE Refcount.ROWID = ?1 AND Refcount.Refcount = 0", - database); - - g_hash_table_iter_init (&iter, graph->refcounts); + for (i = 0; i < graph->refcounts->len; i++) { + RefcountEntry *entry; - while (g_hash_table_iter_next (&iter, &key, &value)) { - id = *(TrackerRowid *) key; - refcount = GPOINTER_TO_INT (value); + entry = &g_array_index (graph->refcounts, RefcountEntry, i); + id = entry->rowid; + refcount = entry->refcount_change; if (refcount > 0) { - stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, - &inner_error, insert_query); - if (inner_error) { - g_propagate_error (error, inner_error); - break; + if (!graph->insert_ref) { + query = g_strdup_printf ("INSERT OR IGNORE INTO \"%s\".Refcount (ROWID, Refcount) VALUES (?1, 0)", + database); + graph->insert_ref = + tracker_db_interface_create_statement (iface, + TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, + &inner_error, query); + g_free (query); + + if (inner_error) { + g_propagate_error (error, inner_error); + break; + } } - tracker_db_statement_bind_int (stmt, 0, id); - tracker_db_statement_execute (stmt, &inner_error); - g_object_unref (stmt); + tracker_db_statement_bind_int (graph->insert_ref, 0, id); + tracker_db_statement_execute (graph->insert_ref, &inner_error); if (inner_error) { g_propagate_error (error, inner_error); @@ -1186,17 +1423,24 @@ tracker_data_flush_graph_refcounts (TrackerData *data, } if (refcount != 0) { - stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, - &inner_error, update_query); - if (inner_error) { - g_propagate_error (error, inner_error); - break; + if (!graph->update_ref) { + query = g_strdup_printf ("UPDATE \"%s\".Refcount SET Refcount = Refcount + ?2 WHERE Refcount.ROWID = ?1", + database); + graph->update_ref = + tracker_db_interface_create_statement (iface, + TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, + &inner_error, query); + g_free (query); + + if (inner_error) { + g_propagate_error (error, inner_error); + break; + } } - tracker_db_statement_bind_int (stmt, 0, id); - tracker_db_statement_bind_int (stmt, 1, refcount); - tracker_db_statement_execute (stmt, &inner_error); - g_object_unref (stmt); + tracker_db_statement_bind_int (graph->update_ref, 0, id); + tracker_db_statement_bind_int (graph->update_ref, 1, refcount); + tracker_db_statement_execute (graph->update_ref, &inner_error); if (inner_error) { g_propagate_error (error, inner_error); @@ -1205,45 +1449,50 @@ tracker_data_flush_graph_refcounts (TrackerData *data, } if (refcount < 0) { - stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, - &inner_error, delete_query); - if (inner_error) { - g_propagate_error (error, inner_error); - break; + if (!graph->delete_ref) { + query = g_strdup_printf ("DELETE FROM \"%s\".Refcount WHERE Refcount.ROWID = ?1 AND Refcount.Refcount = 0", + database); + graph->delete_ref = + tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, + &inner_error, query); + g_free (query); + + if (inner_error) { + g_propagate_error (error, inner_error); + break; + } } - tracker_db_statement_bind_int (stmt, 0, id); - tracker_db_statement_execute (stmt, &inner_error); - g_object_unref (stmt); + tracker_db_statement_bind_int (graph->delete_ref, 0, id); + tracker_db_statement_execute (graph->delete_ref, &inner_error); if (inner_error) { g_propagate_error (error, inner_error); break; } } - - g_hash_table_iter_remove (&iter); } - - g_free (insert_query); - g_free (update_query); - g_free (delete_query); } static void graph_buffer_free (TrackerDataUpdateBufferGraph *graph) { + g_clear_object (&graph->insert_ref); + g_clear_object (&graph->update_ref); + g_clear_object (&graph->delete_ref); + g_clear_object (&graph->query_rdf_types); g_hash_table_unref (graph->resources); - g_hash_table_unref (graph->refcounts); + g_array_unref (graph->refcounts); g_free (graph->graph); + tracker_db_statement_mru_finish (&graph->values_mru); g_slice_free (TrackerDataUpdateBufferGraph, graph); } static void resource_buffer_free (TrackerDataUpdateBufferResource *resource) { - g_hash_table_unref (resource->predicates); - g_hash_table_unref (resource->tables); + g_clear_pointer (&resource->predicates, g_hash_table_unref); + g_list_free (resource->fts_properties); g_ptr_array_free (resource->types, TRUE); resource->types = NULL; @@ -1256,19 +1505,77 @@ tracker_data_update_buffer_flush (TrackerData *data, { TrackerDataUpdateBufferGraph *graph; TrackerDataUpdateBufferResource *resource; + TrackerDBInterface *iface; GHashTableIter iter; GError *actual_error = NULL; + const gchar *database; + GList *l; guint i; + if (data->update_buffer.update_log->len == 0) + return; + + iface = tracker_data_manager_get_writable_db_interface (data->manager); + + for (i = 0; i < data->update_buffer.graphs->len; i++) { + graph = g_ptr_array_index (data->update_buffer.graphs, i); + g_hash_table_iter_init (&iter, graph->resources); + + while (g_hash_table_iter_next (&iter, NULL, (gpointer*) &resource)) { + if (resource->fts_properties && !resource->create) { + GPtrArray *properties; + gboolean retval; + + properties = g_ptr_array_sized_new (8); + database = resource->graph->graph ? resource->graph->graph : "main"; + + for (l = resource->fts_properties; l; l = l->next) + g_ptr_array_add (properties, (gpointer) tracker_property_get_name (l->data)); + + g_ptr_array_add (properties, NULL); + + retval = tracker_db_interface_sqlite_fts_delete_text (iface, + database, + resource->id, + (const gchar **) properties->pdata, + error); + g_ptr_array_free (properties, TRUE); + + if (!retval) + goto out; + } + } + } + + if (!tracker_data_flush_log (data, error)) + goto out; + for (i = 0; i < data->update_buffer.graphs->len; i++) { graph = g_ptr_array_index (data->update_buffer.graphs, i); g_hash_table_iter_init (&iter, graph->resources); while (g_hash_table_iter_next (&iter, NULL, (gpointer*) &resource)) { - tracker_data_resource_buffer_flush (data, resource, &actual_error); - if (actual_error) { - g_propagate_error (error, actual_error); - goto out; + if (resource->fts_properties) { + GPtrArray *properties; + gboolean retval; + + properties = g_ptr_array_sized_new (8); + database = resource->graph->graph ? resource->graph->graph : "main"; + + for (l = resource->fts_properties; l; l = l->next) + g_ptr_array_add (properties, (gpointer) tracker_property_get_name (l->data)); + + g_ptr_array_add (properties, NULL); + + retval = tracker_db_interface_sqlite_fts_update_text (iface, + database, + resource->id, + (const gchar **) properties->pdata, + error); + g_ptr_array_free (properties, TRUE); + + if (!retval) + goto out; } } @@ -1277,11 +1584,16 @@ tracker_data_update_buffer_flush (TrackerData *data, g_propagate_error (error, actual_error); goto out; } + + g_hash_table_remove_all (graph->resources); + g_array_set_size (graph->refcounts, 0); } out: - g_ptr_array_set_size (data->update_buffer.graphs, 0); g_hash_table_remove_all (data->update_buffer.new_resources); + g_hash_table_remove_all (data->update_buffer.class_updates); + g_array_set_size (data->update_buffer.properties, 0); + g_array_set_size (data->update_buffer.update_log, 0); data->resource_buffer = NULL; } @@ -1289,26 +1601,27 @@ void tracker_data_update_buffer_might_flush (TrackerData *data, GError **error) { + if (data->update_buffer.update_log->len > UPDATE_LOG_SIZE - 10) + tracker_data_update_buffer_flush (data, error); +} + +static void +tracker_data_update_buffer_clear (TrackerData *data) +{ TrackerDataUpdateBufferGraph *graph; - guint i, count = 0; + guint i; for (i = 0; i < data->update_buffer.graphs->len; i++) { graph = g_ptr_array_index (data->update_buffer.graphs, i); - count += g_hash_table_size (graph->resources); - - if (count >= 50) { - tracker_data_update_buffer_flush (data, error); - break; - } + g_hash_table_remove_all (graph->resources); + g_array_set_size (graph->refcounts, 0); } -} -static void -tracker_data_update_buffer_clear (TrackerData *data) -{ - g_ptr_array_set_size (data->update_buffer.graphs, 0); g_hash_table_remove_all (data->update_buffer.new_resources); g_hash_table_remove_all (data->update_buffer.resource_cache); + g_hash_table_remove_all (data->update_buffer.class_updates); + g_array_set_size (data->update_buffer.properties, 0); + g_array_set_size (data->update_buffer.update_log, 0); data->resource_buffer = NULL; } @@ -1341,19 +1654,55 @@ cache_create_service_decomposed (TrackerData *data, g_ptr_array_add (data->resource_buffer->types, cl); - g_value_init (&gvalue, G_TYPE_INT64); - - cache_insert_row (data, cl); + log_entry_for_class (data, TRACKER_LOG_CLASS_INSERT, cl); tracker_data_resource_ref (data, data->resource_buffer->id, FALSE); class_id = tracker_class_get_id (cl); ontologies = tracker_data_manager_get_ontologies (data->manager); + g_value_init (&gvalue, G_TYPE_INT64); g_value_set_int64 (&gvalue, class_id); - cache_insert_value (data, "rdfs:Resource_rdf:type", "rdf:type", - &gvalue, TRUE); + log_entry_for_multi_value_property (data, + TRACKER_LOG_MULTIVALUED_PROPERTY_INSERT, + tracker_ontologies_get_rdf_type (ontologies), + &gvalue); + tracker_data_resource_ref (data, class_id, TRUE); + if (!data->resource_buffer->modified) { + /* first modification of this particular resource, update nrl:modified */ + TrackerOntologies *ontologies; + TrackerProperty *modified; + GValue gvalue = { 0 }; + + data->resource_buffer->modified = TRUE; + ontologies = tracker_data_manager_get_ontologies (data->manager); + modified = tracker_ontologies_get_nrl_modified (ontologies); + + g_value_init (&gvalue, G_TYPE_INT64); + g_value_set_int64 (&gvalue, get_transaction_modseq (data)); + log_entry_for_single_value_property (data, + tracker_property_get_domain (modified), + modified, &gvalue); + } + + if (data->resource_buffer->create && + strcmp (tracker_class_get_uri (cl), TRACKER_PREFIX_RDFS "Resource") == 0) { + /* Add nrl:added for the new rdfs:Resource */ + TrackerOntologies *ontologies; + TrackerProperty *added; + GValue gvalue = { 0 }; + + ontologies = tracker_data_manager_get_ontologies (data->manager); + added = tracker_ontologies_get_nrl_added (ontologies); + + g_value_init (&gvalue, G_TYPE_INT64); + g_value_set_int64 (&gvalue, data->resource_time); + log_entry_for_single_value_property (data, + tracker_property_get_domain (added), + added, &gvalue); + } + tracker_data_dispatch_insert_statement_callbacks (data, tracker_property_get_id (tracker_ontologies_get_rdf_type (ontologies)), class_id); @@ -1371,7 +1720,7 @@ cache_create_service_decomposed (TrackerData *data, GArray *old_values; /* read existing property values */ - old_values = get_old_property_values (data, *domain_indexes, &inner_error); + old_values = get_property_values (data, *domain_indexes, &inner_error); if (inner_error) { g_propagate_prefixed_error (error, inner_error, @@ -1393,12 +1742,7 @@ cache_create_service_decomposed (TrackerData *data, tracker_class_get_name (cl)); v = &g_array_index (old_values, GValue, 0); - - cache_insert_value (data, - tracker_class_get_name (cl), - tracker_property_get_name (*domain_indexes), - v, - tracker_property_get_multiple_values (*domain_indexes)); + log_entry_for_single_value_property (data, cl, *domain_indexes, v); } domain_indexes++; @@ -1518,206 +1862,68 @@ get_property_values (TrackerData *data, TrackerProperty *property, GError **error) { - gboolean multiple_values; + TrackerDataUpdateBufferGraph *graph; const gchar *database; GArray *old_values; - multiple_values = tracker_property_get_multiple_values (property); + if (!data->resource_buffer->predicates) { + data->resource_buffer->predicates = + g_hash_table_new_full (NULL, NULL, g_object_unref, + (GDestroyNotify) g_array_unref); + } - old_values = g_array_sized_new (FALSE, TRUE, sizeof (GValue), multiple_values ? 4 : 1); - g_array_set_clear_func (old_values, (GDestroyNotify) g_value_unset); - g_hash_table_insert (data->resource_buffer->predicates, g_object_ref (property), old_values); + old_values = g_hash_table_lookup (data->resource_buffer->predicates, property); + if (old_values != NULL) + return old_values; - database = data->resource_buffer->graph->graph ? - data->resource_buffer->graph->graph : "main"; + graph = data->resource_buffer->graph; + database = graph->graph ? graph->graph : "main"; if (!data->resource_buffer->create) { - TrackerDBInterface *iface; TrackerDBStatement *stmt; - TrackerDBCursor *cursor = NULL; - const gchar *table_name; - const gchar *field_name; - GError *inner_error = NULL; - - table_name = tracker_property_get_table_name (property); - field_name = tracker_property_get_name (property); - iface = tracker_data_manager_get_writable_db_interface (data->manager); - - stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_SELECT, &inner_error, - "SELECT \"%s\" FROM \"%s\".\"%s\" WHERE ID = ?", - field_name, database, table_name); + stmt = tracker_db_statement_mru_lookup (&graph->values_mru, property); if (stmt) { - tracker_db_statement_bind_int (stmt, 0, data->resource_buffer->id); - cursor = tracker_db_statement_start_cursor (stmt, &inner_error); - g_object_unref (stmt); - } - - if (cursor) { - while (tracker_db_cursor_iter_next (cursor, NULL, &inner_error)) { - GValue gvalue = { 0 }; - - tracker_db_cursor_get_value (cursor, 0, &gvalue); - - if (G_VALUE_TYPE (&gvalue)) { - if (tracker_property_get_data_type (property) == TRACKER_PROPERTY_TYPE_DATETIME) { - GDateTime *datetime; - - if (G_VALUE_TYPE (&gvalue) == G_TYPE_INT64) { - datetime = g_date_time_new_from_unix_utc (g_value_get_int64 (&gvalue)); - g_value_unset (&gvalue); - g_value_init (&gvalue, G_TYPE_DATE_TIME); - g_value_take_boxed (&gvalue, datetime); - } else { - datetime = tracker_date_new_from_iso8601 (g_value_get_string (&gvalue), - &inner_error); - g_value_unset (&gvalue); + tracker_db_statement_mru_update (&graph->values_mru, stmt); + g_object_ref (stmt); + } else { + TrackerDBInterface *iface; + const gchar *table_name; + const gchar *field_name; - if (inner_error) { - g_propagate_prefixed_error (error, - inner_error, - "Error in date conversion:"); - return NULL; - } + table_name = tracker_property_get_table_name (property); + field_name = tracker_property_get_name (property); - g_value_init (&gvalue, G_TYPE_DATE_TIME); - g_value_take_boxed (&gvalue, datetime); - } - } + iface = tracker_data_manager_get_writable_db_interface (data->manager); + stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, error, + "SELECT \"%s\" FROM \"%s\".\"%s\" WHERE ID = ?", + field_name, database, table_name); + if (!stmt) + return NULL; - g_array_append_val (old_values, gvalue); - } - } + tracker_db_statement_mru_insert (&graph->values_mru, property, stmt); + } - g_object_unref (cursor); + if (stmt) { + tracker_db_statement_bind_int (stmt, 0, data->resource_buffer->id); + old_values = tracker_db_statement_get_values (stmt, + tracker_property_get_data_type (property), + error); + g_object_unref (stmt); } - if (inner_error) { - g_propagate_error (error, inner_error); + if (!old_values) return NULL; - } } - return old_values; -} - -static GArray * -get_old_property_values (TrackerData *data, - TrackerProperty *property, - GError **error) -{ - GArray *old_values; - const gchar *database; - - database = data->resource_buffer->graph->graph ? - data->resource_buffer->graph->graph : "main"; - - /* read existing property values */ - old_values = g_hash_table_lookup (data->resource_buffer->predicates, property); - if (old_values == NULL) { - if (!check_property_domain (data, property)) { - if (data->implicit_create) { - if (!cache_create_service_decomposed (data, - tracker_property_get_domain (property), - error)) - return NULL; - } else { - TrackerDBInterface *iface; - gchar *resource; - - iface = tracker_data_manager_get_writable_db_interface (data->manager); - resource = tracker_data_query_resource_urn (data->manager, - iface, - data->resource_buffer->id); - - g_set_error (error, TRACKER_SPARQL_ERROR, TRACKER_SPARQL_ERROR_CONSTRAINT, - "%s %s is not is not a %s, cannot have property `%s'", - resource ? "Subject" : "Blank node", - resource ? resource : "", - tracker_class_get_name (tracker_property_get_domain (property)), - tracker_property_get_name (property)); - g_free (resource); - - return NULL; - } - } - - if (tracker_property_get_fulltext_indexed (property)) { - TrackerDBInterface *iface; - - iface = tracker_data_manager_get_writable_db_interface (data->manager); - - if (!data->resource_buffer->fts_updated && !data->resource_buffer->create) { - TrackerOntologies *ontologies; - guint i, n_props; - TrackerProperty **properties, *prop; - GPtrArray *fts_props, *fts_text; - - /* first fulltext indexed property to be modified - * retrieve values of all fulltext indexed properties - */ - ontologies = tracker_data_manager_get_ontologies (data->manager); - properties = tracker_ontologies_get_properties (ontologies, &n_props); - - fts_props = g_ptr_array_new (); - fts_text = g_ptr_array_new_with_free_func (g_free); - - for (i = 0; i < n_props; i++) { - prop = properties[i]; - - if (tracker_property_get_fulltext_indexed (prop) - && check_property_domain (data, prop)) { - const gchar *property_name; - GString *str; - guint j; - - old_values = get_property_values (data, prop, error); - if (!old_values) { - g_ptr_array_unref (fts_props); - g_ptr_array_unref (fts_text); - return NULL; - } - - property_name = tracker_property_get_name (prop); - str = g_string_new (NULL); - - /* delete old fts entries */ - for (j = 0; j < old_values->len; j++) { - GValue *value = &g_array_index (old_values, GValue, j); - if (j != 0) - g_string_append_c (str, ','); - g_string_append (str, g_value_get_string (value)); - } - - g_ptr_array_add (fts_props, (gpointer) property_name); - g_ptr_array_add (fts_text, g_string_free (str, FALSE)); - } - } - - g_ptr_array_add (fts_props, NULL); - g_ptr_array_add (fts_text, NULL); - - tracker_db_interface_sqlite_fts_delete_text (iface, - database, - data->resource_buffer->id, - (const gchar **) fts_props->pdata, - (const gchar **) fts_text->pdata); - - g_ptr_array_unref (fts_props); - g_ptr_array_unref (fts_text); - - old_values = g_hash_table_lookup (data->resource_buffer->predicates, property); - data->resource_buffer->fts_updated = TRUE; - } else { - old_values = get_property_values (data, property, error); - } - - } else { - old_values = get_property_values (data, property, error); - } + if (!old_values) { + old_values = g_array_new (FALSE, FALSE, sizeof (GValue)); + g_array_set_clear_func (old_values, (GDestroyNotify) g_value_unset); } + g_hash_table_insert (data->resource_buffer->predicates, g_object_ref (property), old_values); + return old_values; } @@ -1745,15 +1951,30 @@ get_bnode_id (GHashTable *bnodes, static TrackerRowid get_bnode_for_resource (GHashTable *bnodes, + GHashTable *visited, TrackerData *data, TrackerResource *resource, GError **error) { const gchar *identifier; + TrackerRowid *bnode_id; + + bnode_id = g_hash_table_lookup (visited, resource); + if (bnode_id) + return *bnode_id; - identifier = tracker_resource_get_identifier (resource); + identifier = tracker_resource_get_identifier_internal (resource); - return get_bnode_id (bnodes, data, identifier, error); + if (identifier) { + /* If the resource has a blank node identifier string already, + * then it has been likely referenced in other user SPARQL. + * Make sure this blank node label is cached for future + * references. + */ + return get_bnode_id (bnodes, data, identifier, error); + } else { + return tracker_data_generate_bnode (data, error); + } } static gboolean @@ -1770,21 +1991,37 @@ resource_in_domain_index_class (TrackerData *data, } static void -process_domain_indexes (TrackerData *data, - TrackerProperty *property, - const GValue *gvalue, - const gchar *field_name) +insert_property_domain_indexes (TrackerData *data, + TrackerProperty *property, + const GValue *gvalue) +{ + TrackerClass **domain_index_classes; + + domain_index_classes = tracker_property_get_domain_indexes (property); + while (*domain_index_classes) { + if (resource_in_domain_index_class (data, *domain_index_classes)) { + log_entry_for_single_value_property (data, + *domain_index_classes, + property, + gvalue); + } + domain_index_classes++; + } +} + +static void +delete_property_domain_indexes (TrackerData *data, + TrackerProperty *property, + const GValue *gvalue) { TrackerClass **domain_index_classes; domain_index_classes = tracker_property_get_domain_indexes (property); while (*domain_index_classes) { if (resource_in_domain_index_class (data, *domain_index_classes)) { - cache_insert_value (data, - tracker_class_get_name (*domain_index_classes), - field_name, - gvalue, - FALSE); + log_entry_for_single_value_property (data, + *domain_index_classes, + property, NULL); } domain_index_classes++; } @@ -1820,6 +2057,20 @@ maybe_convert_value (TrackerData *data, return FALSE; } +static void +maybe_append_fts_property (TrackerData *data, + TrackerProperty *property) +{ + if (!tracker_property_get_fulltext_indexed (property)) + return; + + if (g_list_find (data->resource_buffer->fts_properties, property)) + return; + + data->resource_buffer->fts_properties = + g_list_prepend (data->resource_buffer->fts_properties, property); +} + static gboolean cache_insert_metadata_decomposed (TrackerData *data, TrackerProperty *property, @@ -1827,15 +2078,40 @@ cache_insert_metadata_decomposed (TrackerData *data, GError **error) { gboolean multiple_values; - const gchar *table_name; - const gchar *field_name; TrackerProperty **super_properties; GArray *old_values; GError *new_error = NULL; gboolean change = FALSE; + if (!check_property_domain (data, property)) { + if (data->implicit_create) { + if (!cache_create_service_decomposed (data, + tracker_property_get_domain (property), + error)) + return FALSE; + } else { + TrackerDBInterface *iface; + gchar *resource; + + iface = tracker_data_manager_get_writable_db_interface (data->manager); + resource = tracker_data_query_resource_urn (data->manager, + iface, + data->resource_buffer->id); + + g_set_error (error, TRACKER_SPARQL_ERROR, TRACKER_SPARQL_ERROR_CONSTRAINT, + "%s %s is not is not a %s, cannot have property `%s'", + resource ? "Subject" : "Blank node", + resource ? resource : "", + tracker_class_get_name (tracker_property_get_domain (property)), + tracker_property_get_name (property)); + g_free (resource); + + return FALSE; + } + } + /* read existing property values */ - old_values = get_old_property_values (data, property, &new_error); + old_values = get_property_values (data, property, &new_error); if (new_error) { g_propagate_error (error, new_error); return FALSE; @@ -1845,8 +2121,7 @@ cache_insert_metadata_decomposed (TrackerData *data, super_properties = tracker_property_get_super_properties (property); multiple_values = tracker_property_get_multiple_values (property); - data->resource_buffer->fts_updated |= - tracker_property_get_fulltext_indexed (property); + maybe_append_fts_property (data, property); while (*super_properties) { gboolean super_is_multi; @@ -1855,14 +2130,13 @@ cache_insert_metadata_decomposed (TrackerData *data, const GValue *val; super_is_multi = tracker_property_get_multiple_values (*super_properties); - super_old_values = get_old_property_values (data, *super_properties, &new_error); + super_old_values = get_property_values (data, *super_properties, &new_error); if (new_error) { g_propagate_error (error, new_error); return FALSE; } - data->resource_buffer->fts_updated |= - tracker_property_get_fulltext_indexed (*super_properties); + maybe_append_fts_property (data, *super_properties); if (maybe_convert_value (data, tracker_property_get_data_type (property), @@ -1886,9 +2160,6 @@ cache_insert_metadata_decomposed (TrackerData *data, super_properties++; } - table_name = tracker_property_get_table_name (property); - field_name = tracker_property_get_name (property); - if (!value_set_add_value (old_values, object)) { /* value already inserted */ } else if (!multiple_values && old_values->len > 1) { @@ -1924,7 +2195,7 @@ cache_insert_metadata_decomposed (TrackerData *data, g_set_error (error, TRACKER_SPARQL_ERROR, TRACKER_SPARQL_ERROR_CONSTRAINT, "Unable to insert multiple values on single valued property `%s' for %s %s " "(old_value: '%s', new value: '%s')", - field_name, + tracker_property_get_name (property), resource ? "resource" : "blank node", resource ? resource : "", old_value_str ? old_value_str : "<untransformable>", @@ -1936,16 +2207,38 @@ cache_insert_metadata_decomposed (TrackerData *data, g_value_unset (&old_value); g_value_unset (&new_value); } else { - cache_insert_value (data, table_name, field_name, - object, - multiple_values); + if (multiple_values) { + log_entry_for_multi_value_property (data, + TRACKER_LOG_MULTIVALUED_PROPERTY_INSERT, + property, object); + } else { + log_entry_for_single_value_property (data, + tracker_property_get_domain (property), + property, object); + } + + if (!data->resource_buffer->modified) { + /* first modification of this particular resource, update nrl:modified */ + TrackerOntologies *ontologies; + TrackerProperty *modified; + GValue gvalue = { 0 }; + + data->resource_buffer->modified = TRUE; + ontologies = tracker_data_manager_get_ontologies (data->manager); + modified = tracker_ontologies_get_nrl_modified (ontologies); + + g_value_init (&gvalue, G_TYPE_INT64); + g_value_set_int64 (&gvalue, get_transaction_modseq (data)); + log_entry_for_single_value_property (data, + tracker_property_get_domain (modified), + modified, &gvalue); + } if (tracker_property_get_data_type (property) == TRACKER_PROPERTY_TYPE_RESOURCE) tracker_data_resource_ref (data, g_value_get_int64 (object), multiple_values); - if (!multiple_values) { - process_domain_indexes (data, property, object, field_name); - } + if (!multiple_values) + insert_property_domain_indexes (data, property, object); change = TRUE; } @@ -1960,19 +2253,17 @@ delete_metadata_decomposed (TrackerData *data, GError **error) { gboolean multiple_values; - const gchar *table_name; - const gchar *field_name; TrackerProperty **super_properties; GArray *old_values; GError *new_error = NULL; gboolean change = FALSE; multiple_values = tracker_property_get_multiple_values (property); - table_name = tracker_property_get_table_name (property); - field_name = tracker_property_get_name (property); + + maybe_append_fts_property (data, property); /* read existing property values */ - old_values = get_old_property_values (data, property, &new_error); + old_values = get_property_values (data, property, &new_error); if (new_error) { /* no need to error out if statement does not exist for any reason */ g_clear_error (&new_error); @@ -1982,26 +2273,21 @@ delete_metadata_decomposed (TrackerData *data, if (!value_set_remove_value (old_values, object)) { /* value not found */ } else { - cache_delete_value (data, table_name, field_name, - object, multiple_values); + if (multiple_values) { + log_entry_for_multi_value_property (data, + TRACKER_LOG_MULTIVALUED_PROPERTY_DELETE, + property, object); + } else { + log_entry_for_single_value_property (data, + tracker_property_get_domain (property), + property, NULL); + } + if (tracker_property_get_data_type (property) == TRACKER_PROPERTY_TYPE_RESOURCE) tracker_data_resource_unref (data, g_value_get_int64 (object), multiple_values); - if (!multiple_values) { - TrackerClass **domain_index_classes; - - domain_index_classes = tracker_property_get_domain_indexes (property); - - while (*domain_index_classes) { - if (resource_in_domain_index_class (data, *domain_index_classes)) { - cache_delete_value (data, - tracker_class_get_name (*domain_index_classes), - field_name, - object, multiple_values); - } - domain_index_classes++; - } - } + if (!multiple_values) + delete_property_domain_indexes (data, property, object); change = TRUE; } @@ -2035,26 +2321,16 @@ cache_delete_resource_type_full (TrackerData *data, gboolean single_type, GError **error) { - TrackerDBInterface *iface; - TrackerDBStatement *stmt; - TrackerDBCursor *cursor = NULL; - TrackerProperty **properties, *prop; - gboolean found; - guint i, p, n_props; - GError *inner_error = NULL; - TrackerOntologies *ontologies; - const gchar *database; + TrackerProperty **properties, *prop; + gboolean found; + guint i, j, p, n_props; + TrackerOntologies *ontologies; GValue gvalue = G_VALUE_INIT; - iface = tracker_data_manager_get_writable_db_interface (data->manager); ontologies = tracker_data_manager_get_ontologies (data->manager); - database = data->resource_buffer->graph->graph ? - data->resource_buffer->graph->graph : "main"; if (!single_type) { - if (strcmp (tracker_class_get_uri (class), TRACKER_PREFIX_RDFS "Resource") == 0 && - g_hash_table_size (data->resource_buffer->tables) == 0) { - + if (strcmp (tracker_class_get_uri (class), TRACKER_PREFIX_RDFS "Resource") == 0) { /* skip subclass query when deleting whole resource to improve performance */ @@ -2085,50 +2361,39 @@ cache_delete_resource_type_full (TrackerData *data, /* retrieve all subclasses we need to remove from the subject * before we can remove the class specified as object of the statement */ - stmt = tracker_db_interface_create_vstatement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_SELECT, &inner_error, - "SELECT (SELECT Uri FROM Resource WHERE ID = subclass.ID) " - "FROM \"%s\".\"rdfs:Resource_rdf:type\" AS type INNER JOIN \"%s\".\"rdfs:Class_rdfs:subClassOf\" AS subclass ON (type.\"rdf:type\" = subclass.ID) " - "WHERE type.ID = ? AND subclass.\"rdfs:subClassOf\" = (SELECT ID FROM Resource WHERE Uri = ?)", - database, database); + for (i = 0; i < data->resource_buffer->types->len; i++) { + TrackerClass *type, **super_classes; - if (stmt) { - tracker_db_statement_bind_int (stmt, 0, data->resource_buffer->id); - tracker_db_statement_bind_text (stmt, 1, tracker_class_get_uri (class)); - cursor = tracker_db_statement_start_cursor (stmt, &inner_error); - g_object_unref (stmt); - } + type = g_ptr_array_index (data->resource_buffer->types, i); + super_classes = tracker_class_get_super_classes (type); - if (cursor) { - while (tracker_db_cursor_iter_next (cursor, NULL, &inner_error)) { - const gchar *class_uri; + for (j = 0; super_classes[j]; j++) { + if (super_classes[j] != class) + continue; - class_uri = tracker_db_cursor_get_string (cursor, 0, NULL); - if (!cache_delete_resource_type_full (data, tracker_ontologies_get_class_by_uri (ontologies, class_uri), - FALSE, error)) + if (!cache_delete_resource_type_full (data, type, FALSE, error)) return FALSE; } - - g_object_unref (cursor); } - if (inner_error) { - g_propagate_prefixed_error (error, - inner_error, - "Deleting resource:"); - return FALSE; - } + /* Once done deleting subclasses, fall through */ } - /* delete all property values */ - + /* Delete property values, only properties that: + * - Have ?prop rdfs:range rdfs:Resource + * - Are domain indexes in other classes + * + * Do need inspection of the previous content. All multivalued properties + * can be cleared through TRACKER_LOG_MULTIVALUED_PROPERTY_CLEAR, and all + * values for other single-valued properties will eventually disappear when + * deleting the row from the table representing the given TrackerClass. + */ properties = tracker_ontologies_get_properties (ontologies, &n_props); for (p = 0; p < n_props; p++) { - gboolean multiple_values; - const gchar *table_name; - const gchar *field_name; + gboolean multiple_values; GArray *old_values; - gint y; + gint y; prop = properties[p]; @@ -2138,52 +2403,58 @@ cache_delete_resource_type_full (TrackerData *data, continue; multiple_values = tracker_property_get_multiple_values (prop); - table_name = tracker_property_get_table_name (prop); - field_name = tracker_property_get_name (prop); - old_values = get_old_property_values (data, prop, error); - if (!old_values) - return FALSE; + maybe_append_fts_property (data, prop); + + if (*tracker_property_get_domain_indexes (prop) || + tracker_property_get_data_type (prop) == TRACKER_PROPERTY_TYPE_RESOURCE) { + old_values = get_property_values (data, prop, error); + if (!old_values) + return FALSE; + + for (y = old_values->len - 1; y >= 0 ; y--) { + GValue *old_gvalue, copy = G_VALUE_INIT; + + old_gvalue = &g_array_index (old_values, GValue, y); + g_value_init (©, G_VALUE_TYPE (old_gvalue)); + g_value_copy (old_gvalue, ©); - for (y = old_values->len - 1; y >= 0 ; y--) { - GValue *old_gvalue, copy = G_VALUE_INIT; - - old_gvalue = &g_array_index (old_values, GValue, y); - g_value_init (©, G_VALUE_TYPE (old_gvalue)); - g_value_copy (old_gvalue, ©); - - value_set_remove_value (old_values, old_gvalue); - cache_delete_value (data, table_name, field_name, - ©, multiple_values); - if (tracker_property_get_data_type (prop) == TRACKER_PROPERTY_TYPE_RESOURCE) - tracker_data_resource_unref (data, g_value_get_int64 (©), multiple_values); - - if (!multiple_values) { - TrackerClass **domain_index_classes; - - domain_index_classes = tracker_property_get_domain_indexes (prop); - while (*domain_index_classes) { - if (resource_in_domain_index_class (data, *domain_index_classes)) { - cache_delete_value (data, - tracker_class_get_name (*domain_index_classes), - field_name, - ©, multiple_values); - } - domain_index_classes++; + value_set_remove_value (old_values, old_gvalue); + + if (!multiple_values) { + log_entry_for_single_value_property (data, + tracker_property_get_domain (prop), + prop, NULL); } + + if (tracker_property_get_data_type (prop) == TRACKER_PROPERTY_TYPE_RESOURCE) + tracker_data_resource_unref (data, g_value_get_int64 (©), multiple_values); + + if (!multiple_values) + delete_property_domain_indexes (data, prop, ©); + + g_value_unset (©); } + } - g_value_unset (©); + if (multiple_values) { + log_entry_for_multi_value_property (data, + TRACKER_LOG_MULTIVALUED_PROPERTY_CLEAR, + prop, NULL); } + + if (data->resource_buffer->predicates) + g_hash_table_remove (data->resource_buffer->predicates, prop); } g_value_init (&gvalue, G_TYPE_INT64); g_value_set_int64 (&gvalue, tracker_class_get_id (class)); - cache_delete_value (data, "rdfs:Resource_rdf:type", "rdf:type", - &gvalue, TRUE); + log_entry_for_multi_value_property (data, TRACKER_LOG_MULTIVALUED_PROPERTY_DELETE, + tracker_ontologies_get_rdf_type (ontologies), + &gvalue); tracker_data_resource_unref (data, tracker_class_get_id (class), TRUE); - cache_delete_row (data, class); + log_entry_for_class (data, TRACKER_LOG_CLASS_DELETE, class); tracker_data_resource_unref (data, data->resource_buffer->id, FALSE); tracker_data_dispatch_delete_statement_callbacks (data, @@ -2224,19 +2495,18 @@ ensure_graph_buffer (TrackerDataUpdateBuffer *buffer, } graph_buffer = g_slice_new0 (TrackerDataUpdateBufferGraph); - graph_buffer->refcounts = - g_hash_table_new_full (tracker_rowid_hash, tracker_rowid_equal, - (GDestroyNotify) tracker_rowid_free, NULL); + graph_buffer->refcounts = g_array_sized_new (FALSE, FALSE, sizeof (RefcountEntry), UPDATE_LOG_SIZE); graph_buffer->graph = g_strdup (name); - if (graph_buffer->graph) { - graph_buffer->id = tracker_data_manager_find_graph (data->manager, - graph_buffer->graph, - TRUE); - } graph_buffer->resources = g_hash_table_new_full (tracker_rowid_hash, tracker_rowid_equal, NULL, (GDestroyNotify) resource_buffer_free); + + tracker_db_statement_mru_init (&graph_buffer->values_mru, 20, + g_direct_hash, + g_direct_equal, + NULL); + g_ptr_array_add (buffer->graphs, graph_buffer); return graph_buffer; @@ -2282,8 +2552,8 @@ resource_buffer_switch (TrackerData *data, create = g_hash_table_contains (data->update_buffer.new_resources, &subject); if (!create) { - rdf_types = tracker_data_query_rdf_type (data->manager, - graph, + rdf_types = tracker_data_query_rdf_type (data, + graph_buffer, subject, &inner_error); if (!rdf_types) { @@ -2296,14 +2566,11 @@ resource_buffer_switch (TrackerData *data, resource_buffer->id = subject; resource_buffer->create = create; - resource_buffer->fts_updated = FALSE; if (resource_buffer->create) { resource_buffer->types = g_ptr_array_new (); } else { resource_buffer->types = rdf_types; } - resource_buffer->predicates = g_hash_table_new_full (g_direct_hash, g_direct_equal, g_object_unref, (GDestroyNotify) g_array_unref); - resource_buffer->tables = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) cache_table_free); resource_buffer->graph = graph_buffer; g_hash_table_insert (graph_buffer->resources, &resource_buffer->id, resource_buffer); @@ -2381,25 +2648,25 @@ delete_all_helper (TrackerData *data, if (subproperty == property) { if (tracker_property_get_multiple_values (property)) { - cache_delete_all_values (data, - tracker_property_get_table_name (property), - tracker_property_get_name (property)); + log_entry_for_multi_value_property (data, + TRACKER_LOG_MULTIVALUED_PROPERTY_CLEAR, + property, NULL); + if (tracker_property_get_data_type (property) == TRACKER_PROPERTY_TYPE_RESOURCE) { if (!tracker_data_resource_unref_all (data, property, error)) return FALSE; } } else { value = &g_array_index (old_values, GValue, 0); - cache_delete_value (data, - tracker_property_get_table_name (property), - tracker_property_get_name (property), - value, - FALSE); + log_entry_for_single_value_property (data, + tracker_property_get_domain (property), + property, NULL); + if (tracker_property_get_data_type (property) == TRACKER_PROPERTY_TYPE_RESOURCE) tracker_data_resource_unref (data, g_value_get_int64 (value), FALSE); } } else { - super_old_values = get_old_property_values (data, property, error); + super_old_values = get_property_values (data, property, error); if (!super_old_values) return FALSE; @@ -2409,11 +2676,16 @@ delete_all_helper (TrackerData *data, if (!value_set_remove_value (super_old_values, value)) continue; - cache_delete_value (data, - tracker_property_get_table_name (property), - tracker_property_get_name (property), - value, - tracker_property_get_multiple_values (property)); + if (tracker_property_get_multiple_values (property)) { + log_entry_for_multi_value_property (data, + TRACKER_LOG_MULTIVALUED_PROPERTY_DELETE, + property, value); + } else { + log_entry_for_single_value_property (data, + tracker_property_get_domain (property), + property, NULL); + } + if (tracker_property_get_data_type (property) == TRACKER_PROPERTY_TYPE_RESOURCE) { tracker_data_resource_unref (data, g_value_get_int64 (value), tracker_property_get_multiple_values (property)); @@ -2462,7 +2734,7 @@ tracker_data_delete_all (TrackerData *data, ontologies = tracker_data_manager_get_ontologies (data->manager); property = tracker_ontologies_get_property_by_uri (ontologies, predicate); - old_values = get_old_property_values (data, property, &inner_error); + old_values = get_property_values (data, property, &inner_error); if (inner_error) { g_propagate_error (error, inner_error); return FALSE; @@ -2490,9 +2762,10 @@ delete_single_valued (TrackerData *data, multiple_values = tracker_property_get_multiple_values (predicate); if (super_is_single_valued && multiple_values) { - cache_delete_all_values (data, - tracker_property_get_table_name (predicate), - tracker_property_get_name (predicate)); + log_entry_for_multi_value_property (data, + TRACKER_LOG_MULTIVALUED_PROPERTY_CLEAR, + predicate, NULL); + if (tracker_property_get_data_type (predicate) == TRACKER_PROPERTY_TYPE_RESOURCE) { if (!tracker_data_resource_unref_all (data, predicate, error)) return FALSE; @@ -2501,21 +2774,20 @@ delete_single_valued (TrackerData *data, GError *inner_error = NULL; GArray *old_values; - old_values = get_old_property_values (data, predicate, &inner_error); + log_entry_for_single_value_property (data, + tracker_property_get_domain (predicate), + predicate, NULL); - if (old_values && old_values->len == 1) { - GValue *value; + if (tracker_property_get_data_type (predicate) == TRACKER_PROPERTY_TYPE_RESOURCE) { + old_values = get_property_values (data, predicate, &inner_error); - value = &g_array_index (old_values, GValue, 0); - cache_delete_value (data, - tracker_property_get_table_name (predicate), - tracker_property_get_name (predicate), - value, - FALSE); - if (tracker_property_get_data_type (predicate) == TRACKER_PROPERTY_TYPE_RESOURCE) - tracker_data_resource_unref (data, g_value_get_int64 (value), multiple_values); + if (old_values && old_values->len == 1) { + GValue *value; - g_array_remove_index (old_values, 0); + value = &g_array_index (old_values, GValue, 0); + tracker_data_resource_unref (data, g_value_get_int64 (value), multiple_values); + g_array_remove_index (old_values, 0); + } } else { /* no need to error out if statement does not exist for any reason */ g_clear_error (&inner_error); @@ -2555,7 +2827,7 @@ tracker_data_insert_statement (TrackerData *data, } } -void +static void tracker_data_insert_statement_with_uri (TrackerData *data, const gchar *graph, TrackerRowid subject, @@ -2620,7 +2892,7 @@ tracker_data_insert_statement_with_uri (TrackerData *data, } } -void +static void tracker_data_insert_statement_with_string (TrackerData *data, const gchar *graph, TrackerRowid subject, @@ -2698,9 +2970,12 @@ tracker_data_update_statement (TrackerData *data, if (!resource_buffer_switch (data, graph, subject, error)) return; - cache_delete_all_values (data, - tracker_property_get_table_name (predicate), - tracker_property_get_name (predicate)); + maybe_append_fts_property (data, predicate); + + log_entry_for_multi_value_property (data, + TRACKER_LOG_MULTIVALUED_PROPERTY_CLEAR, + predicate, NULL); + if (tracker_property_get_data_type (predicate) == TRACKER_PROPERTY_TYPE_RESOURCE) { if (!tracker_data_resource_unref_all (data, predicate, error)) return; @@ -2709,6 +2984,8 @@ tracker_data_update_statement (TrackerData *data, if (!resource_buffer_switch (data, graph, subject, error)) return; + maybe_append_fts_property (data, predicate); + if (!delete_single_valued (data, graph, subject, predicate, !tracker_property_get_multiple_values (predicate), error)) @@ -2733,6 +3010,14 @@ tracker_data_update_statement (TrackerData *data, } } +static void +clear_property (gpointer data) +{ + TrackerDataPropertyEntry *entry = data; + + g_value_unset (&entry->value); +} + void tracker_data_begin_transaction (TrackerData *data, GError **error) @@ -2765,6 +3050,16 @@ tracker_data_begin_transaction (TrackerData *data, (GDestroyNotify) tracker_rowid_free, NULL); /* used for normal transactions */ data->update_buffer.graphs = g_ptr_array_new_with_free_func ((GDestroyNotify) graph_buffer_free); + + data->update_buffer.properties = g_array_sized_new (FALSE, TRUE, sizeof (TrackerDataPropertyEntry), UPDATE_LOG_SIZE); + g_array_set_clear_func (data->update_buffer.properties, clear_property); + data->update_buffer.update_log = g_array_sized_new (FALSE, TRUE, sizeof (TrackerDataLogEntry), UPDATE_LOG_SIZE); + data->update_buffer.class_updates = g_hash_table_new (tracker_data_log_entry_hash, + tracker_data_log_entry_equal); + tracker_db_statement_mru_init (&data->update_buffer.stmt_mru, 100, + tracker_data_log_entry_schema_hash, + tracker_data_log_entry_schema_equal, + (GDestroyNotify) tracker_data_log_entry_free); } data->resource_buffer = NULL; @@ -2825,7 +3120,6 @@ tracker_data_commit_transaction (TrackerData *data, tracker_db_interface_execute_query (iface, NULL, "PRAGMA cache_size = %d", TRACKER_DB_CACHE_SIZE_DEFAULT); - g_ptr_array_set_size (data->update_buffer.graphs, 0); g_hash_table_remove_all (data->update_buffer.resource_cache); tracker_data_dispatch_commit_statement_callbacks (data); @@ -3061,7 +3355,7 @@ tracker_data_ensure_graph (TrackerData *data, return 0; iface = tracker_data_manager_get_writable_db_interface (data->manager); - stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, error, + stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, error, "INSERT OR IGNORE INTO Graph (ID) VALUES (?)"); if (!stmt) return 0; @@ -3087,7 +3381,7 @@ tracker_data_delete_graph (TrackerData *data, return FALSE; iface = tracker_data_manager_get_writable_db_interface (data->manager); - stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, error, + stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_NONE, error, "DELETE FROM Graph WHERE ID = ?"); if (!stmt) return FALSE; @@ -3109,13 +3403,11 @@ resource_maybe_reset_property (TrackerData *data, GError **error) { GError *inner_error = NULL; - const gchar *subject_name; /* If the subject is a blank node, this is a whole new insertion. * We don't need deleting anything then. */ - subject_name = tracker_resource_get_identifier (resource); - if (g_str_has_prefix (subject_name, "_:")) + if (tracker_resource_is_blank_node (resource)) return TRUE; if (!tracker_data_delete_all (data, @@ -3135,93 +3427,72 @@ update_resource_property (TrackerData *data, const gchar *graph_uri, TrackerResource *resource, TrackerRowid subject, - const gchar *property, + TrackerProperty *property, + const GValue *val, GHashTable *visited, GHashTable *bnodes, GError **error) { - TrackerOntologies *ontologies; - TrackerProperty *predicate; - GList *values, *v; - gchar *property_uri; GError *inner_error = NULL; + GValue free_me = G_VALUE_INIT; + const GValue *value; + TrackerRowid id; - values = tracker_resource_get_values (resource, property); - tracker_data_manager_expand_prefix (data->manager, - property, - NULL, NULL, - &property_uri); - - ontologies = tracker_data_manager_get_ontologies (data->manager); - predicate = tracker_ontologies_get_property_by_uri (ontologies, property_uri); - if (predicate == NULL) { - g_set_error (error, TRACKER_SPARQL_ERROR, - TRACKER_SPARQL_ERROR_UNKNOWN_PROPERTY, - "Property '%s' not found in the ontology", - property_uri); - return FALSE; - } - - for (v = values; v && !inner_error; v = v->next) { - GValue *value, free_me = G_VALUE_INIT; - TrackerRowid id; - - if (G_VALUE_HOLDS (v->data, TRACKER_TYPE_RESOURCE)) { + if (G_VALUE_HOLDS (val, TRACKER_TYPE_RESOURCE)) { + if (!update_resource_single (data, + graph_uri, + g_value_get_object (val), + visited, + bnodes, + &id, + error)) + return FALSE; - if (!update_resource_single (data, - graph_uri, - g_value_get_object (v->data), - visited, - bnodes, - &id, - &inner_error)) - break; + g_value_init (&free_me, G_TYPE_INT64); + g_value_set_int64 (&free_me, id); + value = &free_me; + } else if (tracker_property_get_data_type (property) == TRACKER_PROPERTY_TYPE_RESOURCE && + g_type_is_a (G_VALUE_TYPE (val), G_TYPE_STRING)) { + if (g_str_has_prefix (g_value_get_string (val), "_:")) { + id = get_bnode_id (bnodes, data, g_value_get_string (val), error); + if (id == 0) + return FALSE; g_value_init (&free_me, G_TYPE_INT64); g_value_set_int64 (&free_me, id); - value = &free_me; - } else if (tracker_property_get_data_type (predicate) == TRACKER_PROPERTY_TYPE_RESOURCE && - g_type_is_a (G_VALUE_TYPE (v->data), G_TYPE_STRING)) { + } else { gchar *object_str; + gboolean retval; tracker_data_manager_expand_prefix (data->manager, - g_value_get_string (v->data), + g_value_get_string (val), NULL, NULL, &object_str); - if (g_str_has_prefix (object_str, "_:")) { - id = get_bnode_id (bnodes, data, object_str, error); - if (inner_error) - break; - - g_value_init (&free_me, G_TYPE_INT64); - g_value_set_int64 (&free_me, id); - } else if (!tracker_data_query_string_to_value (data->manager, - object_str, - NULL, - tracker_property_get_data_type (predicate), - &free_me, - &inner_error)) { - break; - } - + retval = tracker_data_query_string_to_value (data->manager, + object_str, + NULL, + tracker_property_get_data_type (property), + &free_me, + error); g_free (object_str); - value = &free_me; - } else { - value = v->data; + + if (!retval) + return FALSE; } - tracker_data_insert_statement (data, - graph_uri, - subject, - predicate, - value, - &inner_error); - g_value_unset (&free_me); + value = &free_me; + } else { + value = val; } - g_list_free (values); - g_free (property_uri); + tracker_data_insert_statement (data, + graph_uri, + subject, + property, + value, + &inner_error); + g_value_unset (&free_me); if (inner_error) { g_propagate_error (error, inner_error); @@ -3233,29 +3504,31 @@ update_resource_property (TrackerData *data, static gboolean update_resource_single (TrackerData *data, - const gchar *graph, + const gchar *graph_uri, TrackerResource *resource, GHashTable *visited, GHashTable *bnodes, TrackerRowid *id, GError **error) { - GList *properties = NULL, *l; + TrackerOntologies *ontologies; + TrackerResourceIterator iter; + GList *types, *l; GError *inner_error = NULL; - const gchar *subject_str; + const gchar *subject_str, *property; TrackerRowid subject; - gchar *graph_uri = NULL; + const GValue *value; gboolean is_bnode = FALSE; - subject_str = tracker_resource_get_identifier (resource); - if (!subject_str || g_str_has_prefix (subject_str, "_:")) { + if (tracker_resource_is_blank_node (resource)) { is_bnode = TRUE; - subject = get_bnode_for_resource (bnodes, data, resource, error); + subject = get_bnode_for_resource (bnodes, visited, data, resource, error); if (!subject) return FALSE; } else { gchar *subject_uri; + subject_str = tracker_resource_get_identifier (resource); tracker_data_manager_expand_prefix (data->manager, subject_str, NULL, NULL, &subject_uri); @@ -3265,85 +3538,65 @@ update_resource_single (TrackerData *data, return FALSE; } - if (g_hash_table_lookup (visited, resource)) - goto out; - - g_hash_table_add (visited, resource); + if (id) + *id = subject; - properties = tracker_resource_get_properties (resource); + if (g_hash_table_lookup (visited, resource)) + return TRUE; - if (graph) { - tracker_data_manager_expand_prefix (data->manager, - graph, NULL, NULL, - &graph_uri); - } + g_hash_table_insert (visited, resource, tracker_rowid_copy (&subject)); + ontologies = tracker_data_manager_get_ontologies (data->manager); /* Handle rdf:type first */ - if (g_list_find_custom (properties, "rdf:type", (GCompareFunc) g_strcmp0)) { - update_resource_property (data, graph_uri, resource, - subject, "rdf:type", - visited, bnodes, - &inner_error); - if (inner_error) - goto out; - } + types = tracker_resource_get_values (resource, "rdf:type"); - if (!is_bnode) { - gboolean need_flush = FALSE; + for (l = types; l; l = l->next) { + if (!update_resource_property (data, graph_uri, resource, + subject, + tracker_ontologies_get_rdf_type (ontologies), + l->data, + visited, bnodes, + error)) { + g_list_free (types); + return FALSE; + } + } - for (l = properties; l; l = l->next) { - const gchar *property = l->data; + g_list_free (types); - if (tracker_resource_get_property_overwrite (resource, property)) { - gchar *property_uri; + tracker_resource_iterator_init (&iter, resource); - tracker_data_manager_expand_prefix (data->manager, - property, - NULL, NULL, - &property_uri); + while (tracker_resource_iterator_next (&iter, &property, &value)) { + TrackerProperty *predicate; - if (resource_maybe_reset_property (data, graph_uri, resource, - subject, property_uri, - bnodes, &inner_error)) { - need_flush = TRUE; - } else if (inner_error) { - g_free (property_uri); - goto out; - } + if (g_str_equal (property, "rdf:type")) + continue; - g_free (property_uri); - } + predicate = tracker_ontologies_get_property_by_uri (ontologies, property); + if (predicate == NULL) { + g_set_error (error, TRACKER_SPARQL_ERROR, + TRACKER_SPARQL_ERROR_UNKNOWN_PROPERTY, + "Property '%s' not found in the ontology", + property); + return FALSE; } - if (need_flush) { - tracker_data_update_buffer_flush (data, &inner_error); + if (!is_bnode && tracker_resource_get_property_overwrite (resource, property)) { + resource_maybe_reset_property (data, graph_uri, resource, + subject, property, + bnodes, &inner_error); + if (inner_error) - goto out; + return FALSE; } - } - - for (l = properties; l; l = l->next) { - if (g_str_equal (l->data, "rdf:type")) - continue; if (!update_resource_property (data, graph_uri, resource, - subject, l->data, + subject, predicate, value, visited, bnodes, - &inner_error)) - break; - } - -out: - g_list_free (properties); - g_free (graph_uri); - - if (inner_error) { - g_propagate_error (error, inner_error); - return FALSE; + error)) + return FALSE; } - if (id) - *id = subject; return TRUE; } @@ -3353,22 +3606,27 @@ tracker_data_update_resource (TrackerData *data, const gchar *graph, TrackerResource *resource, GHashTable *bnodes, + GHashTable *visited, GError **error) { - GHashTable *visited; gboolean retval; - - visited = g_hash_table_new (NULL, NULL); + gchar *graph_uri = NULL; if (bnodes) g_hash_table_ref (bnodes); else bnodes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) tracker_rowid_free); - retval = update_resource_single (data, graph, resource, visited, bnodes, NULL, error); + if (graph) { + tracker_data_manager_expand_prefix (data->manager, + graph, NULL, NULL, + &graph_uri); + } + + retval = update_resource_single (data, graph_uri, resource, visited, bnodes, NULL, error); - g_hash_table_unref (visited); g_hash_table_unref (bnodes); + g_free (graph_uri); return retval; } @@ -3382,26 +3640,20 @@ tracker_data_generate_bnode (TrackerData *data, GError *inner_error = NULL; TrackerRowid id; - iface = tracker_data_manager_get_writable_db_interface (data->manager); - - stmt = tracker_db_interface_create_statement (iface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, - &inner_error, - "INSERT INTO Resource (Uri, BlankNode) VALUES (?, ?)"); - if (!stmt) { - g_propagate_error (error, inner_error); + if (!tracker_data_ensure_insert_resource_stmt (data, error)) return 0; - } + stmt = data->update_buffer.insert_resource; tracker_db_statement_bind_null (stmt, 0); tracker_db_statement_bind_int (stmt, 1, TRUE); tracker_db_statement_execute (stmt, &inner_error); - g_object_unref (stmt); if (inner_error) { g_propagate_error (error, inner_error); return 0; } + iface = tracker_data_manager_get_writable_db_interface (data->manager); id = tracker_db_interface_sqlite_get_last_insert_id (iface); g_hash_table_add (data->update_buffer.new_resources, tracker_rowid_copy (&id)); diff --git a/src/libtracker-sparql/core/tracker-data-update.h b/src/libtracker-sparql/core/tracker-data-update.h index e00d5edcb..bdba752ea 100644 --- a/src/libtracker-sparql/core/tracker-data-update.h +++ b/src/libtracker-sparql/core/tracker-data-update.h @@ -43,14 +43,13 @@ typedef struct _TrackerDataClass TrackerDataClass; typedef struct _TrackerData TrackerDataUpdate; -typedef void (*TrackerStatementCallback) (TrackerRowid graph_id, - const gchar *graph, - TrackerRowid subject_id, - TrackerRowid predicate_id, - TrackerRowid object_id, - GPtrArray *rdf_types, - gpointer user_data); -typedef void (*TrackerCommitCallback) (gpointer user_data); +typedef void (*TrackerStatementCallback) (const gchar *graph, + TrackerRowid subject_id, + TrackerRowid predicate_id, + TrackerRowid object_id, + GPtrArray *rdf_types, + gpointer user_data); +typedef void (*TrackerCommitCallback) (gpointer user_data); GQuark tracker_data_error_quark (void); @@ -139,6 +138,7 @@ gboolean tracker_data_update_resource (TrackerData *data, const gchar *graph, TrackerResource *resource, GHashTable *bnodes, + GHashTable *visited, GError **error); TrackerRowid tracker_data_update_ensure_resource (TrackerData *data, diff --git a/src/libtracker-sparql/core/tracker-db-interface-sqlite.c b/src/libtracker-sparql/core/tracker-db-interface-sqlite.c index 49f6764f4..dd3455221 100644 --- a/src/libtracker-sparql/core/tracker-db-interface-sqlite.c +++ b/src/libtracker-sparql/core/tracker-db-interface-sqlite.c @@ -63,13 +63,6 @@ #define sqlite3_value_text(x) ((const gchar *) sqlite3_value_text(x)) typedef struct { - TrackerDBStatement *head; - TrackerDBStatement *tail; - guint size; - guint max; -} TrackerDBStatementLru; - -typedef struct { GRegex *syntax_check; GRegex *replacement; GRegex *unescape; @@ -82,8 +75,6 @@ struct TrackerDBInterface { gchar *shared_cache_key; sqlite3 *db; - GHashTable *dynamic_statements; - /* Compiled regular expressions */ TrackerDBReplaceFuncChecks replace_func_checks; @@ -93,10 +84,8 @@ struct TrackerDBInterface { guint flags; GCancellable *cancellable; - TrackerDBStatementLru select_stmt_lru; - TrackerDBStatementLru update_stmt_lru; - - gchar *fts_properties; + TrackerDBStatementMru select_stmt_mru; + TrackerDBStatementMru update_stmt_mru; /* Used if TRACKER_DB_INTERFACE_USE_MUTEX is set */ GMutex mutex; @@ -127,9 +116,10 @@ struct TrackerDBStatement { TrackerDBInterface *db_interface; sqlite3_stmt *stmt; guint stmt_is_used : 1; - guint stmt_is_owned : 1; + guint stmt_is_borrowed : 1; TrackerDBStatement *next; TrackerDBStatement *prev; + gconstpointer mru_key; }; struct TrackerDBStatementClass { @@ -2219,10 +2209,8 @@ close_database (TrackerDBInterface *db_interface) { gint rc; - if (db_interface->dynamic_statements) { - g_hash_table_unref (db_interface->dynamic_statements); - db_interface->dynamic_statements = NULL; - } + tracker_db_statement_mru_finish (&db_interface->select_stmt_mru); + tracker_db_statement_mru_finish (&db_interface->update_stmt_mru); if (db_interface->replace_func_checks.syntax_check) g_regex_unref (db_interface->replace_func_checks.syntax_check); @@ -2240,34 +2228,6 @@ close_database (TrackerDBInterface *db_interface) } } -static gchar ** -_fts_create_properties (GHashTable *properties) -{ - GHashTableIter iter; - GPtrArray *cols; - GList *columns; - gchar *table; - - if (g_hash_table_size (properties) == 0) { - return NULL; - } - - g_hash_table_iter_init (&iter, properties); - cols = g_ptr_array_new (); - - while (g_hash_table_iter_next (&iter, (gpointer *) &table, - (gpointer *) &columns)) { - while (columns) { - g_ptr_array_add (cols, g_strdup (columns->data)); - columns = columns->next; - } - } - - g_ptr_array_add (cols, NULL); - - return (gchar **) g_ptr_array_free (cols, FALSE); -} - gboolean tracker_db_interface_sqlite_fts_init (TrackerDBInterface *db_interface, const gchar *database, @@ -2277,7 +2237,6 @@ tracker_db_interface_sqlite_fts_init (TrackerDBInterface *db_interface, GError **error) { GError *inner_error = NULL; - GStrv fts_columns; if (!tracker_fts_init_db (db_interface->db, db_interface, db_interface->flags, properties, error)) @@ -2293,25 +2252,6 @@ tracker_db_interface_sqlite_fts_init (TrackerDBInterface *db_interface, return FALSE; } - fts_columns = _fts_create_properties (properties); - - if (fts_columns) { - GString *fts_properties; - gint i; - - fts_properties = g_string_new (NULL); - - for (i = 0; fts_columns[i] != NULL; i++) { - g_string_append_printf (fts_properties, ", \"%s\"", - fts_columns[i]); - } - - g_free (db_interface->fts_properties); - db_interface->fts_properties = g_string_free (fts_properties, - FALSE); - g_strfreev (fts_columns); - } - return TRUE; } @@ -2335,130 +2275,130 @@ tracker_db_interface_sqlite_fts_alter_table (TrackerDBInterface *db_interface, } static gchar * -tracker_db_interface_sqlite_fts_create_query (TrackerDBInterface *db_interface, - const gchar *database, - gboolean delete, - const gchar **properties) +tracker_db_interface_sqlite_fts_create_update_query (TrackerDBInterface *db_interface, + const gchar *database, + const gchar **properties) { - GString *insert_str, *values_str; - gint i; - - insert_str = g_string_new (NULL); - g_string_append_printf (insert_str, "INSERT INTO \"%s\".fts5 (", database); - values_str = g_string_new (NULL); + GString *props_str; + gchar *query; + gint i; - if (delete) { - g_string_append (insert_str, "fts5,"); - g_string_append (values_str, "'delete',"); - } + props_str = g_string_new (NULL); - g_string_append (insert_str, "rowid"); - g_string_append (values_str, "?"); + for (i = 0; properties[i] != NULL; i++) { + if (i != 0) + g_string_append_c (props_str, ','); - for (i = 0; properties[i] != NULL; i++) { - g_string_append_printf (insert_str, ",\"%s\"", properties[i]); - g_string_append (values_str, ",?"); - } + g_string_append_printf (props_str, "\"%s\"", properties[i]); + } - g_string_append_printf (insert_str, ") VALUES (%s)", values_str->str); - g_string_free (values_str, TRUE); + query = g_strdup_printf ("INSERT INTO \"%s\".fts5 (ROWID, %s) " + "SELECT ROWID, %s FROM \"%s\".fts_view WHERE ROWID = ? AND COALESCE(%s, NULL) IS NOT NULL", + database, + props_str->str, + props_str->str, + database, + props_str->str); + g_string_free (props_str, TRUE); - return g_string_free (insert_str, FALSE); + return query; } gboolean tracker_db_interface_sqlite_fts_update_text (TrackerDBInterface *db_interface, const gchar *database, - int id, + TrackerRowid id, const gchar **properties, - const gchar **text) + GError **error) { TrackerDBStatement *stmt; - GError *error = NULL; + GError *inner_error = NULL; gchar *query; - gint i; - query = tracker_db_interface_sqlite_fts_create_query (db_interface, - database, - FALSE, properties); + query = tracker_db_interface_sqlite_fts_create_update_query (db_interface, + database, + properties); stmt = tracker_db_interface_create_statement (db_interface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, - &error, + error, query); g_free (query); - if (!stmt || error) { - if (error) { - g_warning ("Could not create FTS insert statement: %s\n", - error->message); - g_error_free (error); - } + if (!stmt) return FALSE; - } tracker_db_statement_bind_int (stmt, 0, id); - for (i = 0; text[i] != NULL; i++) { - tracker_db_statement_bind_text (stmt, i + 1, text[i]); - } - - tracker_db_statement_execute (stmt, &error); + tracker_db_statement_execute (stmt, &inner_error); g_object_unref (stmt); - if (error) { - g_warning ("Could not insert FTS text: %s", error->message); - g_error_free (error); + if (inner_error) { + g_propagate_prefixed_error (error, inner_error, "Could not insert FTS text: "); return FALSE; } return TRUE; } +static gchar * +tracker_db_interface_sqlite_fts_create_delete_query (TrackerDBInterface *db_interface, + const gchar *database, + const gchar **properties) +{ + GString *props_str; + gchar *query; + gint i; + + props_str = g_string_new (NULL); + + for (i = 0; properties[i] != NULL; i++) { + if (i != 0) + g_string_append_c (props_str, ','); + + g_string_append_printf (props_str, "\"%s\"", properties[i]); + } + + query = g_strdup_printf ("INSERT INTO \"%s\".fts5 (fts5, ROWID, %s) " + "SELECT 'delete', ROWID, %s FROM \"%s\".fts_view WHERE ROWID = ? AND COALESCE(%s, NULL) IS NOT NULL", + database, + props_str->str, + props_str->str, + database, + props_str->str); + g_string_free (props_str, TRUE); + + return query; +} + gboolean tracker_db_interface_sqlite_fts_delete_text (TrackerDBInterface *db_interface, const gchar *database, - int rowid, + TrackerRowid rowid, const gchar **properties, - const gchar **old_text) + GError **error) { TrackerDBStatement *stmt; - GError *error = NULL; + GError *inner_error = NULL; gchar *query; - gboolean has_text = FALSE; - gint i; - - for (i = 0; old_text[i] != NULL; i++) - has_text |= old_text[i] && *old_text[i]; - - if (!has_text) - return TRUE; - query = tracker_db_interface_sqlite_fts_create_query (db_interface, - database, - TRUE, properties); + query = tracker_db_interface_sqlite_fts_create_delete_query (db_interface, + database, + properties); stmt = tracker_db_interface_create_statement (db_interface, TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE, - &error, + error, query); g_free (query); - if (!stmt || error) { - g_warning ("Could not create FTS delete statement: %s", - error ? error->message : "No error given"); - g_clear_error (&error); + if (!stmt) return FALSE; - } tracker_db_statement_bind_int (stmt, 0, rowid); - for (i = 0; old_text[i] != NULL; i++) - tracker_db_statement_bind_text (stmt, i + 1, old_text[i]); - - tracker_db_statement_execute (stmt, &error); + tracker_db_statement_execute (stmt, &inner_error); g_object_unref (stmt); - if (error) { - g_warning ("Could not delete FTS text: %s", error->message); - g_error_free (error); - return FALSE; + if (inner_error) { + g_propagate_prefixed_error (error, inner_error, "Could not delete FTS text: "); + return FALSE; } return TRUE; @@ -2533,7 +2473,6 @@ tracker_db_interface_sqlite_finalize (GObject *object) db_interface = TRACKER_DB_INTERFACE (object); close_database (db_interface); - g_free (db_interface->fts_properties); TRACKER_NOTE (SQLITE, g_message ("Closed sqlite3 database:'%s'", db_interface->filename)); @@ -2583,9 +2522,10 @@ tracker_db_interface_class_init (TrackerDBInterfaceClass *class) static void tracker_db_interface_init (TrackerDBInterface *db_interface) { - db_interface->dynamic_statements = g_hash_table_new_full (g_str_hash, g_str_equal, - NULL, - (GDestroyNotify) g_object_unref); + tracker_db_statement_mru_init (&db_interface->select_stmt_mru, 100, + g_str_hash, g_str_equal, NULL); + tracker_db_statement_mru_init (&db_interface->update_stmt_mru, 100, + g_str_hash, g_str_equal, NULL); } void @@ -2593,21 +2533,21 @@ tracker_db_interface_set_max_stmt_cache_size (TrackerDBInterface *db_int TrackerDBStatementCacheType cache_type, guint max_size) { - TrackerDBStatementLru *stmt_lru; + TrackerDBStatementMru *stmt_mru; if (cache_type == TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE) { - stmt_lru = &db_interface->update_stmt_lru; + stmt_mru = &db_interface->update_stmt_mru; } else if (cache_type == TRACKER_DB_STATEMENT_CACHE_TYPE_SELECT) { - stmt_lru = &db_interface->select_stmt_lru; + stmt_mru = &db_interface->select_stmt_mru; } else { return; } /* Must be larger than 2 to make sense (to have a tail and head) */ if (max_size > 2) { - stmt_lru->max = max_size; + stmt_mru->max = max_size; } else { - stmt_lru->max = 3; + stmt_mru->max = 3; } } @@ -2641,69 +2581,55 @@ tracker_db_interface_prepare_stmt (TrackerDBInterface *db_interface, return sqlite_stmt; } -static TrackerDBStatement * -tracker_db_interface_lru_lookup (TrackerDBInterface *db_interface, - TrackerDBStatementCacheType *cache_type, - const gchar *full_query) +void +tracker_db_statement_mru_init (TrackerDBStatementMru *mru, + guint size, + GHashFunc hash_func, + GEqualFunc equal_func, + GDestroyNotify key_destroy) { - TrackerDBStatement *stmt; - - g_return_val_if_fail (*cache_type == TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE || - *cache_type == TRACKER_DB_STATEMENT_CACHE_TYPE_SELECT, - NULL); - - /* There are three kinds of queries: - * a) Cached queries: SELECT and UPDATE ones (cache_type) - * b) Non-Cached queries: NONE ones (cache_type) - * c) Forced Non-Cached: in case of a stmt being already in use, we can't - * reuse it (you can't use two different loops on a sqlite3_stmt, of - * course). This happens with recursive uses of a cursor, for example. - */ - - stmt = g_hash_table_lookup (db_interface->dynamic_statements, - full_query); - if (!stmt) { - /* Query not in LRU */ - return NULL; - } - - /* a) Cached */ - - if (stmt && stmt->stmt_is_owned) { - /* c) Forced non-cached - * prepared statement is owned somewhere else, create new uncached one - */ - stmt = NULL; - /* Make sure to set cache_type here, to avoid replacing - * the current statement. - */ - *cache_type = TRACKER_DB_STATEMENT_CACHE_TYPE_NONE; - } + mru->head = mru->tail = NULL; + mru->max = size; + mru->size = 0; + mru->stmts = g_hash_table_new_full (hash_func, equal_func, + key_destroy, g_object_unref); +} - return stmt; +void +tracker_db_statement_mru_finish (TrackerDBStatementMru *stmt_mru) +{ + stmt_mru->head = stmt_mru->tail = NULL; + stmt_mru->size = stmt_mru->max = 0; + g_clear_pointer (&stmt_mru->stmts, g_hash_table_unref); } -static void -tracker_db_interface_lru_insert_unchecked (TrackerDBInterface *db_interface, - TrackerDBStatementCacheType cache_type, - TrackerDBStatement *stmt) +void +tracker_db_statement_mru_clear (TrackerDBStatementMru *stmt_mru) { - TrackerDBStatementLru *stmt_lru; + stmt_mru->head = stmt_mru->tail = NULL; + stmt_mru->size = 0; + g_hash_table_remove_all (stmt_mru->stmts); +} - g_return_if_fail (cache_type == TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE || - cache_type == TRACKER_DB_STATEMENT_CACHE_TYPE_SELECT); +TrackerDBStatement * +tracker_db_statement_mru_lookup (TrackerDBStatementMru *stmt_mru, + gconstpointer key) +{ + return g_hash_table_lookup (stmt_mru->stmts, key); +} - /* LRU holds a reference to the stmt */ - stmt_lru = cache_type == TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE ? - &db_interface->update_stmt_lru : &db_interface->select_stmt_lru; +void +tracker_db_statement_mru_insert (TrackerDBStatementMru *stmt_mru, + gpointer key, + TrackerDBStatement *stmt) +{ + g_return_if_fail (stmt->mru_key == NULL); - /* use replace instead of insert to make sure we store the string that - * belongs to the right sqlite statement to ensure the lifetime of the string + /* use replace instead of insert to make sure we store the key that + * belongs to the right sqlite statement to ensure the lifetime of the key * matches the statement */ - g_hash_table_replace (db_interface->dynamic_statements, - (gpointer) sqlite3_sql (stmt->stmt), - g_object_ref_sink (stmt)); + g_hash_table_replace (stmt_mru->stmts, key, g_object_ref_sink (stmt)); /* So the ring looks a bit like this: * * * @@ -2714,59 +2640,55 @@ tracker_db_interface_lru_insert_unchecked (TrackerDBInterface *db_inter * `- [n-p] <- [n-p] <--------' * * */ - if (stmt_lru->size == 0) { - stmt_lru->head = stmt; - stmt_lru->tail = stmt; - } else if (stmt_lru->size >= stmt_lru->max) { + if (stmt_mru->size == 0) { + stmt_mru->head = stmt; + stmt_mru->tail = stmt; + } else if (stmt_mru->size >= stmt_mru->max) { TrackerDBStatement *new_head; - /* We reached max-size of the LRU stmt cache. Destroy current - * least recently used (stmt_lru.head) and fix the ring. For + /* We reached max-size of the MRU stmt cache. Destroy current + * least recently used (stmt_mru.head) and fix the ring. For * that we take out the current head, and close the ring. * Then we assign head->next as new head. */ - new_head = stmt_lru->head->next; - g_hash_table_remove (db_interface->dynamic_statements, - (gpointer) sqlite3_sql (stmt_lru->head->stmt)); - stmt_lru->size--; - stmt_lru->head = new_head; + new_head = stmt_mru->head->next; + g_hash_table_remove (stmt_mru->stmts, (gpointer) stmt_mru->head->mru_key); + stmt_mru->head->mru_key = NULL; + stmt_mru->head->next = stmt_mru->head->prev = NULL; + stmt_mru->size--; + stmt_mru->head = new_head; } /* Set the current stmt (which is always new here) as the new tail * (new most recent used). We insert current stmt between head and * current tail, and we set tail to current stmt. */ - stmt_lru->size++; - stmt->next = stmt_lru->head; - stmt_lru->head->prev = stmt; + stmt_mru->size++; + stmt->next = stmt_mru->head; + stmt_mru->head->prev = stmt; - stmt_lru->tail->next = stmt; - stmt->prev = stmt_lru->tail; - stmt_lru->tail = stmt; + stmt_mru->tail->next = stmt; + stmt->prev = stmt_mru->tail; + stmt_mru->tail = stmt; + + stmt->mru_key = key; } -static void -tracker_db_interface_lru_update (TrackerDBInterface *db_interface, - TrackerDBStatementCacheType cache_type, - TrackerDBStatement *stmt) +void +tracker_db_statement_mru_update (TrackerDBStatementMru *stmt_mru, + TrackerDBStatement *stmt) { - TrackerDBStatementLru *stmt_lru; - - g_return_if_fail (cache_type == TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE || - cache_type == TRACKER_DB_STATEMENT_CACHE_TYPE_SELECT); - - stmt_lru = cache_type == TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE ? - &db_interface->update_stmt_lru : &db_interface->select_stmt_lru; + g_return_if_fail (stmt->mru_key != NULL); tracker_db_statement_sqlite_reset (stmt); - if (stmt == stmt_lru->head) { + if (stmt == stmt_mru->head) { /* Current stmt is least recently used, shift head and tail * of the ring to efficiently make it most recently used. */ - stmt_lru->head = stmt_lru->head->next; - stmt_lru->tail = stmt_lru->tail->next; - } else if (stmt != stmt_lru->tail) { + stmt_mru->head = stmt_mru->head->next; + stmt_mru->tail = stmt_mru->tail->next; + } else if (stmt != stmt_mru->tail) { /* Current statement isn't most recently used, make it most * recently used now (less efficient way than above). */ @@ -2776,11 +2698,11 @@ tracker_db_interface_lru_update (TrackerDBInterface *db_interface, stmt->next->prev = stmt->prev; /* Put stmt as tail (most recent used) */ - stmt->next = stmt_lru->head; - stmt_lru->head->prev = stmt; - stmt->prev = stmt_lru->tail; - stmt_lru->tail->next = stmt; - stmt_lru->tail = stmt; + stmt->next = stmt_mru->head; + stmt_mru->head->prev = stmt; + stmt->prev = stmt_mru->tail; + stmt_mru->tail->next = stmt; + stmt_mru->tail = stmt; } /* if (stmt == tail), it's already the most recently used in the @@ -2794,14 +2716,25 @@ tracker_db_interface_create_statement (TrackerDBInterface *db_interfac const gchar *query) { TrackerDBStatement *stmt = NULL; + TrackerDBStatementMru *mru = NULL; g_return_val_if_fail (TRACKER_IS_DB_INTERFACE (db_interface), NULL); tracker_db_interface_lock (db_interface); - if (cache_type != TRACKER_DB_STATEMENT_CACHE_TYPE_NONE) { - stmt = tracker_db_interface_lru_lookup (db_interface, &cache_type, - query); + /* MRU holds a reference to the stmt */ + if (cache_type == TRACKER_DB_STATEMENT_CACHE_TYPE_SELECT) + mru = &db_interface->select_stmt_mru; + else if (cache_type == TRACKER_DB_STATEMENT_CACHE_TYPE_UPDATE) + mru = &db_interface->update_stmt_mru; + + if (mru) + stmt = tracker_db_statement_mru_lookup (mru, query); + + if (stmt && stmt->stmt_is_borrowed) { + /* prepared statement is owned somewhere else, create new uncached one */ + stmt = NULL; + mru = NULL; } if (!stmt) { @@ -2818,17 +2751,13 @@ tracker_db_interface_create_statement (TrackerDBInterface *db_interfac stmt = tracker_db_statement_sqlite_new (db_interface, sqlite_stmt); - if (cache_type != TRACKER_DB_STATEMENT_CACHE_TYPE_NONE) { - tracker_db_interface_lru_insert_unchecked (db_interface, - cache_type, - stmt); - } - } else if (cache_type != TRACKER_DB_STATEMENT_CACHE_TYPE_NONE) { - tracker_db_interface_lru_update (db_interface, cache_type, - stmt); + if (mru) + tracker_db_statement_mru_insert (mru, (gpointer) sqlite3_sql (stmt->stmt), stmt); + } else if (mru) { + tracker_db_statement_mru_update (mru, stmt); } - stmt->stmt_is_owned = TRUE; + stmt->stmt_is_borrowed = cache_type != TRACKER_DB_STATEMENT_CACHE_TYPE_NONE; tracker_db_interface_unlock (db_interface); @@ -2937,31 +2866,26 @@ execute_stmt (TrackerDBInterface *interface, return FALSE; } - if (!error) { - g_warning ("Could not perform SQLite operation, error:%d->'%s'", - sqlite3_errcode (interface->db), - sqlite3_errmsg (interface->db)); + if (result == SQLITE_INTERRUPT) { + g_set_error (error, + TRACKER_DB_INTERFACE_ERROR, + TRACKER_DB_INTERRUPTED, + "Interrupted"); + } else if (result == SQLITE_CONSTRAINT) { + g_set_error (error, + TRACKER_DB_INTERFACE_ERROR, + TRACKER_DB_CONSTRAINT, + "Constraint would be broken: %s", + sqlite3_errmsg (interface->db)); } else { - if (result == SQLITE_INTERRUPT) { - g_set_error (error, - TRACKER_DB_INTERFACE_ERROR, - TRACKER_DB_INTERRUPTED, - "Interrupted"); - } else if (result == SQLITE_CONSTRAINT) { - g_set_error (error, - TRACKER_DB_INTERFACE_ERROR, - TRACKER_DB_CONSTRAINT, - "Constraint would be broken"); - } else { - g_set_error (error, - TRACKER_DB_INTERFACE_ERROR, - errno != ENOSPC ? TRACKER_DB_QUERY_ERROR : TRACKER_DB_NO_SPACE, - "%s%s%s%s", - sqlite3_errmsg (interface->db), - errno != 0 ? " (strerror of errno (not necessarily related): " : "", - errno != 0 ? g_strerror (errno) : "", - errno != 0 ? ")" : ""); - } + g_set_error (error, + TRACKER_DB_INTERFACE_ERROR, + errno != ENOSPC ? TRACKER_DB_QUERY_ERROR : TRACKER_DB_NO_SPACE, + "%s%s%s%s", + sqlite3_errmsg (interface->db), + errno != 0 ? " (strerror of errno (not necessarily related): " : "", + errno != 0 ? g_strerror (errno) : "", + errno != 0 ? ")" : ""); } } @@ -3071,7 +2995,6 @@ static TrackerDBStatement * tracker_db_statement_sqlite_grab (TrackerDBStatement *stmt) { g_assert (!stmt->stmt_is_used); - g_assert (stmt->stmt_is_owned); stmt->stmt_is_used = TRUE; g_object_ref (stmt->db_interface); return g_object_ref (stmt); @@ -3082,13 +3005,12 @@ tracker_db_statement_sqlite_release (TrackerDBStatement *stmt) { TrackerDBInterface *iface = stmt->db_interface; - g_assert (stmt->stmt_is_owned); + stmt->stmt_is_borrowed = FALSE; - stmt->stmt_is_owned = FALSE; + tracker_db_statement_sqlite_reset (stmt); if (stmt->stmt_is_used) { stmt->stmt_is_used = FALSE; - tracker_db_statement_sqlite_reset (stmt); g_object_unref (stmt); g_object_unref (iface); } @@ -3428,29 +3350,29 @@ db_cursor_iter_next (TrackerDBCursor *cursor, tracker_db_interface_lock (iface); - if (g_cancellable_is_cancelled (cancellable)) { - result = SQLITE_INTERRUPT; + if (g_cancellable_set_error_if_cancelled (cancellable, error)) { sqlite3_reset (cursor->stmt); + cursor->finished = TRUE; } else { /* only one statement can be active at the same time per interface */ iface->cancellable = cancellable; result = stmt_step (cursor->stmt); iface->cancellable = NULL; - } - if (result == SQLITE_INTERRUPT) { - g_set_error (error, - TRACKER_DB_INTERFACE_ERROR, - TRACKER_DB_INTERRUPTED, - "Interrupted"); - } else if (result != SQLITE_ROW && result != SQLITE_DONE) { - g_set_error (error, - TRACKER_DB_INTERFACE_ERROR, - TRACKER_DB_QUERY_ERROR, - "%s", sqlite3_errmsg (iface->db)); - } + if (result == SQLITE_INTERRUPT) { + g_set_error (error, + TRACKER_DB_INTERFACE_ERROR, + TRACKER_DB_INTERRUPTED, + "Interrupted"); + } else if (result != SQLITE_ROW && result != SQLITE_DONE) { + g_set_error (error, + TRACKER_DB_INTERFACE_ERROR, + TRACKER_DB_QUERY_ERROR, + "%s", sqlite3_errmsg (iface->db)); + } - cursor->finished = (result != SQLITE_ROW); + cursor->finished = (result != SQLITE_ROW); + } tracker_db_interface_unlock (iface); } @@ -3712,15 +3634,132 @@ tracker_db_cursor_get_string (TrackerDBCursor *cursor, return result; } -void +gboolean tracker_db_statement_execute (TrackerDBStatement *stmt, GError **error) { - g_return_if_fail (TRACKER_IS_DB_STATEMENT (stmt)); - g_return_if_fail (!stmt->stmt_is_used); + gboolean retval; + + g_return_val_if_fail (TRACKER_IS_DB_STATEMENT (stmt), FALSE); + g_return_val_if_fail (!stmt->stmt_is_used, FALSE); + + retval = execute_stmt (stmt->db_interface, stmt->stmt, NULL, error); + tracker_db_statement_sqlite_release (stmt); + + return retval; +} + +GArray * +tracker_db_statement_get_values (TrackerDBStatement *stmt, + TrackerPropertyType type, + GError **error) +{ + gint result = SQLITE_OK; + GArray *values; + + tracker_db_interface_lock (stmt->db_interface); + tracker_db_interface_ref_use (stmt->db_interface); + tracker_db_statement_sqlite_grab (stmt); + +#ifdef G_ENABLE_DEBUG + if (TRACKER_DEBUG_CHECK (SQL_STATEMENTS)) { + gchar *full_query; + + full_query = sqlite3_expanded_sql (stmt->stmt); + + if (full_query) { + g_message ("Executing query: '%s'", full_query); + sqlite3_free (full_query); + } else { + g_message ("Executing query: '%s'", + sqlite3_sql (stmt->stmt)); + } + } +#endif + + values = g_array_new (FALSE, TRUE, sizeof (GValue)); + g_array_set_clear_func (values, (GDestroyNotify) g_value_unset); + + while (TRUE) { + GError *inner_error = NULL; + GDateTime *datetime; + GValue gvalue = G_VALUE_INIT; + + result = stmt_step (stmt->stmt); + + if (result == SQLITE_DONE) { + break; + } else if (result != SQLITE_ROW) { + g_set_error (error, + TRACKER_DB_INTERFACE_ERROR, + TRACKER_DB_QUERY_ERROR, + "%s", sqlite3_errmsg (stmt->db_interface->db)); + g_clear_pointer (&values, g_array_unref); + break; + } + + if (sqlite3_column_type (stmt->stmt, 0) == SQLITE_NULL) + continue; + + switch (type) { + case TRACKER_PROPERTY_TYPE_UNKNOWN: + case TRACKER_PROPERTY_TYPE_STRING: + g_value_init (&gvalue, G_TYPE_STRING); + g_value_set_string (&gvalue, (gchar *) sqlite3_column_text (stmt->stmt, 0)); + break; + case TRACKER_PROPERTY_TYPE_LANGSTRING: { + sqlite3_value *val = sqlite3_column_value (stmt->stmt, 0); + gchar *text; + + text = g_strdup ((const gchar *) sqlite3_value_text (val)); + + g_value_init (&gvalue, G_TYPE_BYTES); + g_value_take_boxed (&gvalue, + g_bytes_new_with_free_func (text, + sqlite3_value_bytes (val), + g_free, text)); + break; + } + case TRACKER_PROPERTY_TYPE_DOUBLE: + g_value_init (&gvalue, G_TYPE_DOUBLE); + g_value_set_double (&gvalue, sqlite3_column_double (stmt->stmt, 0)); + break; + case TRACKER_PROPERTY_TYPE_BOOLEAN: + case TRACKER_PROPERTY_TYPE_INTEGER: + case TRACKER_PROPERTY_TYPE_RESOURCE: + g_value_init (&gvalue, G_TYPE_INT64); + g_value_set_int64 (&gvalue, sqlite3_column_int64 (stmt->stmt, 0)); + break; + case TRACKER_PROPERTY_TYPE_DATE: + case TRACKER_PROPERTY_TYPE_DATETIME: + if (sqlite3_column_type (stmt->stmt, 0) == SQLITE_INTEGER) { + datetime = g_date_time_new_from_unix_utc (sqlite3_column_int64 (stmt->stmt, 0)); + } else { + datetime = tracker_date_new_from_iso8601 ((const gchar *) sqlite3_column_text (stmt->stmt, 0), + &inner_error); + if (!datetime) + break; + } + + g_value_init (&gvalue, G_TYPE_DATE_TIME); + g_value_take_boxed (&gvalue, datetime); + break; + } + + if (inner_error) { + g_propagate_error (error, inner_error); + g_clear_pointer (&values, g_array_unref); + break; + } + + g_array_append_val (values, gvalue); + } - execute_stmt (stmt->db_interface, stmt->stmt, NULL, error); tracker_db_statement_sqlite_release (stmt); + tracker_db_interface_unref_use (stmt->db_interface); + tracker_db_interface_unlock (stmt->db_interface); + + return values; } TrackerDBCursor * @@ -3757,8 +3796,6 @@ tracker_db_cursor_init (TrackerDBCursor *cursor) static void tracker_db_statement_sqlite_reset (TrackerDBStatement *stmt) { - g_assert (!stmt->stmt_is_used); - sqlite3_reset (stmt->stmt); sqlite3_clear_bindings (stmt->stmt); } @@ -3871,11 +3908,8 @@ tracker_db_interface_detach_database (TrackerDBInterface *db_interface, gssize tracker_db_interface_sqlite_release_memory (TrackerDBInterface *db_interface) { - db_interface->select_stmt_lru.head = db_interface->select_stmt_lru.tail = NULL; - db_interface->select_stmt_lru.size = 0; - db_interface->update_stmt_lru.head = db_interface->update_stmt_lru.tail = NULL; - db_interface->update_stmt_lru.size = 0; - g_hash_table_remove_all (db_interface->dynamic_statements); + tracker_db_statement_mru_clear (&db_interface->select_stmt_mru); + tracker_db_statement_mru_clear (&db_interface->update_stmt_mru); return (gssize) sqlite3_db_release_memory (db_interface->db); } diff --git a/src/libtracker-sparql/core/tracker-db-interface-sqlite.h b/src/libtracker-sparql/core/tracker-db-interface-sqlite.h index 98fc4d7ee..265919045 100644 --- a/src/libtracker-sparql/core/tracker-db-interface-sqlite.h +++ b/src/libtracker-sparql/core/tracker-db-interface-sqlite.h @@ -69,15 +69,15 @@ gboolean tracker_db_interface_sqlite_fts_alter_table (TrackerD GError **error); gboolean tracker_db_interface_sqlite_fts_update_text (TrackerDBInterface *db_interface, const gchar *database, - int id, + TrackerRowid id, const gchar **properties, - const gchar **text); + GError **error); gboolean tracker_db_interface_sqlite_fts_delete_text (TrackerDBInterface *interface, const gchar *database, - int rowid, + TrackerRowid rowid, const gchar **properties, - const gchar **old_text); + GError **error); gboolean tracker_db_interface_sqlite_fts_rebuild_tokens (TrackerDBInterface *interface, const gchar *database, GError **error); @@ -94,6 +94,10 @@ gssize tracker_db_interface_sqlite_release_memory (TrackerD void tracker_db_interface_ref_use (TrackerDBInterface *db_interface); gboolean tracker_db_interface_unref_use (TrackerDBInterface *db_interface); +GArray * tracker_db_statement_get_values (TrackerDBStatement *stmt, + TrackerPropertyType type, + GError **error); + G_END_DECLS #endif /* __LIBTRACKER_DB_INTERFACE_SQLITE_H__ */ diff --git a/src/libtracker-sparql/core/tracker-db-interface.h b/src/libtracker-sparql/core/tracker-db-interface.h index 7c4e4c497..db8381264 100644 --- a/src/libtracker-sparql/core/tracker-db-interface.h +++ b/src/libtracker-sparql/core/tracker-db-interface.h @@ -76,6 +76,15 @@ typedef struct TrackerDBStatement TrackerDBStatement; typedef struct TrackerDBStatementClass TrackerDBStatementClass; typedef struct TrackerDBCursor TrackerDBCursor; typedef struct TrackerDBCursorClass TrackerDBCursorClass; +typedef struct TrackerDBStatementMru TrackerDBStatementMru; + +struct TrackerDBStatementMru { + TrackerDBStatement *head; + TrackerDBStatement *tail; + GHashTable *stmts; + guint size; + guint max; +}; GQuark tracker_db_interface_error_quark (void); @@ -117,6 +126,7 @@ gboolean tracker_db_interface_end_db_transaction (TrackerDBI GError **error); gboolean tracker_db_interface_get_is_used (TrackerDBInterface *interface); +/* Statements */ void tracker_db_statement_bind_double (TrackerDBStatement *stmt, int index, double value); @@ -134,7 +144,7 @@ void tracker_db_statement_bind_bytes (TrackerDBS void tracker_db_statement_bind_value (TrackerDBStatement *stmt, int index, const GValue *value); -void tracker_db_statement_execute (TrackerDBStatement *stmt, +gboolean tracker_db_statement_execute (TrackerDBStatement *stmt, GError **error); TrackerDBCursor * tracker_db_statement_start_cursor (TrackerDBStatement *stmt, GError **error); @@ -142,6 +152,27 @@ TrackerDBCursor * tracker_db_statement_start_sparql_cursor (TrackerDBS guint n_columns, GError **error); +/* Statement caches */ +void tracker_db_statement_mru_init (TrackerDBStatementMru *mru, + guint size, + GHashFunc hash_func, + GEqualFunc equal_func, + GDestroyNotify key_destroy); + +void tracker_db_statement_mru_finish (TrackerDBStatementMru *mru); + +void tracker_db_statement_mru_clear (TrackerDBStatementMru *mru); + +TrackerDBStatement * tracker_db_statement_mru_lookup (TrackerDBStatementMru *mru, + gconstpointer key); + +void tracker_db_statement_mru_insert (TrackerDBStatementMru *mru, + gpointer key, + TrackerDBStatement *stmt); + +void tracker_db_statement_mru_update (TrackerDBStatementMru *mru, + TrackerDBStatement *stmt); + /* Functions to deal with a cursor */ void tracker_db_cursor_rewind (TrackerDBCursor *cursor); gboolean tracker_db_cursor_iter_next (TrackerDBCursor *cursor, diff --git a/src/libtracker-sparql/core/tracker-db-manager.h b/src/libtracker-sparql/core/tracker-db-manager.h index 941ecad9d..413b6b55d 100644 --- a/src/libtracker-sparql/core/tracker-db-manager.h +++ b/src/libtracker-sparql/core/tracker-db-manager.h @@ -52,10 +52,11 @@ typedef enum { /* Starts at 25 because we forgot to clean up */ TRACKER_DB_VERSION_3_0 = 25, /* 3.0 */ TRACKER_DB_VERSION_3_3, /* Blank nodes */ + TRACKER_DB_VERSION_3_4, /* Fixed FTS view */ } TrackerDBVersion; /* Set current database version we are working with */ -#define TRACKER_DB_VERSION_NOW TRACKER_DB_VERSION_3_3 +#define TRACKER_DB_VERSION_NOW TRACKER_DB_VERSION_3_4 void tracker_db_manager_rollback_db_creation (TrackerDBManager *db_manager); diff --git a/src/libtracker-sparql/core/tracker-fts.c b/src/libtracker-sparql/core/tracker-fts.c index 353685992..8398b6e7a 100644 --- a/src/libtracker-sparql/core/tracker-fts.c +++ b/src/libtracker-sparql/core/tracker-fts.c @@ -91,8 +91,9 @@ tracker_fts_create_table (sqlite3 *db, /* Create view on tables/columns marked as FTS-indexed */ str = g_string_new ("CREATE VIEW "); g_string_append_printf (str, "\"%s\".fts_view AS SELECT \"rdfs:Resource\".ID as rowid ", - database); - from = g_string_new ("FROM \"rdfs:Resource\" "); + database); + from = g_string_new (NULL); + g_string_append_printf (from, "FROM \"%s\".\"rdfs:Resource\" ", database); fts = g_string_new ("CREATE VIRTUAL TABLE "); g_string_append_printf (fts, "\"%s\".%s USING fts5(content=\"fts_view\", ", @@ -107,7 +108,7 @@ tracker_fts_create_table (sqlite3 *db, while (columns) { if (grouped_columns && - g_hash_table_lookup (grouped_columns, columns->data)) { + g_hash_table_lookup (grouped_columns, index_table)) { g_string_append_printf (str, ", group_concat(\"%s\".\"%s\")", index_table, (gchar *) columns->data); @@ -132,6 +133,7 @@ tracker_fts_create_table (sqlite3 *db, g_list_free (keys); + g_string_append (from, "GROUP BY ROWID"); g_string_append (str, from->str); g_string_free (from, TRUE); @@ -177,7 +179,7 @@ tracker_fts_delete_table (sqlite3 *db, gchar *query; int rc; - query = g_strdup_printf ("DROP VIEW IF EXISTS fts_view"); + query = g_strdup_printf ("DROP VIEW IF EXISTS \"%s\".fts_view", database); rc = sqlite3_exec (db, query, NULL, NULL, NULL); g_free (query); diff --git a/src/libtracker-sparql/core/tracker-ontologies.c b/src/libtracker-sparql/core/tracker-ontologies.c index ef0f8306d..c54a42dba 100644 --- a/src/libtracker-sparql/core/tracker-ontologies.c +++ b/src/libtracker-sparql/core/tracker-ontologies.c @@ -65,8 +65,10 @@ struct _TrackerOntologiesPrivate { /* Hash (int id, const gchar *uri) */ GHashTable *id_uri_pairs; - /* rdf:type */ + /* Some fast paths for frequent properties */ TrackerProperty *rdf_type; + TrackerProperty *nrl_added; + TrackerProperty *nrl_modified; GvdbTable *gvdb_table; GvdbTable *gvdb_namespaces_table; @@ -134,9 +136,9 @@ tracker_ontologies_finalize (GObject *object) g_ptr_array_free (priv->properties, TRUE); g_hash_table_unref (priv->property_uris); - if (priv->rdf_type) { - g_object_unref (priv->rdf_type); - } + g_clear_object (&priv->rdf_type); + g_clear_object (&priv->nrl_added); + g_clear_object (&priv->nrl_modified); if (priv->gvdb_table) { gvdb_table_unref (priv->gvdb_properties_table); @@ -172,6 +174,26 @@ tracker_ontologies_get_rdf_type (TrackerOntologies *ontologies) return priv->rdf_type; } +TrackerProperty * +tracker_ontologies_get_nrl_added (TrackerOntologies *ontologies) +{ + TrackerOntologiesPrivate *priv = tracker_ontologies_get_instance_private (ontologies); + + g_return_val_if_fail (priv->nrl_added != NULL, NULL); + + return priv->nrl_added; +} + +TrackerProperty * +tracker_ontologies_get_nrl_modified (TrackerOntologies *ontologies) +{ + TrackerOntologiesPrivate *priv = tracker_ontologies_get_instance_private (ontologies); + + g_return_val_if_fail (priv->nrl_modified != NULL, NULL); + + return priv->nrl_modified; +} + const gchar* tracker_ontologies_get_uri_by_id (TrackerOntologies *ontologies, TrackerRowid id) @@ -350,6 +372,10 @@ tracker_ontologies_add_property (TrackerOntologies *ontologies, if (g_strcmp0 (uri, TRACKER_PREFIX_RDF "type") == 0) { g_set_object (&priv->rdf_type, field); + } else if (g_strcmp0 (uri, TRACKER_PREFIX_NRL "added") == 0) { + g_set_object (&priv->nrl_added, field); + } else if (g_strcmp0 (uri, TRACKER_PREFIX_NRL "modified") == 0) { + g_set_object (&priv->nrl_modified, field); } g_ptr_array_add (priv->properties, g_object_ref (field)); @@ -358,6 +384,9 @@ tracker_ontologies_add_property (TrackerOntologies *ontologies, g_hash_table_insert (priv->property_uris, g_strdup (uri), g_object_ref (field)); + g_hash_table_insert (priv->property_uris, + g_strdup (tracker_property_get_name (field)), + g_object_ref (field)); } void @@ -397,7 +426,12 @@ tracker_ontologies_get_property_by_uri (TrackerOntologies *ontologies, g_hash_table_insert (priv->property_uris, g_strdup (uri), - property); + g_object_ref (property)); + g_hash_table_insert (priv->property_uris, + g_strdup (tracker_property_get_name (property)), + g_object_ref (property)); + + g_object_unref (property); } } diff --git a/src/libtracker-sparql/core/tracker-ontologies.h b/src/libtracker-sparql/core/tracker-ontologies.h index 6abcd5830..a92b7e289 100644 --- a/src/libtracker-sparql/core/tracker-ontologies.h +++ b/src/libtracker-sparql/core/tracker-ontologies.h @@ -64,6 +64,8 @@ TrackerClass ** tracker_ontologies_get_classes (TrackerOntologies *o TrackerProperty ** tracker_ontologies_get_properties (TrackerOntologies *ontologies, guint *length); TrackerProperty * tracker_ontologies_get_rdf_type (TrackerOntologies *ontologies); +TrackerProperty * tracker_ontologies_get_nrl_added (TrackerOntologies *ontologies); +TrackerProperty * tracker_ontologies_get_nrl_modified (TrackerOntologies *ontologies); /* Field mechanics */ void tracker_ontologies_add_property (TrackerOntologies *ontologies, diff --git a/src/libtracker-sparql/direct/tracker-direct-batch.c b/src/libtracker-sparql/direct/tracker-direct-batch.c index ed0d0cdb6..bf3d04b66 100644 --- a/src/libtracker-sparql/direct/tracker-direct-batch.c +++ b/src/libtracker-sparql/direct/tracker-direct-batch.c @@ -190,14 +190,16 @@ tracker_direct_batch_update (TrackerDirectBatch *batch, { TrackerDirectBatchPrivate *priv; GError *inner_error = NULL; - GHashTable *bnodes; + GHashTable *bnodes, *visited; TrackerData *data; + const gchar *last_graph = NULL; guint i; priv = tracker_direct_batch_get_instance_private (batch); data = tracker_data_manager_get_data (data_manager); bnodes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) tracker_rowid_free); + visited = g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify) tracker_rowid_free); tracker_data_begin_transaction (data, &inner_error); if (inner_error) @@ -209,11 +211,20 @@ tracker_direct_batch_update (TrackerDirectBatch *batch, elem = &g_array_index (priv->array, TrackerBatchElem, i); if (elem->type == TRACKER_DIRECT_BATCH_RESOURCE) { + /* Clear the visited resources set on graph changes, there + * might be resources that are referenced from multiple + * graphs. + */ + if (g_strcmp0 (last_graph, elem->d.resource.graph) != 0) + g_hash_table_remove_all (visited); + tracker_data_update_resource (data, elem->d.resource.graph, elem->d.resource.resource, bnodes, + visited, &inner_error); + last_graph = elem->d.resource.graph; } else if (elem->type == TRACKER_DIRECT_BATCH_SPARQL) { TrackerSparql *query; @@ -244,11 +255,13 @@ tracker_direct_batch_update (TrackerDirectBatch *batch, goto error; g_hash_table_unref (bnodes); + g_hash_table_unref (visited); return TRUE; error: g_hash_table_unref (bnodes); + g_hash_table_unref (visited); g_propagate_error (error, inner_error); return FALSE; } diff --git a/src/libtracker-sparql/direct/tracker-direct.c b/src/libtracker-sparql/direct/tracker-direct.c index a8c965520..51e08a32b 100644 --- a/src/libtracker-sparql/direct/tracker-direct.c +++ b/src/libtracker-sparql/direct/tracker-direct.c @@ -172,17 +172,24 @@ update_resource (TrackerData *data, GError **error) { GError *inner_error = NULL; + GHashTable *visited; tracker_data_begin_transaction (data, &inner_error); if (inner_error) goto error; + visited = g_hash_table_new_full (NULL, NULL, NULL, + (GDestroyNotify) tracker_rowid_free); + tracker_data_update_resource (data, graph, resource, NULL, + visited, &inner_error); + g_hash_table_unref (visited); + if (inner_error) { tracker_data_rollback_transaction (data); goto error; @@ -568,9 +575,7 @@ get_event_cache_ht (TrackerNotifier *notifier) events = g_object_get_qdata (G_OBJECT (notifier), tracker_direct_notifier_quark ()); if (!events) { - events = g_hash_table_new_full (tracker_rowid_hash, - tracker_rowid_equal, - (GDestroyNotify) tracker_rowid_free, + events = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, (GDestroyNotify) _tracker_notifier_event_cache_free); g_object_set_qdata_full (G_OBJECT (notifier), tracker_direct_notifier_quark (), events, (GDestroyNotify) g_hash_table_unref); @@ -581,18 +586,22 @@ get_event_cache_ht (TrackerNotifier *notifier) static TrackerNotifierEventCache * lookup_event_cache (TrackerNotifier *notifier, - TrackerRowid graph_id, const gchar *graph) { TrackerNotifierEventCache *cache; GHashTable *events; + if (!graph) + graph = ""; + events = get_event_cache_ht (notifier); - cache = g_hash_table_lookup (events, &graph_id); + cache = g_hash_table_lookup (events, graph); if (!cache) { cache = _tracker_notifier_event_cache_new (notifier, graph); - g_hash_table_insert (events, tracker_rowid_copy (&graph_id), cache); + g_hash_table_insert (events, + (gpointer) tracker_notifier_event_cache_get_graph (cache), + cache); } return cache; @@ -602,13 +611,12 @@ lookup_event_cache (TrackerNotifier *notifier, * (always the same one though), handle with care. */ static void -insert_statement_cb (TrackerRowid graph_id, - const gchar *graph, +insert_statement_cb (const gchar *graph, TrackerRowid subject_id, TrackerRowid predicate_id, TrackerRowid object_id, - GPtrArray *rdf_types, - gpointer user_data) + GPtrArray *rdf_types, + gpointer user_data) { TrackerNotifier *notifier = user_data; TrackerSparqlConnection *conn = _tracker_notifier_get_connection (notifier); @@ -620,7 +628,7 @@ insert_statement_cb (TrackerRowid graph_id, TrackerClass *new_class = NULL; guint i; - cache = lookup_event_cache (notifier, graph_id, graph); + cache = lookup_event_cache (notifier, graph); if (predicate_id == tracker_property_get_id (rdf_type)) { const gchar *uri; @@ -646,13 +654,12 @@ insert_statement_cb (TrackerRowid graph_id, } static void -delete_statement_cb (TrackerRowid graph_id, - const gchar *graph, +delete_statement_cb (const gchar *graph, TrackerRowid subject_id, TrackerRowid predicate_id, TrackerRowid object_id, - GPtrArray *rdf_types, - gpointer user_data) + GPtrArray *rdf_types, + gpointer user_data) { TrackerNotifier *notifier = user_data; TrackerSparqlConnection *conn = _tracker_notifier_get_connection (notifier); @@ -664,7 +671,7 @@ delete_statement_cb (TrackerRowid graph_id, TrackerClass *class_being_removed = NULL; guint i; - cache = lookup_event_cache (notifier, graph_id, graph); + cache = lookup_event_cache (notifier, graph); if (predicate_id == tracker_property_get_id (rdf_type)) { const gchar *uri; diff --git a/src/libtracker-sparql/tracker-endpoint-dbus.c b/src/libtracker-sparql/tracker-endpoint-dbus.c index f609adb52..8ee65a3f4 100644 --- a/src/libtracker-sparql/tracker-endpoint-dbus.c +++ b/src/libtracker-sparql/tracker-endpoint-dbus.c @@ -441,7 +441,8 @@ handle_cursor_reply (GTask *task, g_dbus_method_invocation_return_value (request->invocation, g_variant_new ("(^as)", variable_names)); - if (!write_cursor (request, cursor, &write_error)) + if (!write_cursor (request, cursor, &write_error) && + !g_error_matches (write_error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) g_warning ("Endpoint failed to fully write cursor: %s\n", write_error->message); g_free (variable_names); g_clear_error (&write_error); diff --git a/src/libtracker-sparql/tracker-notifier-private.h b/src/libtracker-sparql/tracker-notifier-private.h index e808be1e2..c9bc73e18 100644 --- a/src/libtracker-sparql/tracker-notifier-private.h +++ b/src/libtracker-sparql/tracker-notifier-private.h @@ -39,6 +39,8 @@ _tracker_notifier_event_cache_push_event (TrackerNotifierEventCache *cache, void _tracker_notifier_event_cache_flush_events (TrackerNotifierEventCache *cache); +const gchar * tracker_notifier_event_cache_get_graph (TrackerNotifierEventCache *cache); + void tracker_notifier_disable_urn_query (TrackerNotifier *notifier); #endif /* __TRACKER_NOTIFIER_PRIVATE_H__ */ diff --git a/src/libtracker-sparql/tracker-notifier.c b/src/libtracker-sparql/tracker-notifier.c index 86329ce9d..78056c2c0 100644 --- a/src/libtracker-sparql/tracker-notifier.c +++ b/src/libtracker-sparql/tracker-notifier.c @@ -262,6 +262,12 @@ _tracker_notifier_event_cache_push_event (TrackerNotifierEventCache *cache, event->type = event_type; } +const gchar * +tracker_notifier_event_cache_get_graph (TrackerNotifierEventCache *cache) +{ + return cache->graph ? cache->graph : ""; +} + static void handle_events (TrackerNotifier *notifier, TrackerNotifierEventCache *cache, diff --git a/src/libtracker-sparql/tracker-private.h b/src/libtracker-sparql/tracker-private.h index 8e3027486..d6263621b 100644 --- a/src/libtracker-sparql/tracker-private.h +++ b/src/libtracker-sparql/tracker-private.h @@ -329,5 +329,7 @@ void tracker_resource_iterator_init (TrackerResourceIterator *iter, gboolean tracker_resource_iterator_next (TrackerResourceIterator *iter, const gchar **property, const GValue **value); +const gchar * tracker_resource_get_identifier_internal (TrackerResource *resource); +gboolean tracker_resource_is_blank_node (TrackerResource *resource); #endif /* __TRACKER_PRIVATE_H__ */ diff --git a/src/libtracker-sparql/tracker-resource.c b/src/libtracker-sparql/tracker-resource.c index a59d98ebb..c027eecee 100644 --- a/src/libtracker-sparql/tracker-resource.c +++ b/src/libtracker-sparql/tracker-resource.c @@ -82,7 +82,6 @@ enum { }; static void dispose (GObject *object); -static void constructed (GObject *object); static void finalize (GObject *object); static void get_property (GObject *object, guint param_id, @@ -100,7 +99,6 @@ tracker_resource_class_init (TrackerResourceClass *klass) GObjectClass *object_class = G_OBJECT_CLASS (klass); object_class->dispose = dispose; - object_class->constructed = constructed; object_class->finalize = finalize; object_class->get_property = get_property; object_class->set_property = set_property; @@ -162,29 +160,13 @@ dispose (GObject *object) } static void -constructed (GObject *object) -{ - TrackerResourcePrivate *priv; - - priv = GET_PRIVATE (TRACKER_RESOURCE (object)); - - if (! priv->identifier) { - priv->identifier = generate_blank_node_identifier (); - } - - G_OBJECT_CLASS (tracker_resource_parent_class)->constructed (object); -} - -static void finalize (GObject *object) { TrackerResourcePrivate *priv; priv = GET_PRIVATE (TRACKER_RESOURCE (object)); - if (priv->identifier) { - g_free (priv->identifier); - } + g_clear_pointer (&priv->identifier, g_free); (G_OBJECT_CLASS (tracker_resource_parent_class)->finalize)(object); } @@ -195,13 +177,9 @@ get_property (GObject *object, GValue *value, GParamSpec *pspec) { - TrackerResourcePrivate *priv; - - priv = GET_PRIVATE (TRACKER_RESOURCE (object)); - switch (param_id) { case PROP_IDENTIFIER: - g_value_set_string (value, priv->identifier); + g_value_set_string (value, tracker_resource_get_identifier (TRACKER_RESOURCE (object))); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, param_id, pspec); @@ -855,6 +833,9 @@ tracker_resource_get_identifier (TrackerResource *self) priv = GET_PRIVATE (self); + if (!priv->identifier) + priv->identifier = generate_blank_node_identifier (); + return priv->identifier; } @@ -881,19 +862,8 @@ tracker_resource_set_identifier (TrackerResource *self, priv = GET_PRIVATE (self); - g_free (priv->identifier); - - if (identifier == NULL) { - /* We take NULL to mean "this is a blank node", and generate a - * unique blank node identifier right away. This is easier than - * leaving it NULL and trying to generate a blank node ID at - * serialization time, and it means that the serialization - * output is stable when called multiple times. - */ - priv->identifier = generate_blank_node_identifier (); - } else { - priv->identifier = g_strdup (identifier); - } + g_clear_pointer (&priv->identifier, g_free); + priv->identifier = g_strdup (identifier); } /** @@ -911,14 +881,10 @@ gint tracker_resource_identifier_compare_func (TrackerResource *resource, const char *identifier) { - TrackerResourcePrivate *priv; - g_return_val_if_fail (TRACKER_IS_RESOURCE (resource), 0); g_return_val_if_fail (identifier != NULL, 0); - priv = GET_PRIVATE (resource); - - return strcmp (priv->identifier, identifier); + return strcmp (tracker_resource_get_identifier (resource), identifier); } /** @@ -938,15 +904,11 @@ gint tracker_resource_compare (TrackerResource *a, TrackerResource *b) { - TrackerResourcePrivate *a_priv, *b_priv; - g_return_val_if_fail (TRACKER_IS_RESOURCE (a), 0); g_return_val_if_fail (TRACKER_IS_RESOURCE (b), 0); - a_priv = GET_PRIVATE (a); - b_priv = GET_PRIVATE (b); - - return strcmp (a_priv->identifier, b_priv->identifier); + return strcmp (tracker_resource_get_identifier (a), + tracker_resource_get_identifier (b)); } /** @@ -1324,7 +1286,8 @@ generate_sparql_delete_queries (TrackerResource *resource, } g_string_append (data->string, " "); - generate_turtle_uri_value (priv->identifier, data->string, data->namespaces, NULL); + generate_turtle_uri_value (tracker_resource_get_identifier (resource), + data->string, data->namespaces, NULL); g_string_append_printf (data->string, " %s ?%s }", property, variable_name); g_free (variable_name); @@ -1349,7 +1312,7 @@ generate_sparql_deletes (TrackerResource *resource, data->done_list = g_list_prepend (data->done_list, resource); - if (! is_blank_node (priv->identifier) && g_hash_table_size (priv->overwrite) > 0) { + if (!tracker_resource_is_blank_node (resource) && g_hash_table_size (priv->overwrite) > 0) { generate_sparql_delete_queries (resource, priv->overwrite, data); } @@ -1377,7 +1340,8 @@ generate_sparql_insert_pattern (TrackerResource *resource, /* First, emit any sub-resources. */ g_hash_table_foreach (priv->properties, generate_sparql_relation_inserts_foreach, data); - generate_turtle_uri_value (priv->identifier, data->string, data->namespaces, NULL); + generate_turtle_uri_value (tracker_resource_get_identifier (resource), + data->string, data->namespaces, NULL); g_string_append_printf (data->string, " "); /* rdf:type needs to be first, otherwise you'll see 'subject x is not in domain y' @@ -1506,7 +1470,7 @@ tracker_resource_generate_jsonld (TrackerResource *self, * (where the caller passed NULL as an identifier) than to emit something * SPARQL-specific like '_:123'. */ - if (strncmp (priv->identifier, "_:", 2) != 0) { + if (!tracker_resource_is_blank_node (self)) { json_builder_set_member_name (builder, "@id"); json_builder_add_string_value (builder, priv->identifier); } @@ -1794,8 +1758,7 @@ tracker_resource_serialize (TrackerResource *resource) g_variant_builder_init (&builder, G_VARIANT_TYPE_ARRAY); - if (priv->identifier && - strncmp (priv->identifier, "_:", 2) != 0) { + if (!tracker_resource_is_blank_node (resource)) { g_variant_builder_add (&builder, "{sv}", "@id", g_variant_new_string (priv->identifier)); } @@ -2034,3 +1997,22 @@ tracker_resource_iterator_next (TrackerResourceIterator *iter, *value = val; return TRUE; } + +const gchar * +tracker_resource_get_identifier_internal (TrackerResource *resource) +{ + TrackerResourcePrivate *priv = GET_PRIVATE (resource); + + return priv->identifier; +} + +gboolean +tracker_resource_is_blank_node (TrackerResource *resource) +{ + TrackerResourcePrivate *priv = GET_PRIVATE (resource); + + if (!priv->identifier) + return TRUE; + + return strncmp (priv->identifier, "_:", 2) == 0; +} diff --git a/tests/core/describe/describe-multiple.out b/tests/core/describe/describe-multiple.out index 8edc0fd45..dadbfbb86 100644 --- a/tests/core/describe/describe-multiple.out +++ b/tests/core/describe/describe-multiple.out @@ -1,8 +1,8 @@ "b" "http://example/relation" "z" "c" "http://example/relation" "x" "d" "http://example/relation" "z" -"z" "http://example/title" "titleZ" "x" "http://example/title" "titleX" +"z" "http://example/title" "titleZ" "b" "http://example/number" "73" "c" "http://example/number" "113" "b" "http://example/date" "2001-01-01T00:00:01Z" @@ -10,13 +10,13 @@ "b" "http://example/name" "nameB" "c" "http://example/name" "nameC" "d" "http://example/name" "nameD" -"z" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://www.w3.org/2000/01/rdf-schema#Resource" -"z" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://example/B" -"x" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://www.w3.org/2000/01/rdf-schema#Resource" -"x" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://example/B" "b" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://www.w3.org/2000/01/rdf-schema#Resource" "b" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://example/A" "c" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://www.w3.org/2000/01/rdf-schema#Resource" "c" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://example/A" "d" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://www.w3.org/2000/01/rdf-schema#Resource" "d" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://example/A" +"x" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://www.w3.org/2000/01/rdf-schema#Resource" +"x" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://example/B" +"z" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://www.w3.org/2000/01/rdf-schema#Resource" +"z" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://example/B" diff --git a/tests/core/tracker-sparql-test.c b/tests/core/tracker-sparql-test.c index b06a26ffa..0da6acafa 100644 --- a/tests/core/tracker-sparql-test.c +++ b/tests/core/tracker-sparql-test.c @@ -468,13 +468,14 @@ check_result (TrackerSparqlCursor *cursor, gchar *diff; quoted_results = g_shell_quote (test_results->str); - command_line = g_strdup_printf ("echo -n %s | diff -u %s -", quoted_results, results_filename); + command_line = g_strdup_printf ("echo -n %s | diff -uZ %s -", quoted_results, results_filename); quoted_command_line = g_shell_quote (command_line); shell = g_strdup_printf ("sh -c %s", quoted_command_line); g_spawn_command_line_sync (shell, &diff, NULL, NULL, &error); g_assert_no_error (error); - g_error ("%s", diff); + if (diff && *diff) + g_error ("%s", diff); g_free (quoted_results); g_free (command_line); diff --git a/tests/libtracker-sparql/serialize/describe-graph-trig.out b/tests/libtracker-sparql/serialize/describe-graph-trig.out index 07165da63..34c832c29 100644 --- a/tests/libtracker-sparql/serialize/describe-graph-trig.out +++ b/tests/libtracker-sparql/serialize/describe-graph-trig.out @@ -33,7 +33,6 @@ GRAPH <A> { } GRAPH <B> { - } GRAPH <A> { diff --git a/utils/benchmark/tracker-benchmark.c b/utils/benchmark/tracker-benchmark.c index aede7d303..d4471863c 100644 --- a/utils/benchmark/tracker-benchmark.c +++ b/utils/benchmark/tracker-benchmark.c @@ -117,6 +117,30 @@ create_resource (void) } static inline gpointer +create_changing_resource (void) +{ + TrackerResource *resource; + gchar buf[2] = { 0, 0 }; + static gint res = 0; + static gint counter = 0; + gchar *uri; + + /* In order to keep large batches, and guaranteeing modifications + * do not get coalesced, create one different URI per slot in the batch. + */ + uri = g_strdup_printf ("http://example.com/%d", res); + resource = tracker_resource_new (uri); + tracker_resource_set_uri (resource, "rdf:type", "rdfs:Resource"); + buf[0] = '0' + counter; + tracker_resource_set_string (resource, "rdfs:label", (const gchar *) buf); + counter = (counter + 1) % 10; + res = (res + 1) % batch_size; + g_free (uri); + + return resource; +} + +static inline gpointer create_query (void) { return g_strdup ("SELECT ?u { ?u a rdfs:Resource } limit 1"); @@ -154,6 +178,40 @@ create_batch (TrackerSparqlConnection *conn, return batch; } +static inline TrackerBatch * +create_batch_insert_delete (TrackerSparqlConnection *conn) +{ + TrackerBatch *batch; + int i; + + batch = tracker_sparql_connection_create_batch (conn); + + for (i = 0; i < batch_size; i++) { + gchar *uri; + + uri = g_strdup_printf ("http://example.com/%d", i >> 1); + + if (i % 2 == 0) { + TrackerResource *resource; + + resource = tracker_resource_new (uri); + tracker_resource_set_uri (resource, "rdf:type", "rdfs:Resource"); + tracker_batch_add_resource (batch, NULL, resource); + g_object_unref (resource); + } else { + gchar *query; + + query = g_strdup_printf ("DELETE DATA { <%s> a rdfs:Resource }", uri); + tracker_batch_add_sparql (batch, query); + g_free (query); + } + + g_free (uri); + } + + return batch; +} + static int consume_cursor (TrackerSparqlCursor *cursor) { @@ -250,6 +308,44 @@ benchmark_update_sparql (TrackerSparqlConnection *conn, } static void +benchmark_update_insert_delete (TrackerSparqlConnection *conn, + DataCreateFunc data_func, + double *elapsed, + int *elems, + double *min, + double *max) +{ + GTimer *timer; + GError *error = NULL; + + timer = g_timer_new (); + + while (*elapsed < duration) { + TrackerBatch *batch; + double batch_elapsed; + + g_timer_reset (timer); + batch = create_batch_insert_delete (conn); + tracker_batch_execute (batch, NULL, &error); + g_assert_no_error (error); + g_object_unref (batch); + + batch_elapsed = g_timer_elapsed (timer, NULL); + *min = MIN (*min, batch_elapsed); + *max = MAX (*max, batch_elapsed); + *elapsed += batch_elapsed; + *elems += 1; + } + + /* We count things by resources, not batches */ + *min /= batch_size; + *max /= batch_size; + *elems *= batch_size; + + g_timer_destroy (timer); +} + +static void benchmark_query_statement (TrackerSparqlConnection *conn, DataCreateFunc data_func, double *elapsed, @@ -334,6 +430,8 @@ struct { } benchmarks[] = { { "Resource batch update (sync)", benchmark_update_batch, create_resource }, { "SPARQL batch update (sync)", benchmark_update_sparql, create_resource }, + { "Resource modification (sync)", benchmark_update_batch, create_changing_resource }, + { "Resource insert+delete (sync)", benchmark_update_insert_delete, NULL }, { "Prepared statement query (sync)", benchmark_query_statement, create_query }, { "SPARQL query (sync)", benchmark_query_sparql, create_query }, }; @@ -354,6 +452,8 @@ run_benchmarks (TrackerSparqlConnection *conn) double elapsed = 0, min = G_MAXDOUBLE, max = -G_MAXDOUBLE, adjusted, avg; int elems = 0; + g_print ("%*s\t\t", max_len, benchmarks[i].desc); + benchmarks[i].func (conn, benchmarks[i].data_func, &elapsed, &elems, &min, &max); @@ -368,8 +468,7 @@ run_benchmarks (TrackerSparqlConnection *conn) } avg = elapsed / elems; - g_print ("%*s\t\t%.3f\t%.3f\t%.3f %s\t%.3f %s\t%3.3f %s\n", - max_len, benchmarks[i].desc, + g_print ("%.3f\t%.3f\t%.3f %s\t%.3f %s\t%3.3f %s\n", adjusted, elems / elapsed, transform_unit (min), unit_string (min), |