diff options
Diffstat (limited to 'src/miners/rss/tracker-miner-rss.c')
| -rw-r--r-- | src/miners/rss/tracker-miner-rss.c | 1483 |
1 files changed, 0 insertions, 1483 deletions
diff --git a/src/miners/rss/tracker-miner-rss.c b/src/miners/rss/tracker-miner-rss.c deleted file mode 100644 index 3d87052cd..000000000 --- a/src/miners/rss/tracker-miner-rss.c +++ /dev/null @@ -1,1483 +0,0 @@ -/* - * Copyright (C) 2009/2010, Roberto Guido <madbob@users.barberaware.org> - * Michele Tameni <michele@amdplanet.it> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ - -#include "config.h" - -#include <stdio.h> - -#include <libgrss.h> - -#include <libtracker-common/tracker-dbus.h> -#include <libtracker-sparql/tracker-ontologies.h> -#include <libtracker-common/tracker-common.h> - -#include <glib/gi18n.h> - -#include "tracker-miner-rss.h" - -#define TRACKER_MINER_RSS_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), TRACKER_TYPE_MINER_RSS, TrackerMinerRSSPrivate)) - -typedef struct _TrackerMinerRSSPrivate TrackerMinerRSSPrivate; - -struct _TrackerMinerRSSPrivate { - gboolean paused; - gboolean stopped; - gchar *last_status; - - GrssFeedsPool *pool; - gint now_fetching; - GDBusConnection *connection; - guint graph_updated_id; - - GHashTable *channel_updates; - GHashTable *channels; - - gint rdf_type_id; - gint mfo_feed_channel_id; -}; - -typedef struct { - TrackerMinerRSS *miner; - GrssFeedChannel *channel; - gint timeout_id; - GCancellable *cancellable; -} FeedChannelUpdateData; - -typedef struct { - TrackerMinerRSS *miner; - GrssFeedItem *item; - GCancellable *cancellable; -} FeedItemInsertData; - -typedef struct { - TrackerMinerRSS *miner; - GrssFeedChannel *channel; - GHashTable *items; -} FeedItemListInsertData; - -static void graph_updated_cb (GDBusConnection *connection, - const gchar *sender_name, - const gchar *object_path, - const gchar *interface_name, - const gchar *signal_name, - GVariant *parameters, - gpointer user_data); -static void miner_started (TrackerMiner *miner); -static void miner_stopped (TrackerMiner *miner); -static void miner_paused (TrackerMiner *miner); -static void miner_resumed (TrackerMiner *miner); -static void retrieve_and_schedule_feeds (TrackerMinerRSS *miner, - GArray *channel_ids); -static gboolean feed_channel_changed_timeout_cb (gpointer user_data); -static void feed_channel_update_data_free (FeedChannelUpdateData *fcud); -static void feed_fetching_cb (GrssFeedsPool *pool, - GrssFeedChannel *feed, - gpointer user_data); -static void feed_ready_cb (GrssFeedsPool *pool, - GrssFeedChannel *feed, - GList *items, - gpointer user_data); -static const gchar *get_message_url (GrssFeedItem *item); - -G_DEFINE_TYPE (TrackerMinerRSS, tracker_miner_rss, TRACKER_TYPE_MINER_ONLINE) - -static void -parser_characters (void *data, - const xmlChar *ch, - int len) -{ - GString *string = data; - const gchar *str, *end; - - str = (gchar *)ch; - g_utf8_validate (str, len, &end); - - if (end > str) { - g_string_append_len (string, str, end - str); - } - - if (string->str[string->len - 1] != ' ') - g_string_append_c (string, ' '); -} - -static gchar * -parse_html_text (const gchar *html) -{ - GString *string; - htmlDocPtr doc; - xmlSAXHandler handler = { - NULL, /* internalSubset */ - NULL, /* isStandalone */ - NULL, /* hasInternalSubset */ - NULL, /* hasExternalSubset */ - NULL, /* resolveEntity */ - NULL, /* getEntity */ - NULL, /* entityDecl */ - NULL, /* notationDecl */ - NULL, /* attributeDecl */ - NULL, /* elementDecl */ - NULL, /* unparsedEntityDecl */ - NULL, /* setDocumentLocator */ - NULL, /* startDocument */ - NULL, /* endDocument */ - NULL, /* startElement */ - NULL, /* endElement */ - NULL, /* reference */ - parser_characters, /* characters */ - NULL, /* ignorableWhitespace */ - NULL, /* processingInstruction */ - NULL, /* comment */ - NULL, /* xmlParserWarning */ - NULL, /* xmlParserError */ - NULL, /* xmlParserError */ - NULL, /* getParameterEntity */ - NULL, /* cdataBlock */ - NULL, /* externalSubset */ - 1, /* initialized */ - NULL, /* private */ - NULL, /* startElementNsSAX2Func */ - NULL, /* endElementNsSAX2Func */ - NULL /* xmlStructuredErrorFunc */ - }; - - string = g_string_new (NULL); - doc = htmlSAXParseDoc ((xmlChar *) html, "UTF-8", &handler, string); - - if (doc) { - xmlFreeDoc (doc); - } - - return g_string_free (string, FALSE); -} - -static void -tracker_miner_rss_finalize (GObject *object) -{ - TrackerMinerRSSPrivate *priv; - - priv = TRACKER_MINER_RSS_GET_PRIVATE (object); - - priv->stopped = TRUE; - g_free (priv->last_status); - g_object_unref (priv->pool); - - g_dbus_connection_signal_unsubscribe (priv->connection, priv->graph_updated_id); - g_object_unref (priv->connection); - - g_hash_table_unref (priv->channel_updates); - g_hash_table_unref (priv->channels); - - G_OBJECT_CLASS (tracker_miner_rss_parent_class)->finalize (object); -} - -static gint -get_class_id (TrackerSparqlConnection *conn, - const gchar *class) -{ - TrackerSparqlCursor *cursor; - GError *error = NULL; - gchar *query; - gint id = -1; - - query = g_strdup_printf ("select tracker:id (%s) {}", class); - cursor = tracker_sparql_connection_query (conn, query, NULL, &error); - g_free (query); - - if (error) { - g_critical ("Could not get class ID for '%s': %s\n", - class, error->message); - g_error_free (error); - return -1; - } - - if (tracker_sparql_cursor_next (cursor, NULL, NULL)) - id = tracker_sparql_cursor_get_integer (cursor, 0); - else - g_critical ("'%s' didn't resolve to a known class ID", class); - - g_object_unref (cursor); - - return id; -} - -static gboolean -miner_connected (TrackerMinerOnline *miner, - TrackerNetworkType network) -{ - return (network == TRACKER_NETWORK_TYPE_LAN); -} - -static void -tracker_miner_rss_class_init (TrackerMinerRSSClass *klass) -{ - GObjectClass *object_class = G_OBJECT_CLASS (klass); - TrackerMinerClass *miner_class = TRACKER_MINER_CLASS (klass); - TrackerMinerOnlineClass *miner_online_class = TRACKER_MINER_ONLINE_CLASS (klass); - - object_class->finalize = tracker_miner_rss_finalize; - - miner_class->started = miner_started; - miner_class->stopped = miner_stopped; - miner_class->paused = miner_paused; - miner_class->resumed = miner_resumed; - - miner_online_class->connected = miner_connected; - - g_type_class_add_private (object_class, sizeof (TrackerMinerRSSPrivate)); -} - -static void -tracker_miner_rss_init (TrackerMinerRSS *object) -{ - GError *error = NULL; - TrackerMinerRSSPrivate *priv; - TrackerDomainOntology *domain_ontology; - gchar *dbus_name; - - g_message ("Initializing..."); - - priv = TRACKER_MINER_RSS_GET_PRIVATE (object); - - priv->connection = g_bus_get_sync (TRACKER_IPC_BUS, NULL, &error); - - if (!priv->connection) { - g_critical ("Could not connect to the D-Bus session bus, %s", - error ? error->message : "no error given."); - g_error_free (error); - return; - } - - /* Key object reference is cleaned up in value destroy func */ - priv->channel_updates = g_hash_table_new_full (g_direct_hash, - g_direct_equal, - NULL, - (GDestroyNotify) feed_channel_update_data_free); - priv->channels = g_hash_table_new_full (NULL, NULL, NULL, - (GDestroyNotify) g_object_unref); - - priv->pool = grss_feeds_pool_new (); - g_signal_connect (priv->pool, "feed-fetching", G_CALLBACK (feed_fetching_cb), object); - g_signal_connect (priv->pool, "feed-ready", G_CALLBACK (feed_ready_cb), object); - priv->now_fetching = 0; - - g_message ("Listening for GraphUpdated changes on D-Bus interface..."); - g_message (" arg0:'%s'", TRACKER_PREFIX_MFO "FeedChannel"); - - domain_ontology = tracker_domain_ontology_new (tracker_sparql_connection_get_domain (), - NULL, NULL); - dbus_name = tracker_domain_ontology_get_domain (domain_ontology, NULL); - - priv->graph_updated_id = - g_dbus_connection_signal_subscribe (priv->connection, - dbus_name, - "org.freedesktop.Tracker1.Resources", - "GraphUpdated", - "/org/freedesktop/Tracker1/Resources", - TRACKER_PREFIX_MFO "FeedChannel", - G_DBUS_SIGNAL_FLAGS_NONE, - graph_updated_cb, - object, - NULL); - g_free (dbus_name); - g_object_unref (domain_ontology); -} - -static void -delete_unbound_messages (TrackerMinerRSS *miner) -{ - tracker_sparql_connection_update_async (tracker_miner_get_connection (TRACKER_MINER (miner)), - "DELETE { ?msg a rdfs:Resource." - " ?encl a rdfs:Resource. }" - "WHERE { ?msg a mfo:FeedMessage ;" - " mfo:enclosureList ?encl ." - " FILTER(!BOUND(nmo:communicationChannel(?msg)))" - "}", - G_PRIORITY_DEFAULT, - NULL, NULL, NULL); -} - -static void -delete_message_channels_cb (GObject *source_object, - GAsyncResult *res, - gpointer user_data) -{ - TrackerMinerRSS *miner = user_data; - TrackerSparqlConnection *connection; - GError *error = NULL; - - connection = TRACKER_SPARQL_CONNECTION (source_object); - tracker_sparql_connection_update_finish (connection, res, &error); - - if (error != NULL) { - g_message ("Could not delete message channels: %s", error->message); - g_error_free (error); - return; - } - - delete_unbound_messages (miner); -} - -static void -delete_message_channels (TrackerMinerRSS *miner, - GArray *channel_ids) -{ - TrackerMinerRSSPrivate *priv; - GString *query, *ids_str; - GrssFeedChannel *channel; - gint i, id; - - priv = TRACKER_MINER_RSS_GET_PRIVATE (miner); - ids_str = g_string_new (NULL); - query = g_string_new (NULL); - - for (i = 0; i < channel_ids->len; i++) { - id = g_array_index (channel_ids, gint, i); - if (i != 0) - g_string_append (ids_str, ","); - g_string_append_printf (ids_str, "%d", id); - - channel = g_hash_table_lookup (priv->channels, - GINT_TO_POINTER (id)); - - if (channel) { - g_hash_table_remove (priv->channel_updates, channel); - g_hash_table_remove (priv->channels, GINT_TO_POINTER (id)); - } - } - - g_string_append_printf (query, - "DELETE { ?msg nmo:communicationChannel ?chan }" - "WHERE { ?msg a mfo:FeedMessage;" - " nmo:communicationChannel ?chan ." - " FILTER (tracker:id(?chan) IN (%s))" - "}", ids_str->str); - - tracker_sparql_connection_update_async (tracker_miner_get_connection (TRACKER_MINER (miner)), - query->str, G_PRIORITY_DEFAULT, - NULL, - delete_message_channels_cb, - miner); - g_string_free (ids_str, TRUE); - g_string_free (query, TRUE); -} - -static void -handle_deletes (TrackerMinerRSS *miner, - GVariantIter *iter) -{ - GArray *deleted = g_array_new (FALSE, FALSE, sizeof (gint)); - gint graph, subject, pred, object; - TrackerMinerRSSPrivate *priv; - - priv = TRACKER_MINER_RSS_GET_PRIVATE (miner); - - while (g_variant_iter_next (iter, "(iiii)", - &graph, &subject, &pred, &object)) { - if (pred == priv->rdf_type_id && - object == priv->mfo_feed_channel_id) { - g_array_append_val (deleted, subject); - } - } - - if (deleted->len > 0) - delete_message_channels (miner, deleted); - - g_array_free (deleted, TRUE); -} - -static void -handle_updates (TrackerMinerRSS *miner, - GVariantIter *iter) -{ - GArray *updated = g_array_new (FALSE, FALSE, sizeof (gint)); - gint graph, subject, pred, object; - TrackerMinerRSSPrivate *priv; - - priv = TRACKER_MINER_RSS_GET_PRIVATE (miner); - - while (g_variant_iter_next (iter, "(iiii)", - &graph, &subject, &pred, &object)) { - if (pred == priv->rdf_type_id && - object == priv->mfo_feed_channel_id) { - g_array_append_val (updated, subject); - } - } - - if (updated->len > 0) - retrieve_and_schedule_feeds (miner, updated); - - g_array_free (updated, TRUE); -} - -static void -graph_updated_cb (GDBusConnection *connection, - const gchar *sender_name, - const gchar *object_path, - const gchar *interface_name, - const gchar *signal_name, - GVariant *parameters, - gpointer user_data) -{ - TrackerMinerRSS *miner = TRACKER_MINER_RSS (user_data); - GVariantIter *deletes, *updates; - - g_variant_get (parameters, "(&sa(iiii)a(iiii))", NULL, &deletes, &updates); - handle_deletes (miner, deletes); - handle_updates (miner, updates); - g_variant_iter_free (deletes); - g_variant_iter_free (updates); -} - -static FeedChannelUpdateData * -feed_channel_update_data_new (TrackerMinerRSS *miner, - GrssFeedChannel *channel) -{ - FeedChannelUpdateData *fcud; - - fcud = g_slice_new0 (FeedChannelUpdateData); - fcud->miner = g_object_ref (miner); - fcud->channel = g_object_ref (channel); - fcud->timeout_id = g_timeout_add_seconds (2, feed_channel_changed_timeout_cb, fcud); - fcud->cancellable = g_cancellable_new (); - - return fcud; -} - -static void -feed_channel_update_data_free (FeedChannelUpdateData *fcud) -{ - if (!fcud) { - return; - } - - if (fcud->cancellable) { - g_cancellable_cancel (fcud->cancellable); - g_object_unref (fcud->cancellable); - } - - if (fcud->timeout_id) { - g_source_remove (fcud->timeout_id); - } - - if (fcud->channel) { - g_object_unref (fcud->channel); - } - - if (fcud->miner) { - g_object_unref (fcud->miner); - } - - g_slice_free (FeedChannelUpdateData, fcud); -} - -static FeedItemListInsertData * -feed_item_list_insert_data_new (TrackerMinerRSS *miner, - GrssFeedChannel *channel, - GList *items) -{ - FeedItemListInsertData *data; - GrssFeedItem *prev; - const gchar *url; - GList *l; - - data = g_slice_new0 (FeedItemListInsertData); - data->channel = channel; - data->miner = miner; - data->items = g_hash_table_new_full (g_str_hash, g_str_equal, - (GDestroyNotify) g_free, - (GDestroyNotify) g_object_unref); - - /* Make items unique, keep most recent */ - for (l = items; l; l = l->next) { - url = get_message_url (l->data); - prev = g_hash_table_lookup (data->items, url); - - if (prev) { - /* Compare publish times */ - if (grss_feed_item_get_publish_time (l->data) <= - grss_feed_item_get_publish_time (prev)) - continue; - } - - g_hash_table_insert (data->items, g_strdup (url), - g_object_ref (l->data)); - } - - return data; -} - -static void -feed_item_list_insert_data_free (FeedItemListInsertData *data) -{ - g_hash_table_destroy (data->items); - g_slice_free (FeedItemListInsertData, data); -} - -static void -feed_channel_change_updated_time_cb (GObject *source, - GAsyncResult *result, - gpointer user_data) -{ - TrackerMinerRSSPrivate *priv; - FeedChannelUpdateData *fcud; - GError *error = NULL; - - fcud = user_data; - priv = TRACKER_MINER_RSS_GET_PRIVATE (fcud->miner); - - tracker_sparql_connection_update_finish (TRACKER_SPARQL_CONNECTION (source), result, &error); - if (error != NULL) { - g_critical ("Could not change feed channel updated time, %s", error->message); - g_error_free (error); - } - - /* This will clean up the fcud data too */ - g_hash_table_remove (priv->channel_updates, fcud->channel); -} - -static gboolean -feed_channel_changed_timeout_cb (gpointer user_data) -{ - TrackerSparqlBuilder *sparql; - FeedChannelUpdateData *fcud; - gchar *uri; - time_t now; - - fcud = user_data; - fcud->timeout_id = 0; - - now = time (NULL); - uri = g_object_get_data (G_OBJECT (fcud->channel), "subject"); - - g_message ("Updating mfo:updatedTime for channel '%s'", grss_feed_channel_get_title (fcud->channel)); - - /* I hope there will be soon a SPARQL command to just update a - * value instead to delete and re-insert it - */ - - sparql = tracker_sparql_builder_new_update (); - tracker_sparql_builder_delete_open (sparql, NULL); - tracker_sparql_builder_subject_iri (sparql, uri); - tracker_sparql_builder_predicate (sparql, "mfo:updatedTime"); - tracker_sparql_builder_object_variable (sparql, "unknown"); - tracker_sparql_builder_delete_close (sparql); - tracker_sparql_builder_where_open (sparql); - tracker_sparql_builder_subject_iri (sparql, uri); - tracker_sparql_builder_predicate (sparql, "mfo:updatedTime"); - tracker_sparql_builder_object_variable (sparql, "unknown"); - tracker_sparql_builder_where_close (sparql); - - tracker_sparql_builder_insert_open (sparql, NULL); - tracker_sparql_builder_subject_iri (sparql, uri); - tracker_sparql_builder_predicate (sparql, "mfo:updatedTime"); - tracker_sparql_builder_object_date (sparql, &now); - tracker_sparql_builder_insert_close (sparql); - - tracker_sparql_connection_update_async (tracker_miner_get_connection (TRACKER_MINER (fcud->miner)), - tracker_sparql_builder_get_result (sparql), - G_PRIORITY_DEFAULT, - fcud->cancellable, - feed_channel_change_updated_time_cb, - fcud); - g_object_unref (sparql); - - return FALSE; -} - -static void -feed_channel_change_updated_time (TrackerMinerRSS *miner, - GrssFeedChannel *channel) -{ - TrackerMinerRSSPrivate *priv; - FeedChannelUpdateData *fcud; - - if (!channel) - return; - - priv = TRACKER_MINER_RSS_GET_PRIVATE (miner); - - fcud = g_hash_table_lookup (priv->channel_updates, channel); - if (fcud) { - /* We already had an update for this channel in - * progress, so we just reset the timeout. - */ - g_source_remove (fcud->timeout_id); - fcud->timeout_id = g_timeout_add_seconds (2, - feed_channel_changed_timeout_cb, - fcud); - } else { - /* This is a new update for this channel */ - fcud = feed_channel_update_data_new (miner, channel); - g_hash_table_insert (priv->channel_updates, - fcud->channel, - fcud); - } -} - -static void -feed_fetching_cb (GrssFeedsPool *pool, - GrssFeedChannel *channel, - gpointer user_data) -{ - gint avail; - gdouble prog; - TrackerMinerRSS *miner; - TrackerMinerRSSPrivate *priv; - - miner = TRACKER_MINER_RSS (user_data); - priv = TRACKER_MINER_RSS_GET_PRIVATE (miner); - avail = grss_feeds_pool_get_listened_num (priv->pool); - - priv->now_fetching++; - - if (priv->now_fetching > avail) - priv->now_fetching = avail; - - g_message ("Fetching channel details, source:'%s' (in progress: %d/%d)", - grss_feed_channel_get_source (channel), - priv->now_fetching, - avail); - - prog = ((gdouble) priv->now_fetching) / ((gdouble) avail); - g_object_set (miner, "progress", prog, "status", "Fetching…", NULL); -} - -static gchar * -sparql_add_website (TrackerSparqlBuilder *sparql, - const gchar *uri) -{ - gchar *website_urn; - - website_urn = tracker_sparql_escape_uri_printf ("urn:website:%s", uri); - - tracker_sparql_builder_insert_silent_open (sparql, NULL); - tracker_sparql_builder_subject_iri (sparql, website_urn); - tracker_sparql_builder_predicate (sparql, "a"); - tracker_sparql_builder_object (sparql, "nie:DataObject"); - tracker_sparql_builder_object (sparql, "nfo:Website"); - - tracker_sparql_builder_predicate (sparql, "nie:url"); - tracker_sparql_builder_object_unvalidated (sparql, uri); - - tracker_sparql_builder_insert_close (sparql); - - return website_urn; -} - -static void -sparql_add_contact (TrackerSparqlBuilder *sparql, - const gchar *alias, - GrssPerson *contact, - const gchar *website_urn, - gboolean is_iri) -{ - const gchar *name = grss_person_get_name (contact); - const gchar *email = grss_person_get_email (contact); - - tracker_sparql_builder_subject (sparql, alias); - tracker_sparql_builder_predicate (sparql, "a"); - tracker_sparql_builder_object (sparql, "nco:Contact"); - - tracker_sparql_builder_predicate (sparql, "nco:fullname"); - tracker_sparql_builder_object_unvalidated (sparql, name); - - if (email != NULL) { - tracker_sparql_builder_predicate (sparql, "nco:hasEmailAddress"); - - tracker_sparql_builder_object_blank_open (sparql); - - tracker_sparql_builder_predicate (sparql, "a"); - tracker_sparql_builder_object (sparql, "nco:EmailAddress"); - - tracker_sparql_builder_predicate (sparql, "nco:emailAddress"); - tracker_sparql_builder_object_unvalidated (sparql, email); - tracker_sparql_builder_object_blank_close (sparql); - } - - if (website_urn) { - tracker_sparql_builder_predicate (sparql, "nco:websiteUrl"); - - if (is_iri) { - tracker_sparql_builder_object_iri (sparql, website_urn); - } else { - tracker_sparql_builder_object (sparql, website_urn); - } - } -} - -static void -sparql_add_enclosure (TrackerSparqlBuilder *sparql, - const gchar *subject, - GrssFeedEnclosure *enclosure) -{ - gchar *url; - gint length; - gchar * format; - - url = grss_feed_enclosure_get_url (enclosure); - length = grss_feed_enclosure_get_length (enclosure); - format = grss_feed_enclosure_get_format (enclosure); - - tracker_sparql_builder_subject (sparql, subject); - tracker_sparql_builder_predicate (sparql, "a"); - tracker_sparql_builder_object (sparql, "mfo:Enclosure"); - tracker_sparql_builder_object (sparql, "nfo:RemoteDataObject"); - - tracker_sparql_builder_predicate (sparql, "mfo:remoteLink"); - tracker_sparql_builder_object_unvalidated (sparql, url); - - tracker_sparql_builder_predicate (sparql, "nfo:fileSize"); - tracker_sparql_builder_object_int64 (sparql, length); - - if (format != NULL) { - tracker_sparql_builder_predicate (sparql, "nie:mimeType"); - tracker_sparql_builder_object_unvalidated (sparql, format); - } - -} - -static gchar * -feed_message_create_update_channel_query (const gchar *item_urn, - GrssFeedItem *item) -{ - GrssFeedChannel *channel; - const gchar *channel_urn; - - channel = grss_feed_item_get_parent (item); - channel_urn = g_object_get_data (G_OBJECT (channel), "subject"); - - return g_strdup_printf ("INSERT SILENT { <%s> nmo:communicationChannel <%s> }", - item_urn, channel_urn); -} - -static gchar * -feed_message_create_delete_properties_query (const gchar *item_urn) -{ - return g_strdup_printf ("DELETE { <%s> ?p ?o }" - "WHERE { <%s> a mfo:FeedMessage ;" - " ?p ?o ." - " FILTER (?p != rdf:type &&" - " ?p != nmo:communicationChannel)" - "}", item_urn, item_urn); -} - -static TrackerSparqlBuilder * -feed_message_create_insert_builder (TrackerMinerRSS *miner, - GrssFeedItem *item, - const gchar *item_urn) -{ - time_t t; - gchar *uri; - const gchar *url; - GrssPerson *author; - gdouble latitude; - gdouble longitude; - const gchar *tmp_string; - TrackerSparqlBuilder *sparql; - GrssFeedChannel *channel; - gboolean has_geolocation; - const GList *contributors; - const GList *enclosures; - const GList *list, *l; - GList *contrib_aliases = NULL; - GList *enclosure_aliases = NULL; - GHashTable *websites; - GHashTable *enclosure_urls; - gboolean is_iri = FALSE; - gint i = 0; - - if (!item_urn) { - item_urn = "_:message"; - } else { - is_iri = TRUE; - } - - url = get_message_url (item); - g_message ("Inserting feed item for '%s'", url); - - websites = g_hash_table_new_full (g_str_hash, g_str_equal, - (GDestroyNotify) g_free, - (GDestroyNotify) g_free); - g_hash_table_insert (websites, g_strdup (url), g_strdup (item_urn)); - - sparql = tracker_sparql_builder_new_update (); - author = grss_feed_item_get_author (item); - contributors = grss_feed_item_get_contributors (item); - channel = grss_feed_item_get_parent (item); - enclosures = grss_feed_item_get_enclosures (item); - - for (l = contributors; l; l = l->next) { - const gchar *person_url; - - person_url = grss_person_get_uri (l->data); - - if (!person_url) - continue; - - if (g_hash_table_lookup (websites, person_url)) - continue; - - g_hash_table_insert (websites, g_strdup (person_url), - sparql_add_website (sparql, person_url)); - } - - if (author && grss_person_get_uri (author)) { - const gchar *person_url; - - person_url = grss_person_get_uri (author); - - if (!g_hash_table_lookup (websites, person_url)) { - g_hash_table_insert (websites, g_strdup (person_url), - sparql_add_website (sparql, person_url)); - } - } - - has_geolocation = grss_feed_item_get_geo_point (item, &latitude, &longitude); - tracker_sparql_builder_insert_open (sparql, NULL); - - if (has_geolocation) { - g_message (" Geolocation, using longitude:%f, latitude:%f", - longitude, latitude); - - tracker_sparql_builder_subject (sparql, "_:location"); - tracker_sparql_builder_predicate (sparql, "a"); - tracker_sparql_builder_object (sparql, "slo:GeoLocation"); - - tracker_sparql_builder_predicate (sparql, "slo:latitude"); - tracker_sparql_builder_object_double (sparql, latitude); - tracker_sparql_builder_predicate (sparql, "slo:longitude"); - tracker_sparql_builder_object_double (sparql, longitude); - } - - if (author != NULL) { - g_message (" Author:'%s'", grss_person_get_name (author)); - - if (grss_person_get_uri (author)) - tmp_string = g_hash_table_lookup (websites, grss_person_get_uri (author)); - else - tmp_string = NULL; - - sparql_add_contact (sparql, "_:author", author, tmp_string, - (tmp_string && tmp_string[0] != '_')); - } - - i = 0; - for (l = contributors; l; l = l->next) { - gchar *subject; - - g_debug (" Contributor:'%s'", grss_person_get_name (l->data)); - - subject = g_strdup_printf ("_:contrib%d", i++); - contrib_aliases = g_list_prepend (contrib_aliases, subject); - - if (grss_person_get_uri (l->data)) - tmp_string = g_hash_table_lookup (websites, grss_person_get_uri (l->data)); - else - tmp_string = NULL; - - sparql_add_contact (sparql, subject, l->data, tmp_string, - (tmp_string && tmp_string[0] != '_')); - - /* subject is not freed right now as it will be freed at the end of the - * method in a foreach loop over the contrib_aliases list */ - } - - enclosure_urls = g_hash_table_new_full (g_str_hash, g_str_equal, - (GDestroyNotify) g_free, - (GDestroyNotify) g_free); - i = 0; - for (l = enclosures; l; l = l->next) { - gchar *subject; - gchar *url; - - url = grss_feed_enclosure_get_url (l->data); - - if (!g_hash_table_contains (enclosure_urls, url)) { - g_debug (" Enclosure:'%s'", url); - subject = g_strdup_printf ("_:enclosure%d", i++); - enclosure_aliases = g_list_prepend (enclosure_aliases, subject); - g_hash_table_insert (enclosure_urls, g_strdup (url), NULL); - - sparql_add_enclosure (sparql, subject, l->data); - - } - - /* subject is not freed right now as it will be freed at the end of the - * method in a foreach loop over the enclosure_aliases list */ - } - - if (is_iri) { - tracker_sparql_builder_subject_iri (sparql, item_urn); - } else { - tracker_sparql_builder_subject (sparql, item_urn); - } - tracker_sparql_builder_predicate (sparql, "a"); - tracker_sparql_builder_object (sparql, "mfo:FeedMessage"); - tracker_sparql_builder_predicate (sparql, "a"); - tracker_sparql_builder_object (sparql, "nfo:RemoteDataObject"); - - if (has_geolocation == TRUE) { - tracker_sparql_builder_predicate (sparql, "slo:location"); - tracker_sparql_builder_object (sparql, "_:location"); - } - - tmp_string = grss_feed_item_get_title (item); - if (tmp_string != NULL) { - g_message (" Title:'%s'", tmp_string); - - tracker_sparql_builder_predicate (sparql, "nie:title"); - tracker_sparql_builder_object_unvalidated (sparql, tmp_string); - } - - if (author != NULL) { - tracker_sparql_builder_predicate (sparql, "nco:creator"); - tracker_sparql_builder_object (sparql, "_:author"); - } - - for (l = contrib_aliases; l; l = l->next) { - tracker_sparql_builder_predicate (sparql, "nco:contributor"); - tracker_sparql_builder_object (sparql, l->data); - } - - for (l = enclosure_aliases; l; l = l->next) { - tracker_sparql_builder_predicate (sparql, "mfo:enclosureList"); - tracker_sparql_builder_object (sparql, l->data); - } - - tmp_string = grss_feed_item_get_description (item); - if (tmp_string != NULL) { - gchar *plain_text; - - plain_text = parse_html_text (tmp_string); - tracker_sparql_builder_predicate (sparql, "nie:plainTextContent"); - tracker_sparql_builder_object_unvalidated (sparql, plain_text); - g_free (plain_text); - - tracker_sparql_builder_predicate (sparql, "nmo:htmlMessageContent"); - tracker_sparql_builder_object_unvalidated (sparql, tmp_string); - } - - if (url != NULL) { - tracker_sparql_builder_predicate (sparql, "nie:url"); - tracker_sparql_builder_object_unvalidated (sparql, url); - } - - /* TODO nmo:receivedDate and mfo:downloadedTime are the same? - * Ask for the MFO maintainer */ - - t = time (NULL); - - tracker_sparql_builder_predicate (sparql, "nmo:receivedDate"); - tracker_sparql_builder_object_date (sparql, &t); - - tracker_sparql_builder_predicate (sparql, "mfo:downloadedTime"); - tracker_sparql_builder_object_date (sparql, &t); - - t = grss_feed_item_get_publish_time (item); - tracker_sparql_builder_predicate (sparql, "nie:contentCreated"); - tracker_sparql_builder_object_date (sparql, &t); - - tracker_sparql_builder_predicate (sparql, "nmo:isRead"); - tracker_sparql_builder_object_boolean (sparql, FALSE); - - uri = g_object_get_data (G_OBJECT (channel), "subject"); - tracker_sparql_builder_predicate (sparql, "nmo:communicationChannel"); - tracker_sparql_builder_object_iri (sparql, uri); - - tmp_string = grss_feed_item_get_copyright (item); - if (tmp_string) { - tracker_sparql_builder_predicate (sparql, "nie:copyright"); - tracker_sparql_builder_object_unvalidated (sparql, tmp_string); - } - - list = grss_feed_item_get_categories (item); - for (l = list; l; l = l->next) { - tracker_sparql_builder_predicate (sparql, "nie:keyword"); - tracker_sparql_builder_object_unvalidated (sparql, l->data); - } - - tracker_sparql_builder_insert_close (sparql); - - g_list_free_full (contrib_aliases, g_free); - g_list_free_full (enclosure_aliases, g_free); - g_hash_table_destroy (websites); - g_hash_table_destroy (enclosure_urls); - - return sparql; -} - -static void -feed_channel_content_update_cb (GObject *source, - GAsyncResult *result, - gpointer user_data) -{ - TrackerSparqlConnection *connection; - GPtrArray *errors, *array = user_data; - GError *error = NULL; - guint i; - - connection = TRACKER_SPARQL_CONNECTION (source); - errors = tracker_sparql_connection_update_array_finish (connection, - result, &error); - - if (error) { - g_warning ("Could not update feed items: %s", - error->message); - g_error_free (error); - } else { - for (i = 0; i < errors->len; i++) { - GError *error = g_ptr_array_index (errors, i); - - if (!error) - continue; - - g_warning ("Error in item %d of update: %s\nQuery: %s", i, - error->message, (gchar *) g_ptr_array_index (array, i)); - } - - g_ptr_array_unref (errors); - } - - g_ptr_array_unref (array); -} - -static void -check_feed_items_cb (GObject *source_object, - GAsyncResult *res, - gpointer user_data) -{ - TrackerSparqlConnection *connection; - TrackerSparqlBuilder *sparql; - FeedItemListInsertData *data; - TrackerSparqlCursor *cursor; - GrssFeedItem *item; - GError *error = NULL; - GHashTableIter iter; - GPtrArray *array; - const gchar *str; - - data = user_data; - connection = TRACKER_SPARQL_CONNECTION (source_object); - cursor = tracker_sparql_connection_query_finish (connection, res, &error); - array = g_ptr_array_new_with_free_func ((GDestroyNotify) g_free); - - while (!error && tracker_sparql_cursor_next (cursor, NULL, &error)) { - const gchar *urn, *url, *date; - time_t time; - - urn = tracker_sparql_cursor_get_string (cursor, 0, NULL); - url = tracker_sparql_cursor_get_string (cursor, 1, NULL); - date = tracker_sparql_cursor_get_string (cursor, 2, NULL); - time = (time_t) tracker_string_to_date (date, NULL, NULL); - - item = g_hash_table_lookup (data->items, url); - - if (!item) - continue; - - if (time <= grss_feed_item_get_publish_time (item)) { - g_debug ("Item '%s' already up to date", url); - g_ptr_array_add (array, feed_message_create_update_channel_query (urn, item)); - } else { - g_debug ("Updating item '%s'", url); - - g_ptr_array_add (array, feed_message_create_delete_properties_query (urn)); - - sparql = feed_message_create_insert_builder (data->miner, - item, urn); - str = tracker_sparql_builder_get_result (sparql); - g_ptr_array_add (array, g_strdup (str)); - g_object_unref (sparql); - } - - g_hash_table_remove (data->items, url); - } - - if (cursor) { - g_object_unref (cursor); - } - - if (error) { - g_message ("Could check feed items, %s", error->message); - g_error_free (error); - feed_item_list_insert_data_free (data); - g_ptr_array_unref (array); - return; - } - - g_hash_table_iter_init (&iter, data->items); - - /* Insert all remaining items as new */ - while (g_hash_table_iter_next (&iter, NULL, (gpointer *) &item)) { - sparql = feed_message_create_insert_builder (data->miner, - item, NULL); - str = tracker_sparql_builder_get_result (sparql); - g_ptr_array_add (array, g_strdup (str)); - g_object_unref (sparql); - } - - if (array->len == 0) { - feed_item_list_insert_data_free (data); - g_ptr_array_unref (array); - return; - } - - tracker_sparql_connection_update_array_async (tracker_miner_get_connection (TRACKER_MINER (data->miner)), - (gchar **) array->pdata, - array->len, - G_PRIORITY_DEFAULT, NULL, - feed_channel_content_update_cb, - array); - feed_channel_change_updated_time (data->miner, data->channel); - feed_item_list_insert_data_free (data); -} - -static void -check_feed_items (TrackerMinerRSS *miner, - GrssFeedChannel *channel, - GList *items) -{ - FeedItemListInsertData *data; - GHashTableIter iter; - GrssFeedItem *item; - gboolean first = TRUE; - const gchar *url; - GString *query; - - data = feed_item_list_insert_data_new (miner, channel, items); - g_hash_table_iter_init (&iter, data->items); - - query = g_string_new ("SELECT ?msg nie:url(?msg)" - " nie:contentCreated(?msg) {" - " ?msg a rdfs:Resource ." - " FILTER (nie:url(?msg) IN ("); - - while (g_hash_table_iter_next (&iter, NULL, (gpointer *) &item)) { - if (!first) - g_string_append_c (query, ','); - - url = get_message_url (item); - g_string_append_printf (query, "\"%s\"", url); - first = FALSE; - } - - g_string_append (query, "))}"); - - tracker_sparql_connection_query_async (tracker_miner_get_connection (TRACKER_MINER (miner)), - query->str, NULL, - check_feed_items_cb, - data); - g_string_free (query, TRUE); -} - -static void -update_feed_channel_info (TrackerMinerRSS *miner, - GrssFeedChannel *channel) -{ - const gchar *subject, *str; - GString *update; - gchar *escaped; - time_t time; - - g_debug ("Updating mfo:FeedChannel for '%s'", - grss_feed_channel_get_title (channel)); - - subject = g_object_get_data (G_OBJECT (channel), "subject"); - update = g_string_new ("INSERT OR REPLACE { "); - - str = grss_feed_channel_get_title (channel); - if (str) { - escaped = tracker_sparql_escape_string (str); - g_string_append_printf (update, "<%s> nie:title \"%s\".", subject, escaped); - g_free (escaped); - } - - str = grss_feed_channel_get_format (channel); - if (str) { - escaped = tracker_sparql_escape_string (str); - g_string_append_printf (update, - "<%s> mfo:type [ a mfo:FeedType ;" - " mfo:name \"%s\"].", - subject, escaped); - g_free (escaped); - } - - str = grss_feed_channel_get_description (channel); - if (str) { - escaped = tracker_sparql_escape_string (str); - g_string_append_printf (update, "<%s> nie:description \"%s\".", subject, escaped); - g_free (escaped); - } - - str = grss_feed_channel_get_image (channel); - if (str) { - g_string_append_printf (update, "<%s> mfo:image \"%s\".", subject, str); - } - - str = grss_feed_channel_get_copyright (channel); - if (str) { - escaped = tracker_sparql_escape_string (str); - g_string_append_printf (update, "<%s> nie:copyright \"%s\".", subject, escaped); - g_free (escaped); - } - - time = grss_feed_channel_get_publish_time (channel); - - if (time != 0) { - escaped = tracker_date_to_string (time); - g_string_append_printf (update, "<%s> nmo:lastMessageDate \"%s\".", subject, escaped); - g_free (escaped); - } - - g_string_append (update, "}"); - - tracker_sparql_connection_update_async (tracker_miner_get_connection (TRACKER_MINER (miner)), - update->str, G_PRIORITY_DEFAULT, - NULL, NULL, NULL); - g_string_free (update, TRUE); -} - -static void -feed_ready_cb (GrssFeedsPool *pool, - GrssFeedChannel *channel, - GList *items, - gpointer user_data) -{ - TrackerMinerRSS *miner; - TrackerMinerRSSPrivate *priv; - - miner = TRACKER_MINER_RSS (user_data); - priv = TRACKER_MINER_RSS_GET_PRIVATE (miner); - - priv->now_fetching--; - - g_debug ("Feed fetched, %d remaining", priv->now_fetching); - - if (priv->now_fetching <= 0) { - priv->now_fetching = 0; - g_object_set (miner, "progress", 1.0, "status", "Idle", NULL); - } - - if (items == NULL) { - return; - } - - update_feed_channel_info (miner, channel); - - g_message ("Verifying channel:'%s' is up to date", - grss_feed_channel_get_title (channel)); - - check_feed_items (miner, channel, items); -} - -static void -feeds_retrieve_cb (GObject *source_object, - GAsyncResult *res, - gpointer user_data) -{ - GList *channels; - TrackerSparqlCursor *cursor; - GError *error = NULL; - TrackerMinerRSSPrivate *priv; - GrssFeedChannel *chan; - - priv = TRACKER_MINER_RSS_GET_PRIVATE (user_data); - cursor = tracker_sparql_connection_query_finish (TRACKER_SPARQL_CONNECTION (source_object), - res, - &error); - - if (error != NULL) { - g_message ("Could not retrieve feeds, %s", error->message); - g_error_free (error); - if (cursor) { - g_object_unref (cursor); - } - return; - } - - while (tracker_sparql_cursor_next (cursor, NULL, NULL)) { - const gchar *source; - const gchar *title; - const gchar *interval; - const gchar *subject; - gint mins, id; - - source = tracker_sparql_cursor_get_string (cursor, 0, NULL); - title = tracker_sparql_cursor_get_string (cursor, 1, NULL); - interval = tracker_sparql_cursor_get_string (cursor, 2, NULL); - subject = tracker_sparql_cursor_get_string (cursor, 3, NULL); - id = tracker_sparql_cursor_get_integer (cursor, 4); - - g_debug ("Indexing channel '%s'", source); - - if (g_hash_table_lookup (priv->channels, GINT_TO_POINTER (id))) - continue; - - chan = grss_feed_channel_new (); - g_object_set_data_full (G_OBJECT (chan), - "subject", - g_strdup (subject), - g_free); - grss_feed_channel_set_source (chan, g_strdup (source)); - - /* TODO How to manage feeds with an update mfo:updateInterval == 0 ? - * Here the interval is forced to be at least 1 minute, but perhaps those - * elements are to be considered "disabled" - */ - mins = strtoull (interval, NULL, 10); - if (mins <= 0) - mins = 1; - grss_feed_channel_set_update_interval (chan, mins); - - g_message (" '%s' (%s) - update interval of %s minutes", - title, - source, - interval); - - g_hash_table_insert (priv->channels, GINT_TO_POINTER (id), chan); - } - - if (g_hash_table_size (priv->channels) == 0) { - g_message ("No feeds set up, nothing more to do"); - } - - channels = g_hash_table_get_values (priv->channels); - grss_feeds_pool_listen (priv->pool, channels); - g_list_free (channels); - - g_object_unref (cursor); - - if (g_hash_table_size (priv->channels) == 0) { - g_object_set (user_data, "progress", 1.0, "status", "Idle", NULL); - } -} - -static void -retrieve_and_schedule_feeds (TrackerMinerRSS *miner, - GArray *channel_ids) -{ - GString *sparql; - gint i, id; - - g_message ("Retrieving and scheduling feeds..."); - - sparql = g_string_new ("SELECT ?url nie:title(?urn) ?interval ?urn tracker:id(?urn)" - "WHERE {" - " ?urn a mfo:FeedChannel ; " - " mfo:feedSettings ?settings ; " - " nie:url ?url . " - " ?settings mfo:updateInterval ?interval "); - - if (channel_ids && channel_ids->len > 0) { - g_string_append (sparql, ". FILTER (tracker:id(?urn) IN ("); - - for (i = 0; i < channel_ids->len; i++) { - id = g_array_index (channel_ids, gint, i); - if (i != 0) - g_string_append (sparql, ","); - g_string_append_printf (sparql, "%d", id); - } - - g_string_append (sparql, "))"); - } - - g_string_append_printf (sparql, "}"); - - tracker_sparql_connection_query_async (tracker_miner_get_connection (TRACKER_MINER (miner)), - sparql->str, - NULL, - feeds_retrieve_cb, - miner); - g_string_free (sparql, TRUE); -} - -static const gchar * -get_message_url (GrssFeedItem *item) -{ - const gchar *url; - - grss_feed_item_get_real_source (item, &url, NULL); - if (url == NULL) - url = grss_feed_item_get_source (item); - return url; -} - -static void -miner_started (TrackerMiner *miner) -{ - TrackerMinerRSSPrivate *priv; - TrackerSparqlConnection *conn; - - g_object_set (miner, "progress", 0.0, "status", "Initializing", NULL); - - priv = TRACKER_MINER_RSS_GET_PRIVATE (miner); - conn = tracker_miner_get_connection (miner); - priv->rdf_type_id = get_class_id (conn, "rdf:type"); - priv->mfo_feed_channel_id = get_class_id (conn, "mfo:FeedChannel"); - retrieve_and_schedule_feeds (TRACKER_MINER_RSS (miner), NULL); - grss_feeds_pool_switch (priv->pool, TRUE); -} - -static void -miner_stopped (TrackerMiner *miner) -{ - TrackerMinerRSSPrivate *priv; - - priv = TRACKER_MINER_RSS_GET_PRIVATE (miner); - grss_feeds_pool_switch (priv->pool, FALSE); - g_object_set (miner, "progress", 1.0, "status", "Idle", NULL); -} - -static void -miner_paused (TrackerMiner *miner) -{ - TrackerMinerRSSPrivate *priv; - - priv = TRACKER_MINER_RSS_GET_PRIVATE (miner); - grss_feeds_pool_switch (priv->pool, FALSE); - - /* Save last status */ - g_free (priv->last_status); - g_object_get (miner, "status", &priv->last_status, NULL); - - /* Set paused */ - g_object_set (miner, "status", "Paused", NULL); -} - -static void -miner_resumed (TrackerMiner *miner) -{ - TrackerMinerRSSPrivate *priv; - - priv = TRACKER_MINER_RSS_GET_PRIVATE (miner); - grss_feeds_pool_switch (priv->pool, TRUE); - - /* Resume */ - g_object_set (miner, "status", priv->last_status ? priv->last_status : "Idle", NULL); -} - -TrackerMinerRSS * -tracker_miner_rss_new (GError **error) -{ - return g_initable_new (TRACKER_TYPE_MINER_RSS, - NULL, - error, - NULL); -} |