From fc16447467cc1e049d3626334e12aa5946f3190c Mon Sep 17 00:00:00 2001 From: Adrien Bustany Date: Thu, 1 Apr 2010 16:30:51 -0300 Subject: Twitter miner: move to miners folder and add data files --- configure.ac | 2 +- data/dbus/Makefile.am | 7 + ....freedesktop.Tracker1.Miner.Identica.service.in | 3 + ...g.freedesktop.Tracker1.Miner.Twitter.service.in | 3 + data/icons/scalable/Makefile.am | 10 + data/icons/scalable/tracker-miner-identica.svg | 207 ++++++++++ data/icons/scalable/tracker-miner-twitter.svg | 185 +++++++++ data/miners/Makefile.am | 15 + data/miners/tracker-miner-flickr.desktop.in.in | 8 + data/miners/tracker-miner-identica.desktop.in.in | 8 + data/miners/tracker-miner-twitter.desktop.in.in | 8 + src/Makefile.am | 4 - src/miners/Makefile.am | 4 + src/miners/twitter/Makefile.am | 64 +++ src/miners/twitter/query-queue.vala | 58 +++ src/miners/twitter/tracker-miner-twitter.vala | 441 +++++++++++++++++++++ src/tracker-miner-twitter/Makefile.am | 64 --- src/tracker-miner-twitter/query-queue.vala | 58 --- .../tracker-miner-twitter.vala | 441 --------------------- 19 files changed, 1022 insertions(+), 568 deletions(-) create mode 100644 data/dbus/org.freedesktop.Tracker1.Miner.Identica.service.in create mode 100644 data/dbus/org.freedesktop.Tracker1.Miner.Twitter.service.in create mode 100644 data/icons/scalable/tracker-miner-identica.svg create mode 100644 data/icons/scalable/tracker-miner-twitter.svg create mode 100644 data/miners/tracker-miner-flickr.desktop.in.in create mode 100644 data/miners/tracker-miner-identica.desktop.in.in create mode 100644 data/miners/tracker-miner-twitter.desktop.in.in create mode 100644 src/miners/twitter/Makefile.am create mode 100644 src/miners/twitter/query-queue.vala create mode 100644 src/miners/twitter/tracker-miner-twitter.vala delete mode 100644 src/tracker-miner-twitter/Makefile.am delete mode 100644 src/tracker-miner-twitter/query-queue.vala delete mode 100644 src/tracker-miner-twitter/tracker-miner-twitter.vala diff --git a/configure.ac b/configure.ac index df0292d16..98792e8a0 100644 --- a/configure.ac +++ b/configure.ac @@ -1707,12 +1707,12 @@ AC_CONFIG_FILES([ src/miners/Makefile src/miners/fs/Makefile src/miners/rss/Makefile + src/miners/twitter/Makefile src/tracker-status-icon/Makefile src/tracker-status-icon/tracker-status-icon.desktop.in src/tracker-store/Makefile src/tracker-control/Makefile src/tracker-extract/Makefile - src/tracker-miner-twitter/Makefile src/tracker-preferences/Makefile src/tracker-preferences/tracker-preferences.desktop.in src/tracker-search-bar/Makefile diff --git a/data/dbus/Makefile.am b/data/dbus/Makefile.am index 64a6f7cd9..f5e2288ec 100644 --- a/data/dbus/Makefile.am +++ b/data/dbus/Makefile.am @@ -23,6 +23,13 @@ service_in_files = \ org.freedesktop.Tracker1.Miner.Applications.service.in \ org.freedesktop.Tracker1.Miner.Files.service.in \ org.freedesktop.Tracker1.Extract.service.in + +if HAVE_MINER_TWITTER +service_in_files += \ + org.freedesktop.Tracker1.Miner.Twitter.service.in \ + org.freedesktop.Tracker1.Miner.Identica.service.in +endif + service_DATA = $(service_in_files:.service.in=.service) %.service: %.service.in diff --git a/data/dbus/org.freedesktop.Tracker1.Miner.Identica.service.in b/data/dbus/org.freedesktop.Tracker1.Miner.Identica.service.in new file mode 100644 index 000000000..3888f8ff6 --- /dev/null +++ b/data/dbus/org.freedesktop.Tracker1.Miner.Identica.service.in @@ -0,0 +1,3 @@ +[D-BUS Service] +Name=org.freedesktop.Tracker1.Miner.Identica +Exec=@libexecdir@/tracker-miner-twitter diff --git a/data/dbus/org.freedesktop.Tracker1.Miner.Twitter.service.in b/data/dbus/org.freedesktop.Tracker1.Miner.Twitter.service.in new file mode 100644 index 000000000..1e4d0c288 --- /dev/null +++ b/data/dbus/org.freedesktop.Tracker1.Miner.Twitter.service.in @@ -0,0 +1,3 @@ +[D-BUS Service] +Name=org.freedesktop.Tracker1.Miner.Twitter +Exec=@libexecdir@/tracker-miner-twitter diff --git a/data/icons/scalable/Makefile.am b/data/icons/scalable/Makefile.am index 558300850..43aa6aef1 100644 --- a/data/icons/scalable/Makefile.am +++ b/data/icons/scalable/Makefile.am @@ -3,4 +3,14 @@ include $(top_srcdir)/Makefile.decl icondir = $(datadir)/icons/hicolor/scalable/apps icon_DATA = tracker.svg +minericonsdir = $(datadir)/tracker/icons +minericons_DATA = + + +if HAVE_MINER_TWITTER +minericons_DATA += \ + tracker-miner-twitter.svg \ + tracker-miner-identica.svg +endif + EXTRA_DIST = $(icon_DATA) diff --git a/data/icons/scalable/tracker-miner-identica.svg b/data/icons/scalable/tracker-miner-identica.svg new file mode 100644 index 000000000..1ea84d73b --- /dev/null +++ b/data/icons/scalable/tracker-miner-identica.svg @@ -0,0 +1,207 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + Jakub Szypulka + + + + + Jakub Szypulka + + + + + Jakub Szypulka + + + + + + + + + + + + + diff --git a/data/icons/scalable/tracker-miner-twitter.svg b/data/icons/scalable/tracker-miner-twitter.svg new file mode 100644 index 000000000..d94d8b5e7 --- /dev/null +++ b/data/icons/scalable/tracker-miner-twitter.svg @@ -0,0 +1,185 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + Jakub Szypulka + + + + + Jakub Szypulka + + + + + Jakub Szypulka + + + + + + + + + + + diff --git a/data/miners/Makefile.am b/data/miners/Makefile.am index e76837538..570d2e9a6 100644 --- a/data/miners/Makefile.am +++ b/data/miners/Makefile.am @@ -5,6 +5,12 @@ desktop_in_files = \ tracker-miner-files.desktop.in \ tracker-miner-rss.desktop.in +if HAVE_MINER_TWITTER +desktop_in_files += \ + tracker-miner-twitter.desktop.in.in \ + tracker-miner-identica.desktop.in.in +endif + tracker_minersdir = $(datadir)/tracker/miners tracker_miners_DATA = \ @@ -15,7 +21,16 @@ if USING_MINER_RSS tracker_miners_DATA += tracker-miner-rss.desktop endif +if HAVE_MINER_TWITTER +tracker_miners_DATA += \ + tracker-miner-twitter.desktop \ + tracker-miner-identica.desktop +endif + @INTLTOOL_DESKTOP_RULE@ +%.desktop.in: %.desktop.in.in + @sed -e "s|@datadir[@]|$(datadir)|" $< > $@ + EXTRA_DIST = $(desktop_in_files) CLEANFILES = $(tracker_miners_DATA) diff --git a/data/miners/tracker-miner-flickr.desktop.in.in b/data/miners/tracker-miner-flickr.desktop.in.in new file mode 100644 index 000000000..a9f1ae9ab --- /dev/null +++ b/data/miners/tracker-miner-flickr.desktop.in.in @@ -0,0 +1,8 @@ +[Desktop Entry] +Encoding=UTF-8 +_Name=Flickr +_Comment=Index your Flickr photo albums +Icon=@datadir@/tracker/icons/tracker-miner-flickr.svg +DBusName=org.freedesktop.Tracker1.Miner.Flickr +DBusPath=/org/freedesktop/Tracker1/Miner/Flickr +AuthenticationMethod=Token diff --git a/data/miners/tracker-miner-identica.desktop.in.in b/data/miners/tracker-miner-identica.desktop.in.in new file mode 100644 index 000000000..5f0a37e71 --- /dev/null +++ b/data/miners/tracker-miner-identica.desktop.in.in @@ -0,0 +1,8 @@ +[Desktop Entry] +Encoding=UTF-8 +_Name=Identi.ca +_Comment=Index your Identi.ca messages +Icon=@datadir@/tracker/icons/tracker-miner-identica.svg +DBusName=org.freedesktop.Tracker1.Miner.Identica +DBusPath=/org/freedesktop/Tracker1/Miner/Identica +AuthenticationMethod=Token diff --git a/data/miners/tracker-miner-twitter.desktop.in.in b/data/miners/tracker-miner-twitter.desktop.in.in new file mode 100644 index 000000000..2ba124a39 --- /dev/null +++ b/data/miners/tracker-miner-twitter.desktop.in.in @@ -0,0 +1,8 @@ +[Desktop Entry] +Encoding=UTF-8 +_Name=Twitter +_Comment=Index your tweets +Icon=@datadir@/tracker/icons/tracker-miner-twitter.svg +DBusName=org.freedesktop.Tracker1.Miner.Twitter +DBusPath=/org/freedesktop/Tracker1/Miner/Twitter +AuthenticationMethod=Token diff --git a/src/Makefile.am b/src/Makefile.am index d0ed2ce82..cc018d0cd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -42,7 +42,3 @@ endif if HAVE_TRACKER_EXPLORER SUBDIRS += tracker-explorer endif - -if HAVE_MINER_TWITTER -SUBDIRS += tracker-miner-twitter -endif diff --git a/src/miners/Makefile.am b/src/miners/Makefile.am index f6373b6e7..03f7f9a43 100644 --- a/src/miners/Makefile.am +++ b/src/miners/Makefile.am @@ -5,3 +5,7 @@ SUBDIRS = fs if USING_MINER_RSS SUBDIRS += rss endif + +if HAVE_MINER_TWITTER +SUBDIRS += twitter +endif diff --git a/src/miners/twitter/Makefile.am b/src/miners/twitter/Makefile.am new file mode 100644 index 000000000..543fd76f4 --- /dev/null +++ b/src/miners/twitter/Makefile.am @@ -0,0 +1,64 @@ +include $(top_srcdir)/Makefile.decl + +INCLUDES = \ + -Wall \ + -DSHAREDIR=\""$(datadir)"\" \ + -DPKGLIBDIR=\""$(libdir)/tracker"\" \ + -DLOCALEDIR=\""$(localedir)"\" \ + -DLIBEXEC_PATH=\""$(libexecdir)"\" \ + -DG_LOG_DOMAIN=\"Tracker\" \ + -DTRACKER_COMPILATION \ + -I$(top_srcdir)/src \ + $(WARN_CFLAGS) \ + $(GMODULE_CFLAGS) \ + $(PANGO_CFLAGS) \ + $(DBUS_CFLAGS) \ + $(MINER_TWITTER_CFLAGS) \ + $(GCOV_CFLAGS) + +VALAFLAGS = \ + --pkg gio-2.0 \ + --pkg twitter-glib-1.0 \ + --pkg posix \ + --thread + +libexec_PROGRAMS = tracker-miner-twitter + +tracker_miner_twitter_VALASOURCES = \ + tracker-miner-twitter.vala \ + query-queue.vala \ + $(top_srcdir)/src/libtracker-client/tracker-client-0.7.vapi + +tracker_miner_twitter_SOURCES = \ + $(tracker_miner_twitter_VALASOURCES:.vala=.c) + +tracker_miner_twitter_LDADD = \ + $(top_builddir)/src/libtracker-miner/libtracker-miner-@TRACKER_API_VERSION@.la \ + $(top_builddir)/src/libtracker-client/libtracker-client-@TRACKER_API_VERSION@.la \ + $(DBUS_LIBS) \ + $(GMODULE_LIBS) \ + $(GTHREAD_LIBS) \ + $(GIO_LIBS) \ + $(GCOV_LIBS) \ + $(GLIB2_LIBS) \ + $(MINER_TWITTER_LIBS) \ + -lz \ + -lm + +vapi_sources = \ + $(top_srcdir)/src/libtracker-miner/tracker-miner-@TRACKER_API_VERSION@.vapi + +tracker-miner-twitter.vala.stamp: $(tracker_miner_twitter_VALASOURCES) $(vapi_sources) + $(AM_V_GEN)$(VALAC) $(GCOV_VALAFLAGS) -C $(VALAFLAGS) $^ + touch $@ + + +BUILT_SOURCES = tracker-miner-twitter.vala.stamp + +MAINTAINERCLEANFILES = \ + tracker-miner-twitter.vala.stamp \ + $(tracker_miner_twitter_VALASOURCES:.vala=.c) + +EXTRA_DIST = \ + $(tracker_miner_twitter_VALASOURCES) \ + tracker-miner-twitter.vala.stamp diff --git a/src/miners/twitter/query-queue.vala b/src/miners/twitter/query-queue.vala new file mode 100644 index 000000000..ea3ac7393 --- /dev/null +++ b/src/miners/twitter/query-queue.vala @@ -0,0 +1,58 @@ +public class QueryQueue : GLib.Object { + /* Holds the pending sparql updates and monitors them */ + private HashTable queue; + private uint cookie; + + private Mutex flush_mutex; + + private Tracker.Miner miner; + + public QueryQueue (Tracker.Miner parent) { + miner = parent; + + queue = new HashTable (direct_hash, direct_equal); + cookie = 0; + + flush_mutex = new Mutex (); + } + + public async void append (string query) { + uint current_cookie = cookie ++; + queue.insert (current_cookie, query); + + message ("SPARQL query: %s", query); + + try { + yield miner.execute_batch_update (query); + } catch (Error tracker_error) { + warning ("BatchUpdate query failed: %s", tracker_error.message); + } + + queue.remove (current_cookie); + } + + /* BLOCKING flush */ + public void flush () { + if (!flush_mutex.trylock ()) { + message ("There's already a flush taking place"); + return; + } + + if (queue.size () > 0) { + MainLoop wait_loop; + try { + wait_loop = new MainLoop (null, false); + miner.commit (null, () => { wait_loop.quit (); }); + wait_loop.run (); + } catch (Error tracker_error) { + warning ("Commit query failed: %s", tracker_error.message); + } + } + + flush_mutex.unlock (); + } + + public uint size () { + return queue.size (); + } +} diff --git a/src/miners/twitter/tracker-miner-twitter.vala b/src/miners/twitter/tracker-miner-twitter.vala new file mode 100644 index 000000000..b919af83b --- /dev/null +++ b/src/miners/twitter/tracker-miner-twitter.vala @@ -0,0 +1,441 @@ +/* + * Copyright (C) 2010, Adrien Bustany + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +namespace Tracker { + +public class MinerTwitter : Tracker.MinerWeb { + public Twitter.Provider provider { get; construct; } + private string friendly_name; /* Human readable name of the provider */ + private string channel_urn = null; + + private Twitter.Client service; + + private static const string DATASOURCE_URN = "urn:103e7d6e-2334-4cd2-b0a5-f1b0c8bb10ef"; + private static const string SERVICE_DESCRIPTION = "Tracker miner for %s"; + + private static const uint PULL_INTERVAL = 60; /* seconds */ + private uint pull_timeout_handle; + + private QueryQueue query_queue; + + /* used to store state information like last pulled tweet*/ + private static const string STATE_FILE_NAME = "tracker-miner-%s.state"; + private static const string STATE_FILE_GROUP = "General"; + private KeyFile state_file; + + private uint last_status_timestamp; + + private static MainLoop main_loop; + + + construct { + if (provider == Twitter.Provider.DEFAULT_PROVIDER) { + set ("name", "Twitter"); + friendly_name = "Twitter"; + } else if (provider == Twitter.Provider.IDENTI_CA) { + set ("name", "Identica"); + friendly_name = "Identi.ca"; + } else { + assert_not_reached (); + } + + set ("association-status", MinerWebAssociationStatus.UNASSOCIATED); + + service = new Twitter.Client.full (provider, null, null, null); + service.status_received.connect (status_received_cb); + service.timeline_complete.connect (timeline_complete_cb); + + state_file = new KeyFile (); + load_state_file (); + + pull_timeout_handle = 0; + this.notify["association-status"].connect (association_status_changed); + + query_queue = new QueryQueue (this); + + init_feed_channel (); + } + + public void shutdown () { + set ("association-status", MinerWebAssociationStatus.UNASSOCIATED); + save_state_file (); + } + + private async void init_feed_channel () { + string sparql; + unowned PtrArray results; + + sparql = "select ?c where { ?c a mfo:FeedChannel ; mfo:type ?type ." + + " ?type mfo:name ?typeName . " + + " FILTER (?typeName = \"%s\") }".printf (friendly_name); + + try { + results = yield execute_sparql (sparql); + + if (results.len == 0) { + /* No optimal, but we're waiting for blank support in TrackerMiner */ + sparql = "insert { _:channel a mfo:FeedChannel ; rdfs:label \"%s home timeline\";".printf (friendly_name) + + " mfo:type [ a mfo:FeedType ; mfo:name \"%s\" ] }".printf (friendly_name); + yield execute_update (sparql); + sparql = "select ?c where { ?c a mfo:FeedChannel ; mfo:type ?type ." + + " ?type mfo:name ?typeName . " + + " FILTER (?typeName = \"%s\") }".printf (friendly_name); + results = yield execute_sparql (sparql); + } + + assert (results.len == 1); + channel_urn = ((string[][])results.pdata)[0][0]; + } catch (Error tracker_error) { + critical ("Couldn't initialize feed channel: %s", tracker_error.message); + } + } + + private void status_received_cb (ulong handle, Twitter.Status? status, Error error) { + SparqlBuilder builder; + string name; + TimeVal tv = TimeVal (); + + get ("name", out name); + + if (error != null) { + if (!(error is Twitter.Error.NOT_MODIFIED)) { + warning ("An error occurred while pulling statuses: %s", error.message); + } + return; + } + + builder = new SparqlBuilder.update (); + + builder.insert_open (status.url); + builder.subject ("_:author"); + builder.predicate ("a"); + builder.object ("nco:PersonContact"); + builder.predicate ("nco:fullname"); + builder.object_string (status.user.name); + builder.predicate ("nco:photo"); + builder.object_blank_open (); + builder.predicate ("a"); + builder.object ("nfo:RemoteDataObject"); + builder.predicate ("nie:url"); + builder.object_string (status.user.profile_image_url); + builder.object_blank_close (); /* nco:photo */ + + builder.subject ("_:message"); + builder.predicate ("a"); + builder.object ("mfo:FeedMessage"); + + builder.predicate ("a"); + builder.object ("nfo:RemoteDataObject"); + builder.predicate ("nie:url"); + builder.object_string (status.url); + + builder.predicate ("nmo:communicationChannel"); + builder.object_iri (channel_urn); + + builder.predicate ("nmo:messageId"); + builder.object_string (rdf_message_id (status.id)); + + builder.predicate ("nmo:from"); + builder.object ("_:author"); + + builder.predicate ("nie:plainTextContent"); + builder.object_string (status.text); + + if (status.reply_to_status != 0) { + builder.predicate ("nmo:inReplyTo"); + builder.object_blank_open (); + builder.predicate ("a"); + builder.object ("mfo:FeedMessage"); + builder.predicate ("nmo:communicationChannel"); + builder.object_iri (channel_urn); + builder.predicate ("nmo:messageId"); + builder.object_string (rdf_message_id (status.reply_to_status)); + builder.object_blank_close (); + } + + if (Twitter.date_to_time_val (status.created_at, out tv)) { + builder.predicate ("nmo:receivedDate"); + builder.object_string (tv.to_iso8601 ()); + + /* We receive the status in chronological order */ + last_status_timestamp = (int)tv.tv_sec; + } + + tv.get_current_time (); + builder.predicate ("mfo:downloadedTime"); + builder.object_string (tv.to_iso8601 ()); + + builder.insert_close (); + + query_queue.append (builder.get_result ()); + } + + private void timeline_complete_cb () { + message ("Timeline downloaded"); + + query_queue.flush (); + state_file.set_integer (STATE_FILE_GROUP, "since", (int)last_status_timestamp); + } + + private string rdf_message_id (uint status_id) + { + string name; + + get ("name", out name); + return "feed:%s:%u".printf (name, status_id); + } + + private void association_status_changed (Object source, ParamSpec pspec) { + MinerWebAssociationStatus status; + + get ("association-status", out status); + + switch (status) { + case MinerWebAssociationStatus.ASSOCIATED: + if (pull_timeout_handle != 0) + return; + + message ("Miner is now associated. Initiating periodic pull."); + pull_timeout_handle = Timeout.add_seconds (PULL_INTERVAL, pull_timeout_cb); + Idle.add ( () => { pull_timeout_cb (); return false; }); + break; + case MinerWebAssociationStatus.UNASSOCIATED: + if (pull_timeout_handle == 0) + return; + + Source.remove (pull_timeout_handle); + break; + } + } + + private bool pull_timeout_cb () { + int since; + + if (channel_urn == null) { + message ("Feed channel not initialized yet, skipping this cycle"); + return true; + } + + message ("Pulling new data"); + try { + since = state_file.get_integer (STATE_FILE_GROUP, "since"); + } catch (Error error) { + critical ("Cannot load config variable: %s", error.message); + return true; + } + + service.get_friends_timeline ("", since); + return true; + } + + private void load_state_file () { + string name; + get ("name", out name); + + try { + state_file.load_from_file (Path.build_filename (Environment.get_user_cache_dir (), + "tracker", + STATE_FILE_NAME.printf (name)), + KeyFileFlags.NONE); + } catch (Error error) { + message ("Couldn't load the state file"); + } + + + try { + state_file.get_integer (STATE_FILE_GROUP, "since"); + } catch (Error error) { + state_file.set_integer (STATE_FILE_GROUP, "since", 0); + } + } + + private void save_state_file () { + string name; + string file_path; + + get ("name", out name); + file_path = Path.build_filename (Environment.get_user_cache_dir (), + "tracker", + STATE_FILE_NAME.printf (name)); + + try { + FileUtils.set_contents (file_path, state_file.to_data ()); + } catch (Error error) { + warning ("Couldn't save state file: %s", error.message); + } + } + + // If we don't protect the function with a mutex, it could be called by the + // inner loop, starting at inner-inner one, and so on... + private Mutex authenticate_mutex = new Mutex (); + public override void authenticate () throws MinerWebError { + PasswordProvider password_provider; + string name; + string username; + string password; + bool verified = false; + Error twitter_error = null; + + if(!authenticate_mutex.trylock ()) { + warning ("authenticate called while it was still running"); + return; + } + + password_provider = PasswordProvider.get (); + get ("name", out name); + + set ("association-status", MinerWebAssociationStatus.UNASSOCIATED); + + try { + password = password_provider.get_password (name, out username); + } catch (Error e) { + authenticate_mutex.unlock (); + if (e is PasswordProviderError.SERVICE) { + throw new MinerWebError.KEYRING (e.message); + } + if (e is PasswordProviderError.NOTFOUND) { + throw new MinerWebError.NO_CREDENTIALS ("Miner is not associated"); + } + + critical ("Internal error: %s", e.message); + return; + } + + message ("Verifying username and password"); + service.set_user (username, password); + service.verify_user (); + + var wait_loop = new MainLoop (null, false); + service.user_verified.connect ( (h, v, e) => { + verified = v; + twitter_error = e; + wait_loop.quit (); }); + wait_loop.run (); + authenticate_mutex.unlock (); + + if (twitter_error != null) { + throw new MinerWebError.SERVICE (twitter_error.message); + } + + if (!verified) { + throw new MinerWebError.WRONG_CREDENTIALS ("Wrong username and/or password"); + } else { + message ("Authentication sucessful"); + set ("association-status", MinerWebAssociationStatus.ASSOCIATED); + } + + return; + } + + public override void dissociate () throws MinerWebError { + var password_provider = PasswordProvider.get (); + string name; + get ("name", out name); + + try { + password_provider.forget_password (name); + } catch (Error e) { + if (e is PasswordProviderError.SERVICE) { + throw new MinerWebError.KEYRING (e.message); + } + + critical ("Internal error: %s", e.message); + return; + } + + set ("association-status", MinerWebAssociationStatus.UNASSOCIATED); + } + + public override void associate (HashTable association_data) throws Tracker.MinerWebError { + assert (association_data.lookup ("username") != null && association_data.lookup ("password") != null); + + var password_provider = PasswordProvider.get (); + string name; + get ("name", out name); + + try { + password_provider.store_password (name, + SERVICE_DESCRIPTION.printf (name), + association_data.lookup ("username"), + association_data.lookup ("password")); + } catch (Error e) { + if (e is PasswordProviderError.SERVICE) { + throw new MinerWebError.KEYRING (e.message); + } + + critical ("Internal error: %s", e.message); + return; + } + } + + public GLib.HashTable get_association_data () throws Tracker.MinerWebError { + return new HashTable(str_hash, str_equal); + } + + private static bool in_loop = false; + private static void signal_handler (int signo) { + if (in_loop) { + Posix.exit (Posix.EXIT_FAILURE); + } + + switch (signo) { + case Posix.SIGINT: + case Posix.SIGTERM: + in_loop = true; + main_loop.quit (); + break; + } + } + + private static void init_signals () { +#if G_OS_WIN32 +#else + Posix.sigaction_t act = Posix.sigaction_t (); + Posix.sigset_t empty_mask = Posix.sigset_t (); + Posix.sigemptyset (empty_mask); + act.sa_handler = signal_handler; + act.sa_mask = empty_mask; + act.sa_flags = 0; + + Posix.sigaction (Posix.SIGTERM, act, null); + Posix.sigaction (Posix.SIGINT, act, null); +#endif + } + + public static void main (string[] args) { + Environment.set_application_name ("Twitter/Identi.ca tracker miner"); + MinerTwitter twitter_miner; + twitter_miner = Object.new (typeof (MinerTwitter), + "provider", Twitter.Provider.DEFAULT_PROVIDER) as MinerTwitter; + + MinerTwitter identica_miner; + identica_miner = Object.new (typeof (MinerTwitter), + "provider", Twitter.Provider.IDENTI_CA) as MinerTwitter; + + init_signals (); + + main_loop = new MainLoop (null, false); + main_loop.run (); + + twitter_miner.shutdown (); + identica_miner.shutdown (); + } +} + +} // End namespace Tracker diff --git a/src/tracker-miner-twitter/Makefile.am b/src/tracker-miner-twitter/Makefile.am deleted file mode 100644 index 543fd76f4..000000000 --- a/src/tracker-miner-twitter/Makefile.am +++ /dev/null @@ -1,64 +0,0 @@ -include $(top_srcdir)/Makefile.decl - -INCLUDES = \ - -Wall \ - -DSHAREDIR=\""$(datadir)"\" \ - -DPKGLIBDIR=\""$(libdir)/tracker"\" \ - -DLOCALEDIR=\""$(localedir)"\" \ - -DLIBEXEC_PATH=\""$(libexecdir)"\" \ - -DG_LOG_DOMAIN=\"Tracker\" \ - -DTRACKER_COMPILATION \ - -I$(top_srcdir)/src \ - $(WARN_CFLAGS) \ - $(GMODULE_CFLAGS) \ - $(PANGO_CFLAGS) \ - $(DBUS_CFLAGS) \ - $(MINER_TWITTER_CFLAGS) \ - $(GCOV_CFLAGS) - -VALAFLAGS = \ - --pkg gio-2.0 \ - --pkg twitter-glib-1.0 \ - --pkg posix \ - --thread - -libexec_PROGRAMS = tracker-miner-twitter - -tracker_miner_twitter_VALASOURCES = \ - tracker-miner-twitter.vala \ - query-queue.vala \ - $(top_srcdir)/src/libtracker-client/tracker-client-0.7.vapi - -tracker_miner_twitter_SOURCES = \ - $(tracker_miner_twitter_VALASOURCES:.vala=.c) - -tracker_miner_twitter_LDADD = \ - $(top_builddir)/src/libtracker-miner/libtracker-miner-@TRACKER_API_VERSION@.la \ - $(top_builddir)/src/libtracker-client/libtracker-client-@TRACKER_API_VERSION@.la \ - $(DBUS_LIBS) \ - $(GMODULE_LIBS) \ - $(GTHREAD_LIBS) \ - $(GIO_LIBS) \ - $(GCOV_LIBS) \ - $(GLIB2_LIBS) \ - $(MINER_TWITTER_LIBS) \ - -lz \ - -lm - -vapi_sources = \ - $(top_srcdir)/src/libtracker-miner/tracker-miner-@TRACKER_API_VERSION@.vapi - -tracker-miner-twitter.vala.stamp: $(tracker_miner_twitter_VALASOURCES) $(vapi_sources) - $(AM_V_GEN)$(VALAC) $(GCOV_VALAFLAGS) -C $(VALAFLAGS) $^ - touch $@ - - -BUILT_SOURCES = tracker-miner-twitter.vala.stamp - -MAINTAINERCLEANFILES = \ - tracker-miner-twitter.vala.stamp \ - $(tracker_miner_twitter_VALASOURCES:.vala=.c) - -EXTRA_DIST = \ - $(tracker_miner_twitter_VALASOURCES) \ - tracker-miner-twitter.vala.stamp diff --git a/src/tracker-miner-twitter/query-queue.vala b/src/tracker-miner-twitter/query-queue.vala deleted file mode 100644 index ea3ac7393..000000000 --- a/src/tracker-miner-twitter/query-queue.vala +++ /dev/null @@ -1,58 +0,0 @@ -public class QueryQueue : GLib.Object { - /* Holds the pending sparql updates and monitors them */ - private HashTable queue; - private uint cookie; - - private Mutex flush_mutex; - - private Tracker.Miner miner; - - public QueryQueue (Tracker.Miner parent) { - miner = parent; - - queue = new HashTable (direct_hash, direct_equal); - cookie = 0; - - flush_mutex = new Mutex (); - } - - public async void append (string query) { - uint current_cookie = cookie ++; - queue.insert (current_cookie, query); - - message ("SPARQL query: %s", query); - - try { - yield miner.execute_batch_update (query); - } catch (Error tracker_error) { - warning ("BatchUpdate query failed: %s", tracker_error.message); - } - - queue.remove (current_cookie); - } - - /* BLOCKING flush */ - public void flush () { - if (!flush_mutex.trylock ()) { - message ("There's already a flush taking place"); - return; - } - - if (queue.size () > 0) { - MainLoop wait_loop; - try { - wait_loop = new MainLoop (null, false); - miner.commit (null, () => { wait_loop.quit (); }); - wait_loop.run (); - } catch (Error tracker_error) { - warning ("Commit query failed: %s", tracker_error.message); - } - } - - flush_mutex.unlock (); - } - - public uint size () { - return queue.size (); - } -} diff --git a/src/tracker-miner-twitter/tracker-miner-twitter.vala b/src/tracker-miner-twitter/tracker-miner-twitter.vala deleted file mode 100644 index b919af83b..000000000 --- a/src/tracker-miner-twitter/tracker-miner-twitter.vala +++ /dev/null @@ -1,441 +0,0 @@ -/* - * Copyright (C) 2010, Adrien Bustany - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ - -namespace Tracker { - -public class MinerTwitter : Tracker.MinerWeb { - public Twitter.Provider provider { get; construct; } - private string friendly_name; /* Human readable name of the provider */ - private string channel_urn = null; - - private Twitter.Client service; - - private static const string DATASOURCE_URN = "urn:103e7d6e-2334-4cd2-b0a5-f1b0c8bb10ef"; - private static const string SERVICE_DESCRIPTION = "Tracker miner for %s"; - - private static const uint PULL_INTERVAL = 60; /* seconds */ - private uint pull_timeout_handle; - - private QueryQueue query_queue; - - /* used to store state information like last pulled tweet*/ - private static const string STATE_FILE_NAME = "tracker-miner-%s.state"; - private static const string STATE_FILE_GROUP = "General"; - private KeyFile state_file; - - private uint last_status_timestamp; - - private static MainLoop main_loop; - - - construct { - if (provider == Twitter.Provider.DEFAULT_PROVIDER) { - set ("name", "Twitter"); - friendly_name = "Twitter"; - } else if (provider == Twitter.Provider.IDENTI_CA) { - set ("name", "Identica"); - friendly_name = "Identi.ca"; - } else { - assert_not_reached (); - } - - set ("association-status", MinerWebAssociationStatus.UNASSOCIATED); - - service = new Twitter.Client.full (provider, null, null, null); - service.status_received.connect (status_received_cb); - service.timeline_complete.connect (timeline_complete_cb); - - state_file = new KeyFile (); - load_state_file (); - - pull_timeout_handle = 0; - this.notify["association-status"].connect (association_status_changed); - - query_queue = new QueryQueue (this); - - init_feed_channel (); - } - - public void shutdown () { - set ("association-status", MinerWebAssociationStatus.UNASSOCIATED); - save_state_file (); - } - - private async void init_feed_channel () { - string sparql; - unowned PtrArray results; - - sparql = "select ?c where { ?c a mfo:FeedChannel ; mfo:type ?type ." - + " ?type mfo:name ?typeName . " - + " FILTER (?typeName = \"%s\") }".printf (friendly_name); - - try { - results = yield execute_sparql (sparql); - - if (results.len == 0) { - /* No optimal, but we're waiting for blank support in TrackerMiner */ - sparql = "insert { _:channel a mfo:FeedChannel ; rdfs:label \"%s home timeline\";".printf (friendly_name) - + " mfo:type [ a mfo:FeedType ; mfo:name \"%s\" ] }".printf (friendly_name); - yield execute_update (sparql); - sparql = "select ?c where { ?c a mfo:FeedChannel ; mfo:type ?type ." - + " ?type mfo:name ?typeName . " - + " FILTER (?typeName = \"%s\") }".printf (friendly_name); - results = yield execute_sparql (sparql); - } - - assert (results.len == 1); - channel_urn = ((string[][])results.pdata)[0][0]; - } catch (Error tracker_error) { - critical ("Couldn't initialize feed channel: %s", tracker_error.message); - } - } - - private void status_received_cb (ulong handle, Twitter.Status? status, Error error) { - SparqlBuilder builder; - string name; - TimeVal tv = TimeVal (); - - get ("name", out name); - - if (error != null) { - if (!(error is Twitter.Error.NOT_MODIFIED)) { - warning ("An error occurred while pulling statuses: %s", error.message); - } - return; - } - - builder = new SparqlBuilder.update (); - - builder.insert_open (status.url); - builder.subject ("_:author"); - builder.predicate ("a"); - builder.object ("nco:PersonContact"); - builder.predicate ("nco:fullname"); - builder.object_string (status.user.name); - builder.predicate ("nco:photo"); - builder.object_blank_open (); - builder.predicate ("a"); - builder.object ("nfo:RemoteDataObject"); - builder.predicate ("nie:url"); - builder.object_string (status.user.profile_image_url); - builder.object_blank_close (); /* nco:photo */ - - builder.subject ("_:message"); - builder.predicate ("a"); - builder.object ("mfo:FeedMessage"); - - builder.predicate ("a"); - builder.object ("nfo:RemoteDataObject"); - builder.predicate ("nie:url"); - builder.object_string (status.url); - - builder.predicate ("nmo:communicationChannel"); - builder.object_iri (channel_urn); - - builder.predicate ("nmo:messageId"); - builder.object_string (rdf_message_id (status.id)); - - builder.predicate ("nmo:from"); - builder.object ("_:author"); - - builder.predicate ("nie:plainTextContent"); - builder.object_string (status.text); - - if (status.reply_to_status != 0) { - builder.predicate ("nmo:inReplyTo"); - builder.object_blank_open (); - builder.predicate ("a"); - builder.object ("mfo:FeedMessage"); - builder.predicate ("nmo:communicationChannel"); - builder.object_iri (channel_urn); - builder.predicate ("nmo:messageId"); - builder.object_string (rdf_message_id (status.reply_to_status)); - builder.object_blank_close (); - } - - if (Twitter.date_to_time_val (status.created_at, out tv)) { - builder.predicate ("nmo:receivedDate"); - builder.object_string (tv.to_iso8601 ()); - - /* We receive the status in chronological order */ - last_status_timestamp = (int)tv.tv_sec; - } - - tv.get_current_time (); - builder.predicate ("mfo:downloadedTime"); - builder.object_string (tv.to_iso8601 ()); - - builder.insert_close (); - - query_queue.append (builder.get_result ()); - } - - private void timeline_complete_cb () { - message ("Timeline downloaded"); - - query_queue.flush (); - state_file.set_integer (STATE_FILE_GROUP, "since", (int)last_status_timestamp); - } - - private string rdf_message_id (uint status_id) - { - string name; - - get ("name", out name); - return "feed:%s:%u".printf (name, status_id); - } - - private void association_status_changed (Object source, ParamSpec pspec) { - MinerWebAssociationStatus status; - - get ("association-status", out status); - - switch (status) { - case MinerWebAssociationStatus.ASSOCIATED: - if (pull_timeout_handle != 0) - return; - - message ("Miner is now associated. Initiating periodic pull."); - pull_timeout_handle = Timeout.add_seconds (PULL_INTERVAL, pull_timeout_cb); - Idle.add ( () => { pull_timeout_cb (); return false; }); - break; - case MinerWebAssociationStatus.UNASSOCIATED: - if (pull_timeout_handle == 0) - return; - - Source.remove (pull_timeout_handle); - break; - } - } - - private bool pull_timeout_cb () { - int since; - - if (channel_urn == null) { - message ("Feed channel not initialized yet, skipping this cycle"); - return true; - } - - message ("Pulling new data"); - try { - since = state_file.get_integer (STATE_FILE_GROUP, "since"); - } catch (Error error) { - critical ("Cannot load config variable: %s", error.message); - return true; - } - - service.get_friends_timeline ("", since); - return true; - } - - private void load_state_file () { - string name; - get ("name", out name); - - try { - state_file.load_from_file (Path.build_filename (Environment.get_user_cache_dir (), - "tracker", - STATE_FILE_NAME.printf (name)), - KeyFileFlags.NONE); - } catch (Error error) { - message ("Couldn't load the state file"); - } - - - try { - state_file.get_integer (STATE_FILE_GROUP, "since"); - } catch (Error error) { - state_file.set_integer (STATE_FILE_GROUP, "since", 0); - } - } - - private void save_state_file () { - string name; - string file_path; - - get ("name", out name); - file_path = Path.build_filename (Environment.get_user_cache_dir (), - "tracker", - STATE_FILE_NAME.printf (name)); - - try { - FileUtils.set_contents (file_path, state_file.to_data ()); - } catch (Error error) { - warning ("Couldn't save state file: %s", error.message); - } - } - - // If we don't protect the function with a mutex, it could be called by the - // inner loop, starting at inner-inner one, and so on... - private Mutex authenticate_mutex = new Mutex (); - public override void authenticate () throws MinerWebError { - PasswordProvider password_provider; - string name; - string username; - string password; - bool verified = false; - Error twitter_error = null; - - if(!authenticate_mutex.trylock ()) { - warning ("authenticate called while it was still running"); - return; - } - - password_provider = PasswordProvider.get (); - get ("name", out name); - - set ("association-status", MinerWebAssociationStatus.UNASSOCIATED); - - try { - password = password_provider.get_password (name, out username); - } catch (Error e) { - authenticate_mutex.unlock (); - if (e is PasswordProviderError.SERVICE) { - throw new MinerWebError.KEYRING (e.message); - } - if (e is PasswordProviderError.NOTFOUND) { - throw new MinerWebError.NO_CREDENTIALS ("Miner is not associated"); - } - - critical ("Internal error: %s", e.message); - return; - } - - message ("Verifying username and password"); - service.set_user (username, password); - service.verify_user (); - - var wait_loop = new MainLoop (null, false); - service.user_verified.connect ( (h, v, e) => { - verified = v; - twitter_error = e; - wait_loop.quit (); }); - wait_loop.run (); - authenticate_mutex.unlock (); - - if (twitter_error != null) { - throw new MinerWebError.SERVICE (twitter_error.message); - } - - if (!verified) { - throw new MinerWebError.WRONG_CREDENTIALS ("Wrong username and/or password"); - } else { - message ("Authentication sucessful"); - set ("association-status", MinerWebAssociationStatus.ASSOCIATED); - } - - return; - } - - public override void dissociate () throws MinerWebError { - var password_provider = PasswordProvider.get (); - string name; - get ("name", out name); - - try { - password_provider.forget_password (name); - } catch (Error e) { - if (e is PasswordProviderError.SERVICE) { - throw new MinerWebError.KEYRING (e.message); - } - - critical ("Internal error: %s", e.message); - return; - } - - set ("association-status", MinerWebAssociationStatus.UNASSOCIATED); - } - - public override void associate (HashTable association_data) throws Tracker.MinerWebError { - assert (association_data.lookup ("username") != null && association_data.lookup ("password") != null); - - var password_provider = PasswordProvider.get (); - string name; - get ("name", out name); - - try { - password_provider.store_password (name, - SERVICE_DESCRIPTION.printf (name), - association_data.lookup ("username"), - association_data.lookup ("password")); - } catch (Error e) { - if (e is PasswordProviderError.SERVICE) { - throw new MinerWebError.KEYRING (e.message); - } - - critical ("Internal error: %s", e.message); - return; - } - } - - public GLib.HashTable get_association_data () throws Tracker.MinerWebError { - return new HashTable(str_hash, str_equal); - } - - private static bool in_loop = false; - private static void signal_handler (int signo) { - if (in_loop) { - Posix.exit (Posix.EXIT_FAILURE); - } - - switch (signo) { - case Posix.SIGINT: - case Posix.SIGTERM: - in_loop = true; - main_loop.quit (); - break; - } - } - - private static void init_signals () { -#if G_OS_WIN32 -#else - Posix.sigaction_t act = Posix.sigaction_t (); - Posix.sigset_t empty_mask = Posix.sigset_t (); - Posix.sigemptyset (empty_mask); - act.sa_handler = signal_handler; - act.sa_mask = empty_mask; - act.sa_flags = 0; - - Posix.sigaction (Posix.SIGTERM, act, null); - Posix.sigaction (Posix.SIGINT, act, null); -#endif - } - - public static void main (string[] args) { - Environment.set_application_name ("Twitter/Identi.ca tracker miner"); - MinerTwitter twitter_miner; - twitter_miner = Object.new (typeof (MinerTwitter), - "provider", Twitter.Provider.DEFAULT_PROVIDER) as MinerTwitter; - - MinerTwitter identica_miner; - identica_miner = Object.new (typeof (MinerTwitter), - "provider", Twitter.Provider.IDENTI_CA) as MinerTwitter; - - init_signals (); - - main_loop = new MainLoop (null, false); - main_loop.run (); - - twitter_miner.shutdown (); - identica_miner.shutdown (); - } -} - -} // End namespace Tracker -- cgit v1.2.1