Commit a8718a5e authored by Martyn Russell's avatar Martyn Russell

Merge branch 'miner-rss'

parents 32818be7 b78eba92
......@@ -158,6 +158,7 @@ LIBSTREAMANALYZER_REQUIRED=0.7.0
GEE_REQUIRED=0.3
ID3LIB_REQUIRED=3.8.3
GNOME_KEYRING_REQUIRED=2.26
LIBGRSS_REQUIRED=0.3
# Library Checks
PKG_CHECK_MODULES(GLIB2, [glib-2.0 >= $GLIB_REQUIRED])
......@@ -810,6 +811,43 @@ AC_ARG_ENABLE([kmail_miner],
AM_CONDITIONAL(USING_KMAIL_MINER, test "x$enable_kmail_miner" = "xyes")
##################################################################
# Miner RSS
##################################################################
AC_ARG_ENABLE([miner_rss],
AS_HELP_STRING([--enable-miner-rss],
[enable miner rss [[default=auto]]]),,
[enable_miner_rss=auto])
if test "x$enable_miner_rss" != "xno" ; then
PKG_CHECK_MODULES(LIBGRSS,
[libgrss-0 >= $LIBGRSS_REQUIRED],
[have_libgrss=yes],
[have_libgrss=no])
AC_SUBST(LIBGRSS_CFLAGS)
AC_SUBST(LIBGRSS_LIBS)
if test "x$enable_miner_rss" = "xauto"; then
if test "x$have_libgrss" = "xyes"; then
enable_miner_rss="yes"
else
enable_miner_rss="no"
fi
fi
if test "x$enable_miner_rss" = "xyes"; then
if test "x$have_libgrss" != "xyes"; then
AC_MSG_ERROR([Couldn't find libgrss >= $LIBGRSS_REQUIRED. and miner rss requested])
fi
fi
else
enable_miner_rss="no (disabled)"
fi
AM_CONDITIONAL(USING_MINER_RSS, test "x$enable_miner_rss" = "xyes")
####################################################################
# Deskbar Applet Handler/Module
####################################################################
......@@ -1678,6 +1716,7 @@ AC_CONFIG_FILES([
src/tracker-control/Makefile
src/tracker-extract/Makefile
src/tracker-miner-fs/Makefile
src/tracker-miner-rss/Makefile
src/tracker-preferences/Makefile
src/tracker-preferences/tracker-preferences.desktop.in
src/tracker-search-bar/Makefile
......@@ -1797,6 +1836,7 @@ Plugins/Extensions:
Nautilus (tagging widget) $have_nautilus_extension
Evolution plugin (data-push): $enable_evolution_miner
KMail plugin (data-push): $enable_kmail_miner
Miner RSS: (data-push): $enable_miner_rss
Writeback:
......
......@@ -15,17 +15,31 @@ tracker-miner-fs.desktop.in: tracker-miner-fs.desktop.in.in
@sed -e "s|@libexecdir[@]|${libexecdir}|" \
-e "s|@VERSION[@]|${VERSION}|" $< > $@
if USING_MINER_RSS
tracker-miner-rss.desktop.in: tracker-miner-rss.desktop.in.in
@$(SED) -e "s|@libexecdir[@]|${libexecdir}|" \
-e "s|@VERSION[@]|${VERSION}|" $< > $@
endif
autostartdir = $(sysconfdir)/xdg/autostart
autostart_in_in_files = \
tracker-status-icon.desktop.in.in \
tracker-store.desktop.in.in \
tracker-miner-fs.desktop.in.in
if USING_MINER_RSS
autostart_in_in_files += tracker-miner-rss.desktop.in.in
endif
autostart_in_files = \
tracker-status-icon.desktop.in \
tracker-store.desktop.in \
tracker-miner-fs.desktop.in
if USING_MINER_RSS
autostart_in_files += tracker-miner-rss.desktop.in
endif
autostart_DATA = \
tracker-store.desktop \
tracker-miner-fs.desktop
......@@ -34,6 +48,10 @@ if HAVE_TRACKER_STATUS_ICON
autostart_DATA += tracker-status-icon.desktop
endif
if USING_MINER_RSS
autostart_DATA += tracker-miner-rss.desktop
endif
@INTLTOOL_DESKTOP_RULE@
pkgconfigdir = $(libdir)/pkgconfig
......
......@@ -4,12 +4,20 @@ desktop_in_files = \
tracker-miner-applications.desktop.in \
tracker-miner-files.desktop.in
if USING_MINER_RSS
desktop_in_files += tracker-miner-rss.desktop.in
endif
tracker_minersdir = $(datadir)/tracker/miners
tracker_miners_DATA = \
tracker-miner-applications.desktop \
tracker-miner-files.desktop
if USING_MINER_RSS
tracker_miners_DATA += tracker-miner-rss.desktop
endif
@INTLTOOL_DESKTOP_RULE@
EXTRA_DIST = $(desktop_in_files)
......
[Desktop Entry]
Encoding=UTF-8
_Name=RSS/ATOM Feeds
_Comment=Fetch RSS/ATOM Feeds
DBusName=org.freedesktop.Tracker1.Miner.RSS
DBusPath=/org/freedesktop/Tracker1/Miner/RSS
\ No newline at end of file
[Desktop Entry]
Encoding=UTF-8
Name=RSS/ATOM Feeds
Comment=Fetch RSS/ATOM Feeds
DBusName=org.freedesktop.Tracker1.Miner.RSS
DBusPath=/org/freedesktop/Tracker1/Miner/RSS
\ No newline at end of file
......@@ -39,3 +39,6 @@ if HAVE_TRACKER_EXPLORER
SUBDIRS += tracker-explorer
endif
if USING_MINER_RSS
SUBDIRS += tracker-miner-rss
endif
include $(top_srcdir)/Makefile.decl
INCLUDES = \
-Wall \
-DSHAREDIR=\""$(datadir)"\" \
-DPKGLIBDIR=\""$(libdir)/tracker"\" \
-DLOCALEDIR=\""$(localedir)"\" \
-DLIBEXEC_PATH=\""$(libexecdir)"\" \
-DG_LOG_DOMAIN=\"Tracker\" \
-DTRACKER_COMPILATION \
-I$(top_srcdir)/src \
$(GMODULE_CFLAGS) \
$(DBUS_CFLAGS) \
$(WARN_CFLAGS) \
$(LIBGRSS_CFLAGS)
libexec_PROGRAMS = tracker-miner-rss
tracker_miner_rss_SOURCES = \
tracker-main.c \
tracker-miner-rss.h \
tracker-miner-rss.c
tracker_miner_rss_LDADD = \
$(top_builddir)/src/libtracker-miner/libtracker-miner-@TRACKER_API_VERSION@.la \
$(GLIB2_LIBS) \
$(DBUS_LIBS) \
$(LIBGRSS_LIBS)
CLEANFILES = $(BUILT_SOURCES)
/*
* Copyright (C) 2009/2010, Roberto Guido <madbob@users.barberaware.org>
* Michele Tameni <michele@amdplanet.it>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#include "config.h"
#include <stdlib.h>
#include <locale.h>
#include <glib/gi18n.h>
#include <libtracker-common/tracker-common.h>
#include "tracker-miner-rss.h"
static gint verbosity = -1;
static GOptionEntry entries[] = {
{ "verbosity", 'v', 0,
G_OPTION_ARG_INT, &verbosity,
N_("Logging, 0 = errors only, "
"1 = minimal, 2 = detailed and 3 = debug (default=0)"),
NULL },
{ NULL }
};
int
main (int argc, char **argv)
{
gchar *log_filename;
GMainLoop *loop;
GOptionContext *context;
TrackerMinerRSS *miner;
g_type_init ();
g_thread_init (NULL);
setlocale (LC_ALL, "");
bindtextdomain (GETTEXT_PACKAGE, LOCALEDIR);
bind_textdomain_codeset (GETTEXT_PACKAGE, "UTF-8");
textdomain (GETTEXT_PACKAGE);
tzset ();
/* Translators: this messagge will apper immediately after the
* usage string - Usage: COMMAND <THIS_MESSAGE>
*/
context = g_option_context_new (_("- start the feeds indexer"));
g_option_context_add_main_entries (context, entries, NULL);
g_option_context_parse (context, &argc, &argv, NULL);
g_option_context_free (context);
tracker_log_init (verbosity, &log_filename);
g_print ("Starting log:\n File:'%s'\n", log_filename);
g_free (log_filename);
miner = g_object_new (TRACKER_TYPE_MINER_RSS, "name", "RSS", NULL);
tracker_miner_start (TRACKER_MINER (miner));
loop = g_main_loop_new (NULL, FALSE);
g_main_loop_run (loop);
tracker_log_shutdown ();
g_main_loop_unref (loop);
g_object_unref (miner);
return 0;
}
/*
* Copyright (C) 2009/2010, Roberto Guido <madbob@users.barberaware.org>
* Michele Tameni <michele@amdplanet.it>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#include <stdio.h>
#include <libgrss.h>
#include <dbus/dbus-glib.h>
#include <glib/gi18n.h>
#include "tracker-miner-rss.h"
#define GET_PRIV(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), TRACKER_TYPE_MINER_RSS, TrackerMinerRSSPrivate))
typedef struct _TrackerMinerRSSPrivate TrackerMinerRSSPrivate;
struct _TrackerMinerRSSPrivate {
gboolean paused;
gboolean stopped;
FeedsPool *pool;
gint now_fetching;
};
static void tracker_miner_rss_started (TrackerMiner *miner);
static void tracker_miner_rss_stopped (TrackerMiner *miner);
static void tracker_miner_rss_paused (TrackerMiner *miner);
static void tracker_miner_rss_resumed (TrackerMiner *miner);
static void retrieve_and_schedule_feeds (TrackerMinerRSS *miner);
static void change_status (FeedsPool *pool,
FeedChannel *feed,
gpointer user_data);
static void feed_fetched (FeedsPool *pool,
FeedChannel *feed,
GList *items,
gpointer user_data);
static const gchar *get_message_url (FeedItem *item);
G_DEFINE_TYPE (TrackerMinerRSS, tracker_miner_rss, TRACKER_TYPE_MINER)
static void
tracker_miner_rss_finalize (GObject *object)
{
TrackerMinerRSSPrivate *priv;
priv = GET_PRIV (object);
priv->stopped = TRUE;
g_object_unref (priv->pool);
G_OBJECT_CLASS (tracker_miner_rss_parent_class)->finalize (object);
}
static void
tracker_miner_rss_class_init (TrackerMinerRSSClass *klass)
{
GObjectClass *object_class = G_OBJECT_CLASS (klass);
TrackerMinerClass *miner_class = TRACKER_MINER_CLASS (klass);
object_class->finalize = tracker_miner_rss_finalize;
miner_class->started = tracker_miner_rss_started;
miner_class->stopped = tracker_miner_rss_stopped;
miner_class->paused = tracker_miner_rss_paused;
miner_class->resumed = tracker_miner_rss_resumed;
g_type_class_add_private (object_class, sizeof (TrackerMinerRSSPrivate));
}
static void
subjects_added_cb (DBusGProxy *proxy,
gchar **subjects,
gpointer user_data)
{
TrackerMinerRSS *miner;
miner = TRACKER_MINER_RSS (user_data);
/* TODO Add only the channels added? */
retrieve_and_schedule_feeds (miner);
}
static void
subjects_removed_cb (DBusGProxy *proxy,
gchar **subjects,
gpointer user_data)
{
TrackerMinerRSS *miner;
miner = TRACKER_MINER_RSS (user_data);
/* TODO Remove only the channels removed? */
retrieve_and_schedule_feeds (miner);
}
static void
tracker_miner_rss_init (TrackerMinerRSS *object)
{
DBusGProxy *wrap;
TrackerMinerRSSPrivate *priv;
wrap = dbus_g_proxy_new_for_name (dbus_g_bus_get (DBUS_BUS_SESSION, NULL),
"org.freedesktop.Tracker1",
"/org/freedesktop/Tracker1/Resources/Classes/mfo/FeedChannel",
"org.freedesktop.Tracker1.Resources.Class");
if (wrap == NULL) {
g_warning ("Unable to listen for added and removed channels");
return;
}
priv = GET_PRIV (object);
priv->pool = feeds_pool_new ();
g_signal_connect (priv->pool, "feed-fetching", G_CALLBACK (change_status), object);
g_signal_connect (priv->pool, "feed-ready", G_CALLBACK (feed_fetched), object);
priv->now_fetching = 0;
g_object_set (object, "progress", 0.0, "status", _("Initializing"), NULL);
dbus_g_proxy_add_signal (wrap, "SubjectsAdded", G_TYPE_STRV, G_TYPE_INVALID);
dbus_g_proxy_connect_signal (wrap, "SubjectsAdded", G_CALLBACK (subjects_added_cb), object, NULL);
dbus_g_proxy_add_signal (wrap, "SubjectsRemoved", G_TYPE_STRV, G_TYPE_INVALID);
dbus_g_proxy_connect_signal (wrap, "SubjectsRemoved", G_CALLBACK (subjects_removed_cb), object, NULL);
}
static void
verify_channel_update (GObject *source,
GAsyncResult *result,
gpointer user_data)
{
GError *error;
error = NULL;
tracker_miner_execute_update_finish (TRACKER_MINER (source), result, &error);
if (error != NULL) {
g_critical ("Unable to update information about channel: %s", error->message);
g_error_free (error);
}
}
static void
update_updated_interval (TrackerMinerRSS *miner,
gchar *uri,
time_t *now)
{
TrackerSparqlBuilder *sparql;
/* I hope there will be soon a SPARQL command to just update a
* value instead to delete and re-insert it
*/
sparql = tracker_sparql_builder_new_update ();
tracker_sparql_builder_delete_open (sparql, NULL);
tracker_sparql_builder_subject_iri (sparql, uri);
tracker_sparql_builder_predicate (sparql, "mfo:updatedTime");
tracker_sparql_builder_object_variable (sparql, "unknown");
tracker_sparql_builder_delete_close (sparql);
tracker_sparql_builder_where_open (sparql);
tracker_sparql_builder_subject_iri (sparql, uri);
tracker_sparql_builder_predicate (sparql, "mfo:updatedTime");
tracker_sparql_builder_object_variable (sparql, "unknown");
tracker_sparql_builder_where_close (sparql);
tracker_sparql_builder_insert_open (sparql, uri);
tracker_sparql_builder_subject_iri (sparql, uri);
tracker_sparql_builder_predicate (sparql, "mfo:updatedTime");
tracker_sparql_builder_object_date (sparql, now);
tracker_sparql_builder_insert_close (sparql);
tracker_miner_execute_update (TRACKER_MINER (miner),
tracker_sparql_builder_get_result (sparql),
NULL,
verify_channel_update,
NULL);
g_object_unref (sparql);
}
static void
change_status (FeedsPool *pool,
FeedChannel *feed,
gpointer user_data)
{
gint avail;
gdouble prog;
TrackerMinerRSS *miner;
TrackerMinerRSSPrivate *priv;
g_message ("Fetching %s", feed_channel_get_source (feed));
miner = TRACKER_MINER_RSS (user_data);
priv = GET_PRIV (miner);
avail = feeds_pool_get_listened_num (priv->pool);
priv->now_fetching++;
if (priv->now_fetching > avail)
priv->now_fetching = avail;
prog = ((gdouble) priv->now_fetching) / ((gdouble) avail);
g_object_set (miner, "progress", prog, "status", _("Fetching..."), NULL);
}
static void
verify_item_insertion (GObject *source,
GAsyncResult *result,
gpointer user_data)
{
GError *error;
error = NULL;
tracker_miner_execute_update_finish (TRACKER_MINER (source), result, &error);
if (error != NULL) {
g_critical ("Unable to create new item: %s", error->message);
g_error_free (error);
}
}
static void
item_verify_reply_cb (GObject *source_object,
GAsyncResult *res,
gpointer user_data)
{
time_t t;
gchar *uri;
gchar **values;
const gchar *url;
gdouble latitude;
gdouble longitude;
const gchar *tmp_string;
const GPtrArray *response;
GError *error;
TrackerSparqlBuilder *sparql;
FeedItem *item;
FeedChannel *feed;
TrackerMinerRSS *miner;
gboolean has_geopoint;
miner = TRACKER_MINER_RSS (source_object);
error = NULL;
response = tracker_miner_execute_sparql_finish (TRACKER_MINER (source_object),
res,
&error);
if (error != NULL) {
g_warning ("Unable to verify item existance: %s\n", error->message);
g_error_free (error);
return;
}
values = g_ptr_array_index (response, 0);
if (g_strcmp0 (values[0], "1") == 0)
return;
item = user_data;
url = get_message_url (item);
sparql = tracker_sparql_builder_new_update ();
has_geopoint = feed_item_get_geo_point (item, &latitude, &longitude);
tracker_sparql_builder_insert_open (sparql, url);
if (has_geopoint) {
tracker_sparql_builder_subject (sparql, "_:location");
tracker_sparql_builder_predicate (sparql, "a");
tracker_sparql_builder_object (sparql, "mlo:GeoLocation");
tracker_sparql_builder_predicate (sparql, "mlo:asGeoPoint");
tracker_sparql_builder_object_blank_open (sparql);
tracker_sparql_builder_predicate (sparql, "a");
tracker_sparql_builder_object (sparql, "mlo:GeoPoint");
tracker_sparql_builder_predicate (sparql, "mlo:latitude");
tracker_sparql_builder_object_double (sparql, latitude);
tracker_sparql_builder_predicate (sparql, "mlo:longitude");
tracker_sparql_builder_object_double (sparql, longitude);
tracker_sparql_builder_object_blank_close (sparql);
}
tracker_sparql_builder_subject (sparql, "_:message");
tracker_sparql_builder_predicate (sparql, "a");
tracker_sparql_builder_object (sparql, "mfo:FeedMessage");
tracker_sparql_builder_predicate (sparql, "a");
tracker_sparql_builder_object (sparql, "nfo:RemoteDataObject");
if (has_geopoint == TRUE) {
tracker_sparql_builder_predicate (sparql, "mlo:location");
tracker_sparql_builder_object (sparql, "_:location");
}
tmp_string = feed_item_get_title (item);
if (tmp_string != NULL) {
tracker_sparql_builder_predicate (sparql, "nie:title");
tracker_sparql_builder_object_unvalidated (sparql, tmp_string);
}
tmp_string = feed_item_get_description (item);
if (tmp_string != NULL) {
tracker_sparql_builder_predicate (sparql, "nmo:plainTextMessageContent");
tracker_sparql_builder_object_unvalidated (sparql, tmp_string);
}
if (url != NULL) {
tracker_sparql_builder_predicate (sparql, "nie:url");
tracker_sparql_builder_object_unvalidated (sparql, url);
}
/* TODO nmo:receivedDate and mfo:downloadedTime are the same?
* Ask for the MFO maintainer */
t = time (NULL);
tracker_sparql_builder_predicate (sparql, "nmo:receivedDate");
tracker_sparql_builder_object_date (sparql, &t);
tracker_sparql_builder_predicate (sparql, "mfo:downloadedTime");
tracker_sparql_builder_object_date (sparql, &t);
t = feed_item_get_publish_time (item);
tracker_sparql_builder_predicate (sparql, "nie:contentCreated");
tracker_sparql_builder_object_date (sparql, &t);
tracker_sparql_builder_predicate (sparql, "nmo:isRead");
tracker_sparql_builder_object_boolean (sparql, FALSE);
feed = feed_item_get_parent (item);
uri = g_object_get_data (G_OBJECT (feed), "subject");
tracker_sparql_builder_predicate (sparql, "nmo:communicationChannel");
tracker_sparql_builder_object_iri (sparql, uri);
g_free (uri);
tracker_sparql_builder_insert_close (sparql);
tracker_miner_execute_update (TRACKER_MINER (miner),
tracker_sparql_builder_get_result (sparql),
NULL,
verify_item_insertion,
NULL);
g_object_unref (sparql);
}
static void
check_if_save (TrackerMinerRSS *miner,
FeedItem *item)
{
FeedChannel *feed;
gchar *query;
gchar *communication_channel;
const gchar *url;
url = get_message_url (item);
feed = feed_item_get_parent (item);
communication_channel = g_object_get_data (G_OBJECT (feed), "subject");
query = g_strdup_printf ("ASK { ?message a mfo:FeedMessage; "
"nie:url \"%s\"; nmo:communicationChannel <%s> }",
url,
communication_channel);
tracker_miner_execute_sparql (TRACKER_MINER (miner),
query,
NULL,
item_verify_reply_cb,
item);
g_free (query);
}
static void
feed_fetched (FeedsPool *pool,
FeedChannel *feed,
GList *items,
gpointer user_data)
{
gchar *uri;
time_t now;
GList *iter;
FeedItem *item;
TrackerMinerRSS *miner;
TrackerMinerRSSPrivate *priv;
miner = TRACKER_MINER_RSS (user_data);
priv = GET_PRIV (miner);
priv->now_fetching--;
if (priv->now_fetching <= 0) {
priv->now_fetching = 0;
g_object_set (miner, "progress", 1.0, "status", "Idle", NULL);
}
if (items == NULL)
return;
now = time (NULL);
uri = g_object_get_data (G_OBJECT (feed), "subject");
update_updated_interval (miner, uri, &now);
for (iter = items; iter; iter = iter->next) {
item = iter->data;
check_if_save (miner, item);
}
}
static void
feeds_retrieve_cb (GObject *source_object,
GAsyncResult *res,
gpointer user_data)
{
gint interval;
guint i;
gchar **values;
GList *channels;
const GPtrArray *response;
GError *error;
TrackerMinerRSSPrivate *priv;
FeedChannel *chan;
error = NULL;
response = tracker_miner_execute_sparql_finish (TRACKER_MINER (source_object),
res,
&error);
if (error != NULL) {
g_warning ("Unable to retrieve list of feeds: %s\n", error->message);
g_error_free (error);
return;
}
channels = NULL;
for (i = 0; i < response->len; i++) {
values = g_ptr_array_index (response, i);
chan = feed_channel_new ();
g_object_set_data_full (G_OBJECT (chan),
"subject",
g_strdup (values [2]),
g_free);
feed_channel_set_source (chan, values [0]);
/* TODO How to manage feeds with an update mfo:updateInterval == 0 ?
* Here the interval is forced to be at least 1 minute, but perhaps those
* elements are to be considered "disabled"
*/
interval = strtoull (values [1], NULL, 10);