Commit ae6ab10c authored by Sam Thursfield's avatar Sam Thursfield

Merge branch 'wip/carlosg/extraction-fixes' into 'master'

Extraction and other fixes

Closes #141

See merge request !273
parents 90bdacda bd3ce694
Pipeline #217892 passed with stages
in 9 minutes and 56 seconds
......@@ -46,8 +46,6 @@ else
tracker_uninstalled_stop_words_dir = tracker_subproject.get_variable('tracker_uninstalled_stop_words_dir')
tracker_uninstalled_testutils_dir = tracker_subproject.get_variable('tracker_uninstalled_testutils_dir')
tracker_ontologies_dir = tracker_subproject.get_variable('tracker_ontologies_dir')
tracker_common_enums_header = tracker_subproject.get_variable('tracker_common_enums_header')
endif
avcodec = dependency('libavcodec', version: '>= 0.8.4', required: false)
......
......@@ -351,19 +351,19 @@ get_gps_coordinate (ExifData *exif,
exif_entry_get_value (entry, buf, sizeof (buf) - 1);
if (buf[0] == '\0') {
g_message ("EXIF GPS coordinate information is non-existent but EXIF tag '%s' was found, "
"possible broken EXIF data?",
exif_tag_get_name (tag));
g_debug ("EXIF GPS coordinate information is non-existent but EXIF tag '%s' was found, "
"possible broken EXIF data?",
exif_tag_get_name (tag));
} else {
g_message ("EXIF GPS coordinate information is partial, "
"got EXIF tag '%s' with value '%s', "
"expected with degrees, minutes and seconds",
exif_tag_get_name (tag),
buf);
g_debug ("EXIF GPS coordinate information is partial, "
"got EXIF tag '%s' with value '%s', "
"expected with degrees, minutes and seconds",
exif_tag_get_name (tag),
buf);
}
g_message ("EXIF GPS coordinate information could not be extracted with tag '%s'",
exif_tag_get_name (tag));
g_debug ("EXIF GPS coordinate information could not be extracted with tag '%s'",
exif_tag_get_name (tag));
}
}
......
......@@ -101,6 +101,7 @@ enum {
enum {
ITEMS_AVAILABLE,
FINISHED,
ERROR,
LAST_SIGNAL
};
......@@ -261,11 +262,9 @@ retry_synchronously (TrackerDecorator *decorator,
&error);
if (error) {
GFile *file = g_file_new_for_uri (update->url);
tracker_error_report (file, error->message, update->sparql);
g_signal_emit (decorator, signals[ERROR], 0,
update->url, error->message, update->sparql);
g_error_free (error);
g_object_unref (file);
}
}
}
......@@ -1069,6 +1068,17 @@ tracker_decorator_class_init (TrackerDecoratorClass *klass)
G_STRUCT_OFFSET (TrackerDecoratorClass, finished),
NULL, NULL, NULL,
G_TYPE_NONE, 0);
signals[ERROR] =
g_signal_new ("error",
G_OBJECT_CLASS_TYPE (object_class),
G_SIGNAL_RUN_LAST,
G_STRUCT_OFFSET (TrackerDecoratorClass, error),
NULL, NULL, NULL,
G_TYPE_NONE, 3,
G_TYPE_STRING,
G_TYPE_STRING,
G_TYPE_STRING);
}
static void
......
......@@ -67,7 +67,10 @@ struct _TrackerDecoratorClass {
void (* items_available) (TrackerDecorator *decorator);
void (* finished) (TrackerDecorator *decorator);
void (* error) (TrackerDecorator *decorator,
const gchar *url,
const gchar *error_message,
const gchar *sparql);
/* <Private> */
gpointer padding[10];
};
......
......@@ -25,6 +25,7 @@
#include <libtracker-miner/tracker-miner.h>
enum {
STATUS,
LOST,
N_SIGNALS
};
......@@ -33,8 +34,10 @@ static guint signals[N_SIGNALS] = { 0, };
struct _TrackerExtractWatchdog {
GObject parent_class;
GDBusConnection *conn;
gchar *domain;
guint extractor_watchdog_id;
guint progress_signal_id;
gboolean initializing;
};
......@@ -46,12 +49,39 @@ G_DEFINE_TYPE (TrackerExtractWatchdog, tracker_extract_watchdog, G_TYPE_OBJECT)
static void
extract_watchdog_stop (TrackerExtractWatchdog *watchdog)
{
if (watchdog->conn && watchdog->progress_signal_id) {
g_dbus_connection_signal_unsubscribe (watchdog->conn,
watchdog->progress_signal_id);
watchdog->progress_signal_id = 0;
watchdog->conn = NULL;
}
if (watchdog->extractor_watchdog_id) {
g_bus_unwatch_name (watchdog->extractor_watchdog_id);
watchdog->extractor_watchdog_id = 0;
}
}
static void
on_extract_progress_cb (GDBusConnection *conn,
const gchar *sender_name,
const gchar *object_path,
const gchar *interface_name,
const gchar *signal_name,
GVariant *parameters,
gpointer user_data)
{
TrackerExtractWatchdog *watchdog = user_data;
const gchar *status;
gdouble progress;
gint32 remaining;
g_variant_get (parameters, "(&sdi)",
&status, &progress, &remaining);
g_signal_emit (watchdog, signals[STATUS], 0,
status, progress, (gint) remaining);
}
static void
extract_watchdog_name_appeared (GDBusConnection *conn,
const gchar *name,
......@@ -62,6 +92,19 @@ extract_watchdog_name_appeared (GDBusConnection *conn,
if (watchdog->initializing)
watchdog->initializing = FALSE;
watchdog->conn = conn;
watchdog->progress_signal_id =
g_dbus_connection_signal_subscribe (watchdog->conn,
"org.freedesktop.Tracker3.Miner.Extract",
"org.freedesktop.Tracker3.Miner",
"Progress",
"/org/freedesktop/Tracker3/Miner/Extract",
NULL,
G_DBUS_SIGNAL_FLAGS_NONE,
on_extract_progress_cb,
watchdog,
NULL);
}
static void
......@@ -89,6 +132,7 @@ extract_watchdog_name_vanished (GDBusConnection *conn,
return;
}
g_signal_emit (watchdog, signals[STATUS], 0, "Idle", 1.0, 0);
g_signal_emit (watchdog, signals[LOST], 0);
}
......@@ -139,6 +183,14 @@ tracker_extract_watchdog_class_init (TrackerExtractWatchdogClass *klass)
object_class->finalize = tracker_extract_watchdog_finalize;
signals[STATUS] = g_signal_new ("status",
G_OBJECT_CLASS_TYPE (object_class),
G_SIGNAL_RUN_LAST,
0, NULL, NULL, NULL,
G_TYPE_NONE, 3,
G_TYPE_STRING,
G_TYPE_DOUBLE,
G_TYPE_INT);
signals[LOST] = g_signal_new ("lost",
G_OBJECT_CLASS_TYPE (object_class),
G_SIGNAL_RUN_LAST,
......
......@@ -304,6 +304,22 @@ on_extractor_lost (TrackerExtractWatchdog *watchdog,
g_timeout_add_seconds (1, extractor_lost_timeout_cb, mf);
}
static void
on_extractor_status (TrackerExtractWatchdog *watchdog,
const gchar *status,
gdouble progress,
gint remaining,
TrackerMinerFiles *mf)
{
if (!tracker_miner_is_paused (TRACKER_MINER (mf))) {
g_object_set (mf,
"status", status,
"progress", progress,
"remaining-time", remaining,
NULL);
}
}
static void
tracker_miner_files_init (TrackerMinerFiles *mf)
{
......@@ -646,6 +662,8 @@ miner_files_initable_init (GInitable *initable,
mf->private->extract_watchdog = tracker_extract_watchdog_new (domain_name);
g_signal_connect (mf->private->extract_watchdog, "lost",
G_CALLBACK (on_extractor_lost), mf);
g_signal_connect (mf->private->extract_watchdog, "status",
G_CALLBACK (on_extractor_status), mf);
g_free (domain_name);
return TRUE;
......
......@@ -47,7 +47,7 @@ get_img_resolution (const GFile *file,
stream = g_file_read ((GFile *)file, NULL, &error);
if (error) {
g_message ("Could not read BMP file, %s", error->message);
g_debug ("Could not read BMP file, %s", error->message);
g_clear_error (&error);
return FALSE;
}
......@@ -55,27 +55,27 @@ get_img_resolution (const GFile *file,
inputstream = G_INPUT_STREAM (stream);
if (!g_input_stream_read (inputstream, bfType, 2, NULL, &error)) {
g_message ("Could not read BMP header from stream, %s", error ? error->message : "No error given");
g_debug ("Could not read BMP header from stream, %s", error ? error->message : "No error given");
g_clear_error (&error);
g_object_unref (stream);
return FALSE;
}
if (bfType[0] != 'B' || bfType[1] != 'M') {
g_message ("Expected BMP header to read 'B' or 'M', can not continue");
g_debug ("Expected BMP header to read 'B' or 'M', can not continue");
g_object_unref (stream);
return FALSE;
}
if (!g_input_stream_skip (inputstream, 16, NULL, &error)) {
g_message ("Could not read 16 bytes from BMP header, %s", error ? error->message : "No error given");
g_debug ("Could not read 16 bytes from BMP header, %s", error ? error->message : "No error given");
g_clear_error (&error);
g_object_unref (stream);
return FALSE;
}
if (!g_input_stream_read (inputstream, &w, sizeof (uint), NULL, &error)) {
g_message ("Could not read width from BMP header, %s", error ? error->message : "No error given");
g_debug ("Could not read width from BMP header, %s", error ? error->message : "No error given");
g_clear_error (&error);
g_object_unref (stream);
return FALSE;
......
......@@ -65,7 +65,8 @@ static void tracker_extract_decorator_initable_iface_init (GInitableIface *iface
static void decorator_ignore_file (GFile *file,
TrackerExtractDecorator *decorator,
const gchar *error_message);
const gchar *error_message,
const gchar *extra_info);
G_DEFINE_TYPE_WITH_CODE (TrackerExtractDecorator, tracker_extract_decorator,
TRACKER_TYPE_DECORATOR_FS,
......@@ -169,7 +170,7 @@ get_metadata_cb (TrackerExtract *extract,
if (error) {
decorator_ignore_file (data->file,
TRACKER_EXTRACT_DECORATOR (data->decorator),
error->message);
error->message, NULL);
tracker_decorator_info_complete_error (data->decorator_info, error);
} else {
gchar *resource_sparql, *sparql;
......@@ -389,6 +390,19 @@ tracker_extract_decorator_finished (TrackerDecorator *decorator)
g_free (time_str);
}
static void
tracker_extract_decorator_error (TrackerDecorator *decorator,
const gchar *url,
const gchar *error_message,
const gchar *sparql)
{
GFile *file;
file = g_file_new_for_uri (url);
decorator_ignore_file (file, TRACKER_EXTRACT_DECORATOR (decorator), error_message, sparql);
g_object_unref (file);
}
static void
tracker_extract_decorator_class_init (TrackerExtractDecoratorClass *klass)
{
......@@ -405,6 +419,7 @@ tracker_extract_decorator_class_init (TrackerExtractDecoratorClass *klass)
decorator_class->items_available = tracker_extract_decorator_items_available;
decorator_class->finished = tracker_extract_decorator_finished;
decorator_class->error = tracker_extract_decorator_error;
g_object_class_install_property (object_class,
PROP_EXTRACTOR,
......@@ -420,7 +435,8 @@ tracker_extract_decorator_class_init (TrackerExtractDecoratorClass *klass)
static void
decorator_ignore_file (GFile *file,
TrackerExtractDecorator *decorator,
const gchar *error_message)
const gchar *error_message,
const gchar *extra_info)
{
TrackerSparqlConnection *conn;
GError *error = NULL;
......@@ -437,7 +453,7 @@ decorator_ignore_file (GFile *file,
NULL, &error);
if (info) {
tracker_error_report (file, error_message, NULL);
tracker_error_report (file, error_message, extra_info);
mimetype = g_file_info_get_attribute_string (info,
G_FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE);
......@@ -485,7 +501,7 @@ persistence_ignore_file (GFile *file,
{
TrackerExtractDecorator *decorator = user_data;
decorator_ignore_file (file, decorator, "Crash/hang handling file");
decorator_ignore_file (file, decorator, "Crash/hang handling file", NULL);
}
static void
......
......@@ -140,7 +140,7 @@ mdvi_init_context (const char *file)
}
if ((arg = fuget1 (p)) != DVI_ID) {
g_message ("Unsupported DVI format (version %u)", arg);
g_debug ("Unsupported DVI format (version %u)", arg);
goto error;
}
......
......@@ -75,9 +75,9 @@ gif_error (const gchar *action, int err)
{
const char *str = GifErrorString (err);
if (str != NULL) {
g_message ("%s, error: '%s'", action, str);
g_debug ("%s, error: '%s'", action, str);
} else {
g_message ("%s, undefined error %d", action, err);
g_debug ("%s, undefined error %d", action, err);
}
}
#else /* GIFLIB_MAJOR >= 5 */
......@@ -86,9 +86,9 @@ static inline void print_gif_error()
#if defined(GIFLIB_MAJOR) && defined(GIFLIB_MINOR) && ((GIFLIB_MAJOR == 4 && GIFLIB_MINOR >= 2) || GIFLIB_MAJOR > 4)
const char *str = GifErrorString ();
if (str != NULL) {
g_message ("GIF, error: '%s'", str);
g_debug ("GIF, error: '%s'", str);
} else {
g_message ("GIF, undefined error");
g_debug ("GIF, undefined error");
}
#else
PrintGifError();
......
......@@ -615,9 +615,25 @@ extractor_get_equipment (MetadataExtractor *extractor,
return equipment;
}
static TrackerResource *
ensure_file_resource (TrackerResource *resource,
const gchar *file_url)
{
TrackerResource *file_resource;
file_resource = tracker_resource_get_first_relation (resource, "nie:isStoredAs");
if (!file_resource) {
file_resource = tracker_resource_new (file_url);
tracker_resource_set_take_relation (resource, "nie:isStoredAs", file_resource);
}
return file_resource;
}
static void
extractor_apply_audio_metadata (MetadataExtractor *extractor,
GstTagList *tag_list,
const gchar *file_url,
TrackerResource *audio,
TrackerResource *artist,
TrackerResource *performer,
......@@ -668,12 +684,13 @@ extractor_apply_audio_metadata (MetadataExtractor *extractor,
have_acoustid_fingerprint = gst_tag_list_copy_value (&acoustid_fingerprint, tag_list, GST_TAG_ACOUSTID_FINGERPRINT);
if (have_acoustid_fingerprint) {
TrackerResource *hash_resource = tracker_resource_new (NULL);
TrackerResource *file_resource = ensure_file_resource (audio, file_url);
tracker_resource_set_uri (hash_resource, "rdf:type", "nfo:FileHash");
tracker_resource_set_gvalue (hash_resource, "nfo:hashValue", &acoustid_fingerprint);
tracker_resource_set_string (hash_resource, "nfo:hashAlgorithm", "chromaprint");
tracker_resource_add_take_relation (audio, "nfo:hasHash", hash_resource);
tracker_resource_add_take_relation (file_resource, "nfo:hasHash", hash_resource);
g_value_unset (&acoustid_fingerprint);
}
#endif
......@@ -754,6 +771,7 @@ extract_track (TrackerResource *track,
extractor_apply_audio_metadata (extractor,
toc_entry->tag_list,
file_url,
track,
track_artist,
track_performer,
......@@ -948,7 +966,7 @@ extract_metadata (MetadataExtractor *extractor,
if (extractor->toc && g_list_length (extractor->toc->entry_list) > 1) {
TrackerResource *file_resource;
file_resource = tracker_resource_new (file_url);
file_resource = ensure_file_resource (resource, file_url);
for (node = extractor->toc->entry_list; node; node = node->next) {
TrackerResource *track;
......@@ -965,11 +983,10 @@ extract_metadata (MetadataExtractor *extractor,
tracker_resource_set_relation (track, "nie:isStoredAs", file_resource);
tracker_resource_add_take_relation (file_resource, "nie:interpretedAs", track);
}
g_object_unref (file_resource);
} else {
extractor_apply_audio_metadata (extractor,
extractor->tagcache,
file_url,
resource,
artist,
performer,
......@@ -1004,8 +1021,7 @@ extract_metadata (MetadataExtractor *extractor,
tracker_resource_set_string (hash_resource, "nfo:hashAlgorithm", "gibest");
file_resource = tracker_resource_new (file_url);
tracker_resource_add_take_relation (resource, "nie:isStoredAs", file_resource);
file_resource = ensure_file_resource (resource, file_url);
tracker_resource_set_relation (file_resource, "nfo:hasHash", hash_resource);
......@@ -1171,8 +1187,8 @@ discoverer_init_and_run (MetadataExtractor *extractor,
if (error) {
if (gst_discoverer_info_get_result(info) == GST_DISCOVERER_MISSING_PLUGINS) {
required_plugins_message = get_discoverer_required_plugins_message (info);
g_message ("Missing a GStreamer plugin for %s. %s", uri,
required_plugins_message);
g_debug ("Missing a GStreamer plugin for %s. %s", uri,
required_plugins_message);
g_free (required_plugins_message);
} else if (error->domain != GST_STREAM_ERROR ||
(error->code != GST_STREAM_ERROR_TYPE_NOT_FOUND &&
......
......@@ -43,9 +43,9 @@ find_max_width_and_height (const gchar *uri,
file = g_file_new_for_uri (uri);
stream = g_file_read (file, NULL, &error);
if (error) {
g_message ("Could not read file '%s': %s",
uri,
error->message);
g_debug ("Could not read file '%s': %s",
uri,
error->message);
g_error_free (error);
g_object_unref (file);
......@@ -65,8 +65,8 @@ find_max_width_and_height (const gchar *uri,
NULL,
NULL,
&error)) {
g_message ("Error reading icon header from stream: '%s'",
error->message);
g_debug ("Error reading icon header from stream: '%s'",
error->message);
g_error_free (error);
g_object_unref (stream);
g_object_unref (file);
......@@ -91,9 +91,9 @@ find_max_width_and_height (const gchar *uri,
NULL,
NULL,
&error)) {
g_message ("Error reading icon image metadata '%u' from stream: '%s'",
i,
error->message);
g_debug ("Error reading icon image metadata '%u' from stream: '%s'",
i,
error->message);
g_error_free (error);
break;
}
......
......@@ -65,8 +65,8 @@ tracker_extract_get_metadata (TrackerExtractInfo *info_)
if (error != NULL) {
if (error->code != OSINFO_MEDIA_ERROR_NOT_BOOTABLE) {
g_object_unref (metadata);
g_message ("Could not extract iso info from '%s': %s",
filename, error->message);
g_debug ("Could not extract iso info from '%s': %s",
filename, error->message);
g_free (filename);
g_error_free (error);
return FALSE;
......
......@@ -2145,7 +2145,7 @@ parse_id3v24 (const gchar *data,
/* We don't handle experimental cases */
if (experimental) {
g_message ("[v24] Experimental MP3s are not extracted, doing nothing");
g_debug ("[v24] Experimental MP3s are not extracted, doing nothing");
return;
}
......@@ -2161,7 +2161,7 @@ parse_id3v24 (const gchar *data,
* bytes, so we check that there is some content AFTER the
* headers. */
if (tsize > size - header_size) {
g_message ("[v24] Expected MP3 tag size and header size to be within file size boundaries");
g_debug ("[v24] Expected MP3 tag size and header size to be within file size boundaries");
return;
}
......@@ -2183,7 +2183,7 @@ parse_id3v24 (const gchar *data,
* the headers, in other words the padding.
*/
if (ext_header_size > size - header_size - tsize) {
g_message ("[v24] Expected MP3 tag size and extended header size to be within file size boundaries");
g_debug ("[v24] Expected MP3 tag size and extended header size to be within file size boundaries");
return;
}
......@@ -2204,10 +2204,10 @@ parse_id3v24 (const gchar *data,
* Flags $xx xx
*/
if (pos + frame_size > tsize + header_size) {
g_message ("[v24] Expected MP3 frame size (%d) to be within tag size (%d) boundaries, position = %d",
frame_size,
tsize + header_size,
pos);
g_debug ("[v24] Expected MP3 frame size (%d) to be within tag size (%d) boundaries, position = %d",
frame_size,
tsize + header_size,
pos);
break;
}
......@@ -2349,7 +2349,7 @@ parse_id3v23 (const gchar *data,
/* We don't handle experimental cases */
if (experimental) {
g_message ("[v23] Experimental MP3s are not extracted, doing nothing");
g_debug ("[v23] Experimental MP3s are not extracted, doing nothing");
return;
}
......@@ -2365,7 +2365,7 @@ parse_id3v23 (const gchar *data,
* bytes, so we check that there is some content AFTER the
* headers. */
if (tsize > size - header_size) {
g_message ("[v23] Expected MP3 tag size and header size to be within file size boundaries");
g_debug ("[v23] Expected MP3 tag size and header size to be within file size boundaries");
return;
}
......@@ -2387,7 +2387,7 @@ parse_id3v23 (const gchar *data,
* the headers, in other words the padding.
*/
if (ext_header_size > size - header_size - tsize) {
g_message ("[v23] Expected MP3 tag size and extended header size to be within file size boundaries");
g_debug ("[v23] Expected MP3 tag size and extended header size to be within file size boundaries");
return;
}
......@@ -2408,10 +2408,10 @@ parse_id3v23 (const gchar *data,
* Flags $xx xx
*/
if (pos + frame_size > tsize + header_size) {
g_message ("[v23] Expected MP3 frame size (%d) to be within tag size (%d) boundaries, position = %d",
frame_size,
tsize + header_size,
pos);
g_debug ("[v23] Expected MP3 frame size (%d) to be within tag size (%d) boundaries, position = %d",
frame_size,
tsize + header_size,
pos);
break;
}
......@@ -2528,7 +2528,7 @@ parse_id3v20 (const gchar *data,
tsize = extract_uint32_7bit (&data[6]);
if (tsize > size - header_size) {
g_message ("[v20] Expected MP3 tag size and header size to be within file size boundaries");
g_debug ("[v20] Expected MP3 tag size and header size to be within file size boundaries");
return;
}
......@@ -2542,10 +2542,10 @@ parse_id3v20 (const gchar *data,
g_assert (pos <= size - frame_size);
if (pos + frame_size > tsize + header_size) {
g_message ("[v20] Expected MP3 frame size (%d) to be within tag size (%d) boundaries, position = %d",
frame_size,
tsize + header_size,
pos);