tracker-miner-fs.c 130 KB
Newer Older
1
/*
2
 * Copyright (C) 2009, Nokia <ivan.frade@nokia.com>
3 4
 *
 * This library is free software; you can redistribute it and/or
Martyn Russell's avatar
Martyn Russell committed
5
 * modify it under the terms of the GNU Lesser General Public
6
 * License as published by the Free Software Foundation; either
Martyn Russell's avatar
Martyn Russell committed
7
 * version 2.1 of the License, or (at your option) any later version.
8 9 10 11
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Martyn Russell's avatar
Martyn Russell committed
12
 * Lesser General Public License for more details.
13
 *
Martyn Russell's avatar
Martyn Russell committed
14
 * You should have received a copy of the GNU Lesser General Public
15 16 17 18 19 20 21
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA  02110-1301, USA.
 */

#include "config.h"

22
#include <libtracker-common/tracker-date-time.h>
23
#include <libtracker-common/tracker-dbus.h>
24
#include <libtracker-common/tracker-file-utils.h>
25 26
#include <libtracker-common/tracker-log.h>
#include <libtracker-common/tracker-utils.h>
27

28
#include "tracker-crawler.h"
29
#include "tracker-miner-fs.h"
30 31
#include "tracker-media-art.h"
#include "tracker-monitor.h"
32
#include "tracker-utils.h"
33
#include "tracker-thumbnailer.h"
34
#include "tracker-priority-queue.h"
35
#include "tracker-task-pool.h"
36
#include "tracker-sparql-buffer.h"
37
#include "tracker-file-notifier.h"
38

39
/* If defined will print the tree from GNode while running */
40
#ifdef CRAWLED_TREE_ENABLE_TRACE
41
#warning Tree debugging traces enabled
42
#endif /* CRAWLED_TREE_ENABLE_TRACE */
43

44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
/* If defined will print push/pop actions on queues */
#ifdef EVENT_QUEUE_ENABLE_TRACE
#warning Event Queue traces enabled
#define EVENT_QUEUE_LOG_PREFIX "[Event Queues] "
#define EVENT_QUEUE_STATUS_TIMEOUT_SECS 30
#define trace_eq(message, ...) g_debug (EVENT_QUEUE_LOG_PREFIX message, ##__VA_ARGS__)
#define trace_eq_action(pushed, queue_name, position, gfile1, gfile2, reason) \
	do { \
		gchar *uri1 = g_file_get_uri (gfile1); \
		gchar *uri2 = gfile2 ? g_file_get_uri (gfile2) : NULL; \
		g_debug ("%s%s '%s%s%s' %s %s of queue '%s'%s%s", \
		         EVENT_QUEUE_LOG_PREFIX, \
		         pushed ? "Pushed" : "Popped", \
		         uri1, \
		         uri2 ? "->" : "", \
		         uri2 ? uri2 : "", \
		         pushed ? "to" : "from", \
		         position, \
		         queue_name, \
		         reason ? ": " : "", \
		         reason ? reason : ""); \
		g_free (uri1); \
		g_free (uri2); \
	} while (0)
#define trace_eq_push_tail(queue_name, gfile, reason)	  \
	trace_eq_action (TRUE, queue_name, "tail", gfile, NULL, reason)
#define trace_eq_push_head(queue_name, gfile, reason)	  \
	trace_eq_action (TRUE, queue_name, "head", gfile, NULL, reason)
#define trace_eq_push_tail_2(queue_name, gfile1, gfile2, reason)	  \
	trace_eq_action (TRUE, queue_name, "tail", gfile1, gfile2, reason)
#define trace_eq_push_head_2(queue_name, gfile1, gfile2, reason)	  \
	trace_eq_action (TRUE, queue_name, "head", gfile1, gfile2, reason)
#define trace_eq_pop_head(queue_name, gfile)	  \
	trace_eq_action (FALSE, queue_name, "head", gfile, NULL, NULL)
#define trace_eq_pop_head_2(queue_name, gfile1, gfile2)	  \
	trace_eq_action (FALSE, queue_name, "head", gfile1, gfile2, NULL)
static gboolean miner_fs_queues_status_trace_timeout_cb (gpointer data);
#else
#define trace_eq(...)
#define trace_eq_push_tail(...)
#define trace_eq_push_head(...)
#define trace_eq_push_tail_2(...)
#define trace_eq_push_head_2(...)
#define trace_eq_pop_head(...)
#define trace_eq_pop_head_2(...)
#endif /* EVENT_QUEUE_ENABLE_TRACE */

91 92 93 94 95
/* Number of times a GFile can be re-queued before it's dropped for
 * whatever reason to avoid infinite loops.
*/
#define REENTRY_MAX 2

96 97
/* Default processing pool limits to be set */
#define DEFAULT_WAIT_POOL_LIMIT 1
98
#define DEFAULT_READY_POOL_LIMIT 1
99

100 101 102 103 104 105
/* Put tasks processing at a lower priority so other events
 * (timeouts, monitor events, etc...) are guaranteed to be
 * dispatched promptly.
 */
#define TRACKER_TASK_PRIORITY G_PRIORITY_DEFAULT_IDLE + 10

106 107 108
/**
 * SECTION:tracker-miner-fs
 * @short_description: Abstract base class for filesystem miners
109
 * @include: libtracker-miner/tracker-miner.h
110 111
 *
 * #TrackerMinerFS is an abstract base class for miners that collect data
112 113 114 115 116 117 118 119
 * from a filesystem where parent/child relationships need to be
 * inserted into the database correctly with queue management.
 *
 * All the filesystem crawling and monitoring is abstracted away,
 * leaving to implementations the decisions of what directories/files
 * should it process, and the actual data extraction.
 *
 * Example creating a TrackerMinerFS with our own file system root and
120
 * data provider.
121 122 123 124 125 126 127 128 129 130
 *
 * First create our class and base it on TrackerMinerFS:
 * |[
 * G_DEFINE_TYPE_WITH_CODE (MyMinerFiles, my_miner_files, TRACKER_TYPE_MINER_FS,
 *                          G_IMPLEMENT_INTERFACE (G_TYPE_INITABLE,
 *                                                 my_miner_files_initable_iface_init))
 * ]|
 *
 * Later in our class creation function, we are supplying the
 * arguments we want. In this case, the 'root' is a #GFile pointing to
131 132 133 134
 * a root URI location (for example 'file:///') and 'data_provider' is a
 * #TrackerDataProvider used to enumerate 'root' and return children it
 * finds. If 'data_provider' is %NULL (the default), then a
 * #TrackerFileDataProvider is created automatically.
135 136 137 138 139 140 141
 * |[
 * // Note that only 'name' is mandatory
 * miner = g_initable_new (MY_TYPE_MINER_FILES,
 *                         NULL,
 *                         error,
 *                         "name", "MyMinerFiles",
 *                         "root", root,
142
 *                         "data-provider", data_provider,
143 144 145 146
 *                         "processing-pool-wait-limit", 10,
 *                         "processing-pool-ready-limit", 100,
 *                         NULL);
 * ]|
147 148
 **/

149
#define TRACKER_MINER_FS_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_MINER_FS, TrackerMinerFSPrivate))
150

151
typedef struct {
152 153
	GFile *file;
	GFile *source_file;
154
} ItemMovedData;
155

156
typedef struct {
157 158
	GFile     *file;
	GPtrArray *results;
159
	GStrv      rdf_types;
160 161
	GCancellable *cancellable;
	guint notified : 1;
162 163
} ItemWritebackData;

164
typedef struct {
165
	GFile *file;
166 167
	gchar *urn;
	gchar *parent_urn;
168
	gint priority;
169
	GCancellable *cancellable;
170
	TrackerSparqlBuilder *builder;
171
	TrackerMiner *miner;
172
} UpdateProcessingTaskContext;
173

174 175 176 177 178
typedef struct {
	GMainLoop *main_loop;
	GString   *sparql;
	const gchar *source_uri;
	const gchar *uri;
179
	TrackerMiner *miner;
180 181
} RecursiveMoveData;

182
struct _TrackerMinerFSPrivate {
183
	/* File queues for indexer */
184 185 186 187
	TrackerPriorityQueue *items_created;
	TrackerPriorityQueue *items_updated;
	TrackerPriorityQueue *items_deleted;
	TrackerPriorityQueue *items_moved;
188
	TrackerPriorityQueue *items_writeback;
189 190 191 192 193

	guint item_queues_handler_id;
	GFile *item_queue_blocker;
	GHashTable *items_ignore_next_update;

194
#ifdef EVENT_QUEUE_ENABLE_TRACE
195
	guint queue_status_timeout_id;
196 197
#endif /* EVENT_QUEUE_ENABLE_TRACE */

198 199 200
	/* Root / tree / index */
	GFile *root;
	TrackerIndexingTree *indexing_tree;
201
	TrackerFileNotifier *file_notifier;
202
	TrackerDataProvider *data_provider;
203

204
	/* Sparql insertion tasks */
205
	TrackerTaskPool *task_pool;
206 207
	TrackerSparqlBuffer *sparql_buffer;
	guint sparql_buffer_limit;
208

209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
	/* File properties */
	GQuark quark_ignore_file;
	GQuark quark_attribute_updated;
	GQuark quark_directory_found_crawling;
	GQuark quark_reentry_counter;

	/* Properties */
	gdouble throttle;
	guint mtime_checking : 1;   /* TRUE if mtime checks should be done
	                             * during initial crawling. */
	guint initial_crawling : 1; /* TRUE if initial crawling should be
	                             * done */

	/* Writeback tasks */
	TrackerTaskPool *writeback_pool;
224

225 226
	TrackerThumbnailer *thumbnailer;

227
	/* Status */
228 229 230 231 232 233 234 235 236 237 238 239
	GTimer *timer;
	GTimer *extraction_timer;

	guint been_started : 1;     /* TRUE if miner has been started */
	guint been_crawled : 1;     /* TRUE if initial crawling has been
	                             * done */
	guint shown_totals : 1;     /* TRUE if totals have been shown */
	guint is_paused : 1;        /* TRUE if miner is paused */

	guint timer_stopped : 1;    /* TRUE if main timer is stopped */
	guint extraction_timer_stopped : 1; /* TRUE if the extraction
	                                     * timer is stopped */
240

241 242 243 244 245 246 247 248 249
	GHashTable *roots_to_notify;        /* Used to signal indexing
	                                     * trees finished */

	/*
	 * Statistics
	 */

	/* How many we found during crawling and how many were black
	 * listed (ignored). Reset to 0 when processing stops. */
250 251 252 253 254
	guint total_directories_found;
	guint total_directories_ignored;
	guint total_files_found;
	guint total_files_ignored;

255
	/* How many we indexed and how many had errors indexing. */
256 257 258
	guint total_files_processed;
	guint total_files_notified;
	guint total_files_notified_error;
259 260
};

261
typedef enum {
262 263 264 265
	QUEUE_NONE,
	QUEUE_CREATED,
	QUEUE_UPDATED,
	QUEUE_DELETED,
266
	QUEUE_MOVED,
267
	QUEUE_IGNORE_NEXT_UPDATE,
268 269
	QUEUE_WAIT,
	QUEUE_WRITEBACK
270
} QueueState;
271

272
enum {
273
	PROCESS_FILE,
274
	PROCESS_FILE_ATTRIBUTES,
275
	IGNORE_NEXT_UPDATE_FILE,
276
	FINISHED,
277
	WRITEBACK_FILE,
278
	FINISHED_ROOT,
279 280 281
	LAST_SIGNAL
};

282 283
enum {
	PROP_0,
284
	PROP_THROTTLE,
285
	PROP_ROOT,
286
	PROP_WAIT_POOL_LIMIT,
287
	PROP_READY_POOL_LIMIT,
288
	PROP_DATA_PROVIDER,
289 290
	PROP_MTIME_CHECKING,
	PROP_INITIAL_CRAWLING
291 292
};

293 294
static void           miner_fs_initable_iface_init        (GInitableIface       *iface);

295
static void           fs_finalize                         (GObject              *object);
296
static void           fs_constructed                      (GObject              *object);
297 298 299 300 301 302 303 304
static void           fs_set_property                     (GObject              *object,
                                                           guint                 prop_id,
                                                           const GValue         *value,
                                                           GParamSpec           *pspec);
static void           fs_get_property                     (GObject              *object,
                                                           guint                 prop_id,
                                                           GValue               *value,
                                                           GParamSpec           *pspec);
305

306 307 308 309 310 311 312 313 314
static void           miner_started                       (TrackerMiner         *miner);
static void           miner_stopped                       (TrackerMiner         *miner);
static void           miner_paused                        (TrackerMiner         *miner);
static void           miner_resumed                       (TrackerMiner         *miner);
static void           miner_ignore_next_update            (TrackerMiner         *miner,
                                                           const GStrv           subjects);
static ItemMovedData *item_moved_data_new                 (GFile                *file,
                                                           GFile                *source_file);
static void           item_moved_data_free                (ItemMovedData        *data);
315
static void           item_writeback_data_free            (ItemWritebackData    *data);
316

317
static void           indexing_tree_directory_removed     (TrackerIndexingTree  *indexing_tree,
318 319
                                                           GFile                *directory,
                                                           gpointer              user_data);
320
static void           file_notifier_file_created          (TrackerFileNotifier  *notifier,
321 322
                                                           GFile                *file,
                                                           gpointer              user_data);
323
static void           file_notifier_file_deleted          (TrackerFileNotifier  *notifier,
324 325
                                                           GFile                *file,
                                                           gpointer              user_data);
326
static void           file_notifier_file_updated          (TrackerFileNotifier  *notifier,
327
                                                           GFile                *file,
328
                                                           gboolean              attributes_only,
329
                                                           gpointer              user_data);
330 331 332 333
static void           file_notifier_file_moved            (TrackerFileNotifier  *notifier,
                                                           GFile                *source,
                                                           GFile                *dest,
                                                           gpointer              user_data);
334 335 336
static void           file_notifier_directory_started     (TrackerFileNotifier *notifier,
                                                           GFile               *directory,
                                                           gpointer             user_data);
337 338 339 340 341 342 343 344 345
static void           file_notifier_directory_finished    (TrackerFileNotifier *notifier,
                                                           GFile               *directory,
                                                           guint                directories_found,
                                                           guint                directories_ignored,
                                                           guint                files_found,
                                                           guint                files_ignored,
                                                           gpointer             user_data);
static void           file_notifier_finished              (TrackerFileNotifier *notifier,
                                                           gpointer             user_data);
346

347 348 349 350 351
static void           item_queue_handlers_set_up          (TrackerMinerFS       *fs);
static void           item_update_children_uri            (TrackerMinerFS       *fs,
                                                           RecursiveMoveData    *data,
                                                           const gchar          *source_uri,
                                                           const gchar          *uri);
352

353
static void           task_pool_cancel_foreach                (gpointer        data,
354
                                                               gpointer        user_data);
355 356 357
static void           task_pool_limit_reached_notify_cb       (GObject        *object,
                                                               GParamSpec     *pspec,
                                                               gpointer        user_data);
358

359
static GQuark quark_file_iri = 0;
360
static GInitableIface* miner_fs_initable_parent_iface;
361 362
static guint signals[LAST_SIGNAL] = { 0, };

363 364
G_DEFINE_QUARK (TrackerMinerFSError, tracker_miner_fs_error)

365 366 367
G_DEFINE_ABSTRACT_TYPE_WITH_CODE (TrackerMinerFS, tracker_miner_fs, TRACKER_TYPE_MINER,
                                  G_IMPLEMENT_INTERFACE (G_TYPE_INITABLE,
                                                         miner_fs_initable_iface_init));
368 369

static void
370
tracker_miner_fs_class_init (TrackerMinerFSClass *klass)
371 372
{
	GObjectClass *object_class = G_OBJECT_CLASS (klass);
Martyn Russell's avatar
Martyn Russell committed
373
	TrackerMinerClass *miner_class = TRACKER_MINER_CLASS (klass);
374

375
	object_class->finalize = fs_finalize;
376
	object_class->constructed = fs_constructed;
377 378
	object_class->set_property = fs_set_property;
	object_class->get_property = fs_get_property;
379

Martyn Russell's avatar
Martyn Russell committed
380 381
	miner_class->started = miner_started;
	miner_class->stopped = miner_stopped;
382 383
	miner_class->paused  = miner_paused;
	miner_class->resumed = miner_resumed;
384
	miner_class->ignore_next_update = miner_ignore_next_update;
385

386
	g_object_class_install_property (object_class,
Martyn Russell's avatar
Martyn Russell committed
387 388 389 390 391 392
	                                 PROP_THROTTLE,
	                                 g_param_spec_double ("throttle",
	                                                      "Throttle",
	                                                      "Modifier for the indexing speed, 0 is max speed",
	                                                      0, 1, 0,
	                                                      G_PARAM_READWRITE));
393 394 395 396 397 398 399
	g_object_class_install_property (object_class,
	                                 PROP_ROOT,
	                                 g_param_spec_object ("root",
	                                                      "Root",
	                                                      "Top level URI for our indexing tree and file notify clases",
	                                                      G_TYPE_FILE,
	                                                      G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
400
	g_object_class_install_property (object_class,
401
	                                 PROP_WAIT_POOL_LIMIT,
402
	                                 g_param_spec_uint ("processing-pool-wait-limit",
403 404 405 406 407 408
	                                                    "Processing pool limit for WAIT tasks",
	                                                    "Maximum number of files that can be concurrently "
	                                                    "processed by the upper layer",
	                                                    1, G_MAXUINT, DEFAULT_WAIT_POOL_LIMIT,
	                                                    G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
	g_object_class_install_property (object_class,
409 410 411
	                                 PROP_READY_POOL_LIMIT,
	                                 g_param_spec_uint ("processing-pool-ready-limit",
	                                                    "Processing pool limit for READY tasks",
412 413
	                                                    "Maximum number of SPARQL updates that can be merged "
	                                                    "in a single connection to the store",
414
	                                                    1, G_MAXUINT, DEFAULT_READY_POOL_LIMIT,
Martyn Russell's avatar
Martyn Russell committed
415
	                                                    G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
416
	g_object_class_install_property (object_class,
417 418 419 420 421
	                                 PROP_DATA_PROVIDER,
	                                 g_param_spec_object ("data-provider",
	                                                      "Data provider",
	                                                      "Data provider populating data, e.g. like GFileEnumerator",
	                                                      TRACKER_TYPE_DATA_PROVIDER,
422
	                                                      G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
423
	g_object_class_install_property (object_class,
424
	                                 PROP_MTIME_CHECKING,
425 426 427 428 429 430 431 432 433 434 435 436 437
	                                 g_param_spec_boolean ("mtime-checking",
	                                                       "Mtime checking",
	                                                       "Whether to perform mtime checks during initial crawling or not",
	                                                       TRUE,
	                                                       G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
	g_object_class_install_property (object_class,
	                                 PROP_INITIAL_CRAWLING,
	                                 g_param_spec_boolean ("initial-crawling",
	                                                       "Initial crawling",
	                                                       "Whether to perform initial crawling or not",
	                                                       TRUE,
	                                                       G_PARAM_READWRITE));

438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
	/**
	 * TrackerMinerFS::process-file:
	 * @miner_fs: the #TrackerMinerFS
	 * @file: a #GFile
	 * @builder: a #TrackerSparqlBuilder
	 * @cancellable: a #GCancellable
	 *
	 * The ::process-file signal is emitted whenever a file should
	 * be processed, and it's metadata extracted.
	 *
	 * @builder is the #TrackerSparqlBuilder where all sparql updates
	 * to be performed for @file will be appended.
	 *
	 * This signal allows both synchronous and asynchronous extraction,
	 * in the synchronous case @cancellable can be safely ignored. In
	 * either case, on successful metadata extraction, implementations
Carlos Garnacho's avatar
Carlos Garnacho committed
454
	 * must call tracker_miner_fs_file_notify() to indicate that
455 456 457 458 459
	 * processing has finished on @file, so the miner can execute
	 * the SPARQL updates and continue processing other files.
	 *
	 * Returns: %TRUE if the file is accepted for processing,
	 *          %FALSE if the file should be ignored.
460 461
	 *
	 * Since: 0.8
462 463 464
	 **/
	signals[PROCESS_FILE] =
		g_signal_new ("process-file",
Martyn Russell's avatar
Martyn Russell committed
465 466 467 468
		              G_OBJECT_CLASS_TYPE (object_class),
		              G_SIGNAL_RUN_LAST,
		              G_STRUCT_OFFSET (TrackerMinerFSClass, process_file),
		              NULL, NULL,
Xavier Claessens's avatar
Xavier Claessens committed
469
		              NULL,
Martyn Russell's avatar
Martyn Russell committed
470
		              G_TYPE_BOOLEAN,
471
		              3, G_TYPE_FILE, TRACKER_SPARQL_TYPE_BUILDER, G_TYPE_CANCELLABLE);
472

473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495
	/**
	 * TrackerMinerFS::process-file-attributes:
	 * @miner_fs: the #TrackerMinerFS
	 * @file: a #GFile
	 * @builder: a #TrackerSparqlBuilder
	 * @cancellable: a #GCancellable
	 *
	 * The ::process-file-attributes signal is emitted whenever a file should
	 * be processed, but only the attribute-related metadata extracted.
	 *
	 * @builder is the #TrackerSparqlBuilder where all sparql updates
	 * to be performed for @file will be appended. For the properties being
	 * updated, the DELETE statements should be included as well.
	 *
	 * This signal allows both synchronous and asynchronous extraction,
	 * in the synchronous case @cancellable can be safely ignored. In
	 * either case, on successful metadata extraction, implementations
	 * must call tracker_miner_fs_file_notify() to indicate that
	 * processing has finished on @file, so the miner can execute
	 * the SPARQL updates and continue processing other files.
	 *
	 * Returns: %TRUE if the file is accepted for processing,
	 *          %FALSE if the file should be ignored.
496 497
	 *
	 * Since: 0.10
498 499 500 501 502 503 504
	 **/
	signals[PROCESS_FILE_ATTRIBUTES] =
		g_signal_new ("process-file-attributes",
		              G_OBJECT_CLASS_TYPE (object_class),
		              G_SIGNAL_RUN_LAST,
		              G_STRUCT_OFFSET (TrackerMinerFSClass, process_file_attributes),
		              NULL, NULL,
Xavier Claessens's avatar
Xavier Claessens committed
505
		              NULL,
506 507 508
		              G_TYPE_BOOLEAN,
		              3, G_TYPE_FILE, TRACKER_SPARQL_TYPE_BUILDER, G_TYPE_CANCELLABLE);

509 510 511 512 513 514 515 516 517 518 519 520 521 522 523
	/**
	 * TrackerMinerFS::ignore-next-update-file:
	 * @miner_fs: the #TrackerMinerFS
	 * @file: a #GFile
	 * @builder: a #TrackerSparqlBuilder
	 * @cancellable: a #GCancellable
	 *
	 * The ::ignore-next-update-file signal is emitted whenever a file should
	 * be marked as to ignore on next update, and it's metadata prepared for that.
	 *
	 * @builder is the #TrackerSparqlBuilder where all sparql updates
	 * to be performed for @file will be appended.
	 *
	 * Returns: %TRUE on success
	 *          %FALSE on failure
524 525
	 *
	 * Since: 0.8
526
	 *
527
	 * Deprecated: 0.12
528 529
	 **/
	signals[IGNORE_NEXT_UPDATE_FILE] =
530
		g_signal_new ("ignore-next-update-file",
531 532 533 534
		              G_OBJECT_CLASS_TYPE (object_class),
		              G_SIGNAL_RUN_LAST,
		              G_STRUCT_OFFSET (TrackerMinerFSClass, ignore_next_update_file),
		              NULL, NULL,
Xavier Claessens's avatar
Xavier Claessens committed
535
		              NULL,
536 537
		              G_TYPE_BOOLEAN,
		              3, G_TYPE_FILE, TRACKER_SPARQL_TYPE_BUILDER, G_TYPE_CANCELLABLE);
538 539 540 541 542 543 544 545 546 547 548 549

	/**
	 * TrackerMinerFS::finished:
	 * @miner_fs: the #TrackerMinerFS
	 * @elapsed: elapsed time since mining was started
	 * @directories_found: number of directories found
	 * @directories_ignored: number of ignored directories
	 * @files_found: number of files found
	 * @files_ignored: number of ignored files
	 *
	 * The ::finished signal is emitted when @miner_fs has finished
	 * all pending processing.
550 551
	 *
	 * Since: 0.8
552 553
	 **/
	signals[FINISHED] =
554
		g_signal_new ("finished",
Martyn Russell's avatar
Martyn Russell committed
555 556 557 558
		              G_TYPE_FROM_CLASS (object_class),
		              G_SIGNAL_RUN_LAST,
		              G_STRUCT_OFFSET (TrackerMinerFSClass, finished),
		              NULL, NULL,
Xavier Claessens's avatar
Xavier Claessens committed
559
		              NULL,
Martyn Russell's avatar
Martyn Russell committed
560 561 562 563 564 565 566
		              G_TYPE_NONE,
		              5,
		              G_TYPE_DOUBLE,
		              G_TYPE_UINT,
		              G_TYPE_UINT,
		              G_TYPE_UINT,
		              G_TYPE_UINT);
567

568 569 570 571
	/**
	 * TrackerMinerFS::writeback-file:
	 * @miner_fs: the #TrackerMinerFS
	 * @file: a #GFile
572
	 * @rdf_types: the set of RDF types
573
	 * @results: (element-type GStrv): a set of results prepared by the preparation query
574
	 * @cancellable: a #GCancellable
575 576 577 578 579 580 581 582 583 584 585 586 587 588 589
	 *
	 * The ::writeback-file signal is emitted whenever a file must be written
	 * back
	 *
	 * Returns: %TRUE on success, %FALSE otherwise
	 *
	 * Since: 0.10.20
	 **/
	signals[WRITEBACK_FILE] =
		g_signal_new ("writeback-file",
		              G_OBJECT_CLASS_TYPE (object_class),
		              G_SIGNAL_RUN_LAST,
		              G_STRUCT_OFFSET (TrackerMinerFSClass, writeback_file),
		              NULL,
		              NULL,
Xavier Claessens's avatar
Xavier Claessens committed
590
		              NULL,
591
		              G_TYPE_BOOLEAN,
592
		              4,
593
		              G_TYPE_FILE,
594
		              G_TYPE_STRV,
595 596
		              G_TYPE_PTR_ARRAY,
		              G_TYPE_CANCELLABLE);
597

598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622
	/**
	 * TrackerMinerFS::finished-root:
	 * @miner_fs: the #TrackerMinerFS
	 * @file: a #GFile
	 *
	 * The ::finished-crawl signal is emitted when @miner_fs has
	 * finished finding all resources that need to be indexed
	 * with the root location of @file. At this point, it's likely
	 * many are still in the queue to be added to the database,
	 * but this gives some indication that a location is
	 * processed.
	 *
	 * Since: 1.2
	 **/
	signals[FINISHED_ROOT] =
		g_signal_new ("finished-root",
		              G_TYPE_FROM_CLASS (object_class),
		              G_SIGNAL_RUN_LAST,
		              G_STRUCT_OFFSET (TrackerMinerFSClass, finished_root),
		              NULL, NULL,
		              NULL,
		              G_TYPE_NONE,
		              1,
		              G_TYPE_FILE);

623
	g_type_class_add_private (object_class, sizeof (TrackerMinerFSPrivate));
624 625

	quark_file_iri = g_quark_from_static_string ("tracker-miner-file-iri");
626 627 628
}

static void
629
tracker_miner_fs_init (TrackerMinerFS *object)
630
{
631
	TrackerMinerFSPrivate *priv;
632

633
	object->priv = TRACKER_MINER_FS_GET_PRIVATE (object);
634

635
	priv = object->priv;
636

637 638 639
	priv->timer = g_timer_new ();
	priv->extraction_timer = g_timer_new ();

640 641 642
	g_timer_stop (priv->timer);
	g_timer_stop (priv->extraction_timer);

643 644 645
	priv->timer_stopped = TRUE;
	priv->extraction_timer_stopped = TRUE;

646 647 648 649
	priv->items_created = tracker_priority_queue_new ();
	priv->items_updated = tracker_priority_queue_new ();
	priv->items_deleted = tracker_priority_queue_new ();
	priv->items_moved = tracker_priority_queue_new ();
650
	priv->items_writeback = tracker_priority_queue_new ();
651 652 653 654 655 656 657

#ifdef EVENT_QUEUE_ENABLE_TRACE
	priv->queue_status_timeout_id = g_timeout_add_seconds (EVENT_QUEUE_STATUS_TIMEOUT_SECS,
	                                                       miner_fs_queues_status_trace_timeout_cb,
	                                                       object);
#endif /* PROCESSING_POOL_ENABLE_TRACE */

658 659 660
	priv->items_ignore_next_update = g_hash_table_new_full (g_str_hash, g_str_equal,
	                                                        (GDestroyNotify) g_free,
	                                                        (GDestroyNotify) NULL);
661

662 663
	/* Create processing pools */
	priv->task_pool = tracker_task_pool_new (DEFAULT_WAIT_POOL_LIMIT);
664 665 666
	g_signal_connect (priv->task_pool, "notify::limit-reached",
	                  G_CALLBACK (task_pool_limit_reached_notify_cb), object);

667
	priv->writeback_pool = tracker_task_pool_new (DEFAULT_WAIT_POOL_LIMIT);
668 669
	g_signal_connect (priv->writeback_pool, "notify::limit-reached",
	                  G_CALLBACK (task_pool_limit_reached_notify_cb), object);
670

671
	priv->quark_ignore_file = g_quark_from_static_string ("tracker-ignore-file");
672
	priv->quark_directory_found_crawling = g_quark_from_static_string ("tracker-directory-found-crawling");
673
	priv->quark_attribute_updated = g_quark_from_static_string ("tracker-attribute-updated");
674
	priv->quark_reentry_counter = g_quark_from_static_string ("tracker-reentry-counter");
675

676 677
	priv->mtime_checking = TRUE;
	priv->initial_crawling = TRUE;
678 679 680 681 682

	priv->roots_to_notify = g_hash_table_new_full (g_file_hash,
	                                               (GEqualFunc) g_file_equal,
	                                               g_object_unref,
	                                               NULL);
683 684
}

685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
static gboolean
miner_fs_initable_init (GInitable     *initable,
                        GCancellable  *cancellable,
                        GError       **error)
{
	TrackerMinerFSPrivate *priv;
	guint limit;

	if (!miner_fs_initable_parent_iface->init (initable, cancellable, error)) {
		return FALSE;
	}

	priv = TRACKER_MINER_FS_GET_PRIVATE (initable);

	g_object_get (initable, "processing-pool-ready-limit", &limit, NULL);
	priv->sparql_buffer = tracker_sparql_buffer_new (tracker_miner_get_connection (TRACKER_MINER (initable)),
	                                                 limit);
702 703

	if (!priv->sparql_buffer) {
704 705 706 707
		g_set_error (error,
		             tracker_miner_fs_error_quark (),
		             TRACKER_MINER_FS_ERROR_INIT,
		             "Could not create TrackerSparqlBuffer needed to process resources");
708 709 710
		return FALSE;
	}

711 712 713 714
	g_signal_connect (priv->sparql_buffer, "notify::limit-reached",
	                  G_CALLBACK (task_pool_limit_reached_notify_cb),
	                  initable);

715
	if (!priv->indexing_tree) {
716 717 718 719
		g_set_error (error,
		             tracker_miner_fs_error_quark (),
		             TRACKER_MINER_FS_ERROR_INIT,
		             "Could not create TrackerIndexingTree needed to manage content indexed");
720 721 722 723 724 725 726 727 728
		return FALSE;
	}

	g_signal_connect (priv->indexing_tree, "directory-removed",
	                  G_CALLBACK (indexing_tree_directory_removed),
	                  initable);

	/* Create the file notifier */
	priv->file_notifier = tracker_file_notifier_new (priv->indexing_tree,
729
	                                                 priv->data_provider);
730 731

	if (!priv->file_notifier) {
732 733 734 735
		g_set_error (error,
		             tracker_miner_fs_error_quark (),
		             TRACKER_MINER_FS_ERROR_INIT,
		             "Could not create TrackerFileNotifier needed to signal new resources to be indexed");
736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760
		return FALSE;
	}

	g_signal_connect (priv->file_notifier, "file-created",
	                  G_CALLBACK (file_notifier_file_created),
	                  initable);
	g_signal_connect (priv->file_notifier, "file-updated",
	                  G_CALLBACK (file_notifier_file_updated),
	                  initable);
	g_signal_connect (priv->file_notifier, "file-deleted",
	                  G_CALLBACK (file_notifier_file_deleted),
	                  initable);
	g_signal_connect (priv->file_notifier, "file-moved",
	                  G_CALLBACK (file_notifier_file_moved),
	                  initable);
	g_signal_connect (priv->file_notifier, "directory-started",
	                  G_CALLBACK (file_notifier_directory_started),
	                  initable);
	g_signal_connect (priv->file_notifier, "directory-finished",
	                  G_CALLBACK (file_notifier_directory_finished),
	                  initable);
	g_signal_connect (priv->file_notifier, "finished",
	                  G_CALLBACK (file_notifier_finished),
	                  initable);

761 762
	priv->thumbnailer = tracker_thumbnailer_new ();

763 764 765 766 767 768 769 770 771 772
	return TRUE;
}

static void
miner_fs_initable_iface_init (GInitableIface *iface)
{
	miner_fs_initable_parent_iface = g_type_interface_peek_parent (iface);
	iface->init = miner_fs_initable_init;
}

773
static void
774
fs_finalize (GObject *object)
775
{
776
	TrackerMinerFSPrivate *priv;
777

778
	priv = TRACKER_MINER_FS_GET_PRIVATE (object);
779

780 781
	g_timer_destroy (priv->timer);
	g_timer_destroy (priv->extraction_timer);
782

783 784 785
	if (priv->item_queues_handler_id) {
		g_source_remove (priv->item_queues_handler_id);
		priv->item_queues_handler_id = 0;
786
	}
787

788 789
	if (priv->item_queue_blocker) {
		g_object_unref (priv->item_queue_blocker);
790 791
	}

792 793 794
	if (priv->file_notifier) {
		tracker_file_notifier_stop (priv->file_notifier);
	}
795

796
	/* Cancel every pending task */
797 798 799 800 801
	tracker_task_pool_foreach (priv->task_pool,
	                           task_pool_cancel_foreach,
	                           NULL);
	g_object_unref (priv->task_pool);

802 803
	g_object_unref (priv->writeback_pool);

804 805 806
	if (priv->sparql_buffer) {
		g_object_unref (priv->sparql_buffer);
	}
807

808 809 810 811
	tracker_priority_queue_foreach (priv->items_moved,
	                                (GFunc) item_moved_data_free,
	                                NULL);
	tracker_priority_queue_unref (priv->items_moved);
812

813 814 815 816
	tracker_priority_queue_foreach (priv->items_deleted,
	                                (GFunc) g_object_unref,
	                                NULL);
	tracker_priority_queue_unref (priv->items_deleted);
817

818 819 820 821
	tracker_priority_queue_foreach (priv->items_updated,
	                                (GFunc) g_object_unref,
	                                NULL);
	tracker_priority_queue_unref (priv->items_updated);
822

823 824 825 826
	tracker_priority_queue_foreach (priv->items_created,
	                                (GFunc) g_object_unref,
	                                NULL);
	tracker_priority_queue_unref (priv->items_created);
827

828
	tracker_priority_queue_foreach (priv->items_writeback,
829 830
	                                (GFunc) item_writeback_data_free,
	                                NULL);
831 832
	tracker_priority_queue_unref (priv->items_writeback);

833
	g_hash_table_unref (priv->items_ignore_next_update);
834

835 836 837 838 839 840 841
	if (priv->indexing_tree) {
		g_object_unref (priv->indexing_tree);
	}

	if (priv->file_notifier) {
		g_object_unref (priv->file_notifier);
	}
842

843
	if (priv->thumbnailer) {
844
		g_object_unref (priv->thumbnailer);
845 846 847 848
	}

	if (priv->roots_to_notify) {
		g_hash_table_unref (priv->roots_to_notify);
849 850 851

		/* Just in case we end up using this AFTER finalize, not expected */
		priv->roots_to_notify = NULL;
852
	}
853

854 855 856 857 858
#ifdef EVENT_QUEUE_ENABLE_TRACE
	if (priv->queue_status_timeout_id)
		g_source_remove (priv->queue_status_timeout_id);
#endif /* PROCESSING_POOL_ENABLE_TRACE */

859
	G_OBJECT_CLASS (tracker_miner_fs_parent_class)->finalize (object);
860 861
}

862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887
static void
fs_constructed (GObject *object)
{
	TrackerMinerFSPrivate *priv;

	/* NOTE: We have to do this in this order because initables
	 * are called _AFTER_ constructed and for subclasses that are
	 * not initables we don't have any other way than to chain
	 * constructed and root/indexing tree must exist at that
	 * point.
	 *
	 * If priv->indexing_tree is NULL after this function, the
	 * initiable functions will fail and this class will not be
	 * created anyway.
	 */
	G_OBJECT_CLASS (tracker_miner_fs_parent_class)->constructed (object);

	priv = TRACKER_MINER_FS_GET_PRIVATE (object);

	/* Create root if one didn't exist */
	if (priv->root == NULL) {
		/* We default to file:/// */
		priv->root = g_file_new_for_uri ("file:///");
	}

	/* Create indexing tree */
888
	priv->indexing_tree = tracker_indexing_tree_new_with_root (priv->root);
889 890
}

891 892
static void
fs_set_property (GObject      *object,
Martyn Russell's avatar
Martyn Russell committed
893 894 895
                 guint         prop_id,
                 const GValue *value,
                 GParamSpec   *pspec)
896
{
897 898
	TrackerMinerFS *fs = TRACKER_MINER_FS (object);

899 900 901
	switch (prop_id) {
	case PROP_THROTTLE:
		tracker_miner_fs_set_throttle (TRACKER_MINER_FS (object),
Martyn Russell's avatar
Martyn Russell committed
902
		                               g_value_get_double (value));
903
		break;
904 905 906 907
	case PROP_ROOT:
		/* We expect this to only occur once, on object construct */
		fs->priv->root = g_value_dup_object (value);
		break;
908
	case PROP_WAIT_POOL_LIMIT:
909 910
		tracker_task_pool_set_limit (fs->priv->task_pool,
		                             g_value_get_uint (value));
911
		break;
912
	case PROP_READY_POOL_LIMIT:
913 914 915 916 917 918
		fs->priv->sparql_buffer_limit = g_value_get_uint (value);

		if (fs->priv->sparql_buffer) {
			tracker_task_pool_set_limit (TRACKER_TASK_POOL (fs->priv->sparql_buffer),
			                             fs->priv->sparql_buffer_limit);
		}
919
		break;
920 921
	case PROP_DATA_PROVIDER:
		fs->priv->data_provider = g_value_dup_object (value);
922
		break;
923
	case PROP_MTIME_CHECKING:
924
		fs->priv->mtime_checking = g_value_get_boolean (value);
925
		break;
926
	case PROP_INITIAL_CRAWLING:
927
		fs->priv->initial_crawling = g_value_get_boolean (value);
928
		break;
929 930 931 932 933 934 935 936
	default:
		G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
		break;
	}
}

static void
fs_get_property (GObject    *object,
Martyn Russell's avatar
Martyn Russell committed
937 938 939
                 guint       prop_id,
                 GValue     *value,
                 GParamSpec *pspec)
940 941 942 943 944 945 946
{
	TrackerMinerFS *fs;

	fs = TRACKER_MINER_FS (object);

	switch (prop_id) {
	case PROP_THROTTLE:
947
		g_value_set_double (value, fs->priv->throttle);
948
		break;
949 950 951
	case PROP_ROOT:
		g_value_set_object (value, fs->priv->root);
		break;
952
	case PROP_WAIT_POOL_LIMIT:
953
		g_value_set_uint (value, tracker_task_pool_get_limit (fs->priv->task_pool));
954
		break;
955
	case PROP_READY_POOL_LIMIT:
956
		g_value_set_uint (value, fs->priv->sparql_buffer_limit);
957
		break;
958
	case PROP_MTIME_CHECKING:
959
		g_value_set_boolean (value, fs->priv->mtime_checking);
960
		break;
961 962
	case PROP_DATA_PROVIDER:
		g_value_set_object (value, fs->priv->data_provider);
963
		break;
964
	case PROP_INITIAL_CRAWLING:
965
		g_value_set_boolean (value, fs->priv->initial_crawling);
966
		break;
967 968 969 970 971 972
	default:
		G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
		break;
	}
}

973 974 975 976 977 978 979 980 981 982
static void
task_pool_limit_reached_notify_cb (GObject    *object,
				   GParamSpec *pspec,
				   gpointer    user_data)
{
	if (!tracker_task_pool_limit_reached (TRACKER_TASK_POOL (object))) {
		item_queue_handlers_set_up (TRACKER_MINER_FS (user_data));
	}
}

983 984 985
static void
miner_started (TrackerMiner *miner)
{
986
	TrackerMinerFS *fs;
987

988
	fs = TRACKER_MINER_FS (miner);
989

990
	fs->priv->been_started = TRUE;
991

992 993
	tracker_info ("Initializing");

994
	g_object_set (miner,
Martyn Russell's avatar
Martyn Russell committed
995
	              "progress", 0.0,
996
	              "status", "Initializing",
997
	              "remaining-time", 0,
Martyn Russell's avatar
Martyn Russell committed
998
	              NULL);
999

1000
	tracker_file_notifier_start (fs->priv->file_notifier);
1001 1002
}

1003 1004 1005
static void
miner_stopped (TrackerMiner *miner)
{
1006 1007
	tracker_info ("Idle");

1008
	g_object_set (miner,
Martyn Russell's avatar
Martyn Russell committed
1009
	              "progress", 1.0,
1010
	              "status", "Idle",
1011
	              "remaining-time", -1,
Martyn Russell's avatar
Martyn Russell committed
1012
	              NULL);
1013 1014
}

1015 1016 1017
static void
miner_paused (TrackerMiner *miner)
{
1018
	TrackerMinerFS *fs;
1019

1020
	fs = TRACKER_MINER_FS (miner);
1021

1022
	fs->priv->is_paused = TRUE;
1023

1024
	tracker_file_notifier_stop (fs->priv->file_notifier);
Carlos Garnacho's avatar