BatchImport.vala 81.5 KB
Newer Older
1
/* Copyright 2016 Software Freedom Conservancy Inc.
2
3
 *
 * This software is licensed under the GNU LGPL (version 2.1 or later).
4
 * See the COPYING file in this distribution.
5
 */
6

7
8
9
10
11
12
13
14
15
16
17
18
19
public enum ImportResult {
    SUCCESS,
    FILE_ERROR,
    DECODE_ERROR,
    DATABASE_ERROR,
    USER_ABORT,
    NOT_A_FILE,
    PHOTO_EXISTS,
    UNSUPPORTED_FORMAT,
    NOT_AN_IMAGE,
    DISK_FAILURE,
    DISK_FULL,
    CAMERA_ERROR,
20
21
    FILE_WRITE_ERROR,
    PIXBUF_CORRUPT_IMAGE;
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
    
    public string to_string() {
        switch (this) {
            case SUCCESS:
                return _("Success");
            
            case FILE_ERROR:
                return _("File error");
            
            case DECODE_ERROR:
                return _("Unable to decode file");
            
            case DATABASE_ERROR:
                return _("Database error");
            
            case USER_ABORT:
                return _("User aborted import");
            
            case NOT_A_FILE:
                return _("Not a file");
            
            case PHOTO_EXISTS:
                return _("File already exists in database");
            
            case UNSUPPORTED_FORMAT:
                return _("Unsupported file format");

            case NOT_AN_IMAGE:
                return _("Not an image file");
            
            case DISK_FAILURE:
                return _("Disk failure");
            
            case DISK_FULL:
                return _("Disk full");
            
            case CAMERA_ERROR:
                return _("Camera error");
            
            case FILE_WRITE_ERROR:
                return _("File write error");
63
64
65

            case PIXBUF_CORRUPT_IMAGE:
                return _("Corrupt image file");
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
            
            default:
                return _("Imported failed (%d)").printf((int) this);
        }
    }
    
    public bool is_abort() {
        switch (this) {
            case ImportResult.DISK_FULL:
            case ImportResult.DISK_FAILURE:
            case ImportResult.USER_ABORT:
                return true;
            
            default:
                return false;
        }
    }
    
    public bool is_nonuser_abort() {
        switch (this) {
            case ImportResult.DISK_FULL:
            case ImportResult.DISK_FAILURE:
                return true;
            
            default:
                return false;
        }
    }
    
    public static ImportResult convert_error(Error err, ImportResult default_result) {
        if (err is FileError) {
            FileError ferr = (FileError) err;
            
            if (ferr is FileError.NOSPC)
                return ImportResult.DISK_FULL;
            else if (ferr is FileError.IO)
                return ImportResult.DISK_FAILURE;
            else if (ferr is FileError.ISDIR)
                return ImportResult.NOT_A_FILE;
105
            else if (ferr is FileError.ACCES)
106
                return ImportResult.FILE_WRITE_ERROR;
107
108
            else if (ferr is FileError.PERM)
                return ImportResult.FILE_WRITE_ERROR;
109
110
111
112
113
114
115
116
117
118
119
120
121
            else
                return ImportResult.FILE_ERROR;
        } else if (err is IOError) {
            IOError ioerr = (IOError) err;
            
            if (ioerr is IOError.NO_SPACE)
                return ImportResult.DISK_FULL;
            else if (ioerr is IOError.FAILED)
                return ImportResult.DISK_FAILURE;
            else if (ioerr is IOError.IS_DIRECTORY)
                return ImportResult.NOT_A_FILE;
            else if (ioerr is IOError.CANCELLED)
                return ImportResult.USER_ABORT;
122
            else if (ioerr is IOError.READ_ONLY)
123
                return ImportResult.FILE_WRITE_ERROR;
124
125
            else if (ioerr is IOError.PERMISSION_DENIED)
                return ImportResult.FILE_WRITE_ERROR;
126
127
128
129
            else
                return ImportResult.FILE_ERROR;
        } else if (err is GPhotoError) {
            return ImportResult.CAMERA_ERROR;
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
        } else if (err is Gdk.PixbufError) {
            Gdk.PixbufError pixbuferr = (Gdk.PixbufError) err;

            if (pixbuferr is Gdk.PixbufError.CORRUPT_IMAGE)
                return ImportResult.PIXBUF_CORRUPT_IMAGE;
            else if (pixbuferr is Gdk.PixbufError.INSUFFICIENT_MEMORY)
                return default_result;
            else if (pixbuferr is Gdk.PixbufError.BAD_OPTION)
                return default_result;
            else if (pixbuferr is Gdk.PixbufError.UNKNOWN_TYPE)
                return ImportResult.UNSUPPORTED_FORMAT;
            else if (pixbuferr is Gdk.PixbufError.UNSUPPORTED_OPERATION)
                return default_result;
            else if (pixbuferr is Gdk.PixbufError.FAILED)
                return default_result;
            else
                return default_result;
147
148
149
150
151
152
        }
        
        return default_result;
    }
}

153
154
155
// A BatchImportJob describes a unit of work the BatchImport object should perform.  It returns
// a file to be imported.  If the file is a directory, it is automatically recursed by BatchImport
// to find all files that need to be imported into the library.
156
//
157
// NOTE: All methods may be called from the context of a background thread or the main GTK thread.
158
159
// Implementations should be able to handle either situation.  The prepare method will always be
// called by the same thread context.
160
public abstract class BatchImportJob {
161
162
163
    public abstract string get_dest_identifier();
    
    public abstract string get_source_identifier();
164
    
165
166
    public abstract bool is_directory();
    
Eric Gregory's avatar
Eric Gregory committed
167
168
169
170
    public abstract string get_basename();
    
    public abstract string get_path();
    
171
172
173
    public virtual DuplicatedFile? get_duplicated_file() {
        return null;
    }
174
175
176
177

    public virtual File? get_associated_file() {
        return null;
    }
178
    
Eric Gregory's avatar
Eric Gregory committed
179
180
181
    // Attaches a sibling job (for RAW+JPEG)
    public abstract void set_associated(BatchImportJob associated);
    
182
183
184
185
186
187
188
    // Returns the file size of the BatchImportJob or returns a file/directory which can be queried
    // by BatchImportJob to determine it.  Returns true if the size is return, false if the File is
    // specified.
    // 
    // filesize should only be returned if BatchImportJob represents a single file.
    public abstract bool determine_file_size(out uint64 filesize, out File file_or_dir);
    
189
    // NOTE: prepare( ) is called from a background thread in the worker pool
190
    public abstract bool prepare(out File file_to_import, out bool copy_to_library) throws Error;
191
192
193
194
195
196
197
    
    // Completes the import for the new library photo once it's been imported.
    // If the job is directory based, this method will be called for each photo
    // discovered in the directory. This method is only called for photographs
    // that have been successfully imported.
    //
    // Returns true if any action was taken, false otherwise.
198
199
    //
    // NOTE: complete( )is called from the foreground thread
200
    public virtual bool complete(MediaSource source, BatchImportRoll import_roll) throws Error {
201
202
        return false;
    }
203
204
205
206
207
208
    
    // returns a non-zero time_t value if this has a valid exposure time override, returns zero
    // otherwise
    public virtual time_t get_exposure_time_override() {
        return 0;
    }
209
210
211
212

    public virtual bool recurse() {
        return true;
    }
213
214
}

215
216
217
public class FileImportJob : BatchImportJob {
    private File file_or_dir;
    private bool copy_to_library;
Eric Gregory's avatar
Eric Gregory committed
218
    private FileImportJob? associated = null;
219
    private bool _recurse;
220
    
221
    public FileImportJob(File file_or_dir, bool copy_to_library, bool recurse) {
222
223
        this.file_or_dir = file_or_dir;
        this.copy_to_library = copy_to_library;
224
        this._recurse = recurse;
225
226
    }
    
227
228
229
230
231
    public override string get_dest_identifier() {
        return file_or_dir.get_path();
    }
    
    public override string get_source_identifier() {
232
233
234
235
236
237
238
        return file_or_dir.get_path();
    }
    
    public override bool is_directory() {
        return query_is_directory(file_or_dir);
    }
    
Eric Gregory's avatar
Eric Gregory committed
239
240
241
242
243
244
245
246
247
248
249
250
    public override string get_basename() {
        return file_or_dir.get_basename();
    }
    
    public override string get_path() {
        return is_directory() ? file_or_dir.get_path() : file_or_dir.get_parent().get_path();
    }
    
    public override void set_associated(BatchImportJob associated) {
        this.associated = associated as FileImportJob;
    }
    
251
    public override bool determine_file_size(out uint64 filesize, out File file) {
252
        filesize = 0;
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
        file = file_or_dir;
        
        return false;
    }
    
    public override bool prepare(out File file_to_import, out bool copy) {
        file_to_import = file_or_dir;
        copy = copy_to_library;
        
        return true;
    }
    
    public File get_file() {
        return file_or_dir;
    }
268
269
270
271

    public override bool recurse() {
        return this._recurse;
    }
272
273
}

274
275
276
277
278
279
280
281
// A BatchImportRoll represents important state for a group of imported media.  If this is shared
// among multiple BatchImport objects, the imported media will appear to have been imported all at
// once.
public class BatchImportRoll {
    public ImportID import_id;
    public ViewCollection generated_events = new ViewCollection("BatchImportRoll generated events");
    
    public BatchImportRoll() {
282
        this.import_id = ImportID.generate();
283
284
285
    }
}

286
287
// A BatchImportResult associates a particular job with a File that an import was performed on
// and the import result.  A BatchImportJob can specify multiple files, so there is not necessarily
288
// a one-to-one relationship between it and this object.
289
290
291
//
// Note that job may be null (in the case of a pre-failed job that must be reported) and file may
// be null (for similar reasons).
292
public class BatchImportResult {
293
    public BatchImportJob job;
294
    public File? file;
295
296
    public string src_identifier;   // Source path
    public string dest_identifier;  // Destination path
297
    public ImportResult result;
298
    public string? errmsg = null;
299
    public DuplicatedFile? duplicate_of;
300
    
301
    public BatchImportResult(BatchImportJob job, File? file, string src_identifier, 
302
        string dest_identifier, DuplicatedFile? duplicate_of, ImportResult result) {
303
304
        this.job = job;
        this.file = file;
305
306
        this.src_identifier = src_identifier;
        this.dest_identifier = dest_identifier;
307
        this.duplicate_of = duplicate_of;
308
309
        this.result = result;
    }
310
    
311
312
    public BatchImportResult.from_error(BatchImportJob job, File? file, string src_identifier,
        string dest_identifier, Error err, ImportResult default_result) {
313
314
        this.job = job;
        this.file = file;
315
316
        this.src_identifier = src_identifier;
        this.dest_identifier = dest_identifier;
317
318
319
        this.result = ImportResult.convert_error(err, default_result);
        this.errmsg = err.message;
    }
320
321
}

322
public class ImportManifest {
323
    public Gee.List<MediaSource> imported = new Gee.ArrayList<MediaSource>();
324
    public Gee.List<BatchImportResult> success = new Gee.ArrayList<BatchImportResult>();
325
    public Gee.List<BatchImportResult> camera_failed = new Gee.ArrayList<BatchImportResult>();
326
    public Gee.List<BatchImportResult> failed = new Gee.ArrayList<BatchImportResult>();
327
    public Gee.List<BatchImportResult> write_failed = new Gee.ArrayList<BatchImportResult>();
328
329
    public Gee.List<BatchImportResult> skipped_photos = new Gee.ArrayList<BatchImportResult>();
    public Gee.List<BatchImportResult> skipped_files = new Gee.ArrayList<BatchImportResult>();
330
331
    public Gee.List<BatchImportResult> aborted = new Gee.ArrayList<BatchImportResult>();
    public Gee.List<BatchImportResult> already_imported = new Gee.ArrayList<BatchImportResult>();
332
    public Gee.List<BatchImportResult> corrupt_files = new Gee.ArrayList<BatchImportResult>();
333
    public Gee.List<BatchImportResult> all = new Gee.ArrayList<BatchImportResult>();
Jens Georg's avatar
Jens Georg committed
334
    public GLib.Timer timer;
335
    
336
337
    public ImportManifest(Gee.List<BatchImportJob>? prefailed = null,
        Gee.List<BatchImportJob>? pre_already_imported = null) {
Jens Georg's avatar
Jens Georg committed
338
        this.timer = new Timer();
339
340
        if (prefailed != null) {
            foreach (BatchImportJob job in prefailed) {
341
                BatchImportResult batch_result = new BatchImportResult(job, null, 
342
343
344
                    job.get_source_identifier(), job.get_dest_identifier(), null,
                    ImportResult.FILE_ERROR);
                    
345
346
347
348
349
350
                add_result(batch_result);
            }
        }
        
        if (pre_already_imported != null) {
            foreach (BatchImportJob job in pre_already_imported) {
351
352
353
354
355
                BatchImportResult batch_result = new BatchImportResult(job,
                    File.new_for_path(job.get_basename()),
                    job.get_source_identifier(), job.get_dest_identifier(),
                    job.get_duplicated_file(), ImportResult.PHOTO_EXISTS);
                
356
357
358
359
360
361
                add_result(batch_result);
            }
        }
    }
    
    public void add_result(BatchImportResult batch_result) {
362
        bool reported = true;
363
364
365
366
        switch (batch_result.result) {
            case ImportResult.SUCCESS:
                success.add(batch_result);
            break;
367

368
            case ImportResult.USER_ABORT:
369
                if (batch_result.file != null && !query_is_directory(batch_result.file))
370
                    aborted.add(batch_result);
371
372
                else
                    reported = false;
373
374
375
            break;

            case ImportResult.UNSUPPORTED_FORMAT:
376
377
378
379
380
381
                skipped_photos.add(batch_result);
            break;

            case ImportResult.NOT_A_FILE:
            case ImportResult.NOT_AN_IMAGE:
                skipped_files.add(batch_result);
382
383
384
385
386
387
            break;
            
            case ImportResult.PHOTO_EXISTS:
                already_imported.add(batch_result);
            break;
            
388
389
390
391
            case ImportResult.CAMERA_ERROR:
                camera_failed.add(batch_result);
            break;
            
392
393
394
395
            case ImportResult.FILE_WRITE_ERROR:
                write_failed.add(batch_result);
            break;
            
396
397
398
399
            case ImportResult.PIXBUF_CORRUPT_IMAGE:
                corrupt_files.add(batch_result);
            break;
            
400
401
402
403
404
            default:
                failed.add(batch_result);
            break;
        }
        
405
406
        if (reported)
            all.add(batch_result);
407
408
409
    }
}

410
// BatchImport performs the work of taking a file (supplied by BatchImportJob's) and properly importing
411
412
// it into the system, including database additions and thumbnail creation.  It can be monitored by
// multiple observers, but only one ImportReporter can be registered.
413
414
415
416
417
//
// TODO: With background threads. the better way to implement this is via a FSM (finite state 
// machine) that exists in states and responds to various events thrown off by the background
// jobs.  However, getting this code to a point that it works with threads is task enough, so it
// will have to wait (especially since we'll want to write a generic FSM engine).
418
public class BatchImport : Object {
419
    private const int WORK_SNIFFER_THROBBER_MSEC = 125;
420
    
421
422
    public const int REPORT_EVERY_N_PREPARED_FILES = 100;
    public const int REPORT_PREPARED_FILES_EVERY_N_MSEC = 3000;
423
424
425
426
427
428
    
    private const int READY_SOURCES_COUNT_OVERFLOW = 10;
    
    private const int DISPLAY_QUEUE_TIMER_MSEC = 125;
    private const int DISPLAY_QUEUE_HYSTERESIS_OVERFLOW = (3 * 1000) / DISPLAY_QUEUE_TIMER_MSEC;
    
429
430
    private static Workers feeder_workers = new Workers(1, false);
    private static Workers import_workers = new Workers(Workers.thread_per_cpu_minus_one(), false);
431
    
432
    private Gee.Iterable<BatchImportJob> jobs;
433
    private BatchImportRoll import_roll;
434
    private string name;
435
436
    private uint64 completed_bytes = 0;
    private uint64 total_bytes = 0;
437
    private unowned ImportReporter reporter;
438
439
    private ImportManifest manifest;
    private bool scheduled = false;
440
441
442
443
    private bool completed = false;
    private int file_imports_to_perform = -1;
    private int file_imports_completed = 0;
    private Cancellable? cancellable = null;
444
    private ulong last_preparing_ms = 0;
445
    private Gee.HashSet<File> skipset;
446
#if !NO_DUPE_DETECTION
447
    private Gee.HashMap<string, File> imported_full_md5_table = new Gee.HashMap<string, File>();
448
#endif
449
    private uint throbber_id = 0;
Jens Georg's avatar
Jens Georg committed
450
    private uint max_outstanding_import_jobs = Workers.thread_per_cpu_minus_one();
451
    private bool untrash_duplicates = true;
452
    private bool mark_duplicates_online = true;
453
454
455
    
    // These queues are staging queues, holding batches of work that must happen in the import
    // process, working on them all at once to minimize overhead.
456
457
458
459
460
    private Gee.List<PreparedFile> ready_files = new Gee.LinkedList<PreparedFile>();
    private Gee.List<CompletedImportObject> ready_thumbnails =
        new Gee.LinkedList<CompletedImportObject>();
    private Gee.List<CompletedImportObject> display_imported_queue =
        new Gee.LinkedList<CompletedImportObject>();
461
    private Gee.List<CompletedImportObject> ready_sources = new Gee.LinkedList<CompletedImportObject>();
462
463
464
    
    // Called at the end of the batched jobs.  Can be used to report the result of the import
    // to the user.  This is called BEFORE import_complete is fired.
465
    public delegate void ImportReporter(ImportManifest manifest, BatchImportRoll import_roll);
466
    
467
    // Called once, when the scheduled task begins
468
469
    public signal void starting();
    
470
471
472
473
474
475
476
    // Called repeatedly while preparing the launched BatchImport
    public signal void preparing();
    
    // Called repeatedly to report the progress of the BatchImport (but only called after the
    // last "preparing" signal)
    public signal void progress(uint64 completed_bytes, uint64 total_bytes);
    
477
478
    // Called for each Photo or Video imported to the system. For photos, the pixbuf is
    // screen-sized and rotated. For videos, the pixbuf is a frame-grab of the first frame.
479
480
481
    //
    // The to_follow number is the number of queued-up sources to expect following this signal
    // in one burst.
482
    public signal void imported(MediaSource source, Gdk.Pixbuf pixbuf, int to_follow);
483
    
484
485
486
487
    // Called when a fatal error occurs that stops the import entirely.  Remaining jobs will be
    // failed and import_complete() is still fired.
    public signal void fatal_error(ImportResult result, string message);
    
488
489
490
491
    // Called when a job fails.  import_complete will also be called at the end of the batch
    public signal void import_job_failed(BatchImportResult result);
    
    // Called at the end of the batched jobs; this will be signalled exactly once for the batch
492
    public signal void import_complete(ImportManifest manifest, BatchImportRoll import_roll);
493
494

    public BatchImport(Gee.Iterable<BatchImportJob> jobs, string name, ImportReporter? reporter,
495
        Gee.ArrayList<BatchImportJob>? prefailed = null,
496
        Gee.ArrayList<BatchImportJob>? pre_already_imported = null,
497
498
        Cancellable? cancellable = null, BatchImportRoll? import_roll = null,
        ImportManifest? skip_manifest = null) {
499
500
501
502
        this.jobs = jobs;
        this.name = name;
        this.reporter = reporter;
        this.manifest = new ImportManifest(prefailed, pre_already_imported);
503
        this.cancellable = (cancellable != null) ? cancellable : new Cancellable();
504
        this.import_roll = import_roll != null ? import_roll : new BatchImportRoll();
505
        
506
507
        if (skip_manifest != null) {
            skipset = new Gee.HashSet<File>(file_hash, file_equal);
508
509
            foreach (MediaSource source in skip_manifest.imported) {
                skipset.add(source.get_file());
510
511
512
            }
        }
        
513
        // watch for user exit in the application
514
        Application.get_instance().exiting.connect(user_halt);
515
516
        
        // Use a timer to report imported photos to observers
517
        Timeout.add(DISPLAY_QUEUE_TIMER_MSEC, display_imported_timer);
518
519
    }
    
520
    ~BatchImport() {
521
522
523
#if TRACE_DTORS
        debug("DTOR: BatchImport (%s)", name);
#endif
524
        Application.get_instance().exiting.disconnect(user_halt);
525
526
    }
    
527
528
529
530
    public string get_name() {
        return name;
    }
    
531
532
    public void user_halt() {
        cancellable.cancel();
533
534
    }
    
535
536
537
538
539
540
541
542
    public bool get_untrash_duplicates() {
        return untrash_duplicates;
    }
    
    public void set_untrash_duplicates(bool untrash_duplicates) {
        this.untrash_duplicates = untrash_duplicates;
    }
    
543
544
545
546
547
548
549
550
    public bool get_mark_duplicates_online() {
        return mark_duplicates_online;
    }
    
    public void set_mark_duplicates_online(bool mark_duplicates_online) {
        this.mark_duplicates_online = mark_duplicates_online;
    }
    
551
552
    private void log_status(string where) {
#if TRACE_IMPORT
553
        debug("%s: to_perform=%d completed=%d ready_files=%d ready_thumbnails=%d display_queue=%d ready_sources=%d",
554
            where, file_imports_to_perform, file_imports_completed, ready_files.size,
555
            ready_thumbnails.size, display_imported_queue.size, ready_sources.size);
556
557
        debug("%s workers: feeder=%d import=%d", where, feeder_workers.get_pending_job_count(),
            import_workers.get_pending_job_count());
558
559
560
#endif
    }
    
561
562
563
564
565
566
567
568
569
570
571
572
573
574
    private bool report_failure(BatchImportResult import_result) {
        bool proceed = true;
        
        manifest.add_result(import_result);
        
        if (import_result.result != ImportResult.SUCCESS) {
            import_job_failed(import_result);
            
            if (import_result.file != null && !import_result.result.is_abort()) {
                uint64 filesize = 0;
                try {
                    // A BatchImportResult file is guaranteed to be a single file
                    filesize = query_total_file_size(import_result.file);
                } catch (Error err) {
575
                    warning("Unable to query file size of %s: %s", import_result.file.get_path(),
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
                        err.message);
                }
                
                report_progress(filesize);
            }
        }
        
        // fire this signal only once, and only on non-user aborts
        if (import_result.result.is_nonuser_abort() && proceed) {
            fatal_error(import_result.result, import_result.errmsg);
            proceed = false;
        }
        
        return proceed;
    }
    
    private void report_progress(uint64 increment_of_progress) {
        completed_bytes += increment_of_progress;
        
        // only report "progress" if progress has been made (and enough time has progressed),
        // otherwise still preparing
        if (completed_bytes == 0) {
            ulong now = now_ms();
            if (now - last_preparing_ms > 250) {
                last_preparing_ms = now;
                preparing();
            }
        } else if (increment_of_progress > 0) {
604
605
606
607
608
            ulong now = now_ms();
            if (now - last_preparing_ms > 250) {
                last_preparing_ms = now;
                progress(completed_bytes, total_bytes);
            }
609
610
611
        }
    }
    
612
613
614
615
    private bool report_failures(BackgroundImportJob background_job) {
        bool proceed = true;
        
        foreach (BatchImportResult import_result in background_job.failed) {
616
            if (!report_failure(import_result))
617
                proceed = false;
618
        }
619
620
        
        return proceed;
621
622
    }
    
623
624
625
626
627
628
    private void report_completed(string where) {
        if (completed)
            error("Attempted to complete already-completed import: %s", where);
        
        completed = true;
        
629
        flush_ready_sources();
630
631
        
        log_status("Import completed: %s".printf(where));
Jens Georg's avatar
Jens Georg committed
632
        debug("Import complete after %f", manifest.timer.elapsed());
633
634
635
        
        // report completed to the reporter (called prior to the "import_complete" signal)
        if (reporter != null)
636
            reporter(manifest, import_roll);
637
        
638
        import_complete(manifest, import_roll);
639
640
641
642
643
644
645
646
647
648
649
650
651
    }
    
    // This should be called whenever a file's import process is complete, successful or otherwise
    private void file_import_complete() {
        // mark this job as completed
        file_imports_completed++;
        if (file_imports_to_perform != -1)
            assert(file_imports_completed <= file_imports_to_perform);
        
        // because notifications can come in after completions, have to watch if this is the
        // last file
        if (file_imports_to_perform != -1 && file_imports_completed == file_imports_to_perform)
            report_completed("completed preparing files, all outstanding imports completed");
652
653
654
655
656
657
658
659
660
    }
    
    public void schedule() {
        assert(scheduled == false);
        scheduled = true;
        
        starting();
        
        // fire off a background job to generate all FileToPrepare work
661
662
663
        feeder_workers.enqueue(new WorkSniffer(this, jobs, on_work_sniffed_out, cancellable,
            on_sniffer_cancelled, skipset));
        throbber_id = Timeout.add(WORK_SNIFFER_THROBBER_MSEC, on_sniffer_working);
664
665
    }
    
666
667
668
669
    //
    // WorkSniffer stage
    //
    
670
    private bool on_sniffer_working() {
671
        report_progress(0);
672
673
        
        return true;
674
675
676
    }
    
    private void on_work_sniffed_out(BackgroundJob j) {
677
678
        assert(!completed);
        
679
680
        WorkSniffer sniffer = (WorkSniffer) j;
        
681
682
        log_status("on_work_sniffed_out");
        
683
684
        if (!report_failures(sniffer) || sniffer.files_to_prepare.size == 0) {
            report_completed("work sniffed out: nothing to do");
685
            
686
            return;
687
        }
688
        
689
690
        total_bytes = sniffer.total_bytes;
        
691
692
693
        // submit single background job to go out and prepare all the files, reporting back when/if
        // they're ready for import; this is important because gPhoto can't handle multiple accesses
        // to a camera without fat locking, and it's just not worth it.  Serializing the imports
694
695
        // also means the user sees the photos coming in in (roughly) the order they selected them
        // on the screen
696
        PrepareFilesJob prepare_files_job = new PrepareFilesJob(this, sniffer.files_to_prepare, 
697
698
            on_file_prepared, on_files_prepared, cancellable, on_file_prepare_cancelled);
        
699
700
701
702
703
704
        feeder_workers.enqueue(prepare_files_job);
        
        if (throbber_id > 0) {
            Source.remove(throbber_id);
            throbber_id = 0;
        }
705
706
    }
    
707
    private void on_sniffer_cancelled(BackgroundJob j) {
708
709
        assert(!completed);
        
710
711
        WorkSniffer sniffer = (WorkSniffer) j;
        
712
713
        log_status("on_sniffer_cancelled");
        
714
715
        report_failures(sniffer);
        report_completed("work sniffer cancelled");
716
717
718
719
720
        
        if (throbber_id > 0) {
            Source.remove(throbber_id);
            throbber_id = 0;
        }
721
    }
722
    
723
724
725
726
    //
    // PrepareFiles stage
    //
    
727
728
729
730
731
    private void flush_import_jobs() {
        // flush ready thumbnails before ready files because PreparedFileImportJob is more intense
        // than ThumbnailWriterJob; reversing this order causes work to back up in ready_thumbnails
        // and takes longer for the user to see progress (which is only reported after the thumbnail
        // has been written)
732
        while (ready_thumbnails.size > 0 && import_workers.get_pending_job_count() < max_outstanding_import_jobs) {
733
734
            import_workers.enqueue(new ThumbnailWriterJob(this, ready_thumbnails.remove_at(0),
                on_thumbnail_writer_completed, cancellable, on_thumbnail_writer_cancelled));
735
736
        }
        
737
        while(ready_files.size > 0 && import_workers.get_pending_job_count() < max_outstanding_import_jobs) {
738
739
740
            import_workers.enqueue(new PreparedFileImportJob(this, ready_files.remove_at(0),
                import_roll.import_id, on_import_files_completed, cancellable,
                on_import_files_cancelled));
741
        }
742
743
744
745
    }
    
    // This checks for duplicates in the current import batch, which may not already be in the
    // library and therefore not detected there.
746
    private File? get_in_current_import(PreparedFile prepared_file) {
747
748
#if !NO_DUPE_DETECTION
        if (prepared_file.full_md5 != null
749
            && imported_full_md5_table.has_key(prepared_file.full_md5)) {
750
            
751
            return imported_full_md5_table.get(prepared_file.full_md5);
752
753
754
755
        }
        
        // add for next one
        if (prepared_file.full_md5 != null)
756
            imported_full_md5_table.set(prepared_file.full_md5, prepared_file.file);
757
#endif
758
        return null;
759
760
    }
    
761
    // Called when a cluster of files are located and deemed proper for import by PrepareFiledJob
762
    private void on_file_prepared(BackgroundJob j, NotificationObject? user) {
763
764
        assert(!completed);
        
765
        PreparedFileCluster cluster = (PreparedFileCluster) user;
766
        
767
768
        log_status("on_file_prepared (%d files)".printf(cluster.list.size));
        
769
770
771
        process_prepared_files.begin(cluster.list);
    }
    
772
773
    // TODO: This logic can be cleaned up.  Attempt to remove all calls to
    // the database, as it's a blocking call (use in-memory lookups whenever possible)
774
775
776
777
778
    private async void process_prepared_files(Gee.List<PreparedFile> list) {
        foreach (PreparedFile prepared_file in list) {
            Idle.add(process_prepared_files.callback);
            yield;
            
779
780
            BatchImportResult import_result = null;
            
781
782
783
784
785
786
787
788
789
790
            // first check if file is already registered as a media object
            
            LibraryPhotoSourceCollection.State photo_state;
            LibraryPhoto? photo = LibraryPhoto.global.get_state_by_file(prepared_file.file,
                out photo_state);
            if (photo != null) {
                switch (photo_state) {
                    case LibraryPhotoSourceCollection.State.ONLINE:
                    case LibraryPhotoSourceCollection.State.OFFLINE:
                    case LibraryPhotoSourceCollection.State.EDITABLE:
791
                    case LibraryPhotoSourceCollection.State.DEVELOPER:
792
793
                        import_result = new BatchImportResult(prepared_file.job, prepared_file.file,
                            prepared_file.file.get_path(), prepared_file.file.get_path(),
794
                            DuplicatedFile.create_from_file(photo.get_master_file()),
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
                            ImportResult.PHOTO_EXISTS);
                        
                        if (photo_state == LibraryPhotoSourceCollection.State.OFFLINE)
                            photo.mark_online();
                    break;
                    
                    case LibraryPhotoSourceCollection.State.TRASH:
                        // let the code below deal with it
                    break;
                    
                    default:
                        error("Unknown LibraryPhotoSourceCollection state: %s", photo_state.to_string());
                }
            }
            
            if (import_result != null) {
                report_failure(import_result);
                file_import_complete();
                
                continue;
            }
            
            VideoSourceCollection.State video_state;
            Video? video = Video.global.get_state_by_file(prepared_file.file, out video_state);
            if (video != null) {
                switch (video_state) {
                    case VideoSourceCollection.State.ONLINE:
                    case VideoSourceCollection.State.OFFLINE:
                        import_result = new BatchImportResult(prepared_file.job, prepared_file.file,
                            prepared_file.file.get_path(), prepared_file.file.get_path(),
825
                            DuplicatedFile.create_from_file(video.get_master_file()),
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
                            ImportResult.PHOTO_EXISTS);
                        
                        if (video_state == VideoSourceCollection.State.OFFLINE)
                            video.mark_online();
                    break;
                    
                    case VideoSourceCollection.State.TRASH:
                        // let the code below deal with it
                    break;
                    
                    default:
                        error("Unknown VideoSourceCollection state: %s", video_state.to_string());
                }
            }
            
            if (import_result != null) {
                report_failure(import_result);
                file_import_complete();
                
                continue;
            }
            
            // now check if the file is a duplicate
            
850
            if (prepared_file.is_video && Video.is_duplicate(prepared_file.file, prepared_file.full_md5)) {
851
852
853
854
855
                VideoID[] duplicate_ids =
                    VideoTable.get_instance().get_duplicate_ids(prepared_file.file,
                    prepared_file.full_md5);
                assert(duplicate_ids.length > 0);
                
856
                DuplicatedFile? duplicated_file =
857
                    DuplicatedFile.create_from_video_id(duplicate_ids[0]);
858
859
860
861
862
863
864
                
                ImportResult result_code = ImportResult.PHOTO_EXISTS;
                if (mark_duplicates_online) {
                    Video? dupe_video =
                        (Video) Video.global.get_offline_bin().fetch_by_master_file(prepared_file.file);
                    if (dupe_video == null)
                        dupe_video = (Video) Video.global.get_offline_bin().fetch_by_md5(prepared_file.full_md5);
865
                    
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
                    if(dupe_video != null) {
                        debug("duplicate video found offline, marking as online: %s",
                            prepared_file.file.get_path());
                        
                        dupe_video.set_master_file(prepared_file.file);
                        dupe_video.mark_online();
                        
                        duplicated_file = null;
                        
                        manifest.imported.add(dupe_video);
                        report_progress(dupe_video.get_filesize());
                        file_import_complete();
                        
                        result_code = ImportResult.SUCCESS;
                    }
                }
                
883
                import_result = new BatchImportResult(prepared_file.job, prepared_file.file, 
884
                    prepared_file.file.get_path(), prepared_file.file.get_path(), duplicated_file,
885
886
887
888
889
890
891
                    result_code);
                
                if (result_code == ImportResult.SUCCESS) {
                    manifest.add_result(import_result);
                    
                    continue;
                }
892
            }
893
            
894
            if (get_in_current_import(prepared_file) != null) {
895
896
897
                // this looks for duplicates within the import set, since Photo.is_duplicate
                // only looks within already-imported photos for dupes
                import_result = new BatchImportResult(prepared_file.job, prepared_file.file,
898
899
                    prepared_file.file.get_path(), prepared_file.file.get_path(),
                    DuplicatedFile.create_from_file(get_in_current_import(prepared_file)),
900
                    ImportResult.PHOTO_EXISTS);
901
902
903
904
905
906
            } else if (Photo.is_duplicate(prepared_file.file, null, prepared_file.full_md5,
                prepared_file.file_format)) {
                if (untrash_duplicates) {
                    // If a file is being linked and has a dupe in the trash, we take it out of the trash
                    // and revert its edits.
                    photo = LibraryPhoto.global.get_trashed_by_file(prepared_file.file);
907
                    
908
909
                    if (photo == null && prepared_file.full_md5 != null)
                        photo = LibraryPhoto.global.get_trashed_by_md5(prepared_file.full_md5);
910
                    
911
912
913
914
                    if (photo != null) {
                        debug("duplicate linked photo found in trash, untrashing and removing transforms for %s",
                            prepared_file.file.get_path());
                        
915
                        photo.set_master_file(prepared_file.file);
916
917
                        photo.untrash();
                        photo.remove_all_transformations();
918
919
920
921
922
923
924
925
926
927
928
929
930
                    }
                }
                
                if (photo == null && mark_duplicates_online) {
                    // if a duplicate is found marked offline, make it online
                    photo = LibraryPhoto.global.get_offline_by_file(prepared_file.file);
                    
                    if (photo == null && prepared_file.full_md5 != null)
                        photo = LibraryPhoto.global.get_offline_by_md5(prepared_file.full_md5);
                    
                    if (photo != null) {
                        debug("duplicate photo found marked offline, marking online: %s",
                            prepared_file.file.get_path());
931
                        
932
933
                        photo.set_master_file(prepared_file.file);
                        photo.mark_online();
934
                    }
935
936
                }
                
937
938
                if (photo != null) {
                    import_result = new BatchImportResult(prepared_file.job, prepared_file.file,
939
                        prepared_file.file.get_path(), prepared_file.file.get_path(), null,
940
941
942
943
944
945
946
947
948
949
950
                        ImportResult.SUCCESS);
                    
                    manifest.imported.add(photo);
                    manifest.add_result(import_result);
                    
                    report_progress(photo.get_filesize());
                    file_import_complete();
                    
                    continue;
                }
                
951
                debug("duplicate photo detected, not importing %s", prepared_file.file.get_path());
952
                
953
954
955
956
957
958
959
                PhotoID[] photo_ids =
                    PhotoTable.get_instance().get_duplicate_ids(prepared_file.file, null,
                    prepared_file.full_md5, prepared_file.file_format);
                assert(photo_ids.length > 0);
                
                DuplicatedFile duplicated_file = DuplicatedFile.create_from_photo_id(photo_ids[0]);
                
960
                import_result = new BatchImportResult(prepared_file.job, prepared_file.file, 
961
962
                    prepared_file.file.get_path(), prepared_file.file.get_path(), duplicated_file,
                    ImportResult.PHOTO_EXISTS); 
963
            }
964
            
965
966
967
968
969
            if (import_result != null) {
                report_failure(import_result);
                file_import_complete();
                
                continue;
970
            }
971
            
972
            report_progress(0);
973
            ready_files.add(prepared_file);
974
975
        }
        
976
        flush_import_jobs();
977
978
    }
    
979
    private void done_preparing_files(BackgroundJob j, string caller) {
980
981
        assert(!completed);
        
982
983
984
985
986
987
        PrepareFilesJob prepare_files_job = (PrepareFilesJob) j;
        
        report_failures(prepare_files_job);
        
        // mark this job as completed and record how many file imports must finish to be complete
        file_imports_to_perform = prepare_files_job.prepared_files;
988
989
        assert(file_imports_to_perform >= file_imports_completed);
        
990
991
        log_status(caller);
        
992
        // this call can result in report_completed() being called, so don't call twice
993
        flush_import_jobs();
994
995
996
        
        // if none prepared, then none outstanding (or will become outstanding, depending on how
        // the notifications are queued)
997
        if (file_imports_to_perform == 0 && !completed)
998
            report_completed("no files prepared for import");
999
        else if (file_imports_completed == file_imports_to_perform && !completed)
1000
1001
1002
            report_completed("completed preparing files, all outstanding imports completed");
    }
    
1003
1004
1005
1006
    private void on_files_prepared(BackgroundJob j) {
        done_preparing_files(j, "on_files_prepared");
    }
    
1007
    private void on_file_prepare_cancelled(BackgroundJob j) {
1008
1009
1010
1011
1012
1013
1014
1015
        done_preparing_files(j, "on_file_prepare_cancelled");
    }
    
    //
    // Files ready for import stage
    //
    
    private void on_import_files_completed(BackgroundJob j) {
1016
1017
        assert(!completed);
        
1018
        PreparedFileImportJob job = (PreparedFileImportJob) j;
1019
        
1020
        log_status("on_import_files_completed");
1021
        
1022
1023
        // should be ready in some form
        assert(job.not_ready == null);
1024
        
1025
1026
1027
        // mark failed photo
        if (job.failed != null) {
            assert(job.failed.result != ImportResult.SUCCESS);
1028
            
1029
            report_failure(job.failed);
1030
1031
            file_import_complete();
        }
1032
        
1033
1034
        // resurrect ready photos before adding to database and rest of system ... this is more
        // efficient than doing them one at a time
1035
1036
1037
        if (job.ready != null) {
            assert(job.ready.batch_result.result == ImportResult.SUCCESS);
            
1038
            Tombstone? tombstone = Tombstone.global.locate(job.ready.final_file);
1039
            if (tombstone != null)
1040
                Tombstone.global.resurrect(tombstone);
1041
        
1042
1043
1044
1045
            // import ready photos into database
            MediaSource? source = null;
            if (job.ready.is_video) {
                job.ready.batch_result.result = Video.import_create(job.ready.video_import_params,
1046
1047
                    out source);
            } else {
1048
                job.ready.batch_result.result = LibraryPhoto.import_create(job.ready.photo_import_params,
1049
                    out source);
1050
                Photo photo = source as Photo;
Eric Gregory's avatar
Eric Gregory committed
1051
1052
1053
1054
1055
1056
                
                if (job.ready.photo_import_params.final_associated_file != null) {
                    // Associate RAW+JPEG in database.
                    BackingPhotoRow bpr = new BackingPhotoRow();
                    bpr.file_format = PhotoFileFormat.JFIF;
                    bpr.filepath = job.ready.photo_import_params.final_associated_file.get_path();
1057
                    debug("Associating %s with sibling %s", ((Photo) source).get_file().get_path(),
Eric Gregory's avatar
Eric Gregory committed
1058
1059
1060
1061
1062
1063
1064
                        bpr.filepath);
                    try {
                        ((Photo) source).add_backing_photo_for_development(RawDeveloper.CAMERA, bpr);
                    } catch (Error e) {
                        warning("Unable to associate JPEG with RAW. File: %s Error: %s", 
                            bpr.filepath, e.message);
                    }
1065
1066
1067
1068
1069
1070
1071
                }
                
                // Set the default developer for raw photos
                if (photo.get_master_file_format() == PhotoFileFormat.RAW) {
                    RawDeveloper d = Config.Facade.get_instance().get_default_raw_developer();
                    if (d == RawDeveloper.CAMERA && !photo.is_raw_developer_available(d))
                        d = RawDeveloper.EMBEDDED;
1072
                    
1073
                    photo.set_default_raw_developer(d);
Jens Georg's avatar
Jens Georg committed
1074
                    photo.set_raw_developer(d, false);
Eric Gregory's avatar
Eric Gregory committed
1075
                }
1076
            }
1077
1078
1079
            
            if (job.ready.batch_result.result != ImportResult.SUCCESS) {
                debug("on_import_file_completed: %s", job.ready.batch_result.result.to_string());
1080
                
1081
                report_failure(job.ready.batch_result);
1082
1083
                file_import_complete();
            } else {
1084
                ready_thumbnails.add(new CompletedImportObject(source, job.ready.get_thumbnails(),
1085
                    job.ready.prepared_file.job, job.ready.batch_result));
1086
1087
1088
            }
        }
        
1089
        flush_import_jobs();
1090
1091
    }
    
1092
    private void on_import_files_cancelled(BackgroundJob j) {
1093
1094
        assert(!completed);
        
1095
        PreparedFileImportJob job = (PreparedFileImportJob) j;
1096
        
1097
        log_status("on_import_files_cancelled");
1098
        
1099
1100
        if (job.not_ready != null) {
            report_failure(new BatchImportResult(job.not_ready.job, job.not_ready.file,