BatchImport.vala 81.5 KB
Newer Older
1
/* Copyright 2016 Software Freedom Conservancy Inc.
2 3
 *
 * This software is licensed under the GNU LGPL (version 2.1 or later).
4
 * See the COPYING file in this distribution.
5
 */
6

7 8 9 10 11 12 13 14 15 16 17 18 19
public enum ImportResult {
    SUCCESS,
    FILE_ERROR,
    DECODE_ERROR,
    DATABASE_ERROR,
    USER_ABORT,
    NOT_A_FILE,
    PHOTO_EXISTS,
    UNSUPPORTED_FORMAT,
    NOT_AN_IMAGE,
    DISK_FAILURE,
    DISK_FULL,
    CAMERA_ERROR,
20 21
    FILE_WRITE_ERROR,
    PIXBUF_CORRUPT_IMAGE;
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
    
    public string to_string() {
        switch (this) {
            case SUCCESS:
                return _("Success");
            
            case FILE_ERROR:
                return _("File error");
            
            case DECODE_ERROR:
                return _("Unable to decode file");
            
            case DATABASE_ERROR:
                return _("Database error");
            
            case USER_ABORT:
                return _("User aborted import");
            
            case NOT_A_FILE:
                return _("Not a file");
            
            case PHOTO_EXISTS:
                return _("File already exists in database");
            
            case UNSUPPORTED_FORMAT:
                return _("Unsupported file format");

            case NOT_AN_IMAGE:
                return _("Not an image file");
            
            case DISK_FAILURE:
                return _("Disk failure");
            
            case DISK_FULL:
                return _("Disk full");
            
            case CAMERA_ERROR:
                return _("Camera error");
            
            case FILE_WRITE_ERROR:
                return _("File write error");
63 64 65

            case PIXBUF_CORRUPT_IMAGE:
                return _("Corrupt image file");
66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
            
            default:
                return _("Imported failed (%d)").printf((int) this);
        }
    }
    
    public bool is_abort() {
        switch (this) {
            case ImportResult.DISK_FULL:
            case ImportResult.DISK_FAILURE:
            case ImportResult.USER_ABORT:
                return true;
            
            default:
                return false;
        }
    }
    
    public bool is_nonuser_abort() {
        switch (this) {
            case ImportResult.DISK_FULL:
            case ImportResult.DISK_FAILURE:
                return true;
            
            default:
                return false;
        }
    }
    
    public static ImportResult convert_error(Error err, ImportResult default_result) {
        if (err is FileError) {
            FileError ferr = (FileError) err;
            
            if (ferr is FileError.NOSPC)
                return ImportResult.DISK_FULL;
            else if (ferr is FileError.IO)
                return ImportResult.DISK_FAILURE;
            else if (ferr is FileError.ISDIR)
                return ImportResult.NOT_A_FILE;
105
            else if (ferr is FileError.ACCES)
106
                return ImportResult.FILE_WRITE_ERROR;
107 108
            else if (ferr is FileError.PERM)
                return ImportResult.FILE_WRITE_ERROR;
109 110 111 112 113 114 115 116 117 118 119 120 121
            else
                return ImportResult.FILE_ERROR;
        } else if (err is IOError) {
            IOError ioerr = (IOError) err;
            
            if (ioerr is IOError.NO_SPACE)
                return ImportResult.DISK_FULL;
            else if (ioerr is IOError.FAILED)
                return ImportResult.DISK_FAILURE;
            else if (ioerr is IOError.IS_DIRECTORY)
                return ImportResult.NOT_A_FILE;
            else if (ioerr is IOError.CANCELLED)
                return ImportResult.USER_ABORT;
122
            else if (ioerr is IOError.READ_ONLY)
123
                return ImportResult.FILE_WRITE_ERROR;
124 125
            else if (ioerr is IOError.PERMISSION_DENIED)
                return ImportResult.FILE_WRITE_ERROR;
126 127 128 129
            else
                return ImportResult.FILE_ERROR;
        } else if (err is GPhotoError) {
            return ImportResult.CAMERA_ERROR;
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
        } else if (err is Gdk.PixbufError) {
            Gdk.PixbufError pixbuferr = (Gdk.PixbufError) err;

            if (pixbuferr is Gdk.PixbufError.CORRUPT_IMAGE)
                return ImportResult.PIXBUF_CORRUPT_IMAGE;
            else if (pixbuferr is Gdk.PixbufError.INSUFFICIENT_MEMORY)
                return default_result;
            else if (pixbuferr is Gdk.PixbufError.BAD_OPTION)
                return default_result;
            else if (pixbuferr is Gdk.PixbufError.UNKNOWN_TYPE)
                return ImportResult.UNSUPPORTED_FORMAT;
            else if (pixbuferr is Gdk.PixbufError.UNSUPPORTED_OPERATION)
                return default_result;
            else if (pixbuferr is Gdk.PixbufError.FAILED)
                return default_result;
            else
                return default_result;
147 148 149 150 151 152
        }
        
        return default_result;
    }
}

153 154 155
// A BatchImportJob describes a unit of work the BatchImport object should perform.  It returns
// a file to be imported.  If the file is a directory, it is automatically recursed by BatchImport
// to find all files that need to be imported into the library.
156
//
157
// NOTE: All methods may be called from the context of a background thread or the main GTK thread.
158 159
// Implementations should be able to handle either situation.  The prepare method will always be
// called by the same thread context.
160
public abstract class BatchImportJob {
161 162 163
    public abstract string get_dest_identifier();
    
    public abstract string get_source_identifier();
164
    
165 166
    public abstract bool is_directory();
    
167 168 169 170
    public abstract string get_basename();
    
    public abstract string get_path();
    
171 172 173
    public virtual DuplicatedFile? get_duplicated_file() {
        return null;
    }
174 175 176 177

    public virtual File? get_associated_file() {
        return null;
    }
178
    
179 180 181
    // Attaches a sibling job (for RAW+JPEG)
    public abstract void set_associated(BatchImportJob associated);
    
182 183 184 185 186 187 188
    // Returns the file size of the BatchImportJob or returns a file/directory which can be queried
    // by BatchImportJob to determine it.  Returns true if the size is return, false if the File is
    // specified.
    // 
    // filesize should only be returned if BatchImportJob represents a single file.
    public abstract bool determine_file_size(out uint64 filesize, out File file_or_dir);
    
189
    // NOTE: prepare( ) is called from a background thread in the worker pool
190
    public abstract bool prepare(out File file_to_import, out bool copy_to_library) throws Error;
191 192 193 194 195 196 197
    
    // Completes the import for the new library photo once it's been imported.
    // If the job is directory based, this method will be called for each photo
    // discovered in the directory. This method is only called for photographs
    // that have been successfully imported.
    //
    // Returns true if any action was taken, false otherwise.
198 199
    //
    // NOTE: complete( )is called from the foreground thread
200
    public virtual bool complete(MediaSource source, BatchImportRoll import_roll) throws Error {
201 202
        return false;
    }
203 204 205 206 207 208
    
    // returns a non-zero time_t value if this has a valid exposure time override, returns zero
    // otherwise
    public virtual time_t get_exposure_time_override() {
        return 0;
    }
209 210 211 212

    public virtual bool recurse() {
        return true;
    }
213 214
}

215 216 217
public class FileImportJob : BatchImportJob {
    private File file_or_dir;
    private bool copy_to_library;
218
    private FileImportJob? associated = null;
219
    private bool _recurse;
220
    
221
    public FileImportJob(File file_or_dir, bool copy_to_library, bool recurse) {
222 223
        this.file_or_dir = file_or_dir;
        this.copy_to_library = copy_to_library;
224
        this._recurse = recurse;
225 226
    }
    
227 228 229 230 231
    public override string get_dest_identifier() {
        return file_or_dir.get_path();
    }
    
    public override string get_source_identifier() {
232 233 234 235 236 237 238
        return file_or_dir.get_path();
    }
    
    public override bool is_directory() {
        return query_is_directory(file_or_dir);
    }
    
239 240 241 242 243 244 245 246 247 248 249 250
    public override string get_basename() {
        return file_or_dir.get_basename();
    }
    
    public override string get_path() {
        return is_directory() ? file_or_dir.get_path() : file_or_dir.get_parent().get_path();
    }
    
    public override void set_associated(BatchImportJob associated) {
        this.associated = associated as FileImportJob;
    }
    
251
    public override bool determine_file_size(out uint64 filesize, out File file) {
252
        filesize = 0;
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267
        file = file_or_dir;
        
        return false;
    }
    
    public override bool prepare(out File file_to_import, out bool copy) {
        file_to_import = file_or_dir;
        copy = copy_to_library;
        
        return true;
    }
    
    public File get_file() {
        return file_or_dir;
    }
268 269 270 271

    public override bool recurse() {
        return this._recurse;
    }
272 273
}

274 275 276 277 278 279 280 281
// A BatchImportRoll represents important state for a group of imported media.  If this is shared
// among multiple BatchImport objects, the imported media will appear to have been imported all at
// once.
public class BatchImportRoll {
    public ImportID import_id;
    public ViewCollection generated_events = new ViewCollection("BatchImportRoll generated events");
    
    public BatchImportRoll() {
282
        this.import_id = ImportID.generate();
283 284 285
    }
}

286 287
// A BatchImportResult associates a particular job with a File that an import was performed on
// and the import result.  A BatchImportJob can specify multiple files, so there is not necessarily
288
// a one-to-one relationship between it and this object.
289 290 291
//
// Note that job may be null (in the case of a pre-failed job that must be reported) and file may
// be null (for similar reasons).
292
public class BatchImportResult {
293
    public BatchImportJob job;
294
    public File? file;
295 296
    public string src_identifier;   // Source path
    public string dest_identifier;  // Destination path
297
    public ImportResult result;
298
    public string? errmsg = null;
299
    public DuplicatedFile? duplicate_of;
300
    
301
    public BatchImportResult(BatchImportJob job, File? file, string src_identifier, 
302
        string dest_identifier, DuplicatedFile? duplicate_of, ImportResult result) {
303 304
        this.job = job;
        this.file = file;
305 306
        this.src_identifier = src_identifier;
        this.dest_identifier = dest_identifier;
307
        this.duplicate_of = duplicate_of;
308 309
        this.result = result;
    }
310
    
311 312
    public BatchImportResult.from_error(BatchImportJob job, File? file, string src_identifier,
        string dest_identifier, Error err, ImportResult default_result) {
313 314
        this.job = job;
        this.file = file;
315 316
        this.src_identifier = src_identifier;
        this.dest_identifier = dest_identifier;
317 318 319
        this.result = ImportResult.convert_error(err, default_result);
        this.errmsg = err.message;
    }
320 321
}

322
public class ImportManifest {
323
    public Gee.List<MediaSource> imported = new Gee.ArrayList<MediaSource>();
324
    public Gee.List<BatchImportResult> success = new Gee.ArrayList<BatchImportResult>();
325
    public Gee.List<BatchImportResult> camera_failed = new Gee.ArrayList<BatchImportResult>();
326
    public Gee.List<BatchImportResult> failed = new Gee.ArrayList<BatchImportResult>();
327
    public Gee.List<BatchImportResult> write_failed = new Gee.ArrayList<BatchImportResult>();
328 329
    public Gee.List<BatchImportResult> skipped_photos = new Gee.ArrayList<BatchImportResult>();
    public Gee.List<BatchImportResult> skipped_files = new Gee.ArrayList<BatchImportResult>();
330 331
    public Gee.List<BatchImportResult> aborted = new Gee.ArrayList<BatchImportResult>();
    public Gee.List<BatchImportResult> already_imported = new Gee.ArrayList<BatchImportResult>();
332
    public Gee.List<BatchImportResult> corrupt_files = new Gee.ArrayList<BatchImportResult>();
333
    public Gee.List<BatchImportResult> all = new Gee.ArrayList<BatchImportResult>();
334
    public GLib.Timer timer;
335
    
336 337
    public ImportManifest(Gee.List<BatchImportJob>? prefailed = null,
        Gee.List<BatchImportJob>? pre_already_imported = null) {
338
        this.timer = new Timer();
339 340
        if (prefailed != null) {
            foreach (BatchImportJob job in prefailed) {
341
                BatchImportResult batch_result = new BatchImportResult(job, null, 
342 343 344
                    job.get_source_identifier(), job.get_dest_identifier(), null,
                    ImportResult.FILE_ERROR);
                    
345 346 347 348 349 350
                add_result(batch_result);
            }
        }
        
        if (pre_already_imported != null) {
            foreach (BatchImportJob job in pre_already_imported) {
351 352 353 354 355
                BatchImportResult batch_result = new BatchImportResult(job,
                    File.new_for_path(job.get_basename()),
                    job.get_source_identifier(), job.get_dest_identifier(),
                    job.get_duplicated_file(), ImportResult.PHOTO_EXISTS);
                
356 357 358 359 360 361
                add_result(batch_result);
            }
        }
    }
    
    public void add_result(BatchImportResult batch_result) {
362
        bool reported = true;
363 364 365 366
        switch (batch_result.result) {
            case ImportResult.SUCCESS:
                success.add(batch_result);
            break;
367

368
            case ImportResult.USER_ABORT:
369
                if (batch_result.file != null && !query_is_directory(batch_result.file))
370
                    aborted.add(batch_result);
371 372
                else
                    reported = false;
373 374 375
            break;

            case ImportResult.UNSUPPORTED_FORMAT:
376 377 378 379 380 381
                skipped_photos.add(batch_result);
            break;

            case ImportResult.NOT_A_FILE:
            case ImportResult.NOT_AN_IMAGE:
                skipped_files.add(batch_result);
382 383 384 385 386 387
            break;
            
            case ImportResult.PHOTO_EXISTS:
                already_imported.add(batch_result);
            break;
            
388 389 390 391
            case ImportResult.CAMERA_ERROR:
                camera_failed.add(batch_result);
            break;
            
392 393 394 395
            case ImportResult.FILE_WRITE_ERROR:
                write_failed.add(batch_result);
            break;
            
396 397 398 399
            case ImportResult.PIXBUF_CORRUPT_IMAGE:
                corrupt_files.add(batch_result);
            break;
            
400 401 402 403 404
            default:
                failed.add(batch_result);
            break;
        }
        
405 406
        if (reported)
            all.add(batch_result);
407 408 409
    }
}

410
// BatchImport performs the work of taking a file (supplied by BatchImportJob's) and properly importing
411 412
// it into the system, including database additions and thumbnail creation.  It can be monitored by
// multiple observers, but only one ImportReporter can be registered.
413 414 415 416 417
//
// TODO: With background threads. the better way to implement this is via a FSM (finite state 
// machine) that exists in states and responds to various events thrown off by the background
// jobs.  However, getting this code to a point that it works with threads is task enough, so it
// will have to wait (especially since we'll want to write a generic FSM engine).
418
public class BatchImport : Object {
419
    private const int WORK_SNIFFER_THROBBER_MSEC = 125;
420
    
421 422
    public const int REPORT_EVERY_N_PREPARED_FILES = 100;
    public const int REPORT_PREPARED_FILES_EVERY_N_MSEC = 3000;
423 424 425 426 427 428
    
    private const int READY_SOURCES_COUNT_OVERFLOW = 10;
    
    private const int DISPLAY_QUEUE_TIMER_MSEC = 125;
    private const int DISPLAY_QUEUE_HYSTERESIS_OVERFLOW = (3 * 1000) / DISPLAY_QUEUE_TIMER_MSEC;
    
429 430
    private static Workers feeder_workers = new Workers(1, false);
    private static Workers import_workers = new Workers(Workers.thread_per_cpu_minus_one(), false);
431
    
432
    private Gee.Iterable<BatchImportJob> jobs;
433
    private BatchImportRoll import_roll;
434
    private string name;
435 436
    private uint64 completed_bytes = 0;
    private uint64 total_bytes = 0;
437
    private unowned ImportReporter reporter;
438 439
    private ImportManifest manifest;
    private bool scheduled = false;
440 441 442 443
    private bool completed = false;
    private int file_imports_to_perform = -1;
    private int file_imports_completed = 0;
    private Cancellable? cancellable = null;
444
    private ulong last_preparing_ms = 0;
445
    private Gee.HashSet<File> skipset;
446
#if !NO_DUPE_DETECTION
447
    private Gee.HashMap<string, File> imported_full_md5_table = new Gee.HashMap<string, File>();
448
#endif
449
    private uint throbber_id = 0;
Jens Georg's avatar
Jens Georg committed
450
    private uint max_outstanding_import_jobs = Workers.thread_per_cpu_minus_one();
451
    private bool untrash_duplicates = true;
452
    private bool mark_duplicates_online = true;
453 454 455
    
    // These queues are staging queues, holding batches of work that must happen in the import
    // process, working on them all at once to minimize overhead.
456 457 458 459 460
    private Gee.List<PreparedFile> ready_files = new Gee.LinkedList<PreparedFile>();
    private Gee.List<CompletedImportObject> ready_thumbnails =
        new Gee.LinkedList<CompletedImportObject>();
    private Gee.List<CompletedImportObject> display_imported_queue =
        new Gee.LinkedList<CompletedImportObject>();
461
    private Gee.List<CompletedImportObject> ready_sources = new Gee.LinkedList<CompletedImportObject>();
462 463 464
    
    // Called at the end of the batched jobs.  Can be used to report the result of the import
    // to the user.  This is called BEFORE import_complete is fired.
465
    public delegate void ImportReporter(ImportManifest manifest, BatchImportRoll import_roll);
466
    
467
    // Called once, when the scheduled task begins
468 469
    public signal void starting();
    
470 471 472 473 474 475 476
    // Called repeatedly while preparing the launched BatchImport
    public signal void preparing();
    
    // Called repeatedly to report the progress of the BatchImport (but only called after the
    // last "preparing" signal)
    public signal void progress(uint64 completed_bytes, uint64 total_bytes);
    
477 478
    // Called for each Photo or Video imported to the system. For photos, the pixbuf is
    // screen-sized and rotated. For videos, the pixbuf is a frame-grab of the first frame.
479 480 481
    //
    // The to_follow number is the number of queued-up sources to expect following this signal
    // in one burst.
482
    public signal void imported(MediaSource source, Gdk.Pixbuf pixbuf, int to_follow);
483
    
484 485 486 487
    // Called when a fatal error occurs that stops the import entirely.  Remaining jobs will be
    // failed and import_complete() is still fired.
    public signal void fatal_error(ImportResult result, string message);
    
488 489 490 491
    // Called when a job fails.  import_complete will also be called at the end of the batch
    public signal void import_job_failed(BatchImportResult result);
    
    // Called at the end of the batched jobs; this will be signalled exactly once for the batch
492
    public signal void import_complete(ImportManifest manifest, BatchImportRoll import_roll);
493 494

    public BatchImport(Gee.Iterable<BatchImportJob> jobs, string name, ImportReporter? reporter,
495
        Gee.ArrayList<BatchImportJob>? prefailed = null,
496
        Gee.ArrayList<BatchImportJob>? pre_already_imported = null,
497 498
        Cancellable? cancellable = null, BatchImportRoll? import_roll = null,
        ImportManifest? skip_manifest = null) {
499 500 501 502
        this.jobs = jobs;
        this.name = name;
        this.reporter = reporter;
        this.manifest = new ImportManifest(prefailed, pre_already_imported);
503
        this.cancellable = (cancellable != null) ? cancellable : new Cancellable();
504
        this.import_roll = import_roll != null ? import_roll : new BatchImportRoll();
505
        
506 507
        if (skip_manifest != null) {
            skipset = new Gee.HashSet<File>(file_hash, file_equal);
508 509
            foreach (MediaSource source in skip_manifest.imported) {
                skipset.add(source.get_file());
510 511 512
            }
        }
        
513
        // watch for user exit in the application
514
        Application.get_instance().exiting.connect(user_halt);
515 516
        
        // Use a timer to report imported photos to observers
517
        Timeout.add(DISPLAY_QUEUE_TIMER_MSEC, display_imported_timer);
518 519
    }
    
520
    ~BatchImport() {
521 522 523
#if TRACE_DTORS
        debug("DTOR: BatchImport (%s)", name);
#endif
524
        Application.get_instance().exiting.disconnect(user_halt);
525 526
    }
    
527 528 529 530
    public string get_name() {
        return name;
    }
    
531 532
    public void user_halt() {
        cancellable.cancel();
533 534
    }
    
535 536 537 538 539 540 541 542
    public bool get_untrash_duplicates() {
        return untrash_duplicates;
    }
    
    public void set_untrash_duplicates(bool untrash_duplicates) {
        this.untrash_duplicates = untrash_duplicates;
    }
    
543 544 545 546 547 548 549 550
    public bool get_mark_duplicates_online() {
        return mark_duplicates_online;
    }
    
    public void set_mark_duplicates_online(bool mark_duplicates_online) {
        this.mark_duplicates_online = mark_duplicates_online;
    }
    
551 552
    private void log_status(string where) {
#if TRACE_IMPORT
553
        debug("%s: to_perform=%d completed=%d ready_files=%d ready_thumbnails=%d display_queue=%d ready_sources=%d",
554
            where, file_imports_to_perform, file_imports_completed, ready_files.size,
555
            ready_thumbnails.size, display_imported_queue.size, ready_sources.size);
556 557
        debug("%s workers: feeder=%d import=%d", where, feeder_workers.get_pending_job_count(),
            import_workers.get_pending_job_count());
558 559 560
#endif
    }
    
561 562 563 564 565 566 567 568 569 570 571 572 573 574
    private bool report_failure(BatchImportResult import_result) {
        bool proceed = true;
        
        manifest.add_result(import_result);
        
        if (import_result.result != ImportResult.SUCCESS) {
            import_job_failed(import_result);
            
            if (import_result.file != null && !import_result.result.is_abort()) {
                uint64 filesize = 0;
                try {
                    // A BatchImportResult file is guaranteed to be a single file
                    filesize = query_total_file_size(import_result.file);
                } catch (Error err) {
575
                    warning("Unable to query file size of %s: %s", import_result.file.get_path(),
576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
                        err.message);
                }
                
                report_progress(filesize);
            }
        }
        
        // fire this signal only once, and only on non-user aborts
        if (import_result.result.is_nonuser_abort() && proceed) {
            fatal_error(import_result.result, import_result.errmsg);
            proceed = false;
        }
        
        return proceed;
    }
    
    private void report_progress(uint64 increment_of_progress) {
        completed_bytes += increment_of_progress;
        
        // only report "progress" if progress has been made (and enough time has progressed),
        // otherwise still preparing
        if (completed_bytes == 0) {
            ulong now = now_ms();
            if (now - last_preparing_ms > 250) {
                last_preparing_ms = now;
                preparing();
            }
        } else if (increment_of_progress > 0) {
604 605 606 607 608
            ulong now = now_ms();
            if (now - last_preparing_ms > 250) {
                last_preparing_ms = now;
                progress(completed_bytes, total_bytes);
            }
609 610 611
        }
    }
    
612 613 614 615
    private bool report_failures(BackgroundImportJob background_job) {
        bool proceed = true;
        
        foreach (BatchImportResult import_result in background_job.failed) {
616
            if (!report_failure(import_result))
617
                proceed = false;
618
        }
619 620
        
        return proceed;
621 622
    }
    
623 624 625 626 627 628
    private void report_completed(string where) {
        if (completed)
            error("Attempted to complete already-completed import: %s", where);
        
        completed = true;
        
629
        flush_ready_sources();
630 631
        
        log_status("Import completed: %s".printf(where));
632
        debug("Import complete after %f", manifest.timer.elapsed());
633 634 635
        
        // report completed to the reporter (called prior to the "import_complete" signal)
        if (reporter != null)
636
            reporter(manifest, import_roll);
637
        
638
        import_complete(manifest, import_roll);
639 640 641 642 643 644 645 646 647 648 649 650 651
    }
    
    // This should be called whenever a file's import process is complete, successful or otherwise
    private void file_import_complete() {
        // mark this job as completed
        file_imports_completed++;
        if (file_imports_to_perform != -1)
            assert(file_imports_completed <= file_imports_to_perform);
        
        // because notifications can come in after completions, have to watch if this is the
        // last file
        if (file_imports_to_perform != -1 && file_imports_completed == file_imports_to_perform)
            report_completed("completed preparing files, all outstanding imports completed");
652 653 654 655 656 657 658 659 660
    }
    
    public void schedule() {
        assert(scheduled == false);
        scheduled = true;
        
        starting();
        
        // fire off a background job to generate all FileToPrepare work
661 662 663
        feeder_workers.enqueue(new WorkSniffer(this, jobs, on_work_sniffed_out, cancellable,
            on_sniffer_cancelled, skipset));
        throbber_id = Timeout.add(WORK_SNIFFER_THROBBER_MSEC, on_sniffer_working);
664 665
    }
    
666 667 668 669
    //
    // WorkSniffer stage
    //
    
670
    private bool on_sniffer_working() {
671
        report_progress(0);
672 673
        
        return true;
674 675 676
    }
    
    private void on_work_sniffed_out(BackgroundJob j) {
677 678
        assert(!completed);
        
679 680
        WorkSniffer sniffer = (WorkSniffer) j;
        
681 682
        log_status("on_work_sniffed_out");
        
683 684
        if (!report_failures(sniffer) || sniffer.files_to_prepare.size == 0) {
            report_completed("work sniffed out: nothing to do");
685
            
686
            return;
687
        }
688
        
689 690
        total_bytes = sniffer.total_bytes;
        
691 692 693
        // submit single background job to go out and prepare all the files, reporting back when/if
        // they're ready for import; this is important because gPhoto can't handle multiple accesses
        // to a camera without fat locking, and it's just not worth it.  Serializing the imports
694 695
        // also means the user sees the photos coming in in (roughly) the order they selected them
        // on the screen
696
        PrepareFilesJob prepare_files_job = new PrepareFilesJob(this, sniffer.files_to_prepare, 
697 698
            on_file_prepared, on_files_prepared, cancellable, on_file_prepare_cancelled);
        
699 700 701 702 703 704
        feeder_workers.enqueue(prepare_files_job);
        
        if (throbber_id > 0) {
            Source.remove(throbber_id);
            throbber_id = 0;
        }
705 706
    }
    
707
    private void on_sniffer_cancelled(BackgroundJob j) {
708 709
        assert(!completed);
        
710 711
        WorkSniffer sniffer = (WorkSniffer) j;
        
712 713
        log_status("on_sniffer_cancelled");
        
714 715
        report_failures(sniffer);
        report_completed("work sniffer cancelled");
716 717 718 719 720
        
        if (throbber_id > 0) {
            Source.remove(throbber_id);
            throbber_id = 0;
        }
721
    }
722
    
723 724 725 726
    //
    // PrepareFiles stage
    //
    
727 728 729 730 731
    private void flush_import_jobs() {
        // flush ready thumbnails before ready files because PreparedFileImportJob is more intense
        // than ThumbnailWriterJob; reversing this order causes work to back up in ready_thumbnails
        // and takes longer for the user to see progress (which is only reported after the thumbnail
        // has been written)
732
        while (ready_thumbnails.size > 0 && import_workers.get_pending_job_count() < max_outstanding_import_jobs) {
733 734
            import_workers.enqueue(new ThumbnailWriterJob(this, ready_thumbnails.remove_at(0),
                on_thumbnail_writer_completed, cancellable, on_thumbnail_writer_cancelled));
735 736
        }
        
737
        while(ready_files.size > 0 && import_workers.get_pending_job_count() < max_outstanding_import_jobs) {
738 739 740
            import_workers.enqueue(new PreparedFileImportJob(this, ready_files.remove_at(0),
                import_roll.import_id, on_import_files_completed, cancellable,
                on_import_files_cancelled));
741
        }
742 743 744 745
    }
    
    // This checks for duplicates in the current import batch, which may not already be in the
    // library and therefore not detected there.
746
    private File? get_in_current_import(PreparedFile prepared_file) {
747 748
#if !NO_DUPE_DETECTION
        if (prepared_file.full_md5 != null
749
            && imported_full_md5_table.has_key(prepared_file.full_md5)) {
750
            
751
            return imported_full_md5_table.get(prepared_file.full_md5);
752 753 754 755
        }
        
        // add for next one
        if (prepared_file.full_md5 != null)
756
            imported_full_md5_table.set(prepared_file.full_md5, prepared_file.file);
757
#endif
758
        return null;
759 760
    }
    
761
    // Called when a cluster of files are located and deemed proper for import by PrepareFiledJob
762
    private void on_file_prepared(BackgroundJob j, NotificationObject? user) {
763 764
        assert(!completed);
        
765
        PreparedFileCluster cluster = (PreparedFileCluster) user;
766
        
767 768
        log_status("on_file_prepared (%d files)".printf(cluster.list.size));
        
769 770 771
        process_prepared_files.begin(cluster.list);
    }
    
772 773
    // TODO: This logic can be cleaned up.  Attempt to remove all calls to
    // the database, as it's a blocking call (use in-memory lookups whenever possible)
774 775 776 777 778
    private async void process_prepared_files(Gee.List<PreparedFile> list) {
        foreach (PreparedFile prepared_file in list) {
            Idle.add(process_prepared_files.callback);
            yield;
            
779 780
            BatchImportResult import_result = null;
            
781 782 783 784 785 786 787 788 789 790
            // first check if file is already registered as a media object
            
            LibraryPhotoSourceCollection.State photo_state;
            LibraryPhoto? photo = LibraryPhoto.global.get_state_by_file(prepared_file.file,
                out photo_state);
            if (photo != null) {
                switch (photo_state) {
                    case LibraryPhotoSourceCollection.State.ONLINE:
                    case LibraryPhotoSourceCollection.State.OFFLINE:
                    case LibraryPhotoSourceCollection.State.EDITABLE:
791
                    case LibraryPhotoSourceCollection.State.DEVELOPER:
792 793
                        import_result = new BatchImportResult(prepared_file.job, prepared_file.file,
                            prepared_file.file.get_path(), prepared_file.file.get_path(),
794
                            DuplicatedFile.create_from_file(photo.get_master_file()),
795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824
                            ImportResult.PHOTO_EXISTS);
                        
                        if (photo_state == LibraryPhotoSourceCollection.State.OFFLINE)
                            photo.mark_online();
                    break;
                    
                    case LibraryPhotoSourceCollection.State.TRASH:
                        // let the code below deal with it
                    break;
                    
                    default:
                        error("Unknown LibraryPhotoSourceCollection state: %s", photo_state.to_string());
                }
            }
            
            if (import_result != null) {
                report_failure(import_result);
                file_import_complete();
                
                continue;
            }
            
            VideoSourceCollection.State video_state;
            Video? video = Video.global.get_state_by_file(prepared_file.file, out video_state);
            if (video != null) {
                switch (video_state) {
                    case VideoSourceCollection.State.ONLINE:
                    case VideoSourceCollection.State.OFFLINE:
                        import_result = new BatchImportResult(prepared_file.job, prepared_file.file,
                            prepared_file.file.get_path(), prepared_file.file.get_path(),
825
                            DuplicatedFile.create_from_file(video.get_master_file()),
826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849
                            ImportResult.PHOTO_EXISTS);
                        
                        if (video_state == VideoSourceCollection.State.OFFLINE)
                            video.mark_online();
                    break;
                    
                    case VideoSourceCollection.State.TRASH:
                        // let the code below deal with it
                    break;
                    
                    default:
                        error("Unknown VideoSourceCollection state: %s", video_state.to_string());
                }
            }
            
            if (import_result != null) {
                report_failure(import_result);
                file_import_complete();
                
                continue;
            }
            
            // now check if the file is a duplicate
            
850
            if (prepared_file.is_video && Video.is_duplicate(prepared_file.file, prepared_file.full_md5)) {
851 852 853 854 855
                VideoID[] duplicate_ids =
                    VideoTable.get_instance().get_duplicate_ids(prepared_file.file,
                    prepared_file.full_md5);
                assert(duplicate_ids.length > 0);
                
856
                DuplicatedFile? duplicated_file =
857
                    DuplicatedFile.create_from_video_id(duplicate_ids[0]);
858 859 860 861 862 863 864
                
                ImportResult result_code = ImportResult.PHOTO_EXISTS;
                if (mark_duplicates_online) {
                    Video? dupe_video =
                        (Video) Video.global.get_offline_bin().fetch_by_master_file(prepared_file.file);
                    if (dupe_video == null)
                        dupe_video = (Video) Video.global.get_offline_bin().fetch_by_md5(prepared_file.full_md5);
865
                    
866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882
                    if(dupe_video != null) {
                        debug("duplicate video found offline, marking as online: %s",
                            prepared_file.file.get_path());
                        
                        dupe_video.set_master_file(prepared_file.file);
                        dupe_video.mark_online();
                        
                        duplicated_file = null;
                        
                        manifest.imported.add(dupe_video);
                        report_progress(dupe_video.get_filesize());
                        file_import_complete();
                        
                        result_code = ImportResult.SUCCESS;
                    }
                }
                
883
                import_result = new BatchImportResult(prepared_file.job, prepared_file.file, 
884
                    prepared_file.file.get_path(), prepared_file.file.get_path(), duplicated_file,
885 886 887 888 889 890 891
                    result_code);
                
                if (result_code == ImportResult.SUCCESS) {
                    manifest.add_result(import_result);
                    
                    continue;
                }
892
            }
893
            
894
            if (get_in_current_import(prepared_file) != null) {
895 896 897
                // this looks for duplicates within the import set, since Photo.is_duplicate
                // only looks within already-imported photos for dupes
                import_result = new BatchImportResult(prepared_file.job, prepared_file.file,
898 899
                    prepared_file.file.get_path(), prepared_file.file.get_path(),
                    DuplicatedFile.create_from_file(get_in_current_import(prepared_file)),
900
                    ImportResult.PHOTO_EXISTS);
901 902 903 904 905 906
            } else if (Photo.is_duplicate(prepared_file.file, null, prepared_file.full_md5,
                prepared_file.file_format)) {
                if (untrash_duplicates) {
                    // If a file is being linked and has a dupe in the trash, we take it out of the trash
                    // and revert its edits.
                    photo = LibraryPhoto.global.get_trashed_by_file(prepared_file.file);
907
                    
908 909
                    if (photo == null && prepared_file.full_md5 != null)
                        photo = LibraryPhoto.global.get_trashed_by_md5(prepared_file.full_md5);
910
                    
911 912 913 914
                    if (photo != null) {
                        debug("duplicate linked photo found in trash, untrashing and removing transforms for %s",
                            prepared_file.file.get_path());
                        
915
                        photo.set_master_file(prepared_file.file);
916 917
                        photo.untrash();
                        photo.remove_all_transformations();
918 919 920 921 922 923 924 925 926 927 928 929 930
                    }
                }
                
                if (photo == null && mark_duplicates_online) {
                    // if a duplicate is found marked offline, make it online
                    photo = LibraryPhoto.global.get_offline_by_file(prepared_file.file);
                    
                    if (photo == null && prepared_file.full_md5 != null)
                        photo = LibraryPhoto.global.get_offline_by_md5(prepared_file.full_md5);
                    
                    if (photo != null) {
                        debug("duplicate photo found marked offline, marking online: %s",
                            prepared_file.file.get_path());
931
                        
932 933
                        photo.set_master_file(prepared_file.file);
                        photo.mark_online();
934
                    }
935 936
                }
                
937 938
                if (photo != null) {
                    import_result = new BatchImportResult(prepared_file.job, prepared_file.file,
939
                        prepared_file.file.get_path(), prepared_file.file.get_path(), null,
940 941 942 943 944 945 946 947 948 949 950
                        ImportResult.SUCCESS);
                    
                    manifest.imported.add(photo);
                    manifest.add_result(import_result);
                    
                    report_progress(photo.get_filesize());
                    file_import_complete();
                    
                    continue;
                }
                
951
                debug("duplicate photo detected, not importing %s", prepared_file.file.get_path());
952
                
953 954 955 956 957 958 959
                PhotoID[] photo_ids =
                    PhotoTable.get_instance().get_duplicate_ids(prepared_file.file, null,
                    prepared_file.full_md5, prepared_file.file_format);
                assert(photo_ids.length > 0);
                
                DuplicatedFile duplicated_file = DuplicatedFile.create_from_photo_id(photo_ids[0]);
                
960
                import_result = new BatchImportResult(prepared_file.job, prepared_file.file, 
961 962
                    prepared_file.file.get_path(), prepared_file.file.get_path(), duplicated_file,
                    ImportResult.PHOTO_EXISTS); 
963
            }
964
            
965 966 967 968 969
            if (import_result != null) {
                report_failure(import_result);
                file_import_complete();
                
                continue;
970
            }
971
            
972
            report_progress(0);
973
            ready_files.add(prepared_file);
974 975
        }
        
976
        flush_import_jobs();
977 978
    }
    
979
    private void done_preparing_files(BackgroundJob j, string caller) {
980 981
        assert(!completed);
        
982 983 984 985 986 987
        PrepareFilesJob prepare_files_job = (PrepareFilesJob) j;
        
        report_failures(prepare_files_job);
        
        // mark this job as completed and record how many file imports must finish to be complete
        file_imports_to_perform = prepare_files_job.prepared_files;
988 989
        assert(file_imports_to_perform >= file_imports_completed);
        
990 991
        log_status(caller);
        
992
        // this call can result in report_completed() being called, so don't call twice
993
        flush_import_jobs();
994 995 996
        
        // if none prepared, then none outstanding (or will become outstanding, depending on how
        // the notifications are queued)
997
        if (file_imports_to_perform == 0 && !completed)
998
            report_completed("no files prepared for import");
999
        else if (file_imports_completed == file_imports_to_perform && !completed)
1000 1001 1002
            report_completed("completed preparing files, all outstanding imports completed");
    }
    
1003 1004 1005 1006
    private void on_files_prepared(BackgroundJob j) {
        done_preparing_files(j, "on_files_prepared");
    }
    
1007
    private void on_file_prepare_cancelled(BackgroundJob j) {
1008 1009 1010 1011 1012 1013 1014 1015
        done_preparing_files(j, "on_file_prepare_cancelled");
    }
    
    //
    // Files ready for import stage
    //
    
    private void on_import_files_completed(BackgroundJob j) {
1016 1017
        assert(!completed);
        
1018
        PreparedFileImportJob job = (PreparedFileImportJob) j;
1019
        
1020
        log_status("on_import_files_completed");
1021
        
1022 1023
        // should be ready in some form
        assert(job.not_ready == null);
1024
        
1025 1026 1027
        // mark failed photo
        if (job.failed != null) {
            assert(job.failed.result != ImportResult.SUCCESS);
1028
            
1029
            report_failure(job.failed);
1030 1031
            file_import_complete();
        }
1032
        
1033 1034
        // resurrect ready photos before adding to database and rest of system ... this is more
        // efficient than doing them one at a time
1035 1036 1037
        if (job.ready != null) {
            assert(job.ready.batch_result.result == ImportResult.SUCCESS);
            
1038
            Tombstone? tombstone = Tombstone.global.locate(job.ready.final_file);
1039
            if (tombstone != null)
1040
                Tombstone.global.resurrect(tombstone);
1041
        
1042 1043 1044 1045
            // import ready photos into database
            MediaSource? source = null;
            if (job.ready.is_video) {
                job.ready.batch_result.result = Video.import_create(job.ready.video_import_params,
1046 1047
                    out source);
            } else {
1048
                job.ready.batch_result.result = LibraryPhoto.import_create(job.ready.photo_import_params,
1049
                    out source);
1050
                Photo photo = source as Photo;
1051 1052 1053 1054 1055 1056
                
                if (job.ready.photo_import_params.final_associated_file != null) {
                    // Associate RAW+JPEG in database.
                    BackingPhotoRow bpr = new BackingPhotoRow();
                    bpr.file_format = PhotoFileFormat.JFIF;
                    bpr.filepath = job.ready.photo_import_params.final_associated_file.get_path();
1057
                    debug("Associating %s with sibling %s", ((Photo) source).get_file().get_path(),
1058 1059 1060 1061 1062 1063 1064
                        bpr.filepath);
                    try {
                        ((Photo) source).add_backing_photo_for_development(RawDeveloper.CAMERA, bpr);
                    } catch (Error e) {
                        warning("Unable to associate JPEG with RAW. File: %s Error: %s", 
                            bpr.filepath, e.message);
                    }
1065 1066 1067 1068 1069 1070 1071
                }
                
                // Set the default developer for raw photos
                if (photo.get_master_file_format() == PhotoFileFormat.RAW) {
                    RawDeveloper d = Config.Facade.get_instance().get_default_raw_developer();
                    if (d == RawDeveloper.CAMERA && !photo.is_raw_developer_available(d))
                        d = RawDeveloper.EMBEDDED;
1072
                    
1073
                    photo.set_default_raw_developer(d);
1074
                    photo.set_raw_developer(d, false);
1075
                }
1076
            }
1077 1078 1079
            
            if (job.ready.batch_result.result != ImportResult.SUCCESS) {
                debug("on_import_file_completed: %s", job.ready.batch_result.result.to_string());
1080
                
1081
                report_failure(job.ready.batch_result);
1082 1083
                file_import_complete();
            } else {
1084
                ready_thumbnails.add(new CompletedImportObject(source, job.ready.get_thumbnails(),
1085
                    job.ready.prepared_file.job, job.ready.batch_result));
1086 1087 1088
            }
        }
        
1089
        flush_import_jobs();
1090 1091
    }
    
1092
    private void on_import_files_cancelled(BackgroundJob j) {
1093 1094
        assert(!completed);
        
1095
        PreparedFileImportJob job = (PreparedFileImportJob) j;
1096
        
1097
        log_status("on_import_files_cancelled");
1098
        
1099 1100
        if (job.not_ready != null) {
            report_failure(new BatchImportResult(job.not_ready.job, job.not_ready.file,
1101
                job.not_ready.file.get_path(), job.not_ready.file.get_path(), null, 
1102
                ImportResult.USER_ABORT));
1103 1104
            file_import_complete();
        }
1105
        
1106 1107
        if (job.failed != null) {
            report_failure(job.failed);
1108 1109
            file_import_complete();
        }
1110
        
1111 1112
        if (job.ready != null) {
            report_failure(job.ready.abort());
1113
            file_import_complete();
1114 1115
        }
        
1116
        flush_import_jobs();
1117 1118
    }
    
1119 1120 1121
    //
    // ThumbnailWriter stage
    //
1122 1123 1124
    // Because the LibraryPhoto has been created at this stage, any cancelled work must also
    // destroy the LibraryPhoto.
    //
1125 1126
    
    private void on_thumbnail_writer_completed(BackgroundJob j) {
1127 1128
        assert(!completed);
        
1129
        ThumbnailWriterJob job = (ThumbnailWriterJob) j;
1130
        CompletedImportObject completed = job.completed_import_source;
1131
        
1132
        log_status("on_thumbnail_writer_completed");
1133
        
1134 1135 1136 1137 1138 1139 1140 1141
        if (completed.batch_result.result != ImportResult.SUCCESS) {
            warning("Failed to import %s: unable to write thumbnails (%s)",
                completed.source.to_string(), completed.batch_result.result.to_string());
            
            if (completed.source is LibraryPhoto)
                LibraryPhoto.import_failed(completed.source as LibraryPhoto);
            else if (completed.source is Video)
                Video.import_failed(completed.source as Video);
1142

1143 1144 1145
            report_failure(completed.batch_result);
            file_import_complete();
        } else {
1146
            manifest.imported.add(completed.source);
1147 1148 1149
            manifest.add_result(completed.batch_result);
            
            display_imported_queue.add(completed);
1150
        }
1151 1152
        
        flush_import_jobs();
1153 1154 1155 1156
    }
    
    private void on_thumbnail_writer_cancelled(BackgroundJob j) {
        assert(!completed);
1157
        
1158
        ThumbnailWriterJob job = (ThumbnailWriterJob) j;
1159
        CompletedImportObject completed = job.completed_import_source;
1160
        
1161
        log_status("on_thumbnail_writer_cancelled");
1162
        
1163 1164 1165 1166
        if (completed.source is LibraryPhoto)
            LibraryPhoto.import_failed(completed.source as LibraryPhoto);
        else if (completed.source is Video)
            Video.import_failed(completed.source as Video);
1167

1168 1169 1170 1171
        report_failure(completed.batch_result);
        file_import_complete();
        
        flush_import_jobs();
1172 1173 1174
    }
    
    //
1175
    // Display imported sources and integrate into system
1176 1177
    //
    
1178 1179
    private void flush_ready_sources() {
        if (ready_sources.size == 0)
1180
            return;
1181
        
1182 1183 1184
        // the user_preview and thumbnails in the CompletedImportObjects are not available at 
        // this stage
        
1185 1186
        log_status("flush_ready_sources (%d)".printf(ready_sources.size));
        
1187 1188 1189 1190 1191 1192 1193
        Gee.ArrayList<MediaSource> all = new Gee.ArrayList<MediaSource>();
        Gee.ArrayList<LibraryPhoto> photos = new Gee.ArrayList<LibraryPhoto>();
        Gee.ArrayList<Video> videos = new Gee.ArrayList<Video>();
        Gee.HashMap<MediaSource, BatchImportJob> completion_list =
            new Gee.HashMap<MediaSource, BatchImportJob>();
        foreach (CompletedImportObject completed in ready_sources) {
            all.add(completed.source);
1194
            
1195 1196 1197 1198
            if (completed.source is LibraryPhoto)
                photos.add((LibraryPhoto) completed.source);
            else if (completed.source is Video)
                videos.add((Video) completed.source);
1199
            
1200
            completion_list.set(completed.source, completed.original_job);
1201
        }
1202
        
1203 1204 1205
        MediaCollectionRegistry.get_instance().begin_transaction_on_all();
        Event.global.freeze_notifications();
        Tag.global.freeze_notifications();
1206
        
1207 1208
        LibraryPhoto.global.import_many(photos);
        Video.global.import_many(videos);
1209
        
1210 1211 1212 1213 1214 1215 1216 1217 1218
        // allow the BatchImportJob to perform final work on the MediaSource
        foreach (MediaSource media in completion_list.keys) {
            try {
                completion_list.get(media).complete(media, import_roll);
            } catch (Error err) {
                warning("Completion error when finalizing import of %s: %s", media.to_string(),
                    err.message);
            }
        }
1219
        
1220 1221
        // generate events for MediaSources not yet assigned
        Event.generate_many_events(all, import_roll.generated_events);
1222
        
1223 1224 1225
        Tag.global.thaw_notifications();
        Event.global.thaw_notifications();
        MediaCollectionRegistry.get_instance().commit_transaction_on_all();
1226
        
1227
        ready_sources.clear();
1228 1229 1230 1231 1232
    }
    
    // This is called throughout the import process to notify watchers of imported photos in such
    // a way that the GTK event queue gets a chance to operate.
    private bool display_imported_timer() {
1233
        if (display_imported_queue.size == 0)
1234 1235 1236 1237 1238 1239 1240
            return !completed;
        
        if (cancellable.is_cancelled())
            debug("Importing %d photos at once", display_imported_queue.size);
        
        log_status("display_imported_timer");
        
1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251
        // only display one at a time, so the user can see them come into the library in order.
        // however, if the queue backs up to the hysteresis point (currently defined as more than
        // 3 seconds wait for the last photo on the queue), then begin doing them in increasingly
        // larger chunks, to stop the queue from growing and then to get ahead of the other
        // import cycles.
        //
        // if cancelled, want to do as many as possible, but want to relinquish the thread to
        // keep the system active
        int total = 1;
        if (!cancellable.is_cancelled()) {
            if (display_imported_queue.size > DISPLAY_QUEUE_HYSTERESIS_OVERFLOW)
1252 1253
                total = 
                    1 << ((display_imported_queue.size / DISPLAY_QUEUE_HYSTERESIS_OVERFLOW) + 2).clamp(0, 16);
1254 1255
        } else {
            // do in overflow-sized chunks
1256
            total = DISPLAY_QUEUE_HYSTERESIS_OVERFLOW;
1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269
        }
        
        total = int.min(total, display_imported_queue.size);
        
#if TRACE_IMPORT
        if (total > 1) {
            debug("DISPLAY IMPORT QUEUE: hysteresis, dumping %d/%d media sources", total,
                display_imported_queue.size);
        }
#endif
        
        // post-decrement because the 0-based total is used when firing "imported"
        while (total-- > 0) {
1270
            CompletedImportObject completed_object = display_imported_queue.remove_at(0);
1271
            
1272 1273 1274 1275 1276 1277 1278
            // stash preview for reporting progress
            Gdk.Pixbuf user_preview = completed_object.user_preview;
            
            // expensive pixbufs no longer needed
            completed_object.user_preview = null;
            completed_object.thumbnails = null;
            
1279 1280
            // Stage the number of ready media objects to incorporate into the system rather than
            // doing them one at a time, to keep the UI thread responsive.
1281
            // NOTE: completed_object must be added prior to file_import_complete()
1282
            ready_sources.add(completed_object);
1283
            
1284
            imported(completed_object.source, user_preview, total);
1285 1286 1287 1288 1289 1290 1291 1292
            // If we have a photo, use master size. For RAW import, we might end up with reporting
            // the size of the (much smaller) JPEG which will look like no progress at all
            if (completed_object.source is PhotoSource) {
                var photo_source = completed_object.source as PhotoSource;
                report_progress(photo_source.get_master_filesize());
            } else {
                report_progress(completed_object.source.get_filesize());
            }
1293
            file_import_complete();
1294
        }
1295
        
1296
        if (ready_sources.size >= READY_SOURCES_COUNT_OVERFLOW || cancellable.is_cancelled())
1297
            flush_ready_sources();
1298
        
1299
        return true;
1300
    }
1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352
} /* class BatchImport */

public class DuplicatedFile : Object {
    private VideoID? video_id;
    private PhotoID? photo_id;
    private File? file;
    
    private DuplicatedFile() {
        this.video_id = null;
        this.photo_id = null;
        this.file = null;
    }
    
    public static DuplicatedFile create_from_photo_id(PhotoID photo_id) {
        assert(photo_id.is_valid());
        
        DuplicatedFile result = new DuplicatedFile();
        result.photo_id = photo_id;
        return result;
    }
    
    public static DuplicatedFile create_from_video_id(VideoID video_id) {
        assert(video_id.is_valid());
        
        DuplicatedFile result = new DuplicatedFile();
        result.video_id = video_id;
        return result;
    }
    
    public static DuplicatedFile create_from_file(File file) {
        DuplicatedFile result = new DuplicatedFile();
        
        result.file = file;
        
        return result;
    }
    
    public File get_file() {
        if (file != null) {
            return file;
        } else if (photo_id != null) {
            Photo photo_object = (Photo) LibraryPhoto.global.fetch(photo_id);
            file = photo_object.get_master_file();
            return file;
        } else if (video_id != null