Commit 0de100a2 authored by Jim Nelson's avatar Jim Nelson

#2924: Duplicates detected during auto-import are now tombstoned so they are...

#2924: Duplicates detected during auto-import are now tombstoned so they are not auto-imported later.  Also various changes to make the system more reponsive during the startup scan.
parent 47dbbd19
......@@ -21,7 +21,7 @@ public class DatabaseTable {
* tables are created on demand and tables and columns are easily ignored when already present.
* However, the change should be noted in upgrade_database() as a comment.
***/
public const int SCHEMA_VERSION = 11;
public const int SCHEMA_VERSION = 12;
protected static Sqlite.Database db;
......@@ -517,9 +517,22 @@ private DatabaseVerifyResult upgrade_database(int version) {
if (!DatabaseTable.add_column("EventTable", "primary_source_id", "INTEGER DEFAULT 0"))
return DatabaseVerifyResult.UPGRADE_ERROR;
}
version = 11;
//
// Version 12:
// * Added reason columnn to TombstoneTable
//
if (!DatabaseTable.has_column("TombstoneTable", "reason")) {
message("upgrade_database: adding reason column to TombstoneTable");
if (!DatabaseTable.add_column("TombstoneTable", "reason", "INTEGER DEFAULT 0"))
return DatabaseVerifyResult.UPGRADE_ERROR;
}
version = 12;
assert(version == DatabaseTable.SCHEMA_VERSION);
VersionTable.get_instance().update_version(version, Resources.APP_VERSION);
......@@ -2243,7 +2256,8 @@ public struct TombstoneRow {
public int64 filesize;
public string? md5;
public time_t time_created;
}
public Tombstone.Reason reason;
}
public class TombstoneTable : DatabaseTable {
private static TombstoneTable instance = null;
......@@ -2259,7 +2273,8 @@ public class TombstoneTable : DatabaseTable {
+ "filepath TEXT NOT NULL, "
+ "filesize INTEGER, "
+ "md5 TEXT, "
+ "time_created INTEGER "
+ "time_created INTEGER, "
+ "reason INTEGER DEFAULT 0 "
+ ")", -1, out stmt);
assert(res == Sqlite.OK);
......@@ -2275,11 +2290,12 @@ public class TombstoneTable : DatabaseTable {
return instance;
}
public TombstoneRow add(string filepath, int64 filesize, string? md5) throws DatabaseError {
public TombstoneRow add(string filepath, int64 filesize, string? md5, Tombstone.Reason reason)
throws DatabaseError {
Sqlite.Statement stmt;
int res = db.prepare_v2("INSERT INTO TombstoneTable "
+ "(filepath, filesize, md5, time_created) "
+ "VALUES (?, ?, ?, ?)",
+ "(filepath, filesize, md5, time_created, reason) "
+ "VALUES (?, ?, ?, ?, ?)",
-1, out stmt);
assert(res == Sqlite.OK);
......@@ -2293,6 +2309,8 @@ public class TombstoneTable : DatabaseTable {
assert(res == Sqlite.OK);
res = stmt.bind_int64(4, (int64) time_created);
assert(res == Sqlite.OK);
res = stmt.bind_int(5, reason.serialize());
assert(res == Sqlite.OK);
res = stmt.step();
if (res != Sqlite.DONE)
......@@ -2304,6 +2322,7 @@ public class TombstoneTable : DatabaseTable {
row.filesize = filesize;
row.md5 = md5;
row.time_created = time_created;
row.reason = reason;
return row;
}
......@@ -2314,7 +2333,7 @@ public class TombstoneTable : DatabaseTable {
return null;
Sqlite.Statement stmt;
int res = db.prepare_v2("SELECT id, filepath, filesize, md5, time_created "
int res = db.prepare_v2("SELECT id, filepath, filesize, md5, time_created, reason "
+ "FROM TombstoneTable", -1, out stmt);
assert(res == Sqlite.OK);
......@@ -2334,6 +2353,7 @@ public class TombstoneTable : DatabaseTable {
row.filesize = stmt.column_int64(2);
row.md5 = stmt.column_text(3);
row.time_created = (time_t) stmt.column_int64(4);
row.reason = Tombstone.Reason.unserialize(stmt.column_int(5));
rows[index++] = row;
}
......
......@@ -898,14 +898,8 @@ public class DirectoryMonitor : Object {
add_monitor(dir, local_dir_info);
// report files in local directory
if (file_map != null) {
foreach (File file in file_map.keys) {
if (in_discovery)
internal_notify_file_discovered(file, file_map.get(file));
else
internal_notify_file_created(file, file_map.get(file));
}
}
if (file_map != null)
yield notify_directory_files(file_map, in_discovery);
// post all the subdirectory traversals, allowing them to report themselves as discovered
if (recurse && dir_map != null) {
......@@ -920,6 +914,19 @@ public class DirectoryMonitor : Object {
explore_directory_completed(in_discovery);
}
private async void notify_directory_files(Gee.Map<File, FileInfo> map, bool in_discovery) {
Gee.MapIterator<File, FileInfo> iter = map.map_iterator();
while (iter.next()) {
if (in_discovery)
internal_notify_file_discovered(iter.get_key(), iter.get_value());
else
internal_notify_file_created(iter.get_key(), iter.get_value());
Idle.add(notify_directory_files.callback, DEFAULT_PRIORITY);
yield;
}
}
// called whenever exploration of a directory is completed, to know when to signal that
// discovery has ended
private void explore_directory_completed(bool in_discovery) {
......
......@@ -100,6 +100,8 @@ public class LibraryMonitor : DirectoryMonitor {
this.file = file;
this.candidates = candidates;
set_completion_priority(Priority.LOW);
}
public override void execute() {
......@@ -122,28 +124,6 @@ public class LibraryMonitor : DirectoryMonitor {
}
}
private class ChecksumJob : BackgroundJob {
public File file;
public string? md5 = null;
public Error? err = null;
public ChecksumJob(LibraryMonitor owner, File file) {
base (owner, owner.on_checksum_completed, owner.cancellable, owner.on_checksum_cancelled);
this.file = file;
set_completion_priority(Priority.LOW);
}
public override void execute() {
try {
md5 = md5_file(file);
} catch (Error err) {
this.err = err;
}
}
}
private class VerifyJob {
public Monitorable monitorable;
public MediaMonitor monitor;
......@@ -175,6 +155,11 @@ public class LibraryMonitor : DirectoryMonitor {
private uint import_queue_timer_id = 0;
private Gee.Queue<VerifyJob> verify_queue = new Gee.LinkedList<VerifyJob>();
private int outstanding_verify_jobs = 0;
private int files_discovered = 0;
private int completed_monitorable_verifies = 0;
private int total_monitorable_verifies = 0;
public signal void discovery_in_progress();
public signal void auto_update_progress(int completed_files, int total_files);
......@@ -261,10 +246,13 @@ public class LibraryMonitor : DirectoryMonitor {
if (representing != null) {
assert(representation != null && !ignore);
add_to_discovered_list(representing, representation);
} else if (!ignore && !Tombstone.global.matches(file, null)) {
} else if (!ignore && !Tombstone.global.matches(file, null) && is_supported_filetype(file)) {
unknown_files.add(file);
}
if ((++files_discovered % 500) == 0)
discovery_in_progress();
base.file_discovered(file, info);
}
......@@ -324,16 +312,6 @@ public class LibraryMonitor : DirectoryMonitor {
// remove all adopted files from the unknown list
unknown_files.remove_all(adopted);
// After the checksumming is complete, the only use of the unknown files is for
// auto-import, so don't bother checksumming the remainder for duplicates/tombstones unless
// going to do that work
if (auto_import) {
foreach (File file in unknown_files) {
checksums_total++;
workers.enqueue(new ChecksumJob(this, file));
}
}
checksums_completed = 0;
if (checksums_total == 0) {
......@@ -380,41 +358,6 @@ public class LibraryMonitor : DirectoryMonitor {
report_checksum_job_completed();
}
private void on_checksum_completed(BackgroundJob j) {
ChecksumJob job = (ChecksumJob) j;
if (job.err != null) {
warning("Unable to checksum %s to verify tombstone; treating as new file: %s",
job.file.get_path(), job.err.message);
}
if (job.md5 != null) {
Tombstone? tombstone = Tombstone.global.locate(job.file, job.md5);
if (tombstone != null) {
// if tombstoned, update if backing file has moved
if (get_file_info(tombstone.get_file()) == null)
tombstone.move(job.file);
else
mdbg("Skipping auto-import of tombstoned file %s".printf(job.file.get_path()));
} else {
foreach (MediaMonitor monitor in monitors) {
if (monitor.get_media_source_collection().get_source_by_master_md5(job.md5) != null) {
mdbg("Skipping auto-import of duplicate file %s".printf(job.file.get_path()));
unknown_files.remove(job.file);
break;
}
}
}
}
report_checksum_job_completed();
}
private void on_checksum_cancelled(BackgroundJob j) {
report_checksum_job_completed();
}
private void discovery_stage_completed() {
foreach (MediaMonitor monitor in monitors) {
Gee.Set<Monitorable>? monitorables = discovered.get(monitor);
......@@ -511,6 +454,12 @@ public class LibraryMonitor : DirectoryMonitor {
monitor.update_backing_file_info(monitorable, file, info);
}
completed_monitorable_verifies++;
auto_update_progress(completed_monitorable_verifies, total_monitorable_verifies);
Idle.add(verify_monitorable.callback, DEFAULT_PRIORITY);
yield;
// finished, move on to the next job in the queue
assert(outstanding_verify_jobs > 0);
outstanding_verify_jobs--;
......@@ -535,15 +484,13 @@ public class LibraryMonitor : DirectoryMonitor {
}
private void enqueue_import(File file) {
if (!pending_imports.contains(file))
if (!pending_imports.contains(file) && is_supported_filetype(file))
import_queue.add(file);
}
private void enqueue_import_many(Gee.Collection<File> files) {
foreach (File file in files) {
if (!pending_imports.contains(file))
import_queue.add(file);
}
foreach (File file in files)
enqueue_import(file);
}
private void remove_queued_import(File file) {
......@@ -662,6 +609,26 @@ public class LibraryMonitor : DirectoryMonitor {
pending_imports.remove(result.file);
}
if (manifest.already_imported.size > 0) {
Gee.ArrayList<TombstonedFile> to_tombstone = new Gee.ArrayList<TombstonedFile>();
foreach (BatchImportResult result in manifest.already_imported) {
FileInfo? info = get_file_info(result.file);
if (info == null) {
warning("Unable to get info for duplicate file %s", result.file.get_path());
continue;
}
to_tombstone.add(new TombstonedFile(result.file, info.get_size(), null));
}
try {
Tombstone.entomb_many_files(to_tombstone, Tombstone.Reason.AUTO_DETECTED_DUPLICATE);
} catch (DatabaseError err) {
AppWindow.database_error(err);
}
}
mdbg("%d files remain pending for auto-import".printf(pending_imports.size));
discard_current_batch_import();
......
......@@ -21,6 +21,7 @@ public class LibraryWindow : AppWindow {
// these values reflect the priority various background operations have when reporting
// progress to the LibraryWindow progress bar ... higher values give priority to those reports
private const int STARTUP_SCAN_PROGRESS_PRIORITY = 35;
private const int REALTIME_UPDATE_PROGRESS_PRIORITY = 40;
private const int REALTIME_IMPORT_PROGRESS_PRIORITY = 50;
private const int METADATA_WRITER_PROGRESS_PRIORITY = 30;
......@@ -269,6 +270,7 @@ public class LibraryWindow : AppWindow {
sync_videos_visibility();
MetadataWriter.get_instance().progress.connect(on_metadata_writer_progress);
LibraryPhoto.library_monitor.discovery_in_progress.connect(on_library_monitor_discovery_in_progress);
LibraryPhoto.library_monitor.auto_update_progress.connect(on_library_monitor_auto_update_progress);
LibraryPhoto.library_monitor.auto_import_preparing.connect(on_library_monitor_auto_import_preparing);
LibraryPhoto.library_monitor.auto_import_progress.connect(on_library_monitor_auto_import_progress);
......@@ -298,6 +300,7 @@ public class LibraryWindow : AppWindow {
media_sources.items_altered.disconnect(on_media_altered);
MetadataWriter.get_instance().progress.disconnect(on_metadata_writer_progress);
LibraryPhoto.library_monitor.discovery_in_progress.disconnect(on_library_monitor_discovery_in_progress);
LibraryPhoto.library_monitor.auto_update_progress.disconnect(on_library_monitor_auto_update_progress);
LibraryPhoto.library_monitor.auto_import_preparing.disconnect(on_library_monitor_auto_import_preparing);
LibraryPhoto.library_monitor.auto_import_progress.disconnect(on_library_monitor_auto_import_progress);
......@@ -1566,6 +1569,10 @@ public class LibraryWindow : AppWindow {
}
}
private void on_library_monitor_discovery_in_progress() {
pulse_background_progress_bar(_("Updating library..."), STARTUP_SCAN_PROGRESS_PRIORITY);
}
private void on_library_monitor_auto_update_progress(int completed_files, int total_files) {
update_background_progress_bar(_("Updating library..."), REALTIME_UPDATE_PROGRESS_PRIORITY,
completed_files, total_files);
......
......@@ -111,6 +111,7 @@ public abstract class MediaSource : ThumbnailSource {
public abstract File get_file();
public abstract File get_master_file();
public abstract uint64 get_master_filesize();
public abstract uint64 get_filesize();
public abstract time_t get_timestamp();
......@@ -627,7 +628,7 @@ public abstract class MediaSourceCollection : DatabaseSourceCollection {
if (to_tombstone != null && to_tombstone.size > 0) {
try {
Tombstone.entomb_many_sources(to_tombstone);
Tombstone.entomb_many_sources(to_tombstone, Tombstone.Reason.REMOVED_BY_USER);
} catch (DatabaseError err) {
AppWindow.database_error(err);
}
......
......@@ -1696,6 +1696,18 @@ public abstract class Photo : PhotoSource {
}
}
public override uint64 get_master_filesize() {
lock (row) {
return row.master.filesize;
}
}
public uint64 get_editable_filesize() {
lock (row) {
return editable.filesize;
}
}
public override time_t get_exposure_time() {
lock (row) {
return row.exposure_time;
......
......@@ -156,16 +156,59 @@ public class TombstoneSourceCollection : DatabaseSourceCollection {
// if not found, resurrect
if (info == null)
marker.mark(tombstone);
Idle.add(async_scan.callback);
yield;
}
if (marker.get_count() > 0) {
debug("Resurrecting %d tombstones with no backing file", marker.get_count());
DatabaseTable.begin_transaction();
destroy_marked(marker, false);
try {
DatabaseTable.commit_transaction();
} catch (DatabaseError err2) {
AppWindow.database_error(err2);
}
}
}
}
public class TombstonedFile {
public File file;
public int64 filesize;
public string? md5;
public TombstonedFile(File file, int64 filesize, string? md5) {
this.file = file;
this.filesize = filesize;
this.md5 = md5;
}
}
public class Tombstone : DataSource {
// These values are persisted. Do not change.
public enum Reason {
REMOVED_BY_USER = 0,
AUTO_DETECTED_DUPLICATE = 1;
public int serialize() {
return (int) this;
}
public static Reason unserialize(int value) {
switch ((Reason) value) {
case AUTO_DETECTED_DUPLICATE:
return AUTO_DETECTED_DUPLICATE;
// 0 is the default in the database, so it should remain so here
case REMOVED_BY_USER:
default:
return REMOVED_BY_USER;
}
}
}
public static TombstoneSourceCollection global = null;
private TombstoneRow row;
......@@ -197,25 +240,33 @@ public class Tombstone : DataSource {
public static void terminate() {
}
public static void entomb_many_sources(Gee.Collection<MediaSource> sources) throws DatabaseError {
public static void entomb_many_sources(Gee.Collection<MediaSource> sources, Reason reason)
throws DatabaseError {
Gee.Collection<TombstonedFile> files = new Gee.ArrayList<TombstonedFile>();
foreach (MediaSource source in sources) {
foreach (BackingFileState state in source.get_backing_files_state())
files.add(new TombstonedFile(state.get_file(), state.filesize, state.md5));
}
entomb_many_files(files, reason);
}
public static void entomb_many_files(Gee.Collection<TombstonedFile> files, Reason reason)
throws DatabaseError {
// destroy any out-of-date tombstones so they may be updated
Marker to_destroy = global.start_marking();
foreach (MediaSource media in sources) {
foreach (BackingFileState state in media.get_backing_files_state()) {
Tombstone? tombstone = global.locate(state.get_file(), state.md5);
if (tombstone != null)
to_destroy.mark(tombstone);
}
foreach (TombstonedFile file in files) {
Tombstone? tombstone = global.locate(file.file, file.md5);
if (tombstone != null)
to_destroy.mark(tombstone);
}
global.destroy_marked(to_destroy, false);
Gee.ArrayList<Tombstone> tombstones = new Gee.ArrayList<Tombstone>();
foreach (MediaSource media in sources) {
foreach (BackingFileState state in media.get_backing_files_state()) {
tombstones.add(new Tombstone(TombstoneTable.get_instance().add(state.filepath,
state.filesize, state.md5)));
}
foreach (TombstonedFile file in files) {
tombstones.add(new Tombstone(TombstoneTable.get_instance().add(file.file.get_path(),
file.filesize, file.md5, reason)));
}
global.add_many(tombstones);
......@@ -252,6 +303,10 @@ public class Tombstone : DataSource {
return is_string_empty(row.md5) ? null : row.md5;
}
public Reason get_reason() {
return row.reason;
}
public void move(File file) {
try {
TombstoneTable.get_instance().update_file(row.id, file.get_path());
......
......@@ -553,15 +553,20 @@ private class AVIChunk {
}
public void skip(uint64 skip_amount) throws GLib.Error {
skip_uint64(input, skip_amount);
advance_section_offset(skip_amount);
skip_uint64(input, skip_amount);
}
public AVIChunk get_first_child_chunk() {
return new AVIChunk.with_input_stream(input, this);
}
private void advance_section_offset(uint64 amount) {
private void advance_section_offset(uint64 amount) throws Error {
if ((section_offset + amount) > section_size) {
throw new IOError.FAILED("Attempted to advance %d bytes past end of section",
(section_offset + amount) - section_size);
}
section_offset += amount;
if (null != parent) {
parent.advance_section_offset(amount);
......
......@@ -697,6 +697,10 @@ public class Video : VideoSource, Flaggable, Monitorable {
}
public override uint64 get_filesize() {
return get_master_filesize();
}
public override uint64 get_master_filesize() {
lock (backing_row) {
return backing_row.filesize;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment