Commit be169563 authored by Jim Nelson's avatar Jim Nelson

#531: Importing now takes place completely in background threads. Thumbnails...

#531: Importing now takes place completely in background threads.  Thumbnails are generated in background, but 
they are written in the foreground; this could be avoided with more work.
parent 68935da0
......@@ -211,10 +211,10 @@ DIST_TAR_BZ2 = $(DIST_TAR).bz2
DIST_TAR_GZ = $(DIST_TAR).gz
PACKAGE_ORIG_GZ = $(PROGRAM)_`parsechangelog | grep Version | sed 's/.*: //'`.orig.tar.gz
VALA_CFLAGS = `pkg-config --cflags $(EXT_PKGS)` $(foreach hdir,$(HEADER_DIRS),-I$(hdir)) \
VALA_CFLAGS = `pkg-config --cflags $(EXT_PKGS) gthread-2.0` $(foreach hdir,$(HEADER_DIRS),-I$(hdir)) \
$(foreach def,$(DEFINES),-D$(def))
VALA_LDFLAGS = `pkg-config --libs $(EXT_PKGS)` -lgthread-2.0
VALA_LDFLAGS = `pkg-config --libs $(EXT_PKGS) gthread-2.0`
ifdef WINDOWS
VALA_DEFINES = -D WINDOWS -D NO_CAMERA -D NO_PUBLISHING -D NO_LIBUNIQUE -D NO_EXTENDED_POSIX
......
......@@ -337,7 +337,6 @@ public abstract class AppWindow : PageWindow {
protected static AppWindow instance = null;
private static bool user_quit = false;
private static FullscreenWindow fullscreen_window = null;
private static CommandManager command_manager = null;
......@@ -369,6 +368,8 @@ public abstract class AppWindow : PageWindow {
command_manager = new CommandManager();
}
public signal void user_quit();
private Gtk.ActionEntry[] create_actions() {
Gtk.ActionEntry[] actions = new Gtk.ActionEntry[0];
......@@ -441,10 +442,6 @@ public abstract class AppWindow : PageWindow {
return yes;
}
public static bool has_user_quit() {
return user_quit;
}
public abstract string get_app_role();
protected void on_about() {
......@@ -465,7 +462,7 @@ public abstract class AppWindow : PageWindow {
}
protected virtual void on_quit() {
user_quit = true;
user_quit();
Gtk.main_quit();
}
......
This diff is collapsed.
......@@ -1237,7 +1237,7 @@ public class CheckerboardLayout : Gtk.DrawingArea {
|| selection_interior.height < visible_band.height) {
selection_interior = new Gdk.Pixbuf(Gdk.Colorspace.RGB, true, 8, visible_band.width,
visible_band.height);
selection_interior.fill(selection_transparency_color);
selection_interior.fill(selection_transparency_color);
}
window.draw_pixbuf(selection_band_gc, selection_interior, 0, 0, visible_band.x,
......
......@@ -884,15 +884,21 @@ class RGBHistogram {
public const int GRAPHIC_WIDTH = 256;
public const int GRAPHIC_HEIGHT = 100;
private int[] red_counts = new int[256];
private int[] green_counts = new int[256];
private int[] blue_counts = new int[256];
private int[] red_counts = null;
private int[] green_counts = null;
private int[] blue_counts = null;
private int[] qualitative_red_counts = null;
private int[] qualitative_green_counts = null;
private int[] qualitative_blue_counts = null;
private Gdk.Pixbuf graphic = null;
public RGBHistogram(Gdk.Pixbuf pixbuf) {
// allocating here rather than at declaration time due to this bug:
// https://bugzilla.gnome.org/show_bug.cgi?id=607714
red_counts = new int[256];
green_counts = new int[256];
blue_counts = new int[256];
for (int i = 0; i < 256; i++)
red_counts[i] = green_counts[i] = blue_counts[i] = 0;
......@@ -1137,11 +1143,17 @@ class RGBHistogram {
}
public class IntensityHistogram {
private int[] counts = new int[256];
private float[] probabilities = new float[256];
private float[] cumulative_probabilities = new float[256];
private int[] counts = null;
private float[] probabilities = null;
private float[] cumulative_probabilities = null;
public IntensityHistogram(Gdk.Pixbuf pixbuf) {
// allocating here rather than at declaration time due to this bug:
// https://bugzilla.gnome.org/show_bug.cgi?id=607714
counts = new int[256];
probabilities = new float[256];
cumulative_probabilities = new float[256];
int n_channels = pixbuf.get_n_channels();
int rowstride = pixbuf.get_rowstride();
int width = pixbuf.get_width();
......
......@@ -171,7 +171,7 @@ public abstract class SourceSnapshot {
public abstract class DataSource : DataObject {
protected delegate void ContactSubscriber(DataView view);
private DataView[] subscribers = new DataView[4];
private DataView[] subscribers = null;
private bool in_contact = false;
private bool marked_for_destroy = false;
private bool is_destroyed = false;
......@@ -183,6 +183,10 @@ public abstract class DataSource : DataObject {
public DataSource(int64 object_id = INVALID_OBJECT_ID) {
base (object_id);
// allocating this here rather than at declaration time due to this bug:
// https://bugzilla.gnome.org/show_bug.cgi?id=607714
subscribers = new DataView[4];
}
~DataSource() {
......
......@@ -462,8 +462,9 @@ public class PhotoTable : DatabaseTable {
return ImportID(id);
}
public PhotoID add(File file, Dimensions dim, int64 filesize, long timestamp, time_t exposure_time,
Orientation orientation, ImportID import_id, string? md5, string? thumbnail_md5, string? exif_md5) {
// PhotoRow.photo_id, event_id, orientation, flags, and time_created are ignored on input. All
// fields are set on exit with values stored in the database.
public PhotoID add(ref PhotoRow photo_row) {
Sqlite.Statement stmt;
int res = db.prepare_v2(
"INSERT INTO PhotoTable (filename, width, height, filesize, timestamp, exposure_time, "
......@@ -472,33 +473,35 @@ public class PhotoTable : DatabaseTable {
-1, out stmt);
assert(res == Sqlite.OK);
res = stmt.bind_text(1, file.get_path());
ulong time_created = now_sec();
res = stmt.bind_text(1, photo_row.filepath);
assert(res == Sqlite.OK);
res = stmt.bind_int(2, dim.width);
res = stmt.bind_int(2, photo_row.dim.width);
assert(res == Sqlite.OK);
res = stmt.bind_int(3, dim.height);
res = stmt.bind_int(3, photo_row.dim.height);
assert(res == Sqlite.OK);
res = stmt.bind_int64(4, filesize);
res = stmt.bind_int64(4, photo_row.filesize);
assert(res == Sqlite.OK);
res = stmt.bind_int64(5, timestamp);
res = stmt.bind_int64(5, photo_row.timestamp);
assert(res == Sqlite.OK);
res = stmt.bind_int64(6, exposure_time);
res = stmt.bind_int64(6, photo_row.exposure_time);
assert(res == Sqlite.OK);
res = stmt.bind_int(7, orientation);
res = stmt.bind_int(7, photo_row.original_orientation);
assert(res == Sqlite.OK);
res = stmt.bind_int(8, orientation);
res = stmt.bind_int(8, photo_row.original_orientation);
assert(res == Sqlite.OK);
res = stmt.bind_int64(9, import_id.id);
res = stmt.bind_int64(9, photo_row.import_id.id);
assert(res == Sqlite.OK);
res = stmt.bind_int64(10, PhotoID.INVALID);
res = stmt.bind_int64(10, EventID.INVALID);
assert(res == Sqlite.OK);
res = stmt.bind_text(11, md5);
res = stmt.bind_text(11, photo_row.md5);
assert(res == Sqlite.OK);
res = stmt.bind_text(12, thumbnail_md5);
res = stmt.bind_text(12, photo_row.thumbnail_md5);
assert(res == Sqlite.OK);
res = stmt.bind_text(13, exif_md5);
res = stmt.bind_text(13, photo_row.exif_md5);
assert(res == Sqlite.OK);
res = stmt.bind_int64(14, now_sec());
res = stmt.bind_int64(14, time_created);
assert(res == Sqlite.OK);
res = stmt.step();
......@@ -508,8 +511,15 @@ public class PhotoTable : DatabaseTable {
return PhotoID();
}
return PhotoID(db.last_insert_rowid());
// fill in ignored fields with database values
photo_row.photo_id = PhotoID(db.last_insert_rowid());
photo_row.orientation = photo_row.original_orientation;
photo_row.event_id = EventID();
photo_row.time_created = (time_t) time_created;
photo_row.flags = 0;
return photo_row.photo_id;
}
public bool update(PhotoID photoID, Dimensions dim, int64 filesize, long timestamp,
......@@ -1235,6 +1245,84 @@ public class PhotoTable : DatabaseTable {
public bool has_exif_md5(string exif_md5) {
return has_hash("exif_md5", exif_md5);
}
public bool has_duplicate(File? file, string? exif_md5, string? thumbnail_md5, string? md5) {
assert(file != null || exif_md5 != null || thumbnail_md5 != null || md5 != null);
string sql = "SELECT id FROM PhotoTable WHERE";
if (file != null)
sql += " filename=?";
if (exif_md5 != null || thumbnail_md5 != null || md5 != null) {
if (file != null)
sql += " OR ";
sql += " (";
bool first = true;
if (exif_md5 != null) {
sql += "exif_md5=?";
first = false;
}
if (thumbnail_md5 != null) {
if (first)
sql += "thumbnail_md5=?";
else
sql += " OR thumbnail_md5=?";
first = false;
}
if (md5 != null) {
if (first)
sql += "md5=?";
else
sql += " OR md5=?";
}
sql += ")";
}
Sqlite.Statement stmt;
int res = db.prepare_v2(sql, -1, out stmt);
assert(res == Sqlite.OK);
int col = 1;
if (file != null) {
res = stmt.bind_text(col++, file.get_path());
assert(res == Sqlite.OK);
}
if (exif_md5 != null) {
res = stmt.bind_text(col++, exif_md5);
assert(res == Sqlite.OK);
}
if (thumbnail_md5 != null) {
res = stmt.bind_text(col++, thumbnail_md5);
assert(res == Sqlite.OK);
}
if (md5 != null) {
res = stmt.bind_text(col++, md5);
assert(res == Sqlite.OK);
}
res = stmt.step();
if (res == Sqlite.DONE) {
// not found
return false;
} else if (res == Sqlite.ROW) {
// at least one found
return true;
} else {
fatal("has_duplicate", res);
return false;
}
}
}
public struct ThumbnailCacheRow {
......
......@@ -10,8 +10,11 @@ namespace Debug {
private bool message_enabled = false;
private bool warning_enabled = false;
private bool critical_enabled = false;
private Mutex log_mutex = null;
public static void init() {
log_mutex = new Mutex();
if (Environment.get_variable("SHOTWELL_LOG") != null) {
info_enabled = true;
debug_enabled = true;
......@@ -37,10 +40,12 @@ namespace Debug {
}
private void log(FileStream stream, string prefix, string message) {
log_mutex.lock();
stream.puts(prefix);
stream.puts(message);
stream.putc('\n');
stream.flush();
log_mutex.unlock();
}
private void info_handler(string? domain, LogLevelFlags flags, string message) {
......
......@@ -319,6 +319,21 @@ public bool report_manifest(ImportManifest manifest, bool list, QuestionParams?
message += generate_import_failure_list(manifest.failed);
}
if (manifest.camera_failed.size > 0) {
if (list && message.length > 0)
message += "\n";
string camera_failed_message =
ngettext("1 photo failed to import due to a camera error.\n",
"%d photos failed to import due to a camera error.\n",
manifest.camera_failed.size).printf(manifest.camera_failed.size);
message += camera_failed_message;
if (list)
message += generate_import_failure_list(manifest.camera_failed);
}
if (manifest.skipped.size > 0) {
if (list && message.length > 0)
message += "\n";
......@@ -347,8 +362,8 @@ public bool report_manifest(ImportManifest manifest, bool list, QuestionParams?
message += generate_import_failure_list(manifest.aborted);
}
int total = manifest.success.size + manifest.failed.size + manifest.skipped.size
+ manifest.already_imported.size + manifest.aborted.size;
int total = manifest.success.size + manifest.failed.size + manifest.camera_failed.size
+ manifest.skipped.size + manifest.already_imported.size + manifest.aborted.size;
assert(total == manifest.all.size);
// if no photos imported at all (i.e. an empty directory attempted), need to at least report
......
......@@ -81,6 +81,10 @@ public struct Dimensions {
return (width - dim.width).abs() <= fudge && (height - dim.height).abs() <= fudge;
}
public bool approx_scaled(int scale, int fudge = 1) {
return (width <= (scale + fudge)) && (height <= (scale + fudge));
}
public int major_axis() {
return int.max(width, height);
}
......
......@@ -47,7 +47,6 @@ namespace GPhoto {
private void on_idle(Context context) {
idle();
spin_event_loop();
}
private void on_error(Context context, string format, void *va_list) {
......@@ -70,7 +69,6 @@ namespace GPhoto {
private void on_progress_update(Context context, uint id, float current) {
progress_update(current);
spin_event_loop();
}
private void on_progress_stop(Context context, uint id) {
......@@ -78,6 +76,23 @@ namespace GPhoto {
}
}
public class SpinIdleWrapper : ContextWrapper {
public SpinIdleWrapper() {
}
public override void idle() {
base.idle();
spin_event_loop();
}
public override void progress_update(float current) {
base.progress_update(current);
spin_event_loop();
}
}
public void get_info(Context context, Camera camera, string folder, string filename,
out CameraFileInfo info) throws Error {
Result res = camera.get_file_info(folder, filename, out info, context);
......
......@@ -103,7 +103,7 @@ class ImportSource : PhotoSource {
debug("Deleting %s", to_string());
GPhoto.Result result = camera.delete_file(get_fulldir(), get_filename(),
ImportPage.null_context.context);
ImportPage.spin_idle_context.context);
if (result != GPhoto.Result.OK)
warning("Error deleting %s: %s", to_string(), result.as_string());
......@@ -137,10 +137,8 @@ class ImportPreview : LayoutItem {
public bool is_already_imported() {
ImportSource source = (ImportSource) get_source();
bool exif_match = PhotoTable.get_instance().has_exif_md5(source.get_exif_md5());
bool thumbnail_match = PhotoTable.get_instance().has_thumbnail_md5(source.get_preview_md5());
return exif_match || thumbnail_match;
return TransformablePhoto.is_duplicate(null, source.get_exif_md5(), source.get_preview_md5(),
null);
}
}
......@@ -166,11 +164,19 @@ public class ImportPage : CheckerboardPage {
private GPhoto.ContextWrapper context;
private ImportSource import_file;
private File? dest_file;
private GPhoto.Camera camera;
private string fulldir;
private string filename;
public CameraImportJob(GPhoto.ContextWrapper context, ImportSource import_file, File? dest_file) {
this.context = context;
this.import_file = import_file;
this.dest_file = dest_file;
// stash everything called in prepare(), as it may/will be called from a separate thread
camera = import_file.get_camera();
fulldir = import_file.get_fulldir();
filename = import_file.get_filename();
}
public time_t get_exposure_time() {
......@@ -178,26 +184,22 @@ public class ImportPage : CheckerboardPage {
}
public override string get_identifier() {
return import_file.get_filename();
return filename;
}
public ImportSource get_source() {
return import_file;
}
public override bool prepare(out File file_to_import, out bool copy_to_library) {
public override bool is_directory() {
return false;
}
public override bool prepare(out File file_to_import, out bool copy_to_library) throws Error {
if (dest_file == null)
return false;
try {
GPhoto.save_image(context.context, import_file.get_camera(), import_file.get_fulldir(),
import_file.get_filename(), dest_file);
} catch (Error err) {
warning("Unable to fetch photo from %s to %s: %s", import_file.to_string(),
dest_file.get_path(), err.message);
return false;
}
GPhoto.save_image(context.context, camera, fulldir, filename, dest_file);
file_to_import = dest_file;
copy_to_library = false;
......@@ -207,6 +209,7 @@ public class ImportPage : CheckerboardPage {
}
public static GPhoto.ContextWrapper null_context = null;
public static GPhoto.SpinIdleWrapper spin_idle_context = null;
private SourceCollection import_sources = null;
private Gtk.Label camera_label = new Gtk.Label(null);
......@@ -245,6 +248,10 @@ public class ImportPage : CheckerboardPage {
if (null_context == null)
null_context = new GPhoto.ContextWrapper();
// same with idle-loop wrapper
if (spin_idle_context == null)
spin_idle_context = new GPhoto.SpinIdleWrapper();
// monitor source collection to add/remove views
get_view().monitor_source_collection(import_sources, new ImportViewManager(this));
......@@ -577,7 +584,7 @@ public class ImportPage : CheckerboardPage {
refreshed = false;
refresh_error = null;
refresh_result = camera.init(null_context.context);
refresh_result = camera.init(spin_idle_context.context);
if (refresh_result != GPhoto.Result.OK) {
warning("Unable to initialize camera: %s (%d)", refresh_result.as_string(), refresh_result);
......@@ -597,7 +604,7 @@ public class ImportPage : CheckerboardPage {
GPhoto.CameraStorageInformation *sifs = null;
int count = 0;
refresh_result = camera.get_storageinfo(&sifs, out count, null_context.context);
refresh_result = camera.get_storageinfo(&sifs, out count, spin_idle_context.context);
if (refresh_result == GPhoto.Result.OK) {
get_view().clear();
......@@ -614,7 +621,7 @@ public class ImportPage : CheckerboardPage {
progress_bar.set_text("");
progress_bar.set_fraction(0.0);
GPhoto.Result res = camera.exit(null_context.context);
GPhoto.Result res = camera.exit(spin_idle_context.context);
if (res != GPhoto.Result.OK) {
// log but don't fail
warning("Unable to unlock camera: %s (%d)", res.as_string(), (int) res);
......@@ -690,7 +697,7 @@ public class ImportPage : CheckerboardPage {
if (refresh_result != GPhoto.Result.OK)
return false;
refresh_result = camera.list_files(fulldir, files, null_context.context);
refresh_result = camera.list_files(fulldir, files, spin_idle_context.context);
if (refresh_result != GPhoto.Result.OK)
return false;
......@@ -702,7 +709,7 @@ public class ImportPage : CheckerboardPage {
try {
GPhoto.CameraFileInfo info;
GPhoto.get_info(null_context.context, camera, fulldir, filename, out info);
GPhoto.get_info(spin_idle_context.context, camera, fulldir, filename, out info);
// at this point, only interested in JPEG files
// TODO: Increase file format support, for TIFF and RAW at least
......@@ -753,7 +760,7 @@ public class ImportPage : CheckerboardPage {
if (refresh_result != GPhoto.Result.OK)
return false;
refresh_result = camera.list_folders(fulldir, folders, null_context.context);
refresh_result = camera.list_folders(fulldir, folders, spin_idle_context.context);
if (refresh_result != GPhoto.Result.OK)
return false;
......@@ -782,7 +789,7 @@ public class ImportPage : CheckerboardPage {
// load EXIF for photo, which will include the preview thumbnail
uint8[] exif_raw;
size_t exif_raw_length;
Exif.Data exif = GPhoto.load_exif(null_context.context, camera, fulldir, filename,
Exif.Data exif = GPhoto.load_exif(spin_idle_context.context, camera, fulldir, filename,
out exif_raw, out exif_raw_length);
// calculate EXIF's fingerprint
......@@ -798,7 +805,7 @@ public class ImportPage : CheckerboardPage {
uint8[] preview_raw;
size_t preview_raw_length;
Gdk.Pixbuf preview = GPhoto.load_preview(null_context.context, camera, fulldir,
Gdk.Pixbuf preview = GPhoto.load_preview(spin_idle_context.context, camera, fulldir,
filename, out preview_raw, out preview_raw_length);
// calculate thumbnail fingerprint
......@@ -865,7 +872,7 @@ public class ImportPage : CheckerboardPage {
}
private void import(Gee.Iterable<DataObject> items) {
GPhoto.Result res = camera.init(null_context.context);
GPhoto.Result res = camera.init(spin_idle_context.context);
if (res != GPhoto.Result.OK) {
AppWindow.error_message(_("Unable to lock camera: %s").printf(res.as_string()));
......@@ -953,7 +960,7 @@ public class ImportPage : CheckerboardPage {
"Delete these %d photos from camera?", manifest.all.size)).printf(manifest.all.size);
ImportUI.QuestionParams question = new ImportUI.QuestionParams(
question_string, Gtk.STOCK_DELETE, _("Keep"));
question_string, Gtk.STOCK_DELETE, _("_Keep"));
if (!ImportUI.report_manifest(manifest, false, question))
return;
......@@ -978,7 +985,7 @@ public class ImportPage : CheckerboardPage {
}
private void close_import() {
GPhoto.Result res = camera.exit(null_context.context);
GPhoto.Result res = camera.exit(spin_idle_context.context);
if (res != GPhoto.Result.OK) {
// log but don't fail
message("Unable to unlock camera: %s (%d)", res.as_string(), (int) res);
......@@ -1110,12 +1117,8 @@ public class ImportQueuePage : SinglePhotoPage {
current_batch = batch_import;
}
private void on_imported(LibraryPhoto photo) {
try {
set_pixbuf(photo.get_pixbuf(get_canvas_scaling()), photo.get_dimensions());
} catch (Error err) {
warning("%s", err.message);
}
private void on_imported(LibraryPhoto photo, Gdk.Pixbuf pixbuf) {
set_pixbuf(pixbuf, Dimensions());
// set the singleton collection to this item
get_view().clear();
......
......@@ -6,8 +6,31 @@
namespace LibraryFiles {
public const int DIRECTORY_DEPTH = 3;
public File? generate_unique_file(string filename, Exif.Data? exif, time_t ts, out bool collision)
// Returns true if the file is claimed, false if it exists, and throws an Error otherwise.
private bool claim_file(File file) throws Error {
try {
file.create(FileCreateFlags.NONE, null);
// created; success
return true;
} catch (Error err) {
// check for file-exists error
if (!(err is IOError.EXISTS)) {
debug("claim_file %s: %s", file.get_path(), err.message);
throw err;
}
return false;
}
}
// This method uses File.create() in order to "claim" a file in the filesystem. Thus, when the
// method returns success a file may exist already, and should be overwritten.
//
// This function is thread safe.
public File? generate_unique_file(string basename, Exif.Data? exif, time_t ts, out bool collision)
throws Error {
File dir = AppDirs.get_photos_dir();
time_t timestamp = ts;
......@@ -28,36 +51,45 @@ public File? generate_unique_file(string filename, Exif.Data? exif, time_t ts, o
dir = dir.get_child("%02u".printf(tm.month + 1));
dir = dir.get_child("%02u".printf(tm.day));
if (!dir.query_exists(null))
try {
dir.make_directory_with_parents(null);
} catch (Error err) {
if (!(err is IOError.EXISTS))
throw err;
// silently ignore not creating a directory that already exists
}
// if file doesn't exist, use that and done
File file = dir.get_child(filename);
if (!file.query_exists(null)) {
// create the file to atomically "claim" it
File file = dir.get_child(basename);
if (claim_file(file)) {
collision = false;
return file;
}
// file exists, collision and keep searching
collision = true;
string name, ext;
disassemble_filename(file.get_basename(), out name, out ext);
disassemble_filename(basename, out name, out ext);
// generate a unique filename
for (int ctr = 1; ctr < int.MAX; ctr++) {
string new_name = (ext != null) ? "%s_%d.%s".printf(name, ctr, ext) : "%s_%d".printf(name, ctr);
file = dir.get_child(new_name);
if (!file.query_exists(null))
file = dir.get_child(new_name);
if (claim_file(file))
return file;
}
debug("generate_unique_filename for %s: unable to claim file", basename);
return null;
}
private File duplicate(File src) throws Error {
// This function is thread-safe.
private File duplicate(File src, FileProgressCallback? progress_callback) throws Error {
time_t timestamp = 0;
try {
timestamp = query_file_modified(src);
......@@ -72,14 +104,8 @@ private File duplicate(File src) throws Error {
if (dest == null)
throw new FileError.FAILED("Unable to generate unique pathname for destination");
debug("Copying %s to %s", src.get_path(), dest.get_path());
src.copy(dest, FileCopyFlags.ALL_METADATA, null, on_copy_progress);
src.copy(dest, FileCopyFlags.ALL_METADATA | FileCopyFlags.OVERWRITE, null, progress_callback);
return dest;
}
private void on_copy_progress(int64 current, int64 total) {
spin_event_loop();
}
}
......@@ -102,6 +102,10 @@ public class LibraryWindow : AppWindow {
return file_or_dir.get_path();
}
public override bool is_directory() {
return query_is_directory(file_or_dir);
}
public override bool prepare(out File file_to_import, out bool copy) {
file_to_import = file_or_dir;
copy = copy_to_library;
......