Commit 23511dc3 authored by Jim Nelson's avatar Jim Nelson

Periodic database & attachments garbage collection: Bug #714134

See the ticket (comment #2) for more information on the thinking and
strategy here, but in a nutshell this will remove from the Geary
database all emails no longer accessible via any folder and not seen
on the server in over 30 days.  It also deletes those messages
attachment(s) and removes any empty directories in the attachment/
directory to prevent clutter.  If enough messages are garbage
collected, Geary will vacuum the database at startup, which will
lower its disk footprint and reduce fragmentation, potentially
increasing performance.
parent a740b3f2
...@@ -23,3 +23,4 @@ install(FILES version-020.sql DESTINATION ${SQL_DEST}) ...@@ -23,3 +23,4 @@ install(FILES version-020.sql DESTINATION ${SQL_DEST})
install(FILES version-021.sql DESTINATION ${SQL_DEST}) install(FILES version-021.sql DESTINATION ${SQL_DEST})
install(FILES version-022.sql DESTINATION ${SQL_DEST}) install(FILES version-022.sql DESTINATION ${SQL_DEST})
install(FILES version-023.sql DESTINATION ${SQL_DEST}) install(FILES version-023.sql DESTINATION ${SQL_DEST})
install(FILES version-024.sql DESTINATION ${SQL_DEST})
--
-- Add the DeleteAttachmentFile table, which allows for attachment files to be deleted (garbage
-- collected) after all references to them have been removed from the database without worrying
-- about deleting them first and the database transaction failing.
--
-- Also add GarbageCollectionTable, a single-row table holding various information about when
-- GC has occurred and when it should occur next.
--
CREATE TABLE DeleteAttachmentFileTable (
id INTEGER PRIMARY KEY,
filename TEXT NOT NULL
);
CREATE TABLE GarbageCollectionTable (
id INTEGER PRIMARY KEY,
last_reap_time_t INTEGER DEFAULT NULL,
last_vacuum_time_t INTEGER DEFAULT NULL,
reaped_messages_since_last_vacuum INTEGER DEFAULT 0
);
-- Insert a single row with a well-known rowid and default values, this will be the row used
-- by the ImapDB.GC class.
INSERT INTO GarbageCollectionTable (id) VALUES (0);
...@@ -168,6 +168,7 @@ engine/imap-db/imap-db-contact.vala ...@@ -168,6 +168,7 @@ engine/imap-db/imap-db-contact.vala
engine/imap-db/imap-db-database.vala engine/imap-db/imap-db-database.vala
engine/imap-db/imap-db-email-identifier.vala engine/imap-db/imap-db-email-identifier.vala
engine/imap-db/imap-db-folder.vala engine/imap-db/imap-db-folder.vala
engine/imap-db/imap-db-gc.vala
engine/imap-db/imap-db-message-addresses.vala engine/imap-db/imap-db-message-addresses.vala
engine/imap-db/imap-db-message-row.vala engine/imap-db/imap-db-message-row.vala
engine/imap-db/imap-db-search-query.vala engine/imap-db/imap-db-search-query.vala
......
...@@ -60,6 +60,7 @@ public class UpgradeDialog : Object { ...@@ -60,6 +60,7 @@ public class UpgradeDialog : Object {
*/ */
public void add_account(Geary.Account account, Cancellable? cancellable = null) { public void add_account(Geary.Account account, Cancellable? cancellable = null) {
monitor.add(account.db_upgrade_monitor); monitor.add(account.db_upgrade_monitor);
monitor.add(account.db_vacuum_monitor);
if (cancellable != null) if (cancellable != null)
cancellables.add(cancellable); cancellables.add(cancellable);
} }
......
...@@ -8,6 +8,7 @@ public abstract class Geary.AbstractAccount : BaseObject, Geary.Account { ...@@ -8,6 +8,7 @@ public abstract class Geary.AbstractAccount : BaseObject, Geary.Account {
public Geary.AccountInformation information { get; protected set; } public Geary.AccountInformation information { get; protected set; }
public Geary.ProgressMonitor search_upgrade_monitor { get; protected set; } public Geary.ProgressMonitor search_upgrade_monitor { get; protected set; }
public Geary.ProgressMonitor db_upgrade_monitor { get; protected set; } public Geary.ProgressMonitor db_upgrade_monitor { get; protected set; }
public Geary.ProgressMonitor db_vacuum_monitor { get; protected set; }
public Geary.ProgressMonitor opening_monitor { get; protected set; } public Geary.ProgressMonitor opening_monitor { get; protected set; }
public Geary.ProgressMonitor sending_monitor { get; protected set; } public Geary.ProgressMonitor sending_monitor { get; protected set; }
......
...@@ -19,6 +19,7 @@ public interface Geary.Account : BaseObject { ...@@ -19,6 +19,7 @@ public interface Geary.Account : BaseObject {
public abstract Geary.ProgressMonitor search_upgrade_monitor { get; protected set; } public abstract Geary.ProgressMonitor search_upgrade_monitor { get; protected set; }
public abstract Geary.ProgressMonitor db_upgrade_monitor { get; protected set; } public abstract Geary.ProgressMonitor db_upgrade_monitor { get; protected set; }
public abstract Geary.ProgressMonitor db_vacuum_monitor { get; protected set; }
public abstract Geary.ProgressMonitor opening_monitor { get; protected set; } public abstract Geary.ProgressMonitor opening_monitor { get; protected set; }
public abstract Geary.ProgressMonitor sending_monitor { get; protected set; } public abstract Geary.ProgressMonitor sending_monitor { get; protected set; }
......
...@@ -11,7 +11,8 @@ public enum Geary.ProgressType { ...@@ -11,7 +11,8 @@ public enum Geary.ProgressType {
AGGREGATED, AGGREGATED,
ACTIVITY, ACTIVITY,
DB_UPGRADE, DB_UPGRADE,
SEARCH_INDEX SEARCH_INDEX,
DB_VACUUM
} }
/** /**
......
...@@ -27,6 +27,8 @@ private class Geary.ImapDB.Account : BaseObject { ...@@ -27,6 +27,8 @@ private class Geary.ImapDB.Account : BaseObject {
default = new IntervalProgressMonitor(ProgressType.SEARCH_INDEX, 0, 0); } default = new IntervalProgressMonitor(ProgressType.SEARCH_INDEX, 0, 0); }
public SimpleProgressMonitor upgrade_monitor { get; private set; default = new SimpleProgressMonitor( public SimpleProgressMonitor upgrade_monitor { get; private set; default = new SimpleProgressMonitor(
ProgressType.DB_UPGRADE); } ProgressType.DB_UPGRADE); }
public SimpleProgressMonitor vacuum_monitor { get; private set; default = new SimpleProgressMonitor(
ProgressType.DB_VACUUM); }
public SimpleProgressMonitor sending_monitor { get; private set; public SimpleProgressMonitor sending_monitor { get; private set;
default = new SimpleProgressMonitor(ProgressType.ACTIVITY); } default = new SimpleProgressMonitor(ProgressType.ACTIVITY); }
...@@ -68,16 +70,18 @@ private class Geary.ImapDB.Account : BaseObject { ...@@ -68,16 +70,18 @@ private class Geary.ImapDB.Account : BaseObject {
if (db != null) if (db != null)
throw new EngineError.ALREADY_OPEN("IMAP database already open"); throw new EngineError.ALREADY_OPEN("IMAP database already open");
db = new ImapDB.Database(user_data_dir, schema_dir, upgrade_monitor, account_information.email); db = new ImapDB.Database(user_data_dir, schema_dir, upgrade_monitor, vacuum_monitor,
account_information.email);
try { try {
db.open( yield db.open_async(
Db.DatabaseFlags.CREATE_DIRECTORY | Db.DatabaseFlags.CREATE_FILE | Db.DatabaseFlags.CHECK_CORRUPTION, Db.DatabaseFlags.CREATE_DIRECTORY | Db.DatabaseFlags.CREATE_FILE | Db.DatabaseFlags.CHECK_CORRUPTION,
cancellable); cancellable);
} catch (Error err) { } catch (Error err) {
warning("Unable to open database: %s", err.message); warning("Unable to open database: %s", err.message);
// close database before exiting // close database before exiting
db.close(null);
db = null; db = null;
throw err; throw err;
......
...@@ -11,14 +11,17 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase { ...@@ -11,14 +11,17 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase {
private const int OPEN_PUMP_EVENT_LOOP_MSEC = 100; private const int OPEN_PUMP_EVENT_LOOP_MSEC = 100;
private ProgressMonitor upgrade_monitor; private ProgressMonitor upgrade_monitor;
private ProgressMonitor vacuum_monitor;
private string account_owner_email; private string account_owner_email;
private bool new_db = false; private bool new_db = false;
private Cancellable gc_cancellable = new Cancellable();
public Database(File db_dir, File schema_dir, ProgressMonitor upgrade_monitor, public Database(File db_dir, File schema_dir, ProgressMonitor upgrade_monitor,
string account_owner_email) { ProgressMonitor vacuum_monitor, string account_owner_email) {
base (get_db_file(db_dir), schema_dir); base (get_db_file(db_dir), schema_dir);
this.upgrade_monitor = upgrade_monitor; this.upgrade_monitor = upgrade_monitor;
this.vacuum_monitor = vacuum_monitor;
this.account_owner_email = account_owner_email; this.account_owner_email = account_owner_email;
} }
...@@ -32,9 +35,67 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase { ...@@ -32,9 +35,67 @@ private class Geary.ImapDB.Database : Geary.Db.VersionedDatabase {
* This should only be done from the main thread, as it is designed to pump the event loop * This should only be done from the main thread, as it is designed to pump the event loop
* while the database is being opened and updated. * while the database is being opened and updated.
*/ */
public new void open(Db.DatabaseFlags flags, Cancellable? cancellable) throws Error { public async void open_async(Db.DatabaseFlags flags, Cancellable? cancellable) throws Error {
open_background(flags, on_prepare_database_connection, pump_event_loop, open_background(flags, on_prepare_database_connection, pump_event_loop,
OPEN_PUMP_EVENT_LOOP_MSEC, cancellable); OPEN_PUMP_EVENT_LOOP_MSEC, cancellable);
// Tie user-supplied Cancellable to internal Cancellable, which is used when close() is
// called
if (cancellable != null)
cancellable.cancelled.connect(cancel_gc);
// Create new garbage collection object for this database
GC gc = new GC(this, Priority.LOW);
// Get recommendations on what GC operations should be executed
GC.RecommendedOperation recommended = yield gc.should_run_async(gc_cancellable);
// VACUUM needs to execute in the foreground with the user given a busy prompt (and cannot
// be run at the same time as REAP)
if ((recommended & GC.RecommendedOperation.VACUUM) != 0) {
if (!vacuum_monitor.is_in_progress)
vacuum_monitor.notify_start();
try {
yield gc.vacuum_async(gc_cancellable);
} catch (Error err) {
message("Vacuum of IMAP database %s failed: %s", db_file.get_path(), err.message);
throw err;
} finally {
if (vacuum_monitor.is_in_progress)
vacuum_monitor.notify_finish();
}
}
// REAP can run in the background while the application is executing
if ((recommended & GC.RecommendedOperation.REAP) != 0) {
// run in the background and allow application to continue running
gc.reap_async.begin(gc_cancellable, on_reap_async_completed);
}
if (cancellable != null)
cancellable.cancelled.disconnect(cancel_gc);
}
private void on_reap_async_completed(Object? object, AsyncResult result) {
GC gc = (GC) object;
try {
gc.reap_async.end(result);
} catch (Error err) {
message("Garbage collection of IMAP database %s failed: %s", db_file.get_path(), err.message);
}
}
private void cancel_gc() {
gc_cancellable.cancel();
gc_cancellable = new Cancellable();
}
public override void close(Cancellable? cancellable) throws Error {
cancel_gc();
base.close(cancellable);
} }
private void pump_event_loop() { private void pump_event_loop() {
......
...@@ -1983,6 +1983,15 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics { ...@@ -1983,6 +1983,15 @@ private class Geary.ImapDB.Folder : BaseObject, Geary.ReferenceSemantics {
File saved_file = ImapDB.Attachment.generate_file(db.db_file.get_parent(), message_id, File saved_file = ImapDB.Attachment.generate_file(db.db_file.get_parent(), message_id,
attachment_id, filename); attachment_id, filename);
// On the off-chance this is marked for deletion, unmark it
stmt = cx.prepare("""
DELETE FROM DeleteAttachmentFileTable
WHERE filename = ?
""");
stmt.bind_string(0, saved_file.get_path());
stmt.exec(cancellable);
debug("Saving attachment to %s", saved_file.get_path()); debug("Saving attachment to %s", saved_file.get_path());
try { try {
......
This diff is collapsed.
...@@ -37,6 +37,7 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount { ...@@ -37,6 +37,7 @@ private abstract class Geary.ImapEngine.GenericAccount : Geary.AbstractAccount {
search_upgrade_monitor = local.search_index_monitor; search_upgrade_monitor = local.search_index_monitor;
db_upgrade_monitor = local.upgrade_monitor; db_upgrade_monitor = local.upgrade_monitor;
db_vacuum_monitor = local.vacuum_monitor;
opening_monitor = new Geary.ReentrantProgressMonitor(Geary.ProgressType.ACTIVITY); opening_monitor = new Geary.ReentrantProgressMonitor(Geary.ProgressType.ACTIVITY);
sending_monitor = local.sending_monitor; sending_monitor = local.sending_monitor;
......
...@@ -54,10 +54,10 @@ ...@@ -54,10 +54,10 @@
</packing> </packing>
</child> </child>
<child> <child>
<object class="GtkLabel" id="label1"> <object class="GtkLabel" id="text_label">
<property name="visible">True</property> <property name="visible">True</property>
<property name="can_focus">False</property> <property name="can_focus">False</property>
<property name="label" translatable="yes">Geary upgrade in progress.</property> <property name="label" translatable="yes">Geary update in progress…</property>
</object> </object>
<packing> <packing>
<property name="left_attach">1</property> <property name="left_attach">1</property>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment