Commit d179cb9b authored by Jim Nelson's avatar Jim Nelson

Persist messages locally: #3742

This completes the heavy lifting of persisting messages locally.  The strategy is that the local database may be sparsely populated, both in the availability of messages in a folder and the fields of a message that is partially stored.  As data is pulled from the remote server it's always stored in the database.  Future requests will always go to the database first, preventing unnecessary network traffic.

Also, this patch will detect when a message is stored in multiple folders on the server.  The database uses soft links from the folder to the message, so the message is stored only once in the database.  This technique relies heavily on the availability and validity of the Message-ID header, but we expect this to be reliable the vast majority of the time.
parent 4ccabcbd
......@@ -20,6 +20,8 @@ ENGINE_SRC := \
src/engine/api/FolderProperties.vala \
src/engine/api/Credentials.vala \
src/engine/api/EngineError.vala \
src/engine/api/RemoteInterfaces.vala \
src/engine/api/LocalInterfaces.vala \
src/engine/sqlite/Database.vala \
src/engine/sqlite/Table.vala \
src/engine/sqlite/Row.vala \
......
......@@ -18,6 +18,7 @@ CREATE INDEX FolderTableParentIndex ON FolderTable(parent_id);
CREATE TABLE MessageTable (
id INTEGER PRIMARY KEY,
fields INTEGER,
date_field TEXT,
date_time_t INTEGER,
......
......@@ -55,6 +55,8 @@ public class MessageListStore : Gtk.TreeStore {
}
// The Email should've been fetched with Geary.Email.Field.ENVELOPE, at least.
//
// TODO: Need to insert email's in their proper position, not merely append.
public void append_envelope(Geary.Email envelope) {
Gtk.TreeIter iter;
append(out iter, null);
......
......@@ -5,13 +5,13 @@
*/
private class Geary.EngineFolder : Object, Geary.Folder {
private NetworkAccount net;
private RemoteAccount remote;
private LocalAccount local;
private Geary.Folder local_folder;
private Geary.Folder net_folder;
private RemoteFolder remote_folder;
private LocalFolder local_folder;
public EngineFolder(NetworkAccount net, LocalAccount local, Geary.Folder local_folder) {
this.net = net;
public EngineFolder(RemoteAccount remote, LocalAccount local, LocalFolder local_folder) {
this.remote = remote;
this.local = local;
this.local_folder = local_folder;
......@@ -30,152 +30,222 @@ private class Geary.EngineFolder : Object, Geary.Folder {
return null;
}
public async void create_email_async(Geary.Email email, Geary.Email.Field fields,
Cancellable? cancellable) throws Error {
public async void create_email_async(Geary.Email email, Cancellable? cancellable) throws Error {
throw new EngineError.READONLY("Engine currently read-only");
}
public async void open_async(bool readonly, Cancellable? cancellable = null) throws Error {
if (net_folder == null) {
net_folder = yield net.fetch_folder_async(null, local_folder.get_name(), cancellable);
net_folder.updated.connect(on_net_updated);
yield local_folder.open_async(readonly, cancellable);
if (remote_folder == null) {
remote_folder = (RemoteFolder) yield remote.fetch_folder_async(null, local_folder.get_name(),
cancellable);
remote_folder.updated.connect(on_remote_updated);
}
yield net_folder.open_async(readonly, cancellable);
yield remote_folder.open_async(readonly, cancellable);
notify_opened();
}
public async void close_async(Cancellable? cancellable = null) throws Error {
if (net_folder != null) {
net_folder.updated.disconnect(on_net_updated);
yield net_folder.close_async(cancellable);
}
yield local_folder.close_async(cancellable);
net_folder = null;
if (remote_folder != null) {
remote_folder.updated.disconnect(on_remote_updated);
yield remote_folder.close_async(cancellable);
remote_folder = null;
notify_closed(CloseReason.FOLDER_CLOSED);
}
}
public int get_message_count() throws Error {
public async int get_email_count(Cancellable? cancellable = null) throws Error {
// TODO
return 0;
}
public async Gee.List<Geary.Email>? list_email_async(int low, int count, Geary.Email.Field fields,
Cancellable? cancellable = null) throws Error {
public async Gee.List<Geary.Email>? list_email_async(int low, int count,
Geary.Email.Field required_fields, Cancellable? cancellable = null) throws Error {
assert(low >= 1);
assert(count >= 0);
if (count == 0)
return null;
Gee.List<Geary.Email>? local_list = yield local_folder.list_email_async(low, count, fields,
cancellable);
Gee.List<Geary.Email>? local_list = yield local_folder.list_email_async(low, count,
required_fields, cancellable);
int local_list_size = (local_list != null) ? local_list.size : 0;
debug("local list found %d", local_list_size);
if (net_folder != null && local_list_size != count) {
// go through the positions from (low) to (low + count) and see if they're not already
// present in local_list; whatever isn't present needs to be fetched
int[] needed_by_position = new int[0];
int position = low;
for (int index = 0; (index < count) && (position <= (low + count - 1)); position++) {
while ((index < local_list_size) && (local_list[index].location.position < position))
index++;
if (index >= local_list_size || local_list[index].location.position != position)
needed_by_position += position;
}
if (remote_folder == null || local_list_size == count)
return local_list;
// go through the positions from (low) to (low + count) and see if they're not already
// present in local_list; whatever isn't present needs to be fetched
int[] needed_by_position = new int[0];
int position = low;
for (int index = 0; (index < count) && (position <= (low + count - 1)); position++) {
while ((index < local_list_size) && (local_list[index].location.position < position))
index++;
if (needed_by_position.length != 0)
background_update_email_list.begin(needed_by_position, fields, cancellable);
if (index >= local_list_size || local_list[index].location.position != position)
needed_by_position += position;
}
return local_list;
if (needed_by_position.length == 0)
return local_list;
Gee.List<Geary.Email>? remote_list = yield remote_list_email(needed_by_position,
required_fields, cancellable);
return combine_lists(local_list, remote_list);
}
public async Gee.List<Geary.Email>? list_email_sparse_async(int[] by_position,
Geary.Email.Field fields, Cancellable? cancellable = null) throws Error {
Geary.Email.Field required_fields, Cancellable? cancellable = null) throws Error {
if (by_position.length == 0)
return null;
Gee.List<Geary.Email>? local_list = yield local_folder.list_email_sparse_async(by_position,
fields, cancellable);
required_fields, cancellable);
int local_list_size = (local_list != null) ? local_list.size : 0;
if (net_folder != null && local_list_size != by_position.length) {
// go through the list looking for anything not already in the sparse by_position list
// to fetch from the server; since by_position is not guaranteed to be sorted, the local
// list needs to be searched each iteration.
//
// TODO: Optimize this, especially if large lists/sparse sets are supplied
int[] needed_by_position = new int[0];
foreach (int position in by_position) {
bool found = false;
if (local_list != null) {
foreach (Geary.Email email in local_list) {
if (email.location.position == position) {
found = true;
break;
}
if (remote_folder == null || local_list_size == by_position.length)
return local_list;
// go through the list looking for anything not already in the sparse by_position list
// to fetch from the server; since by_position is not guaranteed to be sorted, the local
// list needs to be searched each iteration.
//
// TODO: Optimize this, especially if large lists/sparse sets are supplied
int[] needed_by_position = new int[0];
foreach (int position in by_position) {
bool found = false;
if (local_list != null) {
foreach (Geary.Email email in local_list) {
if (email.location.position == position) {
found = true;
break;
}
}
if (!found)
needed_by_position += position;
}
if (needed_by_position.length != 0)
background_update_email_list.begin(needed_by_position, fields, cancellable);
if (!found)
needed_by_position += position;
}
return local_list;
if (needed_by_position.length == 0)
return local_list;
Gee.List<Geary.Email>? remote_list = yield remote_list_email(needed_by_position,
required_fields, cancellable);
return combine_lists(local_list, remote_list);
}
private async void background_update_email_list(int[] needed_by_position, Geary.Email.Field fields,
Cancellable? cancellable) {
private async Gee.List<Geary.Email>? remote_list_email(int[] needed_by_position,
Geary.Email.Field required_fields, Cancellable? cancellable) throws Error {
debug("Background fetching %d emails for %s", needed_by_position.length, get_name());
Gee.List<Geary.Email>? net_list = null;
try {
net_list = yield net_folder.list_email_sparse_async(needed_by_position, fields,
cancellable);
} catch (Error net_err) {
message("Unable to fetch emails from server: %s", net_err.message);
if (net_err is IOError.CANCELLED)
return;
}
Gee.List<Geary.Email>? remote_list = yield remote_folder.list_email_sparse_async(
needed_by_position, required_fields, cancellable);
if (remote_list != null && remote_list.size == 0)
remote_list = null;
if (net_list != null && net_list.size == 0)
net_list = null;
if (net_list != null)
notify_email_added_removed(net_list, null);
if (net_list != null) {
foreach (Geary.Email email in net_list) {
try {
yield local_folder.create_email_async(email, fields, cancellable);
} catch (Error local_err) {
message("Unable to create email in local store: %s", local_err.message);
if (local_err is IOError.CANCELLED)
return;
// if any were fetched, store locally
// TODO: Bulk writing
if (remote_list != null) {
foreach (Geary.Email email in remote_list) {
bool exists_in_system = false;
if (email.message_id != null) {
int count;
exists_in_system = yield local.has_message_id_async(email.message_id, out count,
cancellable);
}
bool exists_in_folder = yield local_folder.is_email_associated_async(email,
cancellable);
// NOTE: Although this looks redundant, this is a complex decision case and laying
// it out like this helps explain the logic. Also, this code relies on the fact
// that update_email_async() is a powerful call which might be broken down in the
// future (requiring a duplicate email be manually associated with the folder,
// for example), and so would like to keep this around to facilitate that.
if (!exists_in_system && !exists_in_folder) {
// This case indicates the email is new to the local store OR has no
// Message-ID and so a new copy must be stored.
yield local_folder.create_email_async(email, cancellable);
} else if (exists_in_system && !exists_in_folder) {
// This case indicates the email has been (partially) stored previously but
// was not associated with this folder; update it (which implies association)
yield local_folder.update_email_async(email, false, cancellable);
} else if (!exists_in_system && exists_in_folder) {
// This case indicates the message doesn't have a Message-ID and can only be
// identified by a folder-specific ID, so it can be updated in the folder
// (This may result in multiple copies of the message stored locally.)
yield local_folder.update_email_async(email, true, cancellable);
} else if (exists_in_system && exists_in_folder) {
// This indicates the message is in the local store and was previously
// associated with this folder, so merely update the local store
yield local_folder.update_email_async(email, false, cancellable);
}
}
}
return remote_list;
}
public async Geary.Email fetch_email_async(int num, Geary.Email.Field fields,
Cancellable? cancellable = null) throws Error {
if (net_folder == null)
if (remote_folder == null)
throw new EngineError.OPEN_REQUIRED("Folder %s not opened", get_name());
return yield net_folder.fetch_email_async(num, fields, cancellable);
try {
return yield local_folder.fetch_email_async(num, fields, cancellable);
} catch (Error err) {
// TODO: Better parsing of error; currently merely falling through and trying network
// for copy
debug("Unable to fetch email from local store: %s", err.message);
}
// To reach here indicates either the local version does not have all the requested fields
// or it's simply not present. If it's not present, want to ensure that the Message-ID
// is requested, as that's a good way to manage duplicate messages in the system
Geary.Email.Field available_fields;
bool is_present = yield local_folder.is_email_present_at(num, out available_fields, cancellable);
if (!is_present)
fields = fields.set(Geary.Email.Field.REFERENCES);
// fetch from network
Geary.Email email = yield remote_folder.fetch_email_async(num, fields, cancellable);
// save to local store
yield local_folder.update_email_async(email, false, cancellable);
return email;
}
private void on_local_updated() {
}
private void on_net_updated() {
private void on_remote_updated() {
}
private Gee.List<Geary.Email>? combine_lists(Gee.List<Geary.Email>? a, Gee.List<Geary.Email>? b) {
if (a == null)
return b;
if (b == null)
return a;
Gee.List<Geary.Email> combined = new Gee.ArrayList<Geary.Email>();
combined.add_all(a);
combined.add_all(b);
return combined;
}
}
......@@ -5,14 +5,20 @@
*/
private class Geary.ImapEngine : Object, Geary.Account {
private NetworkAccount net;
private RemoteAccount remote;
private LocalAccount local;
public ImapEngine(NetworkAccount net, LocalAccount local) {
this.net = net;
public ImapEngine(RemoteAccount remote, LocalAccount local) {
this.remote = remote;
this.local = local;
}
public Geary.Email.Field get_required_fields_for_writing() {
// Return the more restrictive of the two, which is the NetworkAccount's.
// TODO: This could be determined at runtime rather than fixed in stone here.
return Geary.Email.Field.HEADER | Geary.Email.Field.BODY;
}
public async void create_folder_async(Geary.Folder? parent, Geary.Folder folder,
Cancellable? cancellable = null) throws Error {
}
......@@ -27,7 +33,7 @@ private class Geary.ImapEngine : Object, Geary.Account {
Gee.Collection<Geary.Folder> engine_list = new Gee.ArrayList<Geary.Folder>();
foreach (Geary.Folder local_folder in local_list)
engine_list.add(new EngineFolder(net, local, local_folder));
engine_list.add(new EngineFolder(remote, local, (LocalFolder) local_folder));
background_update_folders.begin(parent, engine_list);
......@@ -38,8 +44,9 @@ private class Geary.ImapEngine : Object, Geary.Account {
public async Geary.Folder fetch_folder_async(Geary.Folder? parent, string folder_name,
Cancellable? cancellable = null) throws Error {
Geary.Folder local_folder = yield local.fetch_folder_async(parent, folder_name, cancellable);
Geary.Folder engine_folder = new EngineFolder(net, local, local_folder);
LocalFolder local_folder = (LocalFolder) yield local.fetch_folder_async(parent, folder_name,
cancellable);
Geary.Folder engine_folder = new EngineFolder(remote, local, local_folder);
return engine_folder;
}
......@@ -65,20 +72,20 @@ private class Geary.ImapEngine : Object, Geary.Account {
private async void background_update_folders(Geary.Folder? parent,
Gee.Collection<Geary.Folder> engine_folders) {
Gee.Collection<Geary.Folder> net_folders;
Gee.Collection<Geary.Folder> remote_folders;
try {
net_folders = yield net.list_folders_async(parent);
} catch (Error neterror) {
error("Unable to retrieve folder list from server: %s", neterror.message);
remote_folders = yield remote.list_folders_async(parent);
} catch (Error remote_error) {
error("Unable to retrieve folder list from server: %s", remote_error.message);
}
Gee.Set<string> local_names = get_folder_names(engine_folders);
Gee.Set<string> net_names = get_folder_names(net_folders);
Gee.Set<string> remote_names = get_folder_names(remote_folders);
debug("%d local names, %d net names", local_names.size, net_names.size);
debug("%d local names, %d remote names", local_names.size, remote_names.size);
Gee.List<Geary.Folder>? to_add = get_excluded_folders(net_folders, local_names);
Gee.List<Geary.Folder>? to_remove = get_excluded_folders(engine_folders, net_names);
Gee.List<Geary.Folder>? to_add = get_excluded_folders(remote_folders, local_names);
Gee.List<Geary.Folder>? to_remove = get_excluded_folders(engine_folders, remote_names);
debug("Adding %d, removing %d to/from local store", to_add.size, to_remove.size);
......@@ -98,10 +105,10 @@ private class Geary.ImapEngine : Object, Geary.Account {
Gee.Collection<Geary.Folder> engine_added = null;
if (to_add != null) {
engine_added = new Gee.ArrayList<Geary.Folder>();
foreach (Geary.Folder net_folder in to_add) {
foreach (Geary.Folder remote_folder in to_add) {
try {
engine_added.add(new EngineFolder(net, local,
yield local.fetch_folder_async(parent, net_folder.get_name())));
engine_added.add(new EngineFolder(remote, local,
(LocalFolder) yield local.fetch_folder_async(parent, remote_folder.get_name())));
} catch (Error convert_err) {
error("Unable to fetch local folder: %s", convert_err.message);
}
......
......@@ -13,6 +13,18 @@ public interface Geary.Account : Object {
folders_added_removed(added, removed);
}
/**
* This method returns which Geary.Email.Field fields must be available in a Geary.Email to
* write (or save or store) the message to the backing medium. Different implementations will
* have different requirements, which must be reconciled.
*
* In this case, Geary.Email.Field.NONE means "any".
*
* If a write operation is attempted on an email that does not have all these fields fulfilled,
* an EngineError.INCOMPLETE_MESSAGE will be thrown.
*/
public abstract Geary.Email.Field get_required_fields_for_writing();
public abstract async void create_folder_async(Geary.Folder? parent, Geary.Folder folder,
Cancellable? cancellable = null) throws Error;
......@@ -32,9 +44,3 @@ public interface Geary.Account : Object {
Cancellable? cancellable = null) throws Error;
}
public interface Geary.NetworkAccount : Object, Geary.Account {
}
public interface Geary.LocalAccount : Object, Geary.Account {
}
......@@ -5,19 +5,19 @@
*/
public class Geary.Email : Object {
[Flags]
// THESE VALUES ARE PERSISTED. Change them only if you know what you're doing.
public enum Field {
NONE = 0,
DATE,
ORIGINATORS,
RECEIVERS,
REFERENCES,
SUBJECT,
HEADER,
BODY,
PROPERTIES,
ENVELOPE = DATE | ORIGINATORS | RECEIVERS | REFERENCES | SUBJECT,
ALL = 0xFFFFFFFF;
NONE = 0,
DATE = 1 << 0,
ORIGINATORS = 1 << 1,
RECEIVERS = 1 << 2,
REFERENCES = 1 << 3,
SUBJECT = 1 << 4,
HEADER = 1 << 5,
BODY = 1 << 6,
PROPERTIES = 1 << 7,
ENVELOPE = DATE | ORIGINATORS | RECEIVERS | REFERENCES | SUBJECT,
ALL = 0xFFFFFFFF;
public static Field[] all() {
return {
......@@ -31,43 +31,112 @@ public class Geary.Email : Object {
PROPERTIES
};
}
public inline bool is_set(Field required_fields) {
return (this & required_fields) == required_fields;
}
public inline Field set(Field field) {
return (this | field);
}
public inline Field clear(Field field) {
return (this & ~(field));
}
}
public Geary.EmailLocation location { get; private set; }
// DATE
public Geary.RFC822.Date? date = null;
public Geary.RFC822.Date? date { get; private set; default = null; }
// ORIGINATORS
public Geary.RFC822.MailboxAddresses? from = null;
public Geary.RFC822.MailboxAddresses? sender = null;
public Geary.RFC822.MailboxAddresses? reply_to = null;
public Geary.RFC822.MailboxAddresses? from { get; private set; default = null; }
public Geary.RFC822.MailboxAddresses? sender { get; private set; default = null; }
public Geary.RFC822.MailboxAddresses? reply_to { get; private set; default = null; }
// RECEIVERS
public Geary.RFC822.MailboxAddresses? to = null;
public Geary.RFC822.MailboxAddresses? cc = null;
public Geary.RFC822.MailboxAddresses? bcc = null;
public Geary.RFC822.MailboxAddresses? to { get; private set; default = null; }
public Geary.RFC822.MailboxAddresses? cc { get; private set; default = null; }
public Geary.RFC822.MailboxAddresses? bcc { get; private set; default = null; }
// REFERENCES
public Geary.RFC822.MessageID? message_id = null;
public Geary.RFC822.MessageID? in_reply_to = null;
public Geary.RFC822.MessageID? message_id { get; private set; default = null; }
public Geary.RFC822.MessageID? in_reply_to { get; private set; default = null; }
// SUBJECT
public Geary.RFC822.Subject? subject = null;
public Geary.RFC822.Subject? subject { get; private set; default = null; }
// HEADER
public RFC822.Header? header = null;
public RFC822.Header? header { get; private set; default = null; }
// BODY
public RFC822.Text? body = null;
public RFC822.Text? body { get; private set; default = null; }
// PROPERTIES
public Geary.EmailProperties? properties = null;
public Geary.EmailProperties? properties { get; private set; default = null; }
public Geary.Email.Field fields { get; private set; default = Field.NONE; }
public Email(Geary.EmailLocation location) {
this.location = location;
}
public void set_send_date(Geary.RFC822.Date date) {
this.date = date;
fields |= Field.DATE;
}
public void set_originators(Geary.RFC822.MailboxAddresses? from,
Geary.RFC822.MailboxAddresses? sender, Geary.RFC822.MailboxAddresses? reply_to) {
this.from = from;
this.sender = sender;
this.reply_to = reply_to;
fields |= Field.ORIGINATORS;
}
public void set_receivers(Geary.RFC822.MailboxAddresses? to,
Geary.RFC822.MailboxAddresses? cc, Geary.RFC822.MailboxAddresses? bcc) {
this.to = to;
this.cc = cc;
this.bcc = bcc;
fields |= Field.RECEIVERS;
}
public void set_references(Geary.RFC822.MessageID? message_id, Geary.RFC822.MessageID? in_reply_to) {
this.message_id = message_id;
this.in_reply_to = in_reply_to;
fields |= Field.REFERENCES;
}
public void set_message_subject(Geary.RFC822.Subject subject) {
this.subject = subject;
fields |= Field.SUBJECT;
}
public void set_message_header(Geary.RFC822.Header header) {
this.header = header;
fields |= Field.HEADER;
}
public void set_message_body(Geary.RFC822.Text body) {
this.body = body;
fields |= Field.BODY;
}
public void set_email_properties(Geary.EmailProperties properties) {
this.properties = properties;
fields |= Field.PROPERTIES;
}
public string to_string() {
StringBuilder builder = new StringBuilder();
......
......@@ -7,8 +7,10 @@
public errordomain Geary.EngineError {
OPEN_REQUIRED,
ALREADY_OPEN,
ALREADY_EXISTS,
NOT_FOUND,
READONLY,
BAD_PARAMETERS
BAD_PARAMETERS,
INCOMPLETE_MESSAGE
}
......@@ -11,28 +11,71 @@ public interface Geary.Folder : Object {
FOLDER_CLOSED
}
/**
* This is fired when the Folder is successfully opened by a caller. It will only fire once
* until the Folder is closed.
*/
public signal void opened();
/**
* This is fired when the Folder is successfully closed by a caller. It will only fire once
* until the Folder is re-opened.
*
* The CloseReason enum can be used to inspect why the folder was closed: the connection was
* broken locally or remotely, or the Folder was simply closed (and the underlying connection
* is still available).
*/
public signal void closed(CloseReason reason);
/**
* "email-added-removed" is fired when new email has been detected due to background monitoring
* operations or if an unrelated operation causes or reveals the existence or removal of
* messages.
*
* There are no guarantees of what Geary.Email.Field fields will be available when these are
* reported. If more information is required, use the fetch or list operations.
*/
public signal void email_added_removed(Gee.List<Geary.Email>? added,
Gee.List<Geary.Email>? removed);
/**
* TBD.
*/
public signal void updated();
public virtual void notify_opened() {
/**
* This helper method should be called by implementors of Folder rather than firing the signal
* directly. This allows subclasses and superclasses the opportunity to inspect the email
* and update state before and/or after the signal has been fired.
*/
protected virtual void notify_opened() {
opened();
}
public virtual void notify_closed(CloseReason reason) {
/**
* This helper method should be called by implementors of Folder rather than firing the signal
* directly. This allows subclasses and superclasses the opportunity to inspect the email
* and update state before and/or after the signal has been fired.
*/
protected virtual void notify_closed(CloseReason reason) {
closed(reason);
}