Commit bc0a892a authored by Jim Nelson's avatar Jim Nelson

Correct problems with IMAP parsing when using Turkish: Bug #714892

When connecting to any IMAP server while the local user's locale is
configured to be Turkish, Geary will mis-parse many of the IMAP
server's responses, leading to essentially a failed connection due to
state issues and more.

The problem is that some of the parsing code was using g_utf8_strdown
to convert received text to lowercase to perform case-insensitive
string comparisons.  Turkish has multiple letter I's (dotted and
dotless), and when the UTF-8 code transformed it to lowercase, a
different UTF-8 code point was selected than the English/ASCII 'i'.

The solution is to explicitly use ASCII variants of string
transformation, comparison, and hashing to ensure 7-bit operations are
used throughout the IMAP and RFC822 stack.  Further commits will
follow that enforce this a bit more, but this commit is sufficient to
correct the problem for our Turkish users.
parent f774f3d4
......@@ -272,6 +272,7 @@ engine/state/state-machine-descriptor.vala
engine/state/state-machine.vala
engine/state/state-mapping.vala
engine/util/util-ascii.vala
engine/util/util-collection.vala
engine/util/util-converter.vala
engine/util/util-files.vala
......
......@@ -107,7 +107,7 @@ public class Geary.Imap.Command : RootParameters {
}
public bool has_name(string name) {
return this.name.down() == name.down();
return Ascii.stri_equal(this.name, name);
}
public override void serialize(Serializer ser, Tag tag) throws Error {
......
......@@ -25,7 +25,7 @@ private bool is_special_char(char ch, char[] ar, string? exceptions) {
return true;
if (ch in ar)
return (exceptions != null) ? exceptions.index_of_char(ch) < 0 : true;
return (exceptions != null) ? Ascii.index_of(exceptions, ch) < 0 : true;
return false;
}
......
......@@ -78,7 +78,7 @@ public class Geary.Imap.FetchBodyDataSpecifier : BaseObject, Gee.Hashable<FetchB
if (String.is_empty(value))
return NONE;
switch (value.down()) {
switch (Ascii.strdown(value)) {
case "header":
return HEADER;
......@@ -180,9 +180,9 @@ public class Geary.Imap.FetchBodyDataSpecifier : BaseObject, Gee.Hashable<FetchB
this.is_peek = is_peek;
if (field_names != null && field_names.length > 0) {
this.field_names = new Gee.TreeSet<string>();
this.field_names = new Gee.TreeSet<string>(Ascii.strcmp);
foreach (string field_name in field_names) {
string converted = field_name.strip().down();
string converted = Ascii.strdown(field_name.strip());
if (!String.is_empty(converted))
this.field_names.add(converted);
......@@ -288,7 +288,7 @@ public class Geary.Imap.FetchBodyDataSpecifier : BaseObject, Gee.Hashable<FetchB
* @see deserialize_response
*/
public static bool is_fetch_body_data_specifier(StringParameter stringp) {
string strd = stringp.value.down().strip();
string strd = stringp.as_lower().strip();
return strd.has_prefix("body[") || strd.has_prefix("body.peek[");
}
......@@ -308,7 +308,7 @@ public class Geary.Imap.FetchBodyDataSpecifier : BaseObject, Gee.Hashable<FetchB
// * leading/trailing whitespace stripped
// * Remove quoting (some servers return field names quoted, some don't, Geary never uses them
// when requesting)
string strd = stringp.value.down().replace("\"", "").strip();
string strd = stringp.as_lower().replace("\"", "").strip();
// Convert full form into two sections: "body[SECTION_STRING]<OCTET_STRING>"
// ^^^^^^^^^^^^^^ optional
......@@ -365,7 +365,7 @@ public class Geary.Imap.FetchBodyDataSpecifier : BaseObject, Gee.Hashable<FetchB
// stop treating as numbers when non-digit found (SectionParts contain periods
// too and must be preserved);
if (!no_more && String.is_numeric(token)) {
if (!no_more && Ascii.is_numeric(token)) {
if (part_number == null)
part_number = new int[0];
......
......@@ -83,7 +83,7 @@ public enum Geary.Imap.FetchDataSpecifier {
* @throws ImapError.PARSE_ERROR if not a recognized value.
*/
public static FetchDataSpecifier decode(string value) throws ImapError {
switch (value.down()) {
switch (Ascii.strdown(value)) {
case "uid":
return UID;
......
......@@ -25,7 +25,7 @@ public abstract class Geary.Imap.Flag : BaseObject, Gee.Hashable<Geary.Imap.Flag
}
public bool equals_string(string value) {
return this.value.down() == value.down();
return Ascii.stri_equal(this.value, value);
}
public bool equal_to(Geary.Imap.Flag flag) {
......@@ -40,7 +40,7 @@ public abstract class Geary.Imap.Flag : BaseObject, Gee.Hashable<Geary.Imap.Flag
}
public uint hash() {
return str_hash(value.down());
return Ascii.stri_hash(value);
}
public string to_string() {
......
......@@ -71,7 +71,7 @@ public abstract class Geary.Imap.Flags : Geary.MessageData.AbstractMessageData,
}
public uint hash() {
return to_string().down().hash();
return Ascii.stri_hash(to_string());
}
}
......@@ -70,7 +70,7 @@ public class Geary.Imap.InternalDate : Geary.MessageData.AbstractMessageData, Ge
// check month (this catches localization problems)
int month = -1;
string mon_down = ((string) mon).down();
string mon_down = Ascii.strdown(((string) mon));
for (int ctr = 0; ctr < EN_US_MON_DOWN.length; ctr++) {
if (mon_down == EN_US_MON_DOWN[ctr]) {
month = ctr;
......
......@@ -70,7 +70,7 @@ public class Geary.Imap.MailboxSpecifier : BaseObject, Gee.Hashable<MailboxSpeci
* @see is_canonical_inbox_name
*/
public static bool is_inbox_name(string name) {
return name.up() == CANONICAL_INBOX_NAME;
return Ascii.stri_equal(name, CANONICAL_INBOX_NAME);
}
/**
......@@ -85,7 +85,7 @@ public class Geary.Imap.MailboxSpecifier : BaseObject, Gee.Hashable<MailboxSpeci
* @see is_inbox_name
*/
public static bool is_canonical_inbox_name(string name) {
return name == CANONICAL_INBOX_NAME;
return Ascii.str_equal(name, CANONICAL_INBOX_NAME);
}
/**
......@@ -183,7 +183,7 @@ public class Geary.Imap.MailboxSpecifier : BaseObject, Gee.Hashable<MailboxSpeci
}
public uint hash() {
return is_inbox ? String.stri_hash(name) : name.hash();
return is_inbox ? Ascii.stri_hash(name) : Ascii.str_hash(name);
}
public bool equal_to(MailboxSpecifier other) {
......@@ -191,9 +191,9 @@ public class Geary.Imap.MailboxSpecifier : BaseObject, Gee.Hashable<MailboxSpeci
return true;
if (is_inbox)
return String.stri_equal(name, other.name);
return Ascii.stri_equal(name, other.name);
return name == other.name;
return Ascii.str_equal(name, other.name);
}
public int compare_to(MailboxSpecifier other) {
......@@ -203,7 +203,7 @@ public class Geary.Imap.MailboxSpecifier : BaseObject, Gee.Hashable<MailboxSpeci
if (is_inbox && other.is_inbox)
return 0;
return strcmp(name, other.name);
return Ascii.strcmp(name, other.name);
}
public string to_string() {
......
......@@ -46,7 +46,7 @@ public enum Geary.Imap.StatusDataType {
}
public static StatusDataType decode(string value) throws ImapError {
switch (value.down()) {
switch (Ascii.strdown(value)) {
case "messages":
return MESSAGES;
......
......@@ -99,7 +99,7 @@ public class Geary.Imap.Tag : AtomParameter, Gee.Hashable<Geary.Imap.Tag> {
}
public uint hash() {
return str_hash(value);
return Ascii.str_hash(value);
}
public bool equal_to(Geary.Imap.Tag tag) {
......
......@@ -41,7 +41,7 @@ public class Geary.Imap.NilParameter : Geary.Imap.Parameter {
* list.
*/
public static bool is_nil(StringParameter stringp) {
return String.ascii_equali(VALUE, stringp.value);
return Ascii.stri_equal(VALUE, stringp.value);
}
/**
......
......@@ -96,14 +96,28 @@ public abstract class Geary.Imap.StringParameter : Geary.Imap.Parameter {
* Case-sensitive comparison.
*/
public bool equals_cs(string value) {
return this.value == value;
return Ascii.str_equal(this.value, value);
}
/**
* Case-insensitive comparison.
*/
public bool equals_ci(string value) {
return this.value.down() == value.down();
return Ascii.stri_equal(this.value, value);
}
/**
* Returns the string lowercased.
*/
public string as_lower() {
return Ascii.strdown(value);
}
/**
* Returns the string uppercased.
*/
public string as_upper() {
return Ascii.strup(value);
}
/**
......
......@@ -71,11 +71,11 @@ public class Geary.Imap.ResponseCodeType : BaseObject, Gee.Hashable<ResponseCode
// store lowercased so it's easily compared with const strings above
original = str;
value = str.down();
value = Ascii.strdown(str);
}
public bool is_value(string str) {
return String.stri_equal(value, str);
return Ascii.stri_equal(value, str);
}
public StringParameter to_parameter() {
......@@ -83,11 +83,11 @@ public class Geary.Imap.ResponseCodeType : BaseObject, Gee.Hashable<ResponseCode
}
public bool equal_to(ResponseCodeType other) {
return (this == other) ? true : String.stri_equal(value, other.value);
return (this == other) ? true : Ascii.stri_equal(value, other.value);
}
public uint hash() {
return String.stri_hash(value);
return Ascii.stri_hash(value);
}
public string to_string() {
......
......@@ -64,7 +64,7 @@ public enum Geary.Imap.ServerDataType {
}
public static ServerDataType decode(string value) throws ImapError {
switch (value.down()) {
switch (Ascii.strdown(value)) {
case "capability":
return CAPABILITY;
......@@ -128,7 +128,7 @@ public enum Geary.Imap.ServerDataType {
public static ServerDataType from_response(RootParameters root) throws ImapError {
StringParameter? firstparam = root.get_if_string(1);
if (firstparam != null) {
switch (firstparam.value.down()) {
switch (firstparam.as_lower()) {
case "capability":
return CAPABILITY;
......@@ -158,7 +158,7 @@ public enum Geary.Imap.ServerDataType {
StringParameter? secondparam = root.get_if_string(2);
if (secondparam != null) {
switch (secondparam.value.down()) {
switch (secondparam.as_lower()) {
case "exists":
return EXISTS;
......
......@@ -40,7 +40,7 @@ public enum Geary.Imap.Status {
}
public static Status decode(string value) throws ImapError {
switch (value.down()) {
switch (Ascii.strdown(value)) {
case "ok":
return OK;
......
......@@ -974,13 +974,12 @@ public class Geary.Imap.ClientSession : BaseObject {
// machine
//
// TODO: Convert commands into proper calls to avoid throwing an exception
switch (cmd.name) {
case LoginCommand.NAME:
case LogoutCommand.NAME:
case SelectCommand.NAME:
case ExamineCommand.NAME:
case CloseCommand.NAME:
throw new ImapError.NOT_SUPPORTED("Use direct calls rather than commands for %s", cmd.name);
if (cmd.has_name(LoginCommand.NAME)
|| cmd.has_name(LogoutCommand.NAME)
|| cmd.has_name(SelectCommand.NAME)
|| cmd.has_name(ExamineCommand.NAME)
|| cmd.has_name(CloseCommand.NAME)) {
throw new ImapError.NOT_SUPPORTED("Use direct calls rather than commands for %s", cmd.name);
}
}
......
......@@ -421,7 +421,7 @@ public class Geary.Imap.Deserializer : BaseObject {
if (current_string == null || String.is_empty(current_string.str))
return false;
return String.stri_equal(current_string.str, cmp);
return Ascii.stri_equal(current_string.str, cmp);
}
private void append_to_string(char ch) {
......@@ -432,7 +432,7 @@ public class Geary.Imap.Deserializer : BaseObject {
}
private void save_string_parameter(bool quoted) {
// deal with empty quoted strings
// deal with empty strings
if (!quoted && is_current_string_empty())
return;
......
......@@ -23,7 +23,7 @@ public class Geary.Mime.ContentParameters : BaseObject {
// See get_parameters() for why the keys but not the values are stored case-insensitive
private Gee.HashMap<string, string> params = new Gee.HashMap<string, string>(
String.stri_hash, String.stri_equal);
Ascii.stri_hash, Ascii.stri_equal);
/**
* Create a mapping of content parameters.
......@@ -76,7 +76,7 @@ public class Geary.Mime.ContentParameters : BaseObject {
public bool has_value_ci(string attribute, string value) {
string? stored = params.get(attribute);
return (stored != null) ? String.stri_equal(stored, value) : false;
return (stored != null) ? Ascii.stri_equal(stored, value) : false;
}
/**
......@@ -87,7 +87,7 @@ public class Geary.Mime.ContentParameters : BaseObject {
public bool has_value_cs(string attribute, string value) {
string? stored = params.get(attribute);
return (stored != null) ? (stored == value) : false;
return (stored != null) ? Ascii.str_equal(stored, value) : false;
}
/**
......
......@@ -35,7 +35,7 @@ public enum Geary.Mime.DispositionType {
if (String.is_empty_or_whitespace(str))
return UNSPECIFIED;
switch (str.down()) {
switch (Ascii.strdown(str)) {
case "inline":
return INLINE;
......
......@@ -19,7 +19,7 @@ public class Geary.RFC822.MailboxAddress : Geary.MessageData.SearchableMessageDa
source_route = null;
int atsign = address.index_of_char('@');
int atsign = Ascii.index_of(address, '@');
if (atsign > 0) {
mailbox = address.slice(0, atsign);
domain = address.slice(atsign + 1, address.length);
......
......@@ -77,10 +77,10 @@ public class Geary.RFC822.MessageIDList : Geary.MessageData.AbstractMessageData,
// be a little less liberal in its parsing.
StringBuilder canonicalized = new StringBuilder();
int index = 0;
unichar ch;
char ch;
bool in_message_id = false;
bool bracketed = false;
while (value.get_next_char(ref index, out ch)) {
while (Ascii.get_next_char(value, ref index, out ch)) {
bool add_char = false;
switch (ch) {
case '<':
......@@ -126,7 +126,7 @@ public class Geary.RFC822.MessageIDList : Geary.MessageData.AbstractMessageData,
}
if (add_char)
canonicalized.append_unichar(ch);
canonicalized.append_c(ch);
if (!in_message_id && !String.is_empty(canonicalized.str)) {
list.add(new MessageID(canonicalized.str));
......
......@@ -58,7 +58,7 @@ public enum Geary.Smtp.Command {
}
public static Command deserialize(string str) throws SmtpError {
switch (str.down()) {
switch (Ascii.strdown(str)) {
case "helo":
return HELO;
......
......@@ -27,7 +27,7 @@ public class Geary.Smtp.Greeting : Response {
}
public static ServerFlavor deserialize(string str) {
switch (str.up()) {
switch (Ascii.strup(str)) {
case "SMTP":
return SMTP;
......
......@@ -46,7 +46,7 @@ public class Geary.Smtp.ResponseCode {
}
public Status get_status() {
int i = String.digit_to_int(str[0]);
int i = Ascii.digit_to_int(str[0]);
// This works because of the checks in the constructor; Condition can't be checked so
// easily
......@@ -54,7 +54,7 @@ public class Geary.Smtp.ResponseCode {
}
public Condition get_condition() {
switch (String.digit_to_int(str[1])) {
switch (Ascii.digit_to_int(str[1])) {
case Condition.SYNTAX:
return Condition.SYNTAX;
......
/* Copyright 2014 Yorba Foundation
*
* This software is licensed under the GNU Lesser General Public License
* (version 2.1 or later). See the COPYING file in this distribution.
*/
namespace Geary.Ascii {
public int index_of(string str, char ch) {
char *strptr = str;
int index = 0;
for (;;) {
char strch = *strptr++;
if (strch == String.EOS)
return -1;
if (strch == ch)
return index;
index++;
}
}
public bool get_next_char(string str, ref int index, out char ch) {
ch = str[index++];
return ch != String.EOS;
}
public inline int strcmp(string a, string b) {
return GLib.strcmp(a, b);
}
public int stricmp(string a, string b) {
char *aptr = a;
char *bptr = b;
for (;;) {
int diff = (int) (*aptr).tolower() - (int) (*bptr).tolower();
if (diff != 0)
return diff;
if (*aptr == String.EOS)
return 0;
aptr++;
bptr++;
}
}
public inline bool str_equal(string a, string b) {
return a == b;
}
public inline bool stri_equal(string a, string b) {
return stricmp(a, b) == 0;
}
public bool nullable_stri_equal(string? a, string? b) {
if (a == null)
return (b == null);
// a != null, so always false
if (b == null)
return false;
return stri_equal(a, b);
}
public uint str_hash(string str) {
return Collection.hash_memory_stream((char *) str, String.EOS, null);
}
public uint stri_hash(string str) {
return Collection.hash_memory_stream((char *) str, String.EOS, (b) => {
return ((char) b).tolower();
});
}
public uint nullable_stri_hash(string? str) {
return (str != null) ? stri_hash(str) : 0;
}
public string strdown(string str) {
return str.ascii_down();
}
public string strup(string str) {
return str.ascii_up();
}
/**
* Returns true if the ASCII string contains only whitespace and at least one numeric character.
*/
public bool is_numeric(string str) {
bool numeric_found = false;
char *strptr = str;
for (;;) {
char ch = *strptr++;
if (ch == String.EOS)
break;
if (ch.isdigit())
numeric_found = true;
else if (!ch.isspace())
return false;
}
return numeric_found;
}
/**
* Returns char from 0 to 9 converted to an int. If a non-numeric value, -1 is returned.
*/
public inline int digit_to_int(char ch) {
return ch.isdigit() ? (ch - '0') : -1;
}
}
......@@ -6,6 +6,8 @@
namespace Geary.Collection {
public delegate uint8 ByteTransformer(uint8 b);
// A substitute for ArrayList<G>.wrap() for compatibility with older versions of Gee.
public Gee.ArrayList<G> array_list_wrap<G>(G[] a, owned Gee.EqualDataFunc<G>? equal_func = null) {
Gee.ArrayList<G> list = new Gee.ArrayList<G>(equal_func);
......@@ -145,7 +147,7 @@ public bool int64_equal_func(int64? a, int64? b) {
/**
* A rotating-XOR hash that can be used to hash memory buffers of any size.
*/
public static uint hash_memory(void *ptr, size_t bytes) {
public uint hash_memory(void *ptr, size_t bytes) {
if (bytes == 0)
return 0;
......@@ -159,4 +161,30 @@ public static uint hash_memory(void *ptr, size_t bytes) {
return hash;
}
/**
* A rotating-XOR hash that can be used to hash memory buffers of any size until a terminator byte
* is found.
*
* A {@link ByteTransformer} may be supplied to convert bytes before they are hashed.
*
* Returns zero if the initial byte is the terminator.
*/
public uint hash_memory_stream(void *ptr, uint8 terminator, ByteTransformer? cb) {
uint8 *u8 = (uint8 *) ptr;
uint hash = 0;
for (;;) {
uint8 b = *u8++;
if (b == terminator)
break;
if (cb != null)
b = cb(b);
hash = (hash << 4) ^ (hash >> 28) ^ b;
}
return hash;
}
}
......@@ -4,6 +4,10 @@
* (version 2.1 or later). See the COPYING file in this distribution.
*/
/**
* Stores a map of name-values pairs as ''ASCII'' (i.e. 7-bit) strings.
*/
public class Geary.GenericCapabilities : BaseObject {
public string name_separator { get; private set; }
public string? value_separator { get; private set; }
......@@ -12,7 +16,7 @@ public class Geary.GenericCapabilities : BaseObject {
// This behavior was changed in the following libgee commit:
// https://git.gnome.org/browse/libgee/commit/?id=5a35303cb04154d0e929a7d8895d4a4812ba7a1c
private Gee.HashMultiMap<string, string?> map = new Gee.HashMultiMap<string, string?>(
String.nullable_stri_hash, String.nullable_stri_equal, String.nullable_stri_hash, String.nullable_stri_equal);
Ascii.nullable_stri_hash, Ascii.nullable_stri_equal, Ascii.nullable_stri_hash, Ascii.nullable_stri_equal);
/**
* Creates an empty set of capabilities.
......
......@@ -67,7 +67,7 @@ private void init_breaking_elements() {
// [3]: Can be used as either block or inline; we go for broke
};
breaking_elements = new Gee.HashSet<string>(String.stri_hash, String.stri_equal);
breaking_elements = new Gee.HashSet<string>(Ascii.stri_hash, Ascii.stri_equal);
foreach (string element in elements)
breaking_elements.add(element);
}
......
......@@ -27,81 +27,14 @@ public int count_char(string s, unichar c) {
return count;
}
public int ascii_cmp(string a, string b) {
return strcmp(a, b);
}
public int ascii_cmpi(string a, string b) {
char *aptr = a;
char *bptr = b;
for (;;) {
int diff = (int) (*aptr).tolower() - (int) (*bptr).tolower();
if (diff != 0)
return diff;
if (*aptr == EOS)
return 0;
aptr++;
bptr++;
}
}
public inline bool ascii_equal(string a, string b) {
return ascii_cmp(a, b) == 0;
}
public inline bool ascii_equali(string a, string b) {
return ascii_cmpi(a, b) == 0;
}
public uint stri_hash(string str) {
return str_hash(str.down());
}
public uint nullable_stri_hash(string? str) {
return (str != null) ? stri_hash(str) : 0;
}
public bool stri_equal(string a, string b) {
return str_equal(a.down(), b.down());
}
public bool nullable_stri_equal(string? a, string? b) {
if (a == null)
return (b == null);
// a != null, so always false
if (b == null)
return false;
return stri_equal(a, b);
}
/**
* Returns true if the string contains only whitespace and at least one numeric character.
*/
public bool is_numeric(string str) {
bool numeric_found = false;
unichar ch;
int index = 0;