Commit 7fae774a authored by Jürg Billeter's avatar Jürg Billeter

Add initial SQLite FTS support

parent 9ac34151
......@@ -341,6 +341,23 @@ fi
AM_CONDITIONAL(HAVE_UNIT_TESTS, test "x$have_unit_tests" = "xyes")
##################################################################
# Enable SQLite FTS support?
##################################################################
AC_ARG_ENABLE([sqlite_fts],
AS_HELP_STRING([--enable-sqlite-fts],
[enable SQLite FTS support (default=no)]),,
[enable_sqlite_fts=no])
if test "x$enable_sqlite_fts" != "xno" ; then
AC_DEFINE(HAVE_SQLITE_FTS, 1, [Define if we have SQLite FTS support])
else
enable_sqlite_fts="no (disabled)"
fi
AM_CONDITIONAL(ENABLE_SQLITE_FTS, test "$enable_sqlite_fts" = "yes")
####################################################################
# QDBM check
####################################################################
......@@ -359,7 +376,7 @@ if test "x$enable_external_qdbm" != "xno"; then
if test "x$have_external_qdbm" = "xyes"; then
AC_DEFINE(HAVE_EXTERNAL_QDBM, [], [Define if we have external QDBM])
fi
else
elif test "x$enable_sqlite_fts" != "xyes" ; then
QDBM_CFLAGS="-I\$(top_srcdir)/src/qdbm"
QDBM_LIBS="\$(top_builddir)/src/qdbm/libqdbm-private.la"
AC_DEFINE(HAVE_QDBM, [], [Define if we have QDBM (our imported version)])
......@@ -1310,21 +1327,6 @@ fi
AM_CONDITIONAL(HAVE_EVOLUTION_PLUGIN, test "$have_evolution_plugin" = "yes")
##################################################################
# Enable SQLite FTS support?
##################################################################
AC_ARG_ENABLE([sqlite_fts],
AS_HELP_STRING([--enable-sqlite-fts],
[enable SQLite FTS support (default=no)]),,
[enable_sqlite_fts=no])
if test "x$enable_sqlite_fts" != "xyes" ; then
enable_sqlite_fts="no (disabled)"
fi
AM_CONDITIONAL(ENABLE_SQLITE_FTS, test "$enable_sqlite_fts" = "yes")
####################################################################
# Checks for gtk-doc and docbook-tools
####################################################################
......
......@@ -4,7 +4,6 @@ configdir = $(datadir)/tracker
config_DATA = \
sqlite-contents.sql \
sqlite-fulltext.sql \
sqlite-tracker.sql
EXTRA_DIST = $(config_DATA)
create virtual table FullText using trackerfts (
cat,
col_default,
col_1,
col_2,
col_3,
col_4,
col_5,
col_6,
col_7,
col_8,
col_9,
col_10,
col_11,
col_12,
col_13,
col_14,
col_15,
col_16,
col_17,
col_18,
col_19,
col_20,
col_21,
col_22,
col_23,
col_24,
col_25,
col_26,
col_27,
col_28,
col_29,
col_30,
col_31,
col_32,
col_33,
col_34,
col_35,
col_36,
col_37,
col_38,
col_39,
col_40,
col_41,
col_42,
col_43,
col_44,
col_45,
col_46,
col_47,
col_48,
col_49,
col_50,
col_51,
col_52,
col_53,
col_54,
col_55,
col_56,
col_57,
col_58,
col_59,
col_60,
col_61,
col_62,
col_63,
col_64,
col_65,
col_66,
col_67,
col_68,
col_69,
col_70,
col_71,
col_72,
col_73,
col_74,
col_75,
col_76,
col_77,
col_78,
col_79,
col_80,
col_81,
col_82,
col_83,
col_84,
col_85,
col_86,
col_87,
col_88,
col_89,
col_90,
col_91,
col_92,
col_93,
col_94,
col_95,
col_96,
col_97,
col_98,
col_99,
col_100,
col_101,
col_102,
col_103,
col_104,
col_105,
col_106,
col_107,
col_108,
col_109,
col_110,
col_111,
col_112,
col_113,
col_114,
col_115,
col_116,
col_117,
col_118,
col_119,
col_120,
col_121,
col_122,
col_123,
col_124,
col_125,
col_126,
col_127,
col_128,
col_129,
col_130,
col_131,
col_132,
col_133,
col_134,
col_135,
col_136,
col_137,
col_138,
col_139,
col_140,
col_141,
col_142,
col_143,
col_144,
col_145,
col_146,
col_147,
col_148,
col_149,
col_150,
col_151,
col_152,
col_153,
col_154,
col_155,
col_156,
col_157,
col_158,
col_159,
col_160,
col_161,
col_162,
col_163,
col_164,
col_165,
col_166,
col_167,
col_168,
col_169,
col_170,
col_171,
col_172,
col_173,
col_174,
col_175,
col_176,
col_177,
col_178,
col_179,
col_180,
col_181,
col_182,
col_183,
col_184,
col_185,
col_186,
col_187,
col_188,
col_189,
col_190,
col_191,
col_192,
col_193,
col_194,
col_195,
col_196,
col_197,
col_198,
col_199,
col_200,
col_201,
col_202,
col_203,
col_204,
col_205,
col_206,
col_207,
col_208,
col_209,
col_210,
col_211,
col_212,
col_213,
col_214,
col_215,
col_216,
col_217,
col_218,
col_219,
col_220,
col_221,
col_222,
col_223,
col_224,
col_225,
col_226,
col_227,
col_228,
col_229,
col_230,
col_231,
col_232,
col_233,
col_234,
col_235,
col_236,
col_237,
col_238,
col_239,
col_240,
col_241,
col_242,
col_243,
col_244,
col_245,
col_246,
col_247,
col_248,
col_249,
col_250,
col_251,
col_252,
col_253,
col_254,
col_255);
......@@ -5,8 +5,10 @@ build_libinotify = libinotify
endif
if !HAVE_EXTERNAL_QDBM
if !ENABLE_SQLITE_FTS
build_qdbm = qdbm
endif
endif
if ENABLE_SQLITE_FTS
build_sqlite_fts = tracker-fts
......
......@@ -25,25 +25,35 @@ libtracker_data_la_SOURCES = \
tracker-data-backup.c \
tracker-data-manager.c \
tracker-data-query.c \
tracker-data-search.c \
tracker-data-update.c \
tracker-query-tree.c \
libtracker-data.vala.stamp \
tracker-turtle.c \
libtracker-data.vala.stamp \
$(libtracker_data_la_VALASOURCES:.vala=.c)
noinst_HEADERS = \
tracker-data-backup.h \
tracker-data-manager.h \
tracker-data-query.h \
tracker-data-search.h \
tracker-data-update.h \
tracker-query-tree.h \
tracker-sparql-query.h \
tracker-turtle.h
if ENABLE_SQLITE_FTS
VALAFLAGS = -D HAVE_SQLITE_FTS
endif
if !ENABLE_SQLITE_FTS
libtracker_data_la_SOURCES += \
tracker-data-search.c \
tracker-query-tree.c
noinst_HEADERS += \
tracker-data-search.h \
tracker-query-tree.h
endif
libtracker-data.vala.stamp: $(libtracker_data_la_VALASOURCES)
$(VALAC) -C -H tracker-sparql-query.h ../rasqal/rasqal.vapi ../libtracker-common/libtracker-common.vapi libtracker-data.vapi ../libtracker-db/libtracker-db.vapi $^
$(VALAC) -C $(VALAFLAGS) -H tracker-sparql-query.h ../rasqal/rasqal.vapi ../libtracker-common/libtracker-common.vapi libtracker-data.vapi ../libtracker-db/libtracker-db.vapi $^
touch $@
libtracker_data_la_LDFLAGS = \
......
......@@ -745,6 +745,37 @@ create_decomposed_transient_metadata_tables (TrackerDBInterface *iface)
g_free (properties);
}
#ifdef HAVE_SQLITE_FTS
static void
create_fts_table (TrackerDBInterface *iface)
{
GString *sql;
TrackerProperty **properties, **property;
gboolean first;
sql = g_string_new ("CREATE VIRTUAL TABLE fulltext.fts USING trackerfts (");
first = TRUE;
properties = tracker_ontology_get_properties ();
for (property = properties; *property; property++) {
if (tracker_property_get_data_type (*property) == TRACKER_PROPERTY_TYPE_STRING) {
if (first) {
first = FALSE;
} else {
g_string_append (sql, ", ");
}
g_string_append_printf (sql, "\"%s\"", tracker_property_get_name (*property));
}
}
g_free (properties);
g_string_append (sql, ")");
tracker_db_interface_execute_query (iface, NULL, "%s", sql->str);
g_string_free (sql, TRUE);
}
#endif
gboolean
tracker_data_manager_init (TrackerConfig *config,
TrackerLanguage *language,
......@@ -776,12 +807,16 @@ tracker_data_manager_init (TrackerConfig *config,
private,
private_free);
#ifdef HAVE_SQLITE_FTS
tracker_db_manager_init (flags, &is_first_time_index, FALSE);
#else
tracker_db_manager_init (flags, &is_first_time_index, TRUE);
if (!tracker_db_index_manager_init (index_flags,
tracker_config_get_min_bucket_count (config),
tracker_config_get_max_bucket_count (config))) {
return FALSE;
}
#endif
if (first_time != NULL) {
*first_time = is_first_time_index;
......@@ -855,6 +890,10 @@ tracker_data_manager_init (TrackerConfig *config,
create_decomposed_metadata_tables (iface, *cl, &max_id);
}
#ifdef HAVE_SQLITE_FTS
create_fts_table (iface);
#endif
/* store ontology in database */
for (l = sorted; l; l = l->next) {
import_ontology_file (l->data);
......@@ -887,7 +926,9 @@ tracker_data_manager_shutdown (void)
{
TrackerDBPrivate *private;
#ifndef HAVE_SQLITE_FTS
tracker_db_index_manager_shutdown ();
#endif
tracker_db_manager_shutdown ();
private = g_static_private_get (&private_key);
......
......@@ -58,6 +58,7 @@ struct _TrackerDataUpdateBuffer {
struct _TrackerDataUpdateBufferProperty {
gchar *name;
GValue value;
gboolean fts;
};
struct _TrackerDataUpdateBufferTable {
......@@ -205,13 +206,15 @@ static void
cache_insert_value (const gchar *table_name,
const gchar *field_name,
GValue *value,
gboolean multiple_values)
gboolean multiple_values,
gboolean fts)
{
TrackerDataUpdateBufferTable *table;
TrackerDataUpdateBufferProperty property;
property.name = g_strdup (field_name);
property.value = *value;
property.fts = fts;
table = cache_ensure_table (table_name, multiple_values);
g_array_append_val (table->properties, property);
......@@ -275,6 +278,13 @@ ensure_resource_id (const gchar *uri)
tracker_db_statement_execute (stmt, NULL);
g_object_unref (stmt);
#ifdef HAVE_SQLITE_FTS
stmt = tracker_db_interface_create_statement (iface, "INSERT INTO \"fts\" (rowid) VALUES (?)");
tracker_db_statement_bind_int (stmt, 0, id);
tracker_db_statement_execute (stmt, NULL);
g_object_unref (stmt);
#endif
g_hash_table_insert (update_buffer.resource_cache, g_strdup (uri), GUINT_TO_POINTER (id));
}
......@@ -314,11 +324,16 @@ tracker_data_update_buffer_flush (void)
TrackerDataUpdateBufferProperty *property;
GHashTableIter iter;
const gchar *table_name;
GString *sql;
int i;
GString *sql, *fts_sql;
int i, fts_index;
iface = tracker_db_manager_get_db_interface ();
#ifdef HAVE_SQLITE_FTS
fts_sql = g_string_new ("UPDATE \"fts\" SET ");
fts_index = 0;
#endif
g_hash_table_iter_init (&iter, update_buffer.tables);
while (g_hash_table_iter_next (&iter, (gpointer*) &table_name, (gpointer*) &table)) {
if (table->multiple_values) {
......@@ -375,8 +390,43 @@ tracker_data_update_buffer_flush (void)
g_string_free (sql, TRUE);
}
#ifdef HAVE_SQLITE_FTS
for (i = 0; i < table->properties->len; i++) {
property = &g_array_index (table->properties, TrackerDataUpdateBufferProperty, i);
if (property->fts) {
if (fts_index > 0) {
g_string_append (fts_sql, ", ");
}
g_string_append_printf (fts_sql, "\"%s\" = ?", property->name);
fts_index++;
}
}
#endif
}
#ifdef HAVE_SQLITE_FTS
if (fts_index > 0) {
g_string_append (fts_sql, " WHERE rowid = ?");
stmt = tracker_db_interface_create_statement (iface, "%s", fts_sql->str);
tracker_db_statement_bind_int (stmt, fts_index, update_buffer.id);
fts_index = 0;
for (i = 0; i < table->properties->len; i++) {
property = &g_array_index (table->properties, TrackerDataUpdateBufferProperty, i);
if (property->fts) {
statement_bind_gvalue (stmt, fts_index, &property->value);
fts_index++;
}
}
tracker_db_statement_execute (stmt, NULL);
g_object_unref (stmt);
}
g_string_free (fts_sql, TRUE);
#endif
g_hash_table_remove_all (update_buffer.tables);
g_free (update_buffer.subject);
update_buffer.subject = NULL;
......@@ -465,7 +515,7 @@ cache_create_service_decomposed (TrackerClass *cl)
cache_insert_row (tracker_class_get_name (cl));
g_value_set_int (&gvalue, ensure_resource_id (tracker_class_get_uri (cl)));
cache_insert_value ("rdfs:Resource_rdf:type", "rdf:type", &gvalue, TRUE);
cache_insert_value ("rdfs:Resource_rdf:type", "rdf:type", &gvalue, TRUE, FALSE);
}
guint32
......@@ -523,7 +573,7 @@ cache_set_metadata_decomposed (TrackerProperty *property,
const gchar *value)
{
guint32 object_id;
gboolean multiple_values;
gboolean multiple_values, fts;
gchar *table_name;
const gchar *field_name;
TrackerProperty **super_properties;
......@@ -577,7 +627,9 @@ cache_set_metadata_decomposed (TrackerProperty *property,
return;
}
cache_insert_value (table_name, field_name, &gvalue, multiple_values);
fts = tracker_property_get_fulltext_indexed (property);
cache_insert_value (table_name, field_name, &gvalue, multiple_values, fts);
g_free (table_name);
}
......@@ -591,6 +643,7 @@ send_text_to_index (TrackerConfig *config,
gboolean full_parsing,
gint weight_factor)
{
#ifndef HAVE_SQLITE_FTS
TrackerDBIndex *lindex;
GHashTable *parsed;
GHashTableIter iter;
......@@ -631,6 +684,7 @@ send_text_to_index (TrackerConfig *config,
}
g_hash_table_unref (parsed);
#endif
}
static void
......@@ -1050,7 +1104,7 @@ tracker_data_insert_statement (const gchar *subject,
update_buffer.types = tracker_data_query_rdf_type (update_buffer.id);
g_value_set_int64 (&gvalue, (gint64) time (NULL));
cache_insert_value ("rdfs:Resource", "Modified", &gvalue, FALSE);
cache_insert_value ("rdfs:Resource", "Modified", &gvalue, FALSE, FALSE);
}
if (strcmp (predicate, RDF_PREFIX "type") == 0) {
......
......@@ -681,6 +681,11 @@ public class Tracker.SparqlQuery : Object {
pattern_sql.append (binding.sql_db_column_name);
pattern_sql.append ("\"");
if (binding.is_fts_match) {
#if HAVE_SQLITE_FTS
// parameters do not work with fts MATCH
string escaped_literal = string.joinv ("''", binding.literal.split ("'"));
pattern_sql.append_printf (" IN (SELECT rowid FROM fts WHERE fts MATCH '%s')", escaped_literal);
#else
pattern_sql.append (" IN (");
// include matches from fulltext search
......@@ -695,6 +700,7 @@ public class Tracker.SparqlQuery : Object {
}
pattern_sql.append (")");
#endif
} else {
pattern_sql.append (" = ");
if (binding.is_uri) {
......
......@@ -19,9 +19,6 @@ libtracker_db_la_SOURCES = \
tracker-db-action.c \
tracker-db-dbus.c \
tracker-db-file-info.c \
tracker-db-index.c \
tracker-db-index-item.c \
tracker-db-index-manager.c \
tracker-db-interface.c \
tracker-db-interface-sqlite.c \
tracker-db-manager.c
......@@ -30,13 +27,22 @@ noinst_HEADERS = \
tracker-db-action.h \
tracker-db-dbus.h \
tracker-db-file-info.h \
tracker-db-index.h \
tracker-db-index-item.h \
tracker-db-index-manager.h \
tracker-db-interface.h \
tracker-db-interface-sqlite.h \
tracker-db-manager.h
if !ENABLE_SQLITE_FTS
libtracker_db_la_SOURCES += \
tracker-db-index.c \
tracker-db-index-item.c \
tracker-db-index-manager.c
noinst_HEADERS += \
tracker-db-index.h \
tracker-db-index-item.h \
tracker-db-index-manager.h
endif
libtracker_db_la_LDFLAGS = \
-version-info $(LT_CURRENT):$(LT_REVISION):$(LT_AGE)
......
......@@ -18,6 +18,8 @@
* Boston, MA 02110-1301, USA.
*/
#include "config.h"
#include <glib/gstdio.h>
#include <sqlite3.h>
......@@ -99,6 +101,7 @@ tracker_db_interface_sqlite_constructor (GType type,
{
GObject *object;
TrackerDBInterfaceSqlitePrivate *priv;
gchar *err_msg = NULL;
object = (* G_OBJECT_CLASS (tracker_db_interface_sqlite_parent_class)->constructor) (type,
n_construct_properties,
......@@ -123,6 +126,18 @@ tracker_db_interface_sqlite_constructor (GType type,
sqlite3_extended_result_codes (priv->db, 0);
sqlite3_busy_timeout (priv->db, 10000000);
#ifdef HAVE_SQLITE_FTS
sqlite3_enable_load_extension (priv->db, 1);
sqlite3_load_extension (priv->db, "tracker-fts.so", NULL, &err_msg);
if (err_msg) {
g_critical ("Could not load tracker-fts extension:'%s'", err_msg);
sqlite3_free (err_msg);
} else {
g_message ("Loaded tracker fts extension");
}
#endif
return object;
}
......
......@@ -796,15 +796,6 @@ db_interface_get_fulltext (void)
iface = db_interface_get (TRACKER_DB_FULLTEXT, &create);
/*
* disabled to avoid warnings while not implemented
if (create) {
tracker_db_interface_start_transaction (iface);
load_sql_file (iface, "sqlite-fulltext.sql", NULL);
tracker_db_interface_end_transaction (iface);
}
*/
return iface;
}
......@@ -1214,10 +1205,18 @@ tracker_db_manager_init (TrackerDBManagerFlags flags,
initialized = TRUE;
#ifdef HAVE_SQLITE_FTS
resources_iface = tracker_db_manager_get_db_interfaces (4,
TRACKER_DB_METADATA,
TRACKER_DB_FULLTEXT,
TRACKER_DB_CONTENTS,
TRACKER_DB_COMMON);
#else
resources_iface = tracker_db_manager_get_db_interfaces (3,
TRACKER_DB_METADATA,
TRACKER_DB_CONTENTS,
TRACKER_DB_COMMON);
#endif
}
void
......
......@@ -6,8 +6,7 @@ INCLUDES = \
$(GLIB2_CFLAGS) \
$(GCOV_CFLAGS) \
$(PANGO_CFLAGS) \
$(SQLITE3_CFLAGS) \
-DSTORE_CATEGORY=1