Commit 1f987c5e authored by Jamie McCracken's avatar Jamie McCracken

basic index merging support

svn path=/trunk/; revision=839
parent 5e1c6997
......@@ -88,7 +88,7 @@ cache_free (Cache *cache)
tracker->word_count--;
}
/*
static DBConnection *
create_merge_index (const gchar *name, gboolean update)
{
......@@ -117,7 +117,8 @@ create_merge_index (const gchar *name, gboolean update)
return db_con;
}
*/
static gint
prepend_key_pointer (gpointer key,
......@@ -151,23 +152,57 @@ sort_func (gchar *a, gchar *b)
}
static inline gboolean
needs_merge (DBConnection *db_con)
{
if (tracker_indexer_size (db_con) > tracker->merge_limit) {
if (!db_con->merge_index) {
db_con->merge_index = create_merge_index (db_con->name, FALSE);
db_con->merge_update_index = create_merge_index (db_con->name, TRUE);
}
return TRUE;
}
return FALSE;
}
static void
flush_list (DBConnection *db_con, GSList *list1, GSList *list2, const gchar *word)
{
if (needs_merge (db_con)) {
tracker_indexer_append_word_lists (db_con->merge_index, word, list1, list2);
} else {
tracker_indexer_append_word_lists (db_con, word, list1, list2);
}
}
static GSList *
flush_update_list (DBConnection *db_con, GSList *list, const gchar *word)
{
if (!list) {
return NULL;
}
if (needs_merge (db_con)) {
GSList *ret_list = tracker_indexer_update_word_list (db_con->merge_update_index, word, list);
if (ret_list) {
tracker_indexer_append_word_lists (db_con->merge_update_index, word, ret_list, NULL);
g_slist_free (ret_list);
}
} else {
GSList *ret_list = tracker_indexer_update_word_list (db_con, word, list);
return ret_list;
}
GSList *ret_list = tracker_indexer_update_word_list (db_con, word, list);
return ret_list;
return NULL;
}
static void
flush_list (DBConnection *db_con, GSList *list1, GSList *list2, const gchar *word)
{
tracker_indexer_append_word_lists (db_con, word, list1, list2);
}
static void
......@@ -185,6 +220,7 @@ flush_cache (DBConnection *db_con, Cache *cache, const gchar *word)
} else {
if (new_update_list) {
flush_list (db_con->word_index, new_update_list, NULL, word);
g_slist_free (new_update_list);
}
}
......
......@@ -22,6 +22,7 @@
#define CREATE_INDEX "CREATE TABLE HitIndex (Word Text not null unique, HitCount Integer, HitArraySize Integer, HitArray Blob);"
#define MAX_HIT_BUFFER 480000
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
......@@ -194,6 +195,7 @@ tracker_indexer_open (const gchar *name)
sqlite3_busy_timeout (db_con->db, 10000);
db_con->merge_index = NULL;
db_con->merge_update_index = NULL;
db_con->cache = NULL;
db_con->emails = NULL;
db_con->others = NULL;
......@@ -248,6 +250,19 @@ tracker_indexer_free (DBConnection *db_con, gboolean remove_file)
}
gboolean
tracker_indexer_has_merge_index (DBConnection *db_con)
{
char *dbname = g_build_filename (tracker->data_dir, "tmp-", db_con->name, NULL);
gboolean result = g_file_test (dbname, G_FILE_TEST_IS_REGULAR);
g_free (dbname);
return result;
}
guint32
tracker_indexer_size (DBConnection *db_con)
{
......@@ -432,88 +447,107 @@ close_blob (sqlite3_blob *blob)
}
void
tracker_indexer_merge_index (DBConnection *db_con, gboolean update)
static gboolean
merge_words (DBConnection *db_con, DBConnection *merge_db_con, gboolean update)
{
DBConnection *merge_db_con = db_con->merge_index;
gint sz = sizeof (WordDetails);
sqlite3_blob *blob;
gchar buffer[MAX_HIT_BUFFER];
if (!merge_db_con || !db_con) {
tracker_error ("merge index missing");
return;
if (tracker->shutdown) {
return FALSE;
}
while (TRUE) {
if (tracker->shutdown) {
break;
}
gchar ***res = tracker_exec_proc (merge_db_con, "GetWordBatch", 0);
gchar ***res = tracker_exec_proc (merge_db_con, "GetWordBatch", 0);
if (res) {
gchar **row;
gint k;
if (res) {
gchar **row;
gint k;
k = 0;
k = 0;
tracker_db_start_transaction (db_con);
tracker_db_start_transaction (db_con);
while ((row = tracker_db_get_row (res, k))) {
while ((row = tracker_db_get_row (res, k))) {
k++;
k++;
if (!row || !row[0] || !row[1] || !row[2] || !row[3]) {
continue;
}
if (!row || !row[0] || !row[1] || !row[2] || !row[3]) {
continue;
}
sqlite_int64 id = atoi (row[0]);
gchar *word = row[1];
gint hit_count = atoi (row[2]);
sqlite_int64 id = atoi (row[0]);
gchar *word = row[1];
gint hit_count = atoi (row[2]);
blob = get_blob (merge_db_con, id, TRUE);
blob = get_blob (merge_db_con, id, TRUE);
if (!blob) {
continue;
}
if (!blob) {
continue;
}
if (!read_blob (blob, buffer, hit_count * sz, 0)) {
close_blob (blob);
continue;
}
if (!read_blob (blob, buffer, hit_count * sz, 0)) {
close_blob (blob);
continue;
}
if (!update) {
tracker_indexer_append_word_chunk (db_con, word, (WordDetails *) buffer, hit_count);
} else {
GSList *list = tracker_indexer_update_word_chunk (db_con, word, (WordDetails *) buffer, hit_count);
tracker_indexer_append_word_lists (db_con, word, list, NULL);
g_slist_free (list);
}
if (!update) {
tracker_indexer_append_word_chunk (db_con, word, (WordDetails *) buffer, hit_count);
} else {
GSList *list = tracker_indexer_update_word_chunk (db_con, word, (WordDetails *) buffer, hit_count);
tracker_indexer_append_word_lists (db_con, word, list, NULL);
g_slist_free (list);
}
close_blob (blob);
close_blob (blob);
gchar *str_id = tracker_int_to_str (id);
tracker_exec_proc (merge_db_con, "DeleteWord", 1, str_id);
g_free (str_id);
gchar *str_id = tracker_int_to_str (id);
tracker_exec_proc (merge_db_con, "DeleteWord", 1, str_id);
g_free (str_id);
}
}
tracker_db_end_transaction (db_con);
tracker_db_end_transaction (db_con);
tracker_db_free_result (res);
tracker_db_free_result (res);
if (tracker->shutdown) {
return FALSE;
}
if (tracker->shutdown) {
break;
}
return TRUE;
}
g_usleep (1000*1000);
} else {
break;
}
return FALSE;
}
void
tracker_indexer_merge_index (DBConnection *db_con)
{
DBConnection *merge_db_con = db_con->merge_index;
DBConnection *merge_update_db_con = db_con->merge_update_index;
int throttle;
if (!merge_db_con || !db_con || !merge_update_db_con) {
tracker_error ("merge index missing");
return;
}
guint32 size = tracker_indexer_size (db_con);
div_t q = div (size, tracker->merge_limit);
throttle = q.quot * 1000 * 500;
while (merge_words (db_con, merge_update_db_con, TRUE)) {
g_usleep (throttle);
}
while (merge_words (db_con, merge_db_con, FALSE)) {
g_usleep (throttle);
}
}
......
......@@ -81,11 +81,13 @@ void tracker_index_free_hit_list (GSList *hit_list);
DBConnection * tracker_indexer_open (const gchar *name);
void tracker_indexer_close (DBConnection *db_con);
void tracker_indexer_free (DBConnection *db_con, gboolean remove_file);
gboolean tracker_indexer_has_merge_index (DBConnection *db_con);
guint32 tracker_indexer_size (DBConnection *db_con);
gboolean tracker_indexer_optimize (DBConnection *db_con);
void tracker_indexer_sync (DBConnection *db_con);
void tracker_indexer_merge_index (DBConnection *db_con, gboolean update);
void tracker_indexer_merge_index (DBConnection *db_con);
/* Indexing api */
gboolean tracker_indexer_append_word (DBConnection *db_con, const gchar *word, guint32 id, gint service, gint score);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment