Commit 93fefc73 authored by Martyn James Russell's avatar Martyn James Russell

Added ParentID and 2 indexes to improve GetStats performance. Removed

	* data/db/sqlite-service-types.sql: 
	* data/db/sqlite-service.sql:
	* data/db/sqlite-stored-procs.sql:
	* src/libtracker-db/tracker-db-manager.c: 
	* src/trackerd/tracker-daemon.c: Added ParentID and 2 indexes
	to improve GetStats performance. Removed erroneous indexes which
	were creating warnings when creating the databases. Added stored
	proc to get all parent service type statistics (like Files,
	Emails, Conversations, etc).

	* src/tracker-applet/tracker-applet.c: Fixed stats order. This was
	broken recently with the change to the stored procs.

	* src/trackerd/tracker-main.c: Don't try to reset the stats in the
	database when we reindex, we don't hold stats in the same place in
	the same way now.

svn path=/trunk/; revision=3113
parent eaabadd5
2009-03-20 Martyn Russell <martyn@imendio.com>
* data/db/sqlite-service-types.sql:
* data/db/sqlite-service.sql:
* data/db/sqlite-stored-procs.sql:
* src/libtracker-db/tracker-db-manager.c:
* src/trackerd/tracker-daemon.c:
* src/tracker-applet/tracker-applet.c: Fixed stats order. This was
broken recently with the change to the stored procs.
* src/tracker-indexer/tracker-module-metadata-utils.c:
* src/trackerd/tracker-main.c: Don't try to reset the stats in the
database when we reindex, we don't hold stats in the same place in
the same way now.
2009-03-20 Philip Van Hoof <philip@codeminded.be>
* src/tracker-indexer/modules/evolution.c:
......@@ -47,7 +63,7 @@
2009-03-19 Martyn Russell <martyn@imendio.com>
* configure.ac:
* configure.ac:
* src/libinotify/Makefile.am:
* src/libstemmer/Makefile.am:
* src/libtracker-common/Makefile.am:
......
......@@ -7,6 +7,7 @@ CREATE TABLE ServiceTypes
TypeCount Integer default 0,
DisplayName Text default ' ',
ParentID Integer default 0,
Parent Text default ' ',
PropertyPrefix Text default ' ',
Enabled Integer default 1,
......@@ -65,6 +66,9 @@ CREATE TABLE ServiceTypes
unique (TypeName)
);
CREATE INDEX ServiceTypesStats ON ServiceTypes (TypeName, TypeID);
CREATE INDEX ServiceTypesParentStats ON ServiceTypes (Parent, TypeID, ParentID);
insert into ServiceTypes (TypeName) values ('default');
/* metadata that should appear in a tabular view and/or metadata tile for the service */
......
......@@ -39,9 +39,7 @@ CREATE TABLE Services
unique (Path, Name)
);
CREATE INDEX ServiceMetaDataCompoundIndex ON ServiceMetaData (ServiceTypeID, Enabled, AuxiliaryID, KeyMetadata1);
CREATE INDEX ServiceMetaDataCompoundIndex ON ServiceMetaData (ServiceTypeID, Enabled, AuxiliaryID, KeyMetadata2);
CREATE INDEX ServiceMetaDataCompoundIndex ON ServiceMetaData (ServiceTypeID, Enabled, AuxiliaryID, KeyMetadata3);
CREATE INDEX ServiceTypeIDIndex1 ON Services (ServiceTypeID);
/* child service relationships for a specific group/struct metadata */
CREATE TABLE ChildServices
......@@ -58,13 +56,13 @@ CREATE TABLE ServiceMetaData
(
ID Integer primary key AUTOINCREMENT not null,
ServiceID Integer not null,
MetaDataID Integer not null,
MetaDataID Integer not null,
MetaDataValue Text,
MetaDataDisplay Text,
MetaDataCollation Text
);
CREATE INDEX ServiceMetaDataCompoundIndex ON ServiceMetaData (ServiceID, MetaDataID, MetaDataDisplay, MetaDataCollation);
CREATE INDEX ServiceMetaDataCompoundIndex4 ON ServiceMetaData (ServiceID, MetaDataID, MetaDataDisplay, MetaDataCollation);
/* metadata for all keyword types - keywords are db indexed for fast searching - they are also not processed like other metadata. */
CREATE TABLE ServiceKeywordMetaData
......
......@@ -119,7 +119,9 @@ DeleteSearchResults1 DELETE FROM SearchResults1;
/*
* Statistics queries
*/
GetStats SELECT COUNT(1), T.TypeName FROM Services S, ServiceTypes T WHERE S.AuxilaryID IN (SELECT VolumeID FROM Volumes WHERE Enabled = 1) AND S.Enabled = 1 AND T.TypeID=S.ServiceTypeID GROUP BY ServiceTypeID ORDER BY T.TypeName;
GetStats SELECT T.TypeName, COUNT(1) FROM Services S, ServiceTypes T WHERE S.AuxilaryID IN (SELECT VolumeID FROM Volumes WHERE Enabled = 1) AND S.Enabled = 1 AND T.TypeID=S.ServiceTypeID GROUP BY ServiceTypeID ORDER BY T.TypeName;
GetStatsForParents SELECT T.Parent, COUNT(1) FROM ServiceTypes AS T JOIN Services AS S ON S.ServiceTypeID=T.TypeID WHERE S.AuxilaryID IN (SELECT VolumeID FROM Volumes WHERE Enabled = 1) AND S.Enabled = 1 AND S.ServiceTypeID IN (SELECT TypeID FROM ServiceTypes WHERE ParentID > 0) GROUP BY T.ParentID ORDER BY T.Parent;
GetHitDetails SELECT ROWID, HitCount, HitArraySize FROM HitIndex WHERE word = ?;
/*
......
......@@ -139,7 +139,6 @@ tracker_dbus_str_to_strv (const gchar *str)
return strv;
}
gchar **
tracker_dbus_queue_str_to_strv (GQueue *queue,
gint max)
......
......@@ -48,7 +48,7 @@
#define TRACKER_DB_MAX_FILE_SIZE 2000000000
/* Set current database version we are working with */
#define TRACKER_DB_VERSION_NOW TRACKER_DB_VERSION_3
#define TRACKER_DB_VERSION_NOW TRACKER_DB_VERSION_4
#define TRACKER_DB_VERSION_FILE "db-version.txt"
typedef enum {
......@@ -61,8 +61,9 @@ typedef enum {
TRACKER_DB_VERSION_UNKNOWN, /* Unknown */
TRACKER_DB_VERSION_1, /* Version 0.6.6 (before indexer-split) */
TRACKER_DB_VERSION_2, /* Version 0.6.90 (after indexer-split) */
TRACKER_DB_VERSION_3, /* Version 0.6.91 (current TRUNK) */
TRACKER_DB_VERSION_4 /* Version 0.7 (vstore branch) */
TRACKER_DB_VERSION_3, /* Version 0.6.91 (stable release) */
TRACKER_DB_VERSION_4, /* Version 0.6.92 (current TRUNK) */
TRACKER_DB_VERSION_5 /* Version 0.7 (vstore branch) */
} TrackerDBVersion;
typedef struct {
......@@ -520,9 +521,41 @@ load_service_file (TrackerDBInterface *iface,
new_value = tracker_string_boolean_to_string_gint (value);
esc_value = tracker_escape_string (new_value);
/* Special case "Parent */
if (g_ascii_strcasecmp (keys[j], "parent") == 0) {
TrackerDBResultSet *result_set;
gchar *query;
query = g_strdup_printf ("SELECT TypeId FROM ServiceTypes WHERE TypeName = '%s'",
esc_value);
result_set = tracker_db_interface_execute_query (iface, NULL, "%s", query);
g_free (query);
if (result_set) {
GValue value = {0, };
GValue transform = {0, };
g_value_init (&transform, G_TYPE_STRING);
_tracker_db_result_set_get_value (result_set, 0, &value);
if (g_value_transform (&value, &transform)) {
tracker_db_interface_execute_query (iface,
NULL,
"UPDATE ServiceTypes SET ParentId = '%s' WHERE TypeID = %s",
g_value_get_string (&transform),
str_id);
}
g_value_unset (&value);
g_value_unset (&transform);
g_object_unref (result_set);
}
}
tracker_db_interface_execute_query (iface,
NULL,
"update ServiceTypes set %s = '%s' where TypeID = %s",
"UPDATE ServiceTypes SET %s = '%s' WHERE TypeID = %s",
keys[j],
esc_value,
str_id);
......@@ -685,11 +718,11 @@ load_prepared_queries (void)
continue;
}
g_message (" Loading query:'%s'", details[0]);
g_message (" Adding query:'%s'", g_strstrip (details[0]));
g_hash_table_insert (prepared_queries,
g_strdup (details[0]),
g_strdup (details[1]));
g_strdup (g_strstrip (details[0])),
g_strdup (g_strstrip (details[1])));
g_strfreev (details);
}
......
......@@ -1204,7 +1204,7 @@ update_stats (GPtrArray *new_stats,
p = g_ptr_array_index (new_stats, i);
service_type = p[1];
service_type = p[0];
if (!service_type) {
continue;
......@@ -1212,7 +1212,7 @@ update_stats (GPtrArray *new_stats,
for (j = 0; j < G_N_ELEMENTS (stat_info); j++) {
if (g_strcmp0 (stat_info[j].name, service_type) == 0) {
gtk_label_set_text (GTK_LABEL (stat_info[j].stat_label), p[0]);
gtk_label_set_text (GTK_LABEL (stat_info[j].stat_label), p[1]);
}
}
}
......@@ -1325,16 +1325,14 @@ statistics_menu_activated (GtkMenuItem *item,
label,
0, 1, i + 1, i + 2);
if (g_strcmp0 (stat_info[i].name, "Files") != 0) {
stat_info[i].stat_label = gtk_label_new ("0");
gtk_label_set_selectable (GTK_LABEL (stat_info[i].stat_label), TRUE);
gtk_misc_set_alignment (GTK_MISC (stat_info[i].stat_label), 0, 0);
gtk_table_attach_defaults (GTK_TABLE (table),
stat_info[i].stat_label,
1, 2,
i + 1, i + 2);
}
stat_info[i].stat_label = gtk_label_new ("0");
gtk_label_set_selectable (GTK_LABEL (stat_info[i].stat_label), TRUE);
gtk_misc_set_alignment (GTK_MISC (stat_info[i].stat_label), 0, 0);
gtk_table_attach_defaults (GTK_TABLE (table),
stat_info[i].stat_label,
1, 2,
i + 1, i + 2);
}
/* Set flags and get stats */
......
......@@ -49,6 +49,8 @@ typedef struct {
TrackerConfig *config;
TrackerProcessor *processor;
DBusGProxy *indexer_proxy;
GTimeVal last_stats_time;
GHashTable *last_stats;
} TrackerDaemonPrivate;
......@@ -307,31 +309,115 @@ tracker_daemon_get_stats (TrackerDaemon *object,
DBusGMethodInvocation *context,
GError **error)
{
TrackerDBInterface *iface;
TrackerDBResultSet *result_set;
guint request_id;
GPtrArray *values = NULL;
TrackerDaemonPrivate *priv;
TrackerDBInterface *iface;
TrackerDBResultSet *result_set;
guint request_id;
GPtrArray *values;
GTimeVal now;
gboolean use_cache;
request_id = tracker_dbus_get_next_request_id ();
tracker_dbus_request_new (request_id,
"DBus request to get daemon service stats");
/* Here it doesn't matter which one we ask, as long as it has common.db
* attached. The service ones are cached connections, so we can use
* those instead of asking for an individual-file connection (like what
* the original code had)
*/
priv = TRACKER_DAEMON_GET_PRIVATE (object);
/* iface = tracker_db_manager_get_db_interfaceX (TRACKER_DB_COMMON); */
values = g_ptr_array_new ();
iface = tracker_db_manager_get_db_interface_by_service (TRACKER_DB_FOR_FILE_SERVICE);
/* Use the cache if we have requests stats recently. */
g_get_current_time (&now);
result_set = tracker_data_manager_exec_proc (iface, "GetStats", 0);
values = tracker_dbus_query_result_to_ptr_array (result_set);
if (priv->last_stats_time.tv_sec == 0) {
use_cache = FALSE;
} else {
use_cache = now.tv_sec - priv->last_stats_time.tv_sec < 60;
}
if (result_set) {
g_object_unref (result_set);
if (use_cache) {
GHashTableIter iter;
gpointer key, value;
g_message ("Using cache for stats");
g_hash_table_iter_init (&iter, priv->last_stats);
while (g_hash_table_iter_next (&iter, &key, &value)) {
GStrv strv;
const gchar *service_type;
gint count;
service_type = key;
count = GPOINTER_TO_INT (value);
strv = g_new (gchar*, 3);
strv[0] = g_strdup (service_type);
strv[1] = g_strdup_printf ("%d", count);
strv[2] = NULL;
g_ptr_array_add (values, strv);
}
} else {
GPtrArray *stats, *parent_stats;
gint i;
g_message ("Using database for stats (cache is %ld seconds old)",
now.tv_sec - priv->last_stats_time.tv_sec);
/* Here it doesn't matter which one we ask, as long as
* it has common.db attached. The service ones are
* cached connections, so we can use those instead of
* asking for an individual-file connection (like what
* the original code had).
*/
iface = tracker_db_manager_get_db_interface_by_service (TRACKER_DB_FOR_FILE_SERVICE);
result_set = tracker_data_manager_exec_proc (iface, "GetStats", 0);
stats = tracker_dbus_query_result_to_ptr_array (result_set);
if (result_set) {
g_object_unref (result_set);
}
result_set = tracker_data_manager_exec_proc (iface, "GetStatsForParents", 0);
parent_stats = tracker_dbus_query_result_to_ptr_array (result_set);
if (result_set) {
g_object_unref (result_set);
}
/* Concatenate stats */
for (i = 0; i < stats->len; i++) {
g_ptr_array_add (values, g_ptr_array_index (stats, i));
}
for (i = 0; i < parent_stats->len; i++) {
g_ptr_array_add (values, g_ptr_array_index (parent_stats, i));
}
/* Update local cache */
for (i = 0; i < values->len; i++) {
gchar **p;
const gchar *service_type = NULL;
gint old_count, new_count;
p = g_ptr_array_index (values, i);
service_type = p[0];
new_count = atoi (p[1]);
if (old_count != new_count) {
g_hash_table_replace (priv->last_stats,
g_strdup (service_type),
GINT_TO_POINTER (new_count));
}
}
g_ptr_array_free (parent_stats, TRUE);
g_ptr_array_free (stats, TRUE);
priv->last_stats_time = now;
g_message ("Updated stats cache time to now");
}
dbus_g_method_return (context, values);
......@@ -567,7 +653,10 @@ tracker_daemon_signal_statistics (void)
TrackerDaemonPrivate *priv;
TrackerDBInterface *iface;
TrackerDBResultSet *result_set;
GPtrArray *new_stats;
GPtrArray *stats, *parent_stats;
GPtrArray *values;
GTimeVal now;
gint i;
daemon = tracker_dbus_get_object (TRACKER_TYPE_DAEMON);
priv = TRACKER_DAEMON_GET_PRIVATE (daemon);
......@@ -578,12 +667,35 @@ tracker_daemon_signal_statistics (void)
* lookup the in a to compare in b, just compare index based.
* Maybe we want to change this nonetheless later?
*/
g_message ("Using database for stats for accurate signal");
result_set = tracker_data_manager_exec_proc (iface, "GetStats", 0);
new_stats = tracker_dbus_query_result_to_ptr_array (result_set);
stats = tracker_dbus_query_result_to_ptr_array (result_set);
if (result_set) {
g_object_unref (result_set);
}
result_set = tracker_data_manager_exec_proc (iface, "GetStatsForParents", 0);
parent_stats = tracker_dbus_query_result_to_ptr_array (result_set);
if (result_set) {
g_object_unref (result_set);
}
/* Concatenate stats */
values = g_ptr_array_new ();
for (i = 0; i < stats->len; i++) {
g_ptr_array_add (values, g_ptr_array_index (stats, i));
}
for (i = 0; i < parent_stats->len; i++) {
g_ptr_array_add (values, g_ptr_array_index (parent_stats, i));
}
g_ptr_array_free (parent_stats, TRUE);
g_ptr_array_free (stats, TRUE);
/* There are 3 situations here:
* - 1. No new stats
......@@ -597,7 +709,7 @@ tracker_daemon_signal_statistics (void)
g_message ("Checking for statistics changes and signalling clients...");
/* Situation #1 */
if (!new_stats || new_stats->len < 1) {
if (!values || values->len < 1) {
g_message (" No new statistics, doing nothing");
return;
}
......@@ -608,15 +720,15 @@ tracker_daemon_signal_statistics (void)
/* Situation #2 */
g_message (" No previous statistics");
for (i = 0; i < new_stats->len; i++) {
for (i = 0; i < values->len; i++) {
const gchar **p;
const gchar *service_type = NULL;
gint new_count;
p = g_ptr_array_index (new_stats, i);
p = g_ptr_array_index (values, i);
service_type = p[1];
new_count = atoi (p[0]);
service_type = p[0];
new_count = atoi (p[1]);
if (!service_type) {
continue;
......@@ -631,20 +743,20 @@ tracker_daemon_signal_statistics (void)
}
/* Emit signal */
g_signal_emit (daemon, signals[SERVICE_STATISTICS_UPDATED], 0, new_stats);
g_signal_emit (daemon, signals[SERVICE_STATISTICS_UPDATED], 0, values);
} else {
gint i;
/* Situation #3 */
for (i = 0; i < new_stats->len; i++) {
for (i = 0; i < values->len; i++) {
gchar **p;
const gchar *service_type = NULL;
gpointer data;
gint old_count, new_count;
p = g_ptr_array_index (new_stats, i);
service_type = p[1];
new_count = atoi (p[0]);
p = g_ptr_array_index (values, i);
service_type = p[0];
new_count = atoi (p[1]);
if (!service_type) {
continue;
......@@ -664,20 +776,31 @@ tracker_daemon_signal_statistics (void)
g_strdup (service_type),
GINT_TO_POINTER (new_count));
} else {
/* Remove from new_stats since the value is the same */
/* Remove from values since the value is the same */
g_strfreev (p);
g_ptr_array_remove (new_stats, p);
g_ptr_array_remove (values, p);
/* Decrement i since we are about to
* increment it and we just removed
* an item. Otherwise we miss items.
*/
i--;
}
}
if (new_stats->len > 0) {
g_signal_emit (daemon, signals[SERVICE_STATISTICS_UPDATED], 0, new_stats);
if (values->len > 0) {
g_signal_emit (daemon, signals[SERVICE_STATISTICS_UPDATED], 0, values);
} else {
g_message (" No changes in the statistics");
}
}
g_ptr_array_foreach (new_stats, (GFunc) g_strfreev, NULL);
g_ptr_array_free (new_stats, TRUE);
/* Update time for last stats */
g_get_current_time (&now);
priv->last_stats_time = now;
g_message ("Updated stats cache time to now");
g_ptr_array_foreach (values, (GFunc) g_strfreev, NULL);
g_ptr_array_free (values, TRUE);
}
......@@ -605,25 +605,7 @@ initialize_databases (void)
* Create SQLite databases
*/
if (!tracker_status_get_is_readonly () && force_reindex) {
TrackerDBInterface *iface;
tracker_status_set_is_first_time_index (TRUE);
/* Reset stats for embedded services if they are being reindexed */
/* Here it doesn't matter which one we ask, as long as it has common.db
* attached. The service ones are cached connections, so we can use
* those instead of asking for an individual-file connection (like what
* the original code had) */
/* iface = tracker_db_manager_get_db_interfaceX (TRACKER_DB_COMMON); */
iface = tracker_db_manager_get_db_interface_by_service (TRACKER_DB_FOR_FILE_SERVICE);
g_message ("*** DELETING STATS *** ");
tracker_data_manager_exec_no_reply (iface,
"update ServiceTypes set TypeCount = 0 where Embedded = 1");
}
/* Check db integrity if not previously shut down cleanly */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment