Commit f8d5ebd3 authored by Morten Welinder's avatar Morten Welinder Committed by Morten Welinder

Switch to allocating from a GStringChunk.

2003-10-13  Morten Welinder  <terra@gnome.org>

	* src/stf-parse.c (stf_parse_fixed_cell, stf_parse_csv_cell,
	stf_parse_general, stf_parse_lines): Switch to allocating from a
	GStringChunk.

2003-10-13  Morten Welinder  <terra@gnome.org>

	* dialog-stf-preview.c (stf_preview_set_lines): Take extra
	GStringChunk* argument.
	(stf_preview_set_lines): Be consistent with the column maximum.
parent b2bd6c1f
2003-10-13 Morten Welinder <terra@gnome.org>
* src/stf-parse.c (stf_parse_fixed_cell, stf_parse_csv_cell,
stf_parse_general, stf_parse_lines): Switch to allocating from a
GStringChunk.
2003-10-12 Jody Goldberg <jody@gnome.org>
* configure.in : remove GTK_DISABLE_DEPRECATED for now. gtk head has
......
......@@ -5,6 +5,11 @@ Jody:
* Don't install schemas when building an rpm.
* Support non-assisted CSV export for ssconvert
Morten:
* Fix crash with STF import and gazillions of columns.
* Drastically reduce memory usage during STF import with lots of
small fields.
Rodrigo:
* Bump requirement to libgda/libgnomedb 1.0.1
......
2003-10-13 Morten Welinder <terra@gnome.org>
* src/stf-parse.c (stf_parse_fixed_cell, stf_parse_csv_cell,
stf_parse_general, stf_parse_lines): Switch to allocating from a
GStringChunk.
2003-10-12 Jody Goldberg <jody@gnome.org>
* configure.in : remove GTK_DISABLE_DEPRECATED for now. gtk head has
......
2003-10-13 Morten Welinder <terra@gnome.org>
* src/stf-parse.c (stf_parse_fixed_cell, stf_parse_csv_cell,
stf_parse_general, stf_parse_lines): Switch to allocating from a
GStringChunk.
2003-10-12 Jody Goldberg <jody@gnome.org>
* configure.in : remove GTK_DISABLE_DEPRECATED for now. gtk head has
......
2003-10-13 Morten Welinder <terra@gnome.org>
* dialog-stf-preview.c (stf_preview_set_lines): Take extra
GStringChunk* argument.
(stf_preview_set_lines): Be consistent with the column maximum.
2003-10-08 Jody Goldberg <jody@gnome.org>
* Release 1.2.1
......
......@@ -46,6 +46,7 @@ csv_page_global_change (G_GNUC_UNUSED GtkWidget *widget,
RenderData_t *renderdata = pagedata->csv.renderdata;
GSList *sepstr;
GString *sepc = g_string_new (NULL);
GStringChunk *lines_chunk;
GPtrArray *lines;
StfTrimType_t trim;
......@@ -87,14 +88,16 @@ csv_page_global_change (G_GNUC_UNUSED GtkWidget *widget,
stf_parse_options_csv_set_duplicates (parseoptions,
gtk_toggle_button_get_active (GTK_TOGGLE_BUTTON (pagedata->csv.csv_duplicates)));
lines_chunk = g_string_chunk_new (100 * 1024);
/* Don't trim on this page. */
trim = parseoptions->trim_spaces;
stf_parse_options_set_trim_spaces (parseoptions, TRIM_TYPE_NEVER);
lines = stf_parse_general (parseoptions,
lines = stf_parse_general (parseoptions, lines_chunk,
pagedata->cur, pagedata->cur_end);
stf_parse_options_set_trim_spaces (parseoptions, trim);
stf_preview_set_lines (renderdata, lines);
stf_preview_set_lines (renderdata, lines_chunk, lines);
}
/**
......
......@@ -379,17 +379,20 @@ fixed_page_update_preview (StfDialogData *pagedata)
StfParseOptions_t *parseoptions = pagedata->parseoptions;
RenderData_t *renderdata = pagedata->fixed.renderdata;
int i;
GStringChunk *lines_chunk;
GPtrArray *lines;
StfTrimType_t trim;
lines_chunk = g_string_chunk_new (100 * 1024);
/* Don't trim on this page. */
trim = parseoptions->trim_spaces;
stf_parse_options_set_trim_spaces (parseoptions, TRIM_TYPE_NEVER);
lines = stf_parse_general (parseoptions,
lines = stf_parse_general (parseoptions, lines_chunk,
pagedata->cur, pagedata->cur_end);
stf_parse_options_set_trim_spaces (parseoptions, trim);
stf_preview_set_lines (renderdata, lines);
stf_preview_set_lines (renderdata, lines_chunk, lines);
for (i = 0; i < renderdata->colcount; i++) {
GtkTreeViewColumn *column =
......
......@@ -329,6 +329,7 @@ format_page_update_preview (StfDialogData *pagedata)
unsigned int ui;
int i;
int col_import_array_len_old, old_part;
GStringChunk *lines_chunk;
stf_preview_colformats_clear (renderdata);
for (ui = 0; ui < pagedata->format.formats->len; ui++) {
......@@ -336,8 +337,10 @@ format_page_update_preview (StfDialogData *pagedata)
stf_preview_colformats_add (renderdata, sf);
}
stf_preview_set_lines (renderdata,
lines_chunk = g_string_chunk_new (100 * 1024);
stf_preview_set_lines (renderdata, lines_chunk,
stf_parse_general (pagedata->parseoptions,
lines_chunk,
pagedata->cur,
pagedata->cur_end));
......
......@@ -72,7 +72,9 @@ static void
main_page_update_preview (StfDialogData *pagedata)
{
RenderData_t *renderdata = pagedata->main.renderdata;
GStringChunk *lines_chunk = g_string_chunk_new (100 * 1024);
GPtrArray *lines = stf_parse_lines (pagedata->parseoptions,
lines_chunk,
pagedata->utf8_data, TRUE);
unsigned int ui;
......@@ -83,7 +85,7 @@ main_page_update_preview (StfDialogData *pagedata)
pagedata->longest_line = MAX (pagedata->longest_line, thislen);
}
stf_preview_set_lines (renderdata, lines);
stf_preview_set_lines (renderdata, lines_chunk, lines);
}
......
......@@ -93,6 +93,7 @@ stf_preview_new (GtkWidget *data_container,
renderdata->startrow = 1;
renderdata->colformats = g_ptr_array_new ();
renderdata->ignore_formats = FALSE;
renderdata->lines_chunk = NULL;
renderdata->lines = NULL;
renderdata->date_conv = date_conv;
......@@ -100,8 +101,8 @@ stf_preview_new (GtkWidget *data_container,
renderdata->ll =
gnumeric_lazy_list_new (render_get_value, renderdata, 0);
gnumeric_lazy_list_add_column (renderdata->ll,
#warning FIXME: we should not need to limit the shown columns to 4 times SHEET_MAX_COLS
4*SHEET_MAX_COLS,
#warning "FIXME: we should not need to limit the shown columns to 4 times SHEET_MAX_COLS"
4 * SHEET_MAX_COLS,
G_TYPE_STRING);
renderdata->tree_view =
......@@ -158,29 +159,36 @@ stf_preview_free (RenderData_t *renderdata)
stf_preview_colformats_clear (renderdata);
g_ptr_array_free (renderdata->colformats, TRUE);
stf_preview_set_lines (renderdata, NULL);
stf_preview_set_lines (renderdata, NULL, NULL);
g_object_unref (renderdata->tooltips);
g_free (renderdata);
}
void
stf_preview_set_lines (RenderData_t *renderdata, GPtrArray *lines)
stf_preview_set_lines (RenderData_t *renderdata,
GStringChunk *lines_chunk,
GPtrArray *lines)
{
unsigned int i;
int colcount = 0;
g_return_if_fail (renderdata != NULL);
if (renderdata->lines == lines)
return;
/* Empty the table. */
gnumeric_lazy_list_set_rows (renderdata->ll, 0);
if (renderdata->lines)
stf_parse_general_free (renderdata->lines);
renderdata->lines = lines;
if (renderdata->lines != lines) {
if (renderdata->lines)
stf_parse_general_free (renderdata->lines);
renderdata->lines = lines;
}
if (renderdata->lines_chunk != lines_chunk) {
if (renderdata->lines_chunk)
g_string_chunk_free (renderdata->lines_chunk);
renderdata->lines_chunk = lines_chunk;
}
if (lines == NULL)
return;
......@@ -192,6 +200,8 @@ stf_preview_set_lines (RenderData_t *renderdata, GPtrArray *lines)
if (colcount <= 0)
colcount = 1;
/* See stf_preview_new. */
colcount = MIN (colcount, 4 * SHEET_MAX_COLS);
/* Fix number of columns. */
while (renderdata->colcount > colcount)
......
......@@ -32,6 +32,7 @@
typedef struct {
GtkWidget *data_container;
GStringChunk *lines_chunk;
GPtrArray *lines;
GnumericLazyList *ll;
GtkTreeView *tree_view;
......@@ -52,7 +53,9 @@ RenderData_t* stf_preview_new (GtkWidget *data_contai
void stf_preview_free (RenderData_t *data);
/* These are for manipulation */
void stf_preview_set_lines (RenderData_t *data, GPtrArray *lines);
void stf_preview_set_lines (RenderData_t *data,
GStringChunk *lines_chunk,
GPtrArray *lines);
void stf_preview_set_startrow (RenderData_t *data, int startrow);
void stf_preview_colformats_clear (RenderData_t *renderdata);
......
......@@ -107,7 +107,7 @@ next_clicked (G_GNUC_UNUSED GtkWidget *widget, StfDialogData *data)
switch (gtk_notebook_get_current_page (data->notebook)) {
case DPG_MAIN:
stf_preview_set_lines (data->main.renderdata, NULL);
stf_preview_set_lines (data->main.renderdata, NULL, NULL);
if (gtk_toggle_button_get_active (GTK_TOGGLE_BUTTON (data->main.main_separated))) {
newpos = DPG_CSV;
} else {
......@@ -116,12 +116,12 @@ next_clicked (G_GNUC_UNUSED GtkWidget *widget, StfDialogData *data)
break;
case DPG_CSV:
stf_preview_set_lines (data->csv.renderdata, NULL);
stf_preview_set_lines (data->csv.renderdata, NULL, NULL);
newpos = DPG_FORMAT;
break;
case DPG_FIXED:
stf_preview_set_lines (data->fixed.renderdata, NULL);
stf_preview_set_lines (data->fixed.renderdata, NULL, NULL);
newpos = DPG_FORMAT;
break;
......@@ -143,7 +143,7 @@ back_clicked (G_GNUC_UNUSED GtkWidget *widget, StfDialogData *data)
switch (gtk_notebook_get_current_page (data->notebook)) {
case DPG_FORMAT:
stf_preview_set_lines (data->format.renderdata, NULL);
stf_preview_set_lines (data->format.renderdata, NULL, NULL);
if (data->parseoptions->parsetype == PARSE_TYPE_CSV)
newpos = DPG_CSV;
else
......@@ -151,12 +151,12 @@ back_clicked (G_GNUC_UNUSED GtkWidget *widget, StfDialogData *data)
break;
case DPG_FIXED:
stf_preview_set_lines (data->fixed.renderdata, NULL);
stf_preview_set_lines (data->fixed.renderdata, NULL, NULL);
newpos = DPG_MAIN;
break;
case DPG_CSV:
stf_preview_set_lines (data->csv.renderdata, NULL);
stf_preview_set_lines (data->csv.renderdata, NULL, NULL);
newpos = DPG_MAIN;
break;
......
......@@ -59,6 +59,7 @@ g_free (oldlocale);}
/* Source_t struct, used for interchanging parsing information between the low level parse functions */
typedef struct {
GStringChunk *chunk;
char const *position; /* Indicates the current position within data */
/* Used internally for fixed width parsing */
......@@ -179,7 +180,7 @@ void
stf_parse_options_free (StfParseOptions_t *parseoptions)
{
g_return_if_fail (parseoptions != NULL);
if (parseoptions->col_import_array)
g_free (parseoptions->col_import_array);
if (parseoptions->locale)
......@@ -547,12 +548,13 @@ stf_parse_csv_is_separator (char const *character, char const *chr, GSList const
*
* returns a pointer to the parsed cell contents. (has to be freed by the calling routine)
**/
static inline char *
static char *
stf_parse_csv_cell (Source_t *src, StfParseOptions_t *parseoptions)
{
char const *cur;
char const *next = NULL;
GString *res;
char *res;
GString *text;
StfTokenType_t ttype;
g_return_val_if_fail (src != NULL, NULL);
......@@ -561,7 +563,7 @@ stf_parse_csv_cell (Source_t *src, StfParseOptions_t *parseoptions)
cur = src->position;
g_return_val_if_fail (cur != NULL, NULL);
res = g_string_sized_new (30);
text = g_string_sized_new (30);
while (cur && *cur) {
char const *here, *there;
......@@ -578,7 +580,7 @@ stf_parse_csv_cell (Source_t *src, StfParseOptions_t *parseoptions)
/* break */ /* fall through */
case STF_TOKEN_CHAR:
if (here && there)
res = g_string_append_len (res, here, there - here);
g_string_append_len (text, here, there - here);
break;
case STF_TOKEN_SEPARATOR:
cur = next;
......@@ -598,19 +600,27 @@ stf_parse_csv_cell (Source_t *src, StfParseOptions_t *parseoptions)
if (parseoptions->indicator_2x_is_single) {
gboolean second = TRUE;
gunichar quote = parseoptions->stringindicator;
int len = res->len;
int len = text->len;
char *found;
while ((found = g_utf8_strrchr (res->str, len, quote))) {
len = found - res->str;
while ((found = g_utf8_strrchr (text->str, len, quote))) {
len = found - text->str;
if (second) {
res = g_string_erase (res, len, g_utf8_next_char(found)-found);
g_string_erase (text, len, g_utf8_next_char(found) - found);
second = FALSE;
} else
second = TRUE;
}
}
return g_string_free (res, FALSE);
#ifdef HAVE_G_STRING_CHUNK_INSERT_LEN
res = g_string_chunk_insert_len (src->chunk, text->str, text->len);
#else
res = g_string_chunk_insert (src->chunk, text->str);
#endif
g_string_free (text, TRUE);
return res;
}
/*
......@@ -663,8 +673,11 @@ stf_parse_csv_line (Source_t *src, StfParseOptions_t *parseoptions)
trim_spaces_inplace (field, parseoptions);
g_ptr_array_add (line, field);
}
#if 0
g_print ("Got a line of %d fields.\n", line->len);
#endif
return line;
}
......@@ -672,42 +685,46 @@ stf_parse_csv_line (Source_t *src, StfParseOptions_t *parseoptions)
/**
* stf_parse_fixed_cell:
*
* returns a pointer to the parsed cell contents. (has to be freed by the calling routine)
* returns a pointer to the parsed cell contents.
**/
static inline char *
static char *
stf_parse_fixed_cell (Source_t *src, StfParseOptions_t *parseoptions)
{
GString *res;
char *res;
char const *cur;
int splitval;
int len = 0;
g_return_val_if_fail (src != NULL, NULL);
g_return_val_if_fail (parseoptions != NULL, NULL);
cur = src->position;
res = g_string_new (NULL);
if (src->splitpos < my_garray_len (parseoptions->splitpositions))
splitval = (int) g_array_index (parseoptions->splitpositions, int, src->splitpos);
else
splitval = -1;
while (*cur != '\0' && !compare_terminator (cur, parseoptions) && splitval != src->linepos) {
g_string_append_unichar (res, g_utf8_get_char (cur));
while (*cur != 0 && !compare_terminator (cur, parseoptions) && splitval != src->linepos) {
src->linepos++;
cur = g_utf8_next_char (cur);
len++;
}
src->position = cur;
#ifdef HAVE_G_STRING_CHUNK_INSERT_LEN
res = g_string_chunk_insert_len (src->chunk,
src->position,
cur - src->position);
#else
{
char save = *cur;
*(char *)cur = 0; /* Ugh! */
res = g_string_chunk_insert (src->chunk, src->position);
*(char *)cur = save;
}
#endif
if (len != 0)
return g_string_free (res, FALSE);
src->position = cur;
g_string_free (res, TRUE);
return NULL;
return res;
}
/**
......@@ -748,11 +765,7 @@ stf_parse_general_free (GPtrArray *lines)
unsigned lineno;
for (lineno = 0; lineno < lines->len; lineno++) {
GPtrArray *line = g_ptr_array_index (lines, lineno);
unsigned field;
for (field = 0; field < line->len; field++) {
char *text = g_ptr_array_index (line, field);
g_free (text);
}
/* Fields are not free here. */
g_ptr_array_free (line, TRUE);
}
g_ptr_array_free (lines, TRUE);
......@@ -770,6 +783,7 @@ stf_parse_general_free (GPtrArray *lines)
**/
GPtrArray *
stf_parse_general (StfParseOptions_t *parseoptions,
GStringChunk *lines_chunk,
char const *data, char const *data_end)
{
GPtrArray *lines;
......@@ -782,6 +796,7 @@ stf_parse_general (StfParseOptions_t *parseoptions,
g_return_val_if_fail (stf_parse_options_valid (parseoptions), NULL);
g_return_val_if_fail (g_utf8_validate (data, -1, NULL), NULL);
src.chunk = lines_chunk;
src.position = data;
row = 0;
......@@ -806,7 +821,9 @@ stf_parse_general (StfParseOptions_t *parseoptions,
}
GPtrArray *
stf_parse_lines (StfParseOptions_t *parseoptions, const char *data, gboolean with_lineno)
stf_parse_lines (StfParseOptions_t *parseoptions,
GStringChunk *lines_chunk,
const char *data, gboolean with_lineno)
{
GPtrArray *lines;
int lineno = 1;
......@@ -818,13 +835,29 @@ stf_parse_lines (StfParseOptions_t *parseoptions, const char *data, gboolean wit
const char *data0 = data;
GPtrArray *line = g_ptr_array_new ();
if (with_lineno)
g_ptr_array_add (line, g_strdup_printf ("%d", lineno++));
if (with_lineno) {
char buf[4 * sizeof (int)];
sprintf (buf, "%d", lineno++);
g_ptr_array_add (line,
g_string_chunk_insert (lines_chunk, buf));
}
while (1) {
int termlen = compare_terminator (data, parseoptions);
if (termlen > 0 || *data == 0) {
g_ptr_array_add (line, g_strndup (data0, data - data0));
#ifdef HAVE_G_STRING_CHUNK_INSERT_LEN
g_ptr_array_add (line,
g_string_chunk_insert_len (lines_chunk,
data0,
data - data0));
#else
char save = *data;
*(char *)data = 0; /* Ugh! */
g_ptr_array_add (line,
g_string_chunk_insert (lines_chunk,
data0));
*(char *)data = save;
#endif
data += termlen;
break;
} else
......@@ -1121,6 +1154,7 @@ stf_parse_sheet (StfParseOptions_t *parseoptions,
int row;
unsigned int lrow;
GnmDateConventions const *date_conv;
GStringChunk *lines_chunk;
GPtrArray *lines;
SETUP_LOCALE_SWITCH;
......@@ -1134,7 +1168,8 @@ stf_parse_sheet (StfParseOptions_t *parseoptions,
if (!data_end)
data_end = data + strlen (data);
lines = stf_parse_general (parseoptions, data, data_end);
lines_chunk = g_string_chunk_new (100 * 1024);
lines = stf_parse_general (parseoptions, lines_chunk, data, data_end);
for (row = start_row, lrow = 0; lrow < lines->len ; row++, lrow++) {
unsigned int lcol, lcol_target = 0;
GPtrArray *line = g_ptr_array_index (lines, lrow);
......@@ -1145,16 +1180,15 @@ stf_parse_sheet (StfParseOptions_t *parseoptions,
char *text = g_ptr_array_index (line, lcol);
if (text) {
GnmValue *v;
StyleFormat *fmt = mstyle_get_format
StyleFormat *fmt = mstyle_get_format
(sheet_style_get (sheet,
start_col + lcol_target,
start_col + lcol_target,
row));
v = format_match (text, fmt, date_conv);
if (v == NULL) {
v = value_new_string_nocopy (text);
g_ptr_array_index (line, lcol) = NULL;
v = value_new_string (text);
}
cell_set_value (sheet_cell_fetch
cell_set_value (sheet_cell_fetch
(sheet, start_col + lcol_target, row), v);
}
lcol_target++;
......@@ -1163,6 +1197,7 @@ stf_parse_sheet (StfParseOptions_t *parseoptions,
}
stf_parse_general_free (lines);
g_string_chunk_free (lines_chunk);
END_LOCALE_SWITCH;
return TRUE;
}
......@@ -1173,6 +1208,7 @@ stf_parse_region (StfParseOptions_t *parseoptions, char const *data, char const
CellRegion *cr;
CellCopyList *content = NULL;
unsigned int row, colhigh = 0;
GStringChunk *lines_chunk;
GPtrArray *lines;
SETUP_LOCALE_SWITCH;
......@@ -1183,11 +1219,12 @@ stf_parse_region (StfParseOptions_t *parseoptions, char const *data, char const
if (!data_end)
data_end = data + strlen (data);
lines = stf_parse_general (parseoptions, data, data_end);
lines_chunk = g_string_chunk_new (100 * 1024);
lines = stf_parse_general (parseoptions, lines_chunk, data, data_end);
for (row = 0; row < lines->len; row++) {
GPtrArray *line = g_ptr_array_index (lines, row);
unsigned int col, targetcol = 0;
#warning FIXME: We should not just assume the 1900 convention
#warning "FIXME: We should not just assume the 1900 convention "
GnmDateConventions date_conv = {FALSE};
for (col = 0; col < line->len; col++) {
......@@ -1198,13 +1235,12 @@ stf_parse_region (StfParseOptions_t *parseoptions, char const *data, char const
if (text) {
CellCopy *ccopy;
GnmValue *v;
StyleFormat *fmt = g_ptr_array_index
StyleFormat *fmt = g_ptr_array_index
(parseoptions->formats, col);
v = format_match (text, fmt, &date_conv);
if (v == NULL) {
v = value_new_string_nocopy (text);
g_ptr_array_index (line, col) = NULL;
v = value_new_string (text);
}
ccopy = g_new (CellCopy, 1);
......@@ -1224,6 +1260,7 @@ stf_parse_region (StfParseOptions_t *parseoptions, char const *data, char const
}
}
stf_parse_general_free (lines);
g_string_chunk_free (lines_chunk);
END_LOCALE_SWITCH;
......@@ -1347,6 +1384,7 @@ StfParseOptions_t *
stf_parse_options_guess (const char *data)
{
StfParseOptions_t *res;
GStringChunk *lines_chunk;
GPtrArray *lines;
int tabcount;
int sepcount;
......@@ -1355,7 +1393,8 @@ stf_parse_options_guess (const char *data)
g_return_val_if_fail (data != NULL, NULL);
res = stf_parse_options_new ();
lines = stf_parse_lines (res, data, FALSE);
lines_chunk = g_string_chunk_new (100 * 1024);
lines = stf_parse_lines (res, lines_chunk, data, FALSE);
tabcount = count_character (lines, '\t', 0.2);
sepcount = count_character (lines, sepchar, 0.2);
......@@ -1401,6 +1440,7 @@ stf_parse_options_guess (const char *data)
}
stf_parse_general_free (lines);
g_string_chunk_free (lines_chunk);
return res;
}
......@@ -96,10 +96,12 @@ int stf_parse_options_fixed_splitpositions_nth (StfParseOptions_t *parse
/* USING the stf structs to actually do some parsing, these are the lower-level functions and utility functions */
GPtrArray *stf_parse_general (StfParseOptions_t *parseoptions,
GStringChunk *lines_chunk,
char const *data,
char const *data_end);
void stf_parse_general_free (GPtrArray *lines);
GPtrArray *stf_parse_lines (StfParseOptions_t *parseoptions,
GStringChunk *lines_chunk,
const char *data,
gboolean with_lineno);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment