Commit add89973 authored by Murray Cumming's avatar Murray Cumming
Browse files

Base_DB: Move export_data_to_stream() to DbUtils.

Also moving get_table_fields_to_show_for_sequence() to Utils.

This lets it be tested separately from the UI widgets.
parent 3c1d01f1
......@@ -35,6 +35,7 @@
#include <glom/glade_utils.h>
#include <libglom/algorithms_utils.h>
#include <libglom/db_utils.h>
#include <libglom/db_utils_export.h>
#include <libglom/privs.h>
#include <glom/python_embed/python_ui_callbacks.h>
#include <glom/python_embed/glom_python.h>
......@@ -2027,7 +2028,7 @@ void AppWindow::on_menu_file_save_as_example()
Document::type_example_rows example_rows;
FoundSet found_set;
found_set.m_table_name = table_name;
m_pFrame->export_data_to_vector(example_rows, found_set, sequence);
DbUtils::export_data_to_vector(document, example_rows, found_set, sequence);
//std::cout << " debug after row_text=" << row_text << std::endl;
document->set_table_example_data(table_name, example_rows);
......
......@@ -504,31 +504,6 @@ std::shared_ptr<LayoutItem_Image> Base_DB::offer_imageobject(const std::shared_p
#endif // !GLOM_ENABLE_CLIENT_ONLY
//static:
bool Base_DB::get_field_primary_key_index_for_fields(const type_vec_fields& fields, guint& field_column)
{
//Initialize input parameter:
field_column = 0;
//TODO_performance: Cache the primary key?
guint col = 0;
guint cols_count = fields.size();
while(col < cols_count)
{
if(fields[col]->get_primary_key())
{
field_column = col;
return true;
}
else
{
++col;
}
}
return false; //Not found.
}
std::shared_ptr<Field> Base_DB::get_field_primary_key_for_table(const Glib::ustring& table_name) const
{
const auto document = get_document();
......@@ -551,161 +526,6 @@ std::shared_ptr<Field> Base_DB::get_field_primary_key_for_table(const Glib::ustr
return std::shared_ptr<Field>();
}
void Base_DB::get_table_fields_to_show_for_sequence_add_group(const Glib::ustring& table_name, const Privileges& table_privs, const type_vec_fields& all_db_fields, const std::shared_ptr<LayoutGroup>& group, Base_DB::type_vecConstLayoutFields& vecFields) const
{
const auto document = std::dynamic_pointer_cast<const Document>(get_document());
//g_warning("Box_Data::get_table_fields_to_show_for_sequence_add_group(): table_name=%s, all_db_fields.size()=%d, group->name=%s", table_name.c_str(), all_db_fields.size(), group->get_name().c_str());
for(const auto& item : group->get_items())
{
auto item_field = std::dynamic_pointer_cast<LayoutItem_Field>(item);
if(item_field)
{
//Get the field info:
const auto field_name = item->get_name();
if(item_field->get_has_relationship_name()) //If it's a field in a related table.
{
//TODO_Performance: get_fields_for_table_one_field() is probably very inefficient
auto field = DbUtils::get_fields_for_table_one_field(document, item_field->get_table_used(table_name), item->get_name());
if(field)
{
auto layout_item = item_field;
layout_item->set_full_field_details(field); //Fill in the full field information for later.
//TODO_Performance: We do this once for each related field, even if there are 2 from the same table:
const auto privs_related = Privs::get_current_privs(item_field->get_table_used(table_name));
layout_item->m_priv_view = privs_related.m_view;
layout_item->m_priv_edit = privs_related.m_edit;
vecFields.emplace_back(layout_item);
}
else
{
std::cerr << G_STRFUNC << ": related field not found: field=" << item->get_layout_display_name() << std::endl;
}
}
else //It's a regular field in the table:
{
const auto iterFind = find_if_same_name(all_db_fields, field_name);
//If the field does not exist anymore then we won't try to show it:
if(iterFind != all_db_fields.end() )
{
auto layout_item = item_field;
layout_item->set_full_field_details(*iterFind); //Fill the LayoutItem with the full field information.
//std::cout << "debug: " << G_STRFUNC << ": name=" << layout_item->get_name() << std::endl;
//Prevent editing of the field if the user may not edit this table:
layout_item->m_priv_view = table_privs.m_view;
layout_item->m_priv_edit = table_privs.m_edit;
vecFields.emplace_back(layout_item);
}
}
}
else
{
auto item_group = std::dynamic_pointer_cast<LayoutGroup>(item);
if(item_group)
{
auto item_portal = std::dynamic_pointer_cast<LayoutItem_Portal>(item);
if(!item_portal) //Do not recurse into portals. They are filled by means of a separate SQL query.
{
//Recurse:
get_table_fields_to_show_for_sequence_add_group(table_name, table_privs, all_db_fields, item_group, vecFields);
}
}
}
}
if(vecFields.empty())
{
//std::cerr << G_STRFUNC << ": Returning empty list.\n";
}
}
Base_DB::type_vecConstLayoutFields Base_DB::get_table_fields_to_show_for_sequence(const Glib::ustring& table_name, const Document::type_list_layout_groups& mapGroupSequence) const
{
const auto pDoc = std::dynamic_pointer_cast<const Document>(get_document());
//Get field definitions from the database, with corrections from the document:
type_vec_fields all_fields = DbUtils::get_fields_for_table(pDoc, table_name);
const auto table_privs = Privs::get_current_privs(table_name);
//Get fields that the document says we should show:
type_vecConstLayoutFields result;
if(pDoc)
{
if(mapGroupSequence.empty())
{
//No field sequence has been saved in the document, so we use all fields by default, so we start with something visible:
//Start with the Primary Key as the first field:
guint iPrimaryKey = 0;
bool bPrimaryKeyFound = get_field_primary_key_index_for_fields(all_fields, iPrimaryKey);
Glib::ustring primary_key_field_name;
if(bPrimaryKeyFound)
{
auto layout_item = std::make_shared<LayoutItem_Field>();
layout_item->set_full_field_details(all_fields[iPrimaryKey]);
//Don't use thousands separators with ID numbers:
layout_item->m_formatting.m_numeric_format.m_use_thousands_separator = false;
layout_item->set_editable(true); //A sensible default.
//Prevent editing of the field if the user may not edit this table:
layout_item->m_priv_view = table_privs.m_view;
layout_item->m_priv_edit = table_privs.m_edit;
result.emplace_back(layout_item);
}
//Add the rest:
for(const auto& field_info : all_fields)
{
if(field_info->get_name() != primary_key_field_name) //We already added the primary key.
{
auto layout_item = std::make_shared<LayoutItem_Field>();
layout_item->set_full_field_details(field_info);
layout_item->set_editable(true); //A sensible default.
//Prevent editing of the field if the user may not edit this table:
layout_item->m_priv_view = table_privs.m_view;
layout_item->m_priv_edit = table_privs.m_edit;
result.emplace_back(layout_item);
}
}
}
else
{
//We will show the fields that the document says we should:
for(const auto& group : mapGroupSequence)
{
if(true) //!group->get_hidden())
{
//Get the fields:
get_table_fields_to_show_for_sequence_add_group(table_name, table_privs, all_fields, group, result);
}
}
}
}
if(result.empty())
{
//std::cerr << G_STRFUNC << ": Returning empty list.\n";
}
return result;
}
void Base_DB::calculate_field_in_all_records(const Glib::ustring& table_name, const std::shared_ptr<const Field>& field)
{
auto primary_key = get_field_primary_key_for_table(table_name);
......
......@@ -275,9 +275,6 @@ protected:
virtual void on_userlevel_changed(AppState::userlevels userlevel);
type_vecConstLayoutFields get_table_fields_to_show_for_sequence(const Glib::ustring& table_name, const Document::type_list_layout_groups& mapGroupSequence) const;
void get_table_fields_to_show_for_sequence_add_group(const Glib::ustring& table_name, const Privileges& table_privs, const type_vec_fields& all_db_fields, const std::shared_ptr<LayoutGroup>& group, type_vecConstLayoutFields& vecFields) const;
bool get_primary_key_is_in_foundset(const FoundSet& found_set, const Gnome::Gda::Value& primary_key_value);
......@@ -291,8 +288,6 @@ protected:
static Glib::RefPtr<Gnome::Gda::Connection> get_connection();
static bool get_field_primary_key_index_for_fields(const type_vec_fields& fields, guint& field_column);
typedef std::vector<Glib::ustring> type_vec_strings;
static type_vec_strings util_vecStrings_from_Fields(const type_vec_fields& fields);
......
......@@ -26,6 +26,7 @@
#include <glom/import_csv/dialog_import_csv.h>
#include <glom/import_csv/dialog_import_csv_progress.h>
#include <libglom/appstate.h>
#include <libglom/db_utils_export.h>
#include <libglom/connectionpool.h>
......@@ -64,6 +65,7 @@
#include <glom/filechooser_export.h>
#include <libglom/privs.h>
#include <libglom/db_utils.h>
#include <libglom/db_utils_export.h>
#include <sstream> //For stringstream.
#include <fstream>
#include <glibmm/i18n.h>
......@@ -581,147 +583,7 @@ void Frame_Glom::on_menu_file_export()
return;
}
export_data_to_stream(the_stream, found_set, mapGroupSequence);
}
//TODO: Reduce copy/pasting in these export_data_to_*() methods:
void Frame_Glom::export_data_to_vector(Document::type_example_rows& the_vector, const FoundSet& found_set, const Document::type_list_layout_groups& sequence)
{
type_vecConstLayoutFields fieldsSequence = get_table_fields_to_show_for_sequence(found_set.m_table_name, sequence);
if(fieldsSequence.empty())
{
std::cerr << G_STRFUNC << ": No fields in sequence.\n";
return;
}
auto query = Utils::build_sql_select_with_where_clause(found_set.m_table_name, fieldsSequence, found_set.m_where_clause, found_set.m_extra_join, found_set.m_sort_clause);
//TODO: Lock the database (prevent changes) during export.
auto result = DbUtils::query_execute_select(query);
guint rows_count = 0;
if(result)
rows_count = result->get_n_rows();
if(rows_count)
{
const guint columns_count = result->get_n_columns();
for(guint row_index = 0; row_index < rows_count; ++row_index)
{
Document::type_row_data row_data;
for(guint col_index = 0; col_index < columns_count; ++col_index)
{
const auto value = result->get_value_at(col_index, row_index);
auto layout_item = fieldsSequence[col_index];
//if(layout_item->m_field.get_glom_type() != Field::glom_field_type::IMAGE) //This is too much data.
//{
//Output data in canonical SQL format, ignoring the user's locale, and ignoring the layout formatting:
row_data.emplace_back(value); //TODO_Performance: reserve the size.
//if(layout_item->m_field.get_glom_type() == Field::glom_field_type::IMAGE) //This is too much data.
//{
//std::cout << " field name=" << layout_item->get_name() << ", value=" << layout_item->m_field.sql(value) << std::endl;
//}
}
//std::cout << " row_string=" << row_string << std::endl;
the_vector.emplace_back(row_data); //TODO_Performance: Reserve the size.
}
}
}
void Frame_Glom::export_data_to_stream(std::ostream& the_stream, const FoundSet& found_set, const Document::type_list_layout_groups& sequence)
{
type_vecConstLayoutFields fieldsSequence = get_table_fields_to_show_for_sequence(found_set.m_table_name, sequence);
if(fieldsSequence.empty())
{
std::cerr << G_STRFUNC << ": No fields in sequence.\n";
return;
}
auto query = Utils::build_sql_select_with_where_clause(found_set.m_table_name, fieldsSequence, found_set.m_where_clause, found_set.m_extra_join, found_set.m_sort_clause);
//TODO: Lock the database (prevent changes) during export.
auto result = DbUtils::query_execute_select(query);
guint rows_count = 0;
if(result)
rows_count = result->get_n_rows();
if(rows_count)
{
const guint columns_count = result->get_n_columns();
for(guint row_index = 0; row_index < rows_count; ++row_index)
{
std::string row_string;
for(guint col_index = 0; col_index < columns_count; ++col_index)
{
const auto value = result->get_value_at(col_index, row_index);
auto layout_item = fieldsSequence[col_index];
//if(layout_item->m_field.get_glom_type() != Field::glom_field_type::IMAGE) //This is too much data.
//{
if(!row_string.empty())
row_string += ",";
//Output data in canonical SQL format, ignoring the user's locale, and ignoring the layout formatting:
auto field = layout_item->get_full_field_details();
if(!field)
{
std::cerr << G_STRFUNC << ": A field was null.\n";
return;
}
const auto field_text = field->to_file_format(value);
if(layout_item->get_glom_type() == Field::glom_field_type::IMAGE) //This is too much data.
{
// Some extra debug checks,
// though we believe that all these problems are now fixed in File::to_file_format():
const char* newline_to_find = "\r\n";
size_t pos = field_text.find_first_of(newline_to_find);
if(pos != std::string::npos)
{
std::cerr << G_STRFUNC << ": export: binary data field text contains an unexpected newline: " << field_text << std::endl;
continue;
}
const char* quote_to_find = "\"";
pos = field_text.find_first_of(quote_to_find);
if(pos != std::string::npos)
{
std::cerr << G_STRFUNC << ": export: binary data field text contains an unexpected quote: " << field_text << std::endl;
continue;
}
}
if(layout_item->get_glom_type() == Field::glom_field_type::TEXT)
{
//The CSV RFC says text may be quoted and should be if it has newlines:
//TODO: Escape the text?
row_string += ("\"" + field_text + "\"");
}
else
row_string += field_text;
//std::cout << " field name=" << layout_item->get_name() << ", value=" << layout_item->m_field.sql(value) << std::endl;
//}
}
//std::cout << " row_string=" << row_string << std::endl;
the_stream << row_string << std::endl;
}
}
DbUtils::export_data_to_stream(document, the_stream, found_set, mapGroupSequence);
}
void Frame_Glom::on_menu_file_import()
......
......@@ -171,9 +171,6 @@ public:
void set_enable_layout_drag_and_drop(bool enable = true);
#endif // !GLOM_ENABLE_CLIENT_ONLY
void export_data_to_vector(Document::type_example_rows& the_vector, const FoundSet& found_set, const Document::type_list_layout_groups& sequence);
void export_data_to_stream(std::ostream& the_stream, const FoundSet& found_set, const Document::type_list_layout_groups& sequence);
/** Show the table again. For instance, if the document has changed, or we want to display it differently.
*/
void show_table_refresh();
......
/* Glom
*
* Copyright (C) 2001-2010 Murray Cumming
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program; if not, write to the
* Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301 USA.
*/
#include <libglom/db_utils_export.h>
#include <libglom/db_utils.h>
#include <libglom/utils.h>
#include <libglom/algorithms_utils.h>
#include <libgdamm/metastore.h>
#include <iostream>
namespace Glom
{
namespace DbUtils
{
//TODO: Reduce copy/pasting in these export_data_to_*() methods:
void export_data_to_vector(const std::shared_ptr<Document>& document, Document::type_example_rows& the_vector, const FoundSet& found_set, const Document::type_list_layout_groups& sequence)
{
auto fieldsSequence = Utils::get_table_fields_to_show_for_sequence(document, found_set.m_table_name, sequence);
if(fieldsSequence.empty())
{
std::cerr << G_STRFUNC << ": No fields in sequence.\n";
return;
}
auto query = Utils::build_sql_select_with_where_clause(found_set.m_table_name, fieldsSequence, found_set.m_where_clause, found_set.m_extra_join, found_set.m_sort_clause);
//TODO: Lock the database (prevent changes) during export.
auto result = DbUtils::query_execute_select(query);
guint rows_count = 0;
if(result)
rows_count = result->get_n_rows();
if(rows_count)
{
const guint columns_count = result->get_n_columns();
for(guint row_index = 0; row_index < rows_count; ++row_index)
{
Document::type_row_data row_data;
for(guint col_index = 0; col_index < columns_count; ++col_index)
{
const auto value = result->get_value_at(col_index, row_index);
auto layout_item = fieldsSequence[col_index];
//if(layout_item->m_field.get_glom_type() != Field::glom_field_type::IMAGE) //This is too much data.
//{
//Output data in canonical SQL format, ignoring the user's locale, and ignoring the layout formatting:
row_data.emplace_back(value); //TODO_Performance: reserve the size.
//if(layout_item->m_field.get_glom_type() == Field::glom_field_type::IMAGE) //This is too much data.
//{
//std::cout << " field name=" << layout_item->get_name() << ", value=" << layout_item->m_field.sql(value) << std::endl;
//}
}
//std::cout << " row_string=" << row_string << std::endl;
the_vector.emplace_back(row_data); //TODO_Performance: Reserve the size.
}
}
}
void export_data_to_stream(const std::shared_ptr<Document>& document, std::ostream& the_stream, const FoundSet& found_set, const Document::type_list_layout_groups& sequence)
{
auto fieldsSequence = Utils::get_table_fields_to_show_for_sequence(document, found_set.m_table_name, sequence);
if(fieldsSequence.empty())
{
std::cerr << G_STRFUNC << ": No fields in sequence.\n";
return;
}
auto query = Utils::build_sql_select_with_where_clause(found_set.m_table_name, fieldsSequence, found_set.m_where_clause, found_set.m_extra_join, found_set.m_sort_clause);
//TODO: Lock the database (prevent changes) during export.
auto result = DbUtils::query_execute_select(query);
guint rows_count = 0;
if(result)
rows_count = result->get_n_rows();
if(rows_count)
{
const guint columns_count = result->get_n_columns();
for(guint row_index = 0; row_index < rows_count; ++row_index)
{
std::string row_string;
for(guint col_index = 0; col_index < columns_count; ++col_index)
{
const auto value = result->get_value_at(col_index, row_index);
auto layout_item = fieldsSequence[col_index];
//if(layout_item->m_field.get_glom_type() != Field::glom_field_type::IMAGE) //This is too much data.
//{
if(!row_string.empty())
row_string += ",";
//Output data in canonical SQL format, ignoring the user's locale, and ignoring the layout formatting:
auto field = layout_item->get_full_field_details();
if(!field)
{
std::cerr << G_STRFUNC << ": A field was null.\n";
return;
}
const auto field_text = field->to_file_format(value);
if(layout_item->get_glom_type() == Field::glom_field_type::IMAGE) //This is too much data.
{
// Some extra debug checks,
// though we believe that all these problems are now fixed in File::to_file_format():
const char* newline_to_find = "\r\n";
size_t pos = field_text.find_first_of(newline_to_find);
if(pos != std::string::npos)
{
std::cerr << G_STRFUNC << ": export: binary data field text contains an unexpected newline: " << field_text << std::endl;
continue;
}
const char* quote_to_find = "\"";
pos = field_text.find_first_of(quote_to_find);
if(pos != std::string::npos)
{
std::cerr << G_STRFUNC << ": export: binary data field text contains an unexpected quote: " << field_text << std::endl;
continue;
}
}
if(layout_item->get_glom_type() == Field::glom_field_type::TEXT)
{
//The CSV RFC says text may be quoted and should be if it has newlines:
//TODO: Escape the text?
row_string += ("\"" + field_text + "\"");
}
else
row_string += field_text;
//std::cout << " field name=" << layout_item->get_name() << ", value=" << layout_item->m_field.sql(value) << std::endl;
//}
}
//std::cout << " row_string=" << row_string << std::endl;
the_stream << row_string << std::endl;
}
}
}
} //namespace DbUtils
} //namespace Glom
/* Glom
*
* Copyright (C) 2001-2016 Murray Cumming
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program; if not, write to the
* Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301 USA.
*/
#ifndef GLOM_DB_UTILS_EXPORT_H
#define GLOM_DB_UTILS_EXPORT_H
#include <libglom/document/document.h>
#include <libglom/data_structure/system_prefs.h>
#include <memory> //For shared_ptr<>.
namespace Glom
{
namespace DbUtils
{
void export_data_to_vector(const std::<