30 #include <glib/gi18n.h> 45 #include <boost/regex.hpp> 46 #include <boost/regex/icu.hpp> 49 #include "gnc-imp-props-price.hpp" 54 G_GNUC_UNUSED
static QofLogModule log_module = GNC_MOD_IMPORT;
56 const int num_currency_formats_price = 3;
57 const gchar* currency_format_user_price[] = {N_(
"Locale"),
58 N_(
"Period: 123,456.78"),
59 N_(
"Comma: 123.456,78")
70 m_skip_errors =
false;
89 if (
m_tokenizer && m_settings.m_file_format == format)
92 auto new_encoding = std::string(
"UTF-8");
93 auto new_imp_file = std::string();
100 if (
file_format() == GncImpFileFormat::FIXED_WIDTH)
103 if (!fwtok->get_columns().empty())
104 m_settings.m_column_widths = fwtok->get_columns();
108 m_settings.m_file_format = format;
109 m_tokenizer = gnc_tokenizer_factory(m_settings.m_file_format);
119 if (!m_settings.m_separators.empty())
120 separators (m_settings.m_separators);
121 enable_escape (m_settings.m_enable_escape);
123 else if ((
file_format() == GncImpFileFormat::FIXED_WIDTH)
124 && !m_settings.m_column_widths.empty())
127 fwtok->columns (m_settings.m_column_widths);
133 return m_settings.m_file_format;
136 void GncPriceImport::over_write (
bool over)
140 bool GncPriceImport::over_write () {
return m_over_write; }
149 m_settings.m_from_commodity = from_commodity;
150 if (m_settings.m_from_commodity)
152 auto col_type_sym = std::find (m_settings.m_column_types_price.begin(),
153 m_settings.m_column_types_price.end(), GncPricePropType::FROM_SYMBOL);
155 if (col_type_sym != m_settings.m_column_types_price.end())
156 set_column_type_price (col_type_sym -m_settings.m_column_types_price.begin(),
157 GncPricePropType::NONE);
159 auto col_type_name = std::find (m_settings.m_column_types_price.begin(),
160 m_settings.m_column_types_price.end(), GncPricePropType::FROM_NAMESPACE);
162 if (col_type_name != m_settings.m_column_types_price.end())
163 set_column_type_price (col_type_name -m_settings.m_column_types_price.begin(),
164 GncPricePropType::NONE);
167 std::vector<GncPricePropType> commodities = { GncPricePropType::TO_CURRENCY };
168 reset_formatted_column (commodities);
171 gnc_commodity *GncPriceImport::from_commodity () {
return m_settings.m_from_commodity; }
180 m_settings.m_to_currency = to_currency;
181 if (m_settings.m_to_currency)
183 auto col_type_currency = std::find (m_settings.m_column_types_price.begin(),
184 m_settings.m_column_types_price.end(), GncPricePropType::TO_CURRENCY);
186 if (col_type_currency != m_settings.m_column_types_price.end())
187 set_column_type_price (col_type_currency -m_settings.m_column_types_price.begin(),
188 GncPricePropType::NONE);
192 std::vector<GncPricePropType> commodities = { GncPricePropType::FROM_SYMBOL };
193 reset_formatted_column (commodities);
196 gnc_commodity *GncPriceImport::to_currency () {
return m_settings.m_to_currency; }
198 void GncPriceImport::reset_formatted_column (std::vector<GncPricePropType>& col_types)
200 for (
auto col_type: col_types)
202 auto col = std::find (m_settings.m_column_types_price.begin(),
203 m_settings.m_column_types_price.end(), col_type);
204 if (col != m_settings.m_column_types_price.end())
205 set_column_type_price (col - m_settings.m_column_types_price.begin(), col_type,
true);
209 void GncPriceImport::currency_format (
int currency_format)
211 m_settings.m_currency_format = currency_format;
214 std::vector<GncPricePropType> commodities = { GncPricePropType::AMOUNT };
215 reset_formatted_column (commodities);
217 int GncPriceImport::currency_format () {
return m_settings.m_currency_format; }
219 void GncPriceImport::date_format (
int date_format)
221 m_settings.m_date_format = date_format;
224 std::vector<GncPricePropType> dates = { GncPricePropType::DATE };
225 reset_formatted_column (dates);
227 int GncPriceImport::date_format () {
return m_settings.m_date_format; }
248 m_settings.m_encoding = encoding;
251 std::string GncPriceImport::encoding () {
return m_settings.m_encoding; }
253 void GncPriceImport::update_skipped_lines(std::optional<uint32_t> start, std::optional<uint32_t> end,
254 std::optional<bool> alt, std::optional<bool> errors)
257 m_settings.m_skip_start_lines = *start;
259 m_settings.m_skip_end_lines = *end;
261 m_settings.m_skip_alt_lines = *alt;
263 m_skip_errors = *errors;
268 ((i < skip_start_lines()) ||
270 (((i - skip_start_lines()) % 2 == 1) &&
272 (m_skip_errors && !std::get<PL_ERROR>(
m_parsed_lines[i]).empty()));
276 uint32_t GncPriceImport::skip_start_lines () {
return m_settings.m_skip_start_lines; }
277 uint32_t GncPriceImport::skip_end_lines () {
return m_settings.m_skip_end_lines; }
278 bool GncPriceImport::skip_alt_lines () {
return m_settings.m_skip_alt_lines; }
279 bool GncPriceImport::skip_err_lines () {
return m_skip_errors; }
281 void GncPriceImport::separators (std::string separators)
286 m_settings.m_separators = separators;
288 csvtok->set_separators (separators);
291 std::string GncPriceImport::separators () {
return m_settings.m_separators; }
293 void GncPriceImport::enable_escape(
bool enable)
297 m_settings.m_enable_escape = enable;
299 csvtok->set_enable_escape (enable);
301 bool GncPriceImport::enable_escape() {
return m_settings.m_enable_escape; }
308 m_settings = settings;
314 separators (m_settings.m_separators);
315 else if (
file_format() == GncImpFileFormat::FIXED_WIDTH)
318 fwtok->columns (m_settings.m_column_widths);
330 std::copy_n (settings.m_column_types_price.begin(),
331 std::min (m_settings.m_column_types_price.size(), settings.m_column_types_price.size()),
332 m_settings.m_column_types_price.begin());
335 bool GncPriceImport::save_settings ()
344 if (
file_format() == GncImpFileFormat::FIXED_WIDTH)
347 m_settings.m_column_widths = fwtok->get_columns();
349 return m_settings.
save();
352 void GncPriceImport::settings_name (std::string name) { m_settings.m_name = name; }
353 std::string GncPriceImport::settings_name () {
return m_settings.m_name; }
369 catch (std::ifstream::failure& ios_err)
372 PWARN (
"Error: %s", ios_err.what());
393 uint32_t max_cols = 0;
396 for (
auto tokenized_line :
m_tokenizer->get_tokens())
398 auto length = tokenized_line.size();
400 m_parsed_lines.push_back (std::make_tuple (tokenized_line, std::string(),
401 std::make_shared<GncImportPrice>(date_format(), currency_format()),
403 if (length > max_cols)
410 throw (std::range_error (
"Tokenizing failed."));
414 m_settings.m_column_types_price.resize(max_cols, GncPricePropType::NONE);
417 for (uint32_t i = 0; i < m_settings.m_column_types_price.size(); i++)
418 set_column_type_price (i, m_settings.m_column_types_price[i],
true);
431 void add_error (std::string msg);
433 bool empty() {
return m_error.empty(); }
438 void ErrorListPrice::add_error (std::string msg)
440 m_error +=
"- " + msg +
"\n";
443 std::string ErrorListPrice::str()
445 return m_error.substr(0, m_error.size() - 1);
452 void GncPriceImport::verify_column_selections (
ErrorListPrice& error_msg)
456 if (!check_for_column_type(GncPricePropType::DATE))
457 error_msg.add_error( _(
"Please select a date column."));
461 if (!check_for_column_type(GncPricePropType::AMOUNT))
462 error_msg.add_error( _(
"Please select an amount column."));
466 if (!check_for_column_type(GncPricePropType::TO_CURRENCY))
468 if (!m_settings.m_to_currency)
469 error_msg.add_error( _(
"Please select a 'Currency to' column or set a Currency in the 'Currency To' field."));
474 if (!check_for_column_type(GncPricePropType::FROM_SYMBOL))
476 if (!m_settings.m_from_commodity)
477 error_msg.add_error( _(
"Please select a 'From Symbol' column or set a Commodity in the 'Commodity From' field."));
482 if (!check_for_column_type(GncPricePropType::FROM_NAMESPACE))
484 if (!m_settings.m_from_commodity)
485 error_msg.add_error( _(
"Please select a 'From Namespace' column or set a Commodity in the 'Commodity From' field."));
490 if ((m_settings.m_to_currency) && (m_settings.m_from_commodity))
493 error_msg.add_error( _(
"'Commodity From' can not be the same as 'Currency To'."));
505 std::string GncPriceImport::verify ()
507 auto newline = std::string();
513 error_msg.add_error(_(
"No valid data found in the selected file. It may be empty or the selected encoding is wrong."));
514 return error_msg.str();
518 auto skip_alt_offset = m_settings.m_skip_alt_lines ? 1 : 0;
519 if (m_settings.m_skip_start_lines + m_settings.m_skip_end_lines + skip_alt_offset >=
m_parsed_lines.size())
521 error_msg.add_error(_(
"No lines are selected for importing. Please reduce the number of lines to skip."));
522 return error_msg.str();
525 verify_column_selections (error_msg);
527 update_skipped_lines (std::nullopt, std::nullopt, std::nullopt, std::nullopt);
529 auto have_line_errors =
false;
532 if (!std::get<PL_SKIP>(line) && !std::get<PL_ERROR>(line).empty())
534 have_line_errors =
true;
539 if (have_line_errors)
540 error_msg.add_error( _(
"Not all fields could be parsed. Please correct the issues reported for each line or adjust the lines to skip."));
542 return error_msg.str();
549 static void price_properties_verify_essentials (std::vector<parse_line_t>::iterator& parsed_line)
551 std::string error_message;
552 std::shared_ptr<GncImportPrice> price_props;
553 std::tie(std::ignore, error_message, price_props, std::ignore) = *parsed_line;
555 auto price_error = price_props->verify_essentials();
557 error_message.clear();
558 if (!price_error.empty())
560 error_message += price_error;
561 error_message +=
"\n";
564 if (!error_message.empty())
565 throw std::invalid_argument(error_message);
568 void GncPriceImport::create_price (std::vector<parse_line_t>::iterator& parsed_line)
571 std::string error_message;
572 std::shared_ptr<GncImportPrice> price_props =
nullptr;
573 bool skip_line =
false;
574 std::tie(line, error_message, price_props, skip_line) = *parsed_line;
579 error_message.clear();
582 auto line_to_currency = price_props->get_to_currency();
583 if (!line_to_currency)
585 if (m_settings.m_to_currency)
586 price_props->set_to_currency(m_settings.m_to_currency);
591 error_message = _(
"No 'Currency to' column selected and no selected Currency specified either.\n" 592 "This should never happen. Please report this as a bug.");
593 PINFO(
"User warning: %s", error_message.c_str());
594 throw std::invalid_argument(error_message);
599 auto line_from_commodity = price_props->get_from_commodity();
600 if (!line_from_commodity)
602 if (m_settings.m_from_commodity)
603 price_props->set_from_commodity(m_settings.m_from_commodity);
608 error_message = _(
"No 'From Namespace/From Symbol' columns selected and no selected Commodity From specified either.\n" 609 "This should never happen. Please report this as a bug.");
610 PINFO(
"User warning: %s", error_message.c_str());
611 throw std::invalid_argument(error_message);
618 price_properties_verify_essentials (parsed_line);
620 QofBook* book = gnc_get_current_book();
624 auto price_created = price_props->create_price (book, pdb, m_over_write);
625 if (price_created == ADDED)
627 else if (price_created == DUPLICATED)
628 m_prices_duplicated++;
629 else if (price_created == REPLACED)
632 catch (
const std::invalid_argument& e)
634 error_message = e.what();
635 PINFO(
"User warning: %s", error_message.c_str());
649 auto verify_result = verify();
650 if (!verify_result.empty())
651 throw std::invalid_argument (verify_result);
654 m_prices_duplicated = 0;
655 m_prices_replaced = 0;
663 if ((std::get<PL_SKIP>(*parsed_lines_it)))
667 create_price (parsed_lines_it);
669 PINFO(
"Number of lines is %d, added %d, duplicated %d, replaced %d",
670 (
int)
m_parsed_lines.size(), m_prices_added, m_prices_duplicated, m_prices_replaced);
674 GncPriceImport::check_for_column_type (GncPricePropType type)
676 return (std::find (m_settings.m_column_types_price.begin(),
677 m_settings.m_column_types_price.end(), type)
678 != m_settings.m_column_types_price.end());
682 void GncPriceImport::update_price_props (uint32_t row, uint32_t col, GncPricePropType prop_type)
684 if (prop_type == GncPricePropType::NONE)
687 auto price_props = std::make_shared<GncImportPrice> (*(std::get<PL_PREPRICE>(
m_parsed_lines[row])).get());
690 price_props->reset (prop_type);
694 bool enable_test_empty =
true;
698 if (prop_type == GncPricePropType::TO_CURRENCY)
700 if (m_settings.m_from_commodity)
701 price_props->set_from_commodity (m_settings.m_from_commodity);
703 if (m_settings.m_to_currency)
704 enable_test_empty =
false;
707 if (prop_type == GncPricePropType::FROM_SYMBOL)
709 if (m_settings.m_to_currency)
710 price_props->set_to_currency (m_settings.m_to_currency);
712 if (m_settings.m_from_commodity)
713 enable_test_empty =
false;
715 price_props->set(prop_type, value, enable_test_empty);
717 catch (
const std::exception& e)
723 PINFO(
"User warning: %s", e.what());
731 GncPriceImport::set_column_type_price (uint32_t position, GncPricePropType type,
bool force)
733 if (position >= m_settings.m_column_types_price.size())
736 auto old_type = m_settings.m_column_types_price[position];
737 if ((type == old_type) && !force)
741 std::replace(m_settings.m_column_types_price.begin(), m_settings.m_column_types_price.end(),
742 type, GncPricePropType::NONE);
744 m_settings.m_column_types_price.at (position) = type;
747 if (type == GncPricePropType::FROM_NAMESPACE)
751 if (type == GncPricePropType::FROM_SYMBOL)
755 if (type == GncPricePropType::TO_CURRENCY)
766 std::get<PL_PREPRICE>(*parsed_lines_it)->set_date_format (m_settings.m_date_format);
767 std::get<PL_PREPRICE>(*parsed_lines_it)->set_currency_format (m_settings.m_currency_format);
774 if (old_type != type)
776 auto old_col = std::get<PL_INPUT>(*parsed_lines_it).size();
777 if ((old_type > GncPricePropType::NONE)
778 && (old_type <= GncPricePropType::PRICE_PROPS))
779 update_price_props (row, old_col, old_type);
782 if ((type > GncPricePropType::NONE)
783 && (type <= GncPricePropType::PRICE_PROPS))
784 update_price_props (row, position, type);
787 auto price_errors = std::get<PL_PREPRICE>(*parsed_lines_it)->errors();
788 std::get<PL_ERROR>(*parsed_lines_it) =
790 (price_errors.empty() ? std::string() :
"\n");
794 std::vector<GncPricePropType> GncPriceImport::column_types_price ()
796 return m_settings.m_column_types_price;
void create_prices()
This function will attempt to convert all tokenized lines into prices using the column types the user...
GncPriceImport(GncImpFileFormat format=GncImpFileFormat::UNKNOWN)
Constructor for GncPriceImport.
a simple price database for gnucash
~GncPriceImport()
Destructor for GncPriceImport.
utility functions for the GnuCash UI
#define PINFO(format, args...)
Print an informational note.
void tokenize(bool guessColTypes)
Splits a file into cells.
Class to convert a csv file into vector of string vectors.
gboolean gnc_commodity_equal(const gnc_commodity *a, const gnc_commodity *b)
This routine returns TRUE if the two commodities are equal.
std::unique_ptr< GncTokenizer > m_tokenizer
Will handle file loading/encoding conversion/splitting into fields.
std::vector< parse_line_t > m_parsed_lines
source file parsed into a two-dimensional array of strings.
GNCPriceDB * gnc_pricedb_get_db(QofBook *book)
Return the pricedb associated with the book.
#define PWARN(format, args...)
Log a warning.
void to_currency(gnc_commodity *to_currency)
Sets a to currency.
bool preset_is_reserved_name(const std::string &name)
Check whether name can be used as a preset name.
GncImpFileFormat
Enumeration for file formats supported by this importer.
void from_commodity(gnc_commodity *from_commodity)
Sets a from commodity.
Class to import prices from CSV or fixed width files.
Class convert a file with fixed with delimited contents into vector of string vectors.
void file_format(GncImpFileFormat format)
Sets the file format for the file to import, which may cause the file to be reloaded as well if the p...
void load_file(const std::string &filename)
Loads a file into a GncPriceImport.
void encoding(const std::string &encoding)
Converts raw file data using a new encoding.
bool save(void)
Save the gathered widget properties to a key File.
Commodity handling public routines.