diff --git a/Gemfile b/Gemfile index 7277a8a6..4b96e89e 100644 --- a/Gemfile +++ b/Gemfile @@ -6,7 +6,7 @@ ruby "3.3.2" gem "activesupport" gem "camt_parser", git: "https://github.com/railslove/camt_parser.git" -gem "cmxl", git: "https://github.com/railslove/cmxl" +gem "cmxl" gem "epics" gem "faraday" gem "grape" diff --git a/Gemfile.lock b/Gemfile.lock index 0b4a9055..6a82ea22 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -5,13 +5,6 @@ GIT camt_parser (1.0.2) nokogiri -GIT - remote: https://github.com/railslove/cmxl - revision: b92aea3de958426119d76d043a886233a313f43f - specs: - cmxl (1.5.0) - rchardet - GIT remote: https://github.com/railslove/king_dtaus_ruby_3.git revision: 28b077aa6b2cf5590e322d0d207236adccf2d543 @@ -43,11 +36,13 @@ GEM multi_json (~> 1) statsd-ruby (~> 1.1) base64 (0.2.0) - bigdecimal (3.1.8) + bigdecimal (3.1.9) builder (3.3.0) + cmxl (2.0) + rchardet coderay (1.1.3) concurrent-ruby (1.3.4) - connection_pool (2.4.1) + connection_pool (2.5.0) crack (1.0.0) bigdecimal rexml @@ -150,7 +145,7 @@ GEM nio4r (~> 2.0) raabro (1.4.0) racc (1.8.0) - rack (2.2.9) + rack (2.2.10) rack-accept (0.4.5) rack (>= 0.4) rack-cors (2.0.2) @@ -160,7 +155,7 @@ GEM rack (>= 1.0, < 3) rainbow (3.1.1) rake (13.2.1) - rchardet (1.8.0) + rchardet (1.9.0) redis (4.8.1) regexp_parser (2.9.2) rest-client (2.1.0) @@ -204,11 +199,11 @@ GEM rubyzip (2.3.2) rufus-scheduler (3.9.1) fugit (~> 1.1, >= 1.1.6) - sentry-ruby (5.18.0) + sentry-ruby (5.22.1) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) - sentry-sidekiq (5.18.0) - sentry-ruby (~> 5.18.0) + sentry-sidekiq (5.22.1) + sentry-ruby (~> 5.22.1) sidekiq (>= 3.0) sepa_king (0.12.0) activemodel (>= 3.1) @@ -261,7 +256,7 @@ DEPENDENCIES airborne barnes camt_parser! - cmxl! + cmxl database_cleaner-sequel dotenv epics diff --git a/box/business_processes/import_bank_statement.rb b/box/business_processes/import_bank_statement.rb index 50963b63..b1e3d590 100644 --- a/box/business_processes/import_bank_statement.rb +++ b/box/business_processes/import_bank_statement.rb @@ -1,14 +1,9 @@ # frozen_string_literal: true -require "cmxl" - require_relative "../models/account" require_relative "../models/bank_statement" require_relative "../../lib/checksum_generator" -# more general matching regex that covers both newlines and newlines with dashes -Cmxl.config[:statement_separator] = /(\n-?)(?=:20)/m - module Box module BusinessProcesses class ImportBankStatement diff --git a/box/business_processes/import_statements.rb b/box/business_processes/import_statements.rb index b89fb973..5846952a 100644 --- a/box/business_processes/import_statements.rb +++ b/box/business_processes/import_statements.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require "cmxl" require "camt_parser" require_relative "../models/account" @@ -33,6 +32,7 @@ def self.from_bank_statement(bank_statement, upcoming = false) def self.create_statement(bank_statement, bank_transaction, upcoming = false) account = bank_statement.account + Box.logger.info("statement is expected") if !!bank_transaction.try(:expected?) trx = statement_attributes_from_bank_transaction(bank_transaction, bank_statement) if (statement = account.statements_dataset.first(sha: trx[:sha])) diff --git a/box/jobs/fetch_statements.rb b/box/jobs/fetch_statements.rb index 9215b976..4d4ce31a 100644 --- a/box/jobs/fetch_statements.rb +++ b/box/jobs/fetch_statements.rb @@ -3,7 +3,6 @@ require "sidekiq-scheduler" require "active_support/all" require "camt_parser" -require "cmxl" require "epics" require "sequel" diff --git a/box/jobs/fetch_upcoming_statements.rb b/box/jobs/fetch_upcoming_statements.rb index 6116c4aa..8c2610fa 100644 --- a/box/jobs/fetch_upcoming_statements.rb +++ b/box/jobs/fetch_upcoming_statements.rb @@ -3,7 +3,6 @@ require "sidekiq-scheduler" require "active_support/all" require "camt_parser" -require "cmxl" require "epics" require "sequel" @@ -38,6 +37,7 @@ def fetch_for_account(account) vmk_data = account.transport_client.VMK(safe_from.to_s(:db), safe_to.to_s(:db)) return unless vmk_data + vmk_data = vmk_data.encode(vmk_data.encoding, universal_newline: true) chunks = Cmxl.parse(vmk_data) import_stats = import_to_database(chunks, account) @@ -48,6 +48,10 @@ def fetch_for_account(account) rescue Sequel::NoMatchingRow => _ex Box.logger.error("[Jobs::FetchUpcomingStatements] Could not find Account ##{account.id}") rescue Epics::Error::BusinessError => ex + if ENV["SENTRY_DSN"] + Sentry.add_attachment(filename: "#{account.id}_vmk_mt942_#{SecureRandom.uuid}", bytes: vmk_data) if !vmk_data.empty? + Sentry.capture_exception(ex) + end # The BusinessError can occur when no new statements are available Box.logger.error("[Jobs::FetchUpcomingStatements] EBICS error. id=#{account.id} reason='#{ex.message}'") end diff --git a/box/jobs/queue_fetch_statements.rb b/box/jobs/queue_fetch_statements.rb index d50d4ea9..fdbeb77f 100644 --- a/box/jobs/queue_fetch_statements.rb +++ b/box/jobs/queue_fetch_statements.rb @@ -3,7 +3,6 @@ require "sidekiq-scheduler" require "active_support/all" require "camt_parser" -require "cmxl" require "epics" require "sequel" diff --git a/box/jobs/queue_fetch_upcoming_statements.rb b/box/jobs/queue_fetch_upcoming_statements.rb index bdfa8299..b9d8094a 100644 --- a/box/jobs/queue_fetch_upcoming_statements.rb +++ b/box/jobs/queue_fetch_upcoming_statements.rb @@ -3,7 +3,6 @@ require "sidekiq-scheduler" require "active_support/all" require "camt_parser" -require "cmxl" require "epics" require "sequel" diff --git a/config/bootstrap.rb b/config/bootstrap.rb index 5b1ef5e3..8548da3e 100644 --- a/config/bootstrap.rb +++ b/config/bootstrap.rb @@ -11,6 +11,7 @@ require_relative "configuration" require_relative "version" +require_relative "initializers/cmxl" # Setup box namespace module Box diff --git a/config/initializers/cmxl.rb b/config/initializers/cmxl.rb new file mode 100644 index 00000000..b0f68d6d --- /dev/null +++ b/config/initializers/cmxl.rb @@ -0,0 +1,11 @@ +require "cmxl" + +Cmxl.config[:statement_separator] = ENV.fetch("CMXL_STATEMENT_SEPARATOR_REGREX", /\n-\s*\n/m) +Cmxl.config[:strip_headers] = true + +# /\n-\s*\n/m +# /(\n-?)(?=:20)/m +# /^:(\d{2})(\w)?:(.*)$/ + +# /(:20:.*?)(?=\n-|\Z)/m +# /\{1:.*?\n-?\}/m diff --git a/spec/business_processes/import_bank_statement_spec.rb b/spec/business_processes/import_bank_statement_spec.rb index e6553364..6facf9ff 100644 --- a/spec/business_processes/import_bank_statement_spec.rb +++ b/spec/business_processes/import_bank_statement_spec.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "active_support/all" -require "cmxl" require_relative "../../box/models/account" require_relative "../../box/business_processes/import_bank_statement" diff --git a/spec/business_processes/import_statement_spec.rb b/spec/business_processes/import_statement_spec.rb index 8d541537..4c081e61 100644 --- a/spec/business_processes/import_statement_spec.rb +++ b/spec/business_processes/import_statement_spec.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "active_support/all" -require "cmxl" require_relative "../../box/models/account" require_relative "../../box/models/organization" @@ -266,6 +265,22 @@ def exec_link_action expect(described_class.create_statement(mt940_bank_statement, transaction)).to be_truthy end end + + context "importing vmk transaction that is expected" do + it "logs an info message" do + allow(Box.logger).to receive(:info) + vmk_data = File.read("spec/fixtures/expected_vmk_data.txt") + bank_statement = ImportBankStatement.from_mt940(vmk_data, account) + bank_transactions = described_class.parse_bank_statement(bank_statement) + transaction = bank_transactions.first + + described_class.create_statement(bank_statement, transaction) + + expect(Box.logger).to( + have_received(:info).with("statement is expected") + ) + end + end end end end diff --git a/spec/fixtures/expected_vmk_data.txt b/spec/fixtures/expected_vmk_data.txt new file mode 100644 index 00000000..2cb08532 --- /dev/null +++ b/spec/fixtures/expected_vmk_data.txt @@ -0,0 +1,12 @@ +RLNWATWWAMS 00001 +942 01 +ELBA-INT +:20:20241212100006 +:25://10020030/1234567 +:28C:24241/001 +:34F:EUR0, +:13D:2412121000+0100 +:61:2412121212EC43374,55NTRFNONREF//950 +:86:999Mustermann GmbH +RNR 176080 23781,44 0,00 RNR 175584 19593,11 0,00 +:90C:1EUR43374,55 \ No newline at end of file