railslove · janz93 · Jan 14, 2025 · Jan 14, 2025 · Dec 20, 2024 · Jan 22, 2025
diff --git a/Gemfile b/Gemfile
@@ -6,7 +6,7 @@ ruby "3.3.2"
 
 gem "activesupport"
 gem "camt_parser", git: "https://github.com/railslove/camt_parser.git"
-gem "cmxl", git: "https://github.com/railslove/cmxl"
+gem "cmxl"
 gem "epics"
 gem "faraday"
 gem "grape"

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -5,13 +5,6 @@ GIT
     camt_parser (1.0.2)
       nokogiri
 
-GIT
-  remote: https://github.com/railslove/cmxl
-  revision: b92aea3de958426119d76d043a886233a313f43f
-  specs:
-    cmxl (1.5.0)
-      rchardet
-
 GIT
   remote: https://github.com/railslove/king_dtaus_ruby_3.git
   revision: 28b077aa6b2cf5590e322d0d207236adccf2d543
@@ -43,11 +36,13 @@ GEM
       multi_json (~> 1)
       statsd-ruby (~> 1.1)
     base64 (0.2.0)
-    bigdecimal (3.1.8)
+    bigdecimal (3.1.9)
     builder (3.3.0)
+    cmxl (2.0)
+      rchardet
     coderay (1.1.3)
     concurrent-ruby (1.3.4)
-    connection_pool (2.4.1)
+    connection_pool (2.5.0)
     crack (1.0.0)
       bigdecimal
       rexml
@@ -150,7 +145,7 @@ GEM
       nio4r (~> 2.0)
     raabro (1.4.0)
     racc (1.8.0)
-    rack (2.2.9)
+    rack (2.2.10)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rack-cors (2.0.2)
@@ -160,7 +155,7 @@ GEM
       rack (>= 1.0, < 3)
     rainbow (3.1.1)
     rake (13.2.1)
-    rchardet (1.8.0)
+    rchardet (1.9.0)
     redis (4.8.1)
     regexp_parser (2.9.2)
     rest-client (2.1.0)
@@ -204,11 +199,11 @@ GEM
     rubyzip (2.3.2)
     rufus-scheduler (3.9.1)
       fugit (~> 1.1, >= 1.1.6)
-    sentry-ruby (5.18.0)
+    sentry-ruby (5.22.1)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
-    sentry-sidekiq (5.18.0)
-      sentry-ruby (~> 5.18.0)
+    sentry-sidekiq (5.22.1)
+      sentry-ruby (~> 5.22.1)
       sidekiq (>= 3.0)
     sepa_king (0.12.0)
       activemodel (>= 3.1)
@@ -261,7 +256,7 @@ DEPENDENCIES
   airborne
   barnes
   camt_parser!
-  cmxl!
+  cmxl
   database_cleaner-sequel
   dotenv
   epics

diff --git a/box/business_processes/import_bank_statement.rb b/box/business_processes/import_bank_statement.rb
@@ -1,14 +1,9 @@
 # frozen_string_literal: true
 
-require "cmxl"
-
 require_relative "../models/account"
 require_relative "../models/bank_statement"
 require_relative "../../lib/checksum_generator"
 
-# more general matching regex that covers both newlines and newlines with dashes
-Cmxl.config[:statement_separator] = /(\n-?)(?=:20)/m
-
 module Box
   module BusinessProcesses
     class ImportBankStatement

diff --git a/box/business_processes/import_statements.rb b/box/business_processes/import_statements.rb
@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "cmxl"
 require "camt_parser"
 
 require_relative "../models/account"
@@ -33,6 +32,7 @@ def self.from_bank_statement(bank_statement, upcoming = false)
 
       def self.create_statement(bank_statement, bank_transaction, upcoming = false)
         account = bank_statement.account
+        Box.logger.info("statement is expected") if !!bank_transaction.try(:expected?)
         trx = statement_attributes_from_bank_transaction(bank_transaction, bank_statement)
 
         if (statement = account.statements_dataset.first(sha: trx[:sha]))

diff --git a/box/jobs/fetch_statements.rb b/box/jobs/fetch_statements.rb
@@ -3,7 +3,6 @@
 require "sidekiq-scheduler"
 require "active_support/all"
 require "camt_parser"
-require "cmxl"
 require "epics"
 require "sequel"
 

diff --git a/box/jobs/fetch_upcoming_statements.rb b/box/jobs/fetch_upcoming_statements.rb
@@ -3,7 +3,6 @@
 require "sidekiq-scheduler"
 require "active_support/all"
 require "camt_parser"
-require "cmxl"
 require "epics"
 require "sequel"
 
@@ -38,6 +37,7 @@ def fetch_for_account(account)
 
         vmk_data = account.transport_client.VMK(safe_from.to_s(:db), safe_to.to_s(:db))
         return unless vmk_data
+        vmk_data = vmk_data.encode(vmk_data.encoding, universal_newline: true)
 
         chunks = Cmxl.parse(vmk_data)
         import_stats = import_to_database(chunks, account)
@@ -48,6 +48,10 @@ def fetch_for_account(account)
       rescue Sequel::NoMatchingRow => _ex
         Box.logger.error("[Jobs::FetchUpcomingStatements] Could not find Account ##{account.id}")
       rescue Epics::Error::BusinessError => ex
+        if ENV["SENTRY_DSN"]
+          Sentry.add_attachment(filename: "#{account.id}_vmk_mt942_#{SecureRandom.uuid}", bytes: vmk_data) if !vmk_data.empty?
+          Sentry.capture_exception(ex)
+        end
         # The BusinessError can occur when no new statements are available
         Box.logger.error("[Jobs::FetchUpcomingStatements] EBICS error. id=#{account.id} reason='#{ex.message}'")
       end

diff --git a/box/jobs/queue_fetch_statements.rb b/box/jobs/queue_fetch_statements.rb
@@ -3,7 +3,6 @@
 require "sidekiq-scheduler"
 require "active_support/all"
 require "camt_parser"
-require "cmxl"
 require "epics"
 require "sequel"
 

diff --git a/box/jobs/queue_fetch_upcoming_statements.rb b/box/jobs/queue_fetch_upcoming_statements.rb
@@ -3,7 +3,6 @@
 require "sidekiq-scheduler"
 require "active_support/all"
 require "camt_parser"
-require "cmxl"
 require "epics"
 require "sequel"
 

diff --git a/config/bootstrap.rb b/config/bootstrap.rb
@@ -11,6 +11,7 @@
 
 require_relative "configuration"
 require_relative "version"
+require_relative "initializers/cmxl"
 
 # Setup box namespace
 module Box

diff --git a/config/initializers/cmxl.rb b/config/initializers/cmxl.rb
@@ -0,0 +1,11 @@
+require "cmxl"
+
+Cmxl.config[:statement_separator] = ENV.fetch("CMXL_STATEMENT_SEPARATOR_REGREX", /\n-\s*\n/m)
+Cmxl.config[:strip_headers] = true
+
+# /\n-\s*\n/m
+# /(\n-?)(?=:20)/m
+# /^:(\d{2})(\w)?:(.*)$/
+
+# /(:20:.*?)(?=\n-|\Z)/m
+# /\{1:.*?\n-?\}/m
diff --git a/spec/business_processes/import_bank_statement_spec.rb b/spec/business_processes/import_bank_statement_spec.rb
@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "active_support/all"
-require "cmxl"
 
 require_relative "../../box/models/account"
 require_relative "../../box/business_processes/import_bank_statement"

diff --git a/spec/business_processes/import_statement_spec.rb b/spec/business_processes/import_statement_spec.rb
@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "active_support/all"
-require "cmxl"
 
 require_relative "../../box/models/account"
 require_relative "../../box/models/organization"
@@ -266,6 +265,22 @@ def exec_link_action
             expect(described_class.create_statement(mt940_bank_statement, transaction)).to be_truthy
           end
         end
+
+        context "importing vmk transaction that is expected" do
+          it "logs an info message" do
+            allow(Box.logger).to receive(:info)
+            vmk_data = File.read("spec/fixtures/expected_vmk_data.txt")
+            bank_statement = ImportBankStatement.from_mt940(vmk_data, account)
+            bank_transactions = described_class.parse_bank_statement(bank_statement)
+            transaction = bank_transactions.first
+
+            described_class.create_statement(bank_statement, transaction)
+
+            expect(Box.logger).to(
+              have_received(:info).with("statement is expected")
+            )
+          end
+        end
       end
     end
   end

diff --git a/spec/fixtures/expected_vmk_data.txt b/spec/fixtures/expected_vmk_data.txt
@@ -0,0 +1,12 @@
+RLNWATWWAMS  00001
+942 01
+ELBA-INT    
+:20:20241212100006
+:25://10020030/1234567
+:28C:24241/001
+:34F:EUR0,
+:13D:2412121000+0100
+:61:2412121212EC43374,55NTRFNONREF//950
+:86:999Mustermann GmbH
+RNR 176080 23781,44 0,00 RNR 175584 19593,11 0,00
+:90C:1EUR43374,55