Fix Statement parsing

.rspec

@@ -1 +1,2 @@
---color
+--color
+--require 'pry'

lib/cmxl.rb

@@ -11,7 +11,11 @@ def self.config
     @config
   end
   @config = {
-    statement_separator: /\n-\s*\n/m,
+    # One or more newlines
+    # followed by `-` at the beginning of a line.
+    # "Eats up" but does not require characters until the end of the line and more newlines.
+    # \R is a platform independent newline but in the negated group `[^\n\r]` that did not seem to work.
+    statement_separator: /\R+-[^\n\r]*\R*/m,
     raise_line_format_errors: true,
     strip_headers: false
   }

lib/cmxl/statement.rb

@@ -41,8 +41,8 @@ def parse!
     end
 
     def strip_headers!
-      source.gsub!(/\A.+?(?=^:)/m, '') # beginning: strip every line in the beginning that does not start with a :
-      source.gsub!(/^[^:]+\z/, '') # end: strip every line in the end that does not start with a :
+      source.gsub!(/\A.*?(?=^:)/m, '') # beginning: strip every line in the beginning that does not start with a :
+      source.gsub!(/^[^:]*\z/, '') # end: strip every line in the end that does not start with a :
       source.strip!
     end
 

spec/fixtures/mt940-abnamro.txt

@@ -0,0 +1,44 @@
+ABNANL2A
+940
+ABNANL2A
+:20:ABN AMRO BANK NV
+:25:517852257
+:28:19321/1
+:60F:C110522EUR3236,28
+:61:1105240524D9,N192NONREF
+:86:GIRO   428428 KPN - DIGITENNE    BETALINGSKENM.  000000042188659
+5314606715                       BETREFT FACTUUR D.D. 20-05-2011
+INCL. 1,44 BTW
+:61:1105210523D11,59N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.54 DIRCKIII FIL2500 KATWIJK,PAS999
+:61:1105230523D11,63N426NONREF
+:86:BEA   NR:XXX1234   23.05.11/09.08 DIGROS FIL1015 KATWIJK Z,PAS999
+:61:1105220523D11,8N426NONREF
+:86:BEA   NR:XXX1234   22.05.11/14.25 MC DONALDS A44 LEIDEN,PAS999
+:61:1105210523D13,45N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.09 PRINCE FIL. 55 KATWIJK Z,PAS999
+:61:1105210523D15,49N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.55 DIRX FIL6017 KATWIJK ZH ,PAS999
+
+:61:1105210523D107,N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.04 HANS ANDERS OPT./056 KAT,PAS999
+:61:1105220523D141,48N426NONREF
+:86:BEA   NR:XXX1234   22.05.11/13.45 MYCOM DEN HAAG  S-GRAVEN,PAS999
+:62F:C110523EUR876,84
+
+-
+
+ABNANL2A
+940
+ABNANL2A
+:20:ABN AMRO BANK NV
+:25:517852257
+:28:19322/1
+:60F:C110523EUR2876,84
+:61:1105240524D9,49N426NONREF
+:86:BEA   NR:XXX1234   24.05.11/09.18 PETS PLACE KATWIJK KATWI,PAS999
+:61:1105240524D15,N426NONREF
+:86:52.89.39.882 MYCOM DEN HAAG  S-GRAVEN,PAS999
+:62F:C110524EUR1849,75
+
+-

spec/fixtures/mt940-handelsbank.txt

@@ -0,0 +1,16 @@
+ä4:
+:20:5566778899100112
+:25:10020030/1234567
+:28C:188/1
+:60F:C130928SEK0,
+:62F:C130930SEK0,
+:64:C130930SEK0,
+-å
+ä4:
+:20:5566778899100169
+:25:10020030/1234567
+:28C:188/1
+:60F:C130928SEK0,
+:62F:C130930SEK0,
+:64:C130930SEK0,
+-å

spec/fixtures/mt940-windows-line-breaks.txt

@@ -0,0 +1,16 @@
+{1:D02AASDISLNETAXXXXXXXXXXXXX}
+{2:E623XXXXXXXXAXXXN}
+{4:
+:20:1234567
+:21:9876543210
+:25:10020030/1234567
+:28C:5/1
+:60F:C160314EUR2187,95
+:61:0211011102DR800,NSTONONREF//55555
+:86:008?00DAUERAUFTRAG?100599?20Miete November?3010020030?31234567?32MUELLER?34339
+:61:0211021102CR3000,NTRFNONREF//55555
+:86:051?00UEBERWEISUNG?100599?20Gehalt Oktober?21Firma
+Mustermann GmbH?3050060400?310847564700?32MUELLER?34339
+:62F:C160315EUR4387,95
+:86:Some random data
+-}

spec/mt940_parsing_spec.rb

@@ -95,4 +95,115 @@
       expect(subject[0].transactions.count).to eql(1)
     end
   end
+
+  describe "MT940 abnamro" do
+    it "splits the file into on statement with the headers included" do
+      expected_data = <<~MT940.chomp
+        ABNANL2A
+        940
+        ABNANL2A
+        :20:ABN AMRO BANK NV
+        :25:517852257
+        :28:19321/1
+        :60F:C110522EUR3236,28
+        :61:1105240524D9,N192NONREF
+        :86:GIRO   428428 KPN - DIGITENNE    BETALINGSKENM.  000000042188659
+        5314606715                       BETREFT FACTUUR D.D. 20-05-2011
+        INCL. 1,44 BTW
+        :61:1105210523D11,59N426NONREF
+        :86:BEA   NR:XXX1234   21.05.11/12.54 DIRCKIII FIL2500 KATWIJK,PAS999
+        :61:1105230523D11,63N426NONREF
+        :86:BEA   NR:XXX1234   23.05.11/09.08 DIGROS FIL1015 KATWIJK Z,PAS999
+        :61:1105220523D11,8N426NONREF
+        :86:BEA   NR:XXX1234   22.05.11/14.25 MC DONALDS A44 LEIDEN,PAS999
+        :61:1105210523D13,45N426NONREF
+        :86:BEA   NR:XXX1234   21.05.11/12.09 PRINCE FIL. 55 KATWIJK Z,PAS999
+        :61:1105210523D15,49N426NONREF
+        :86:BEA   NR:XXX1234   21.05.11/12.55 DIRX FIL6017 KATWIJK ZH ,PAS999
+
+        :61:1105210523D107,N426NONREF
+        :86:BEA   NR:XXX1234   21.05.11/12.04 HANS ANDERS OPT./056 KAT,PAS999
+        :61:1105220523D141,48N426NONREF
+        :86:BEA   NR:XXX1234   22.05.11/13.45 MYCOM DEN HAAG  S-GRAVEN,PAS999
+        :62F:C110523EUR876,84
+      MT940
+      stub = instance_double(Cmxl::Statement)
+      allow(Cmxl::Statement).to receive(:new).and_return(stub)
+
+      Cmxl.parse(mt940_file('mt940-abnamro'))
+
+      expect(Cmxl::Statement).to have_received(:new).with(expected_data)
+    end
+
+    it 'splits the file into two statements' do
+      allow(Cmxl::Statement).to receive(:new)
+
+      Cmxl.parse(mt940_file('mt940-abnamro'))
+
+      expect(Cmxl::Statement).to have_received(:new).twice
+    end
+  end
+
+  describe 'MT940 handelsbank' do
+    it 'splits the file with the special characters correctly' do
+      expected_data = <<~MT940.chomp
+        채4:
+        :20:5566778899100112
+        :25:10020030/1234567
+        :28C:188/1
+        :60F:C130928SEK0,
+        :62F:C130930SEK0,
+        :64:C130930SEK0,
+      MT940
+      stub = instance_double(Cmxl::Statement)
+      allow(Cmxl::Statement).to receive(:new).and_return(stub)
+
+      Cmxl.parse(mt940_file('mt940-handelsbank'))
+
+      expect(Cmxl::Statement).to have_received(:new).with(expected_data)
+    end
+
+    it 'splits the file into two statements' do
+      allow(Cmxl::Statement).to receive(:new)
+
+      Cmxl.parse(mt940_file('mt940-handelsbank'))
+
+      expect(Cmxl::Statement).to have_received(:new).twice
+    end
+  end
+
+  describe "MT940 windows line breaks" do
+    it 'splits the file with the special characters correctly' do
+      expected_data =
+        "{1:D02AASDISLNETAXXXXXXXXXXXXX}\r\n"\
+        "{2:E623XXXXXXXXAXXXN}\r\n"\
+        "{4:\r\n"\
+        ":20:1234567\r\n"\
+        ":21:9876543210\r\n"\
+        ":25:10020030/1234567\r\n"\
+        ":28C:5/1\r\n"\
+        ":60F:C160314EUR2187,95\r\n"\
+        ":61:0211011102DR800,NSTONONREF//55555\r\n"\
+        ":86:008?00DAUERAUFTRAG?100599?20Miete November?3010020030?31234567?32MUELLER?34339\r\n"\
+        ":61:0211021102CR3000,NTRFNONREF//55555\r\n"\
+        ":86:051?00UEBERWEISUNG?100599?20Gehalt Oktober?21Firma\r\n"\
+        "Mustermann GmbH?3050060400?310847564700?32MUELLER?34339\r\n"\
+        ":62F:C160315EUR4387,95\r\n"\
+        ":86:Some random data"
+      stub = instance_double(Cmxl::Statement)
+      allow(Cmxl::Statement).to receive(:new).and_return(stub)
+
+      Cmxl.parse(mt940_file('mt940-windows-line-breaks'))
+
+      expect(Cmxl::Statement).to have_received(:new).with(expected_data)
+    end
+
+    it 'splits the file into two statements' do
+      allow(Cmxl::Statement).to receive(:new)
+
+      Cmxl.parse(mt940_file('mt940-windows-line-breaks'))
+
+      expect(Cmxl::Statement).to have_received(:new).once
+    end
+  end
 end

spec/statement_spec.rb

@@ -101,7 +101,7 @@
       end
     end
 
-    describe 'statement issued over a years boudary' do
+    describe 'statement issued over a years boundary' do
       subject { Cmxl.parse(mt940_file('statement-mt940')).first.transactions.last }
 
       it { expect(subject.mt942?).to be_falsey }
@@ -167,4 +167,88 @@
     it { expect(statement.generation_date).to eql(Date.new(2019, 1, 9)) }
   end
 
+  describe "header parsing" do
+    context "when strip_headers is enabled" do
+      around do |example|
+        existing_value = Cmxl.config[:strip_headers]
+        Cmxl.config[:strip_headers] = true
+        example.run
+        Cmxl.config[:strip_headers] = existing_value
+      end
+
+      it "removes any headers" do
+        data = <<~MT940.chomp
+          {1:D02AASDISLNETAXXXXXXXXXXXXX}
+          {2:E623XXXXXXXXAXXXN}
+          {4:
+          :20:MT940/78374
+          :25:xxxxxxxxxxxxxx
+          :28C:3/1
+          :60F:C160201INR0,00
+          :61:3622687806CR1368378,92NMSC37935531
+          :86:-TX TRN-REF NO.1156ADS5601187 EUR 13456/TSV
+          :62F:C141387INR11 27421,94
+          -}
+        MT940
+
+        result = Cmxl::Statement.new(data)
+
+        expect(result.fields.count).to eq(6)
+      end
+
+      it "does nothing if there are no headers" do
+        data = <<~MT940.chomp
+          :20:MT940/78374
+          :25:xxxxxxxxxxxxxx
+          :28C:3/1
+          :60F:C160201INR0,00
+          :61:3622687806CR1368378,92NMSC37935531
+          :86:-TX TRN-REF NO.1156ADS5601187 EUR 13456/TSV
+          :62F:C141387INR11 27421,94
+        MT940
+
+        result = Cmxl::Statement.new(data)
+
+        expect(result.fields.count).to eq(6)
+      end
+    end
+
+    context "when strip_headers is disabled" do
+      it "raise an parsing error exception if headers are present" do
+        data = <<~MT940.chomp
+          {1:D02AASDISLNETAXXXXXXXXXXXXX}
+          {2:E623XXXXXXXXAXXXN}
+          {4:
+          :20:MT940/78374
+          :25:xxxxxxxxxxxxxx
+          :28C:3/1
+          :60F:C160201INR0,00
+          :61:3622687806CR1368378,92NMSC37935531
+          :86:-TX TRN-REF NO.1156ADS5601187 EUR 13456/TSV
+          :62F:C141387INR11 27421,94
+          -}
+        MT940
+
+        expect{
+          Cmxl::Statement.new(data)
+        }.to raise_error(Cmxl::Field::LineFormatError)
+      end
+
+      it "extracts the field" do
+        data = <<~MT940.chomp
+          :20:MT940/78374
+          :25:xxxxxxxxxxxxxx
+          :28C:3/1
+          :60F:C160201INR0,00
+          :61:3622687806CR1368378,92NMSC37935531
+          :86:-TX TRN-REF NO.1156ADS5601187 EUR 13456/TSV
+          :62F:C141387INR11 27421,94
+        MT940
+
+        result = Cmxl::Statement.new(data)
+
+        expect(result.fields.count).to eq(6)
+      end
+    end
+  end
 end