From 73f3af4c9524c9f1a2132aef5a6688f26b0436fc Mon Sep 17 00:00:00 2001 From: Moritz Wiesinger Date: Wed, 13 Nov 2024 07:55:01 +0100 Subject: [PATCH] [chore] enhance log body filtering (#370) --- config_examples/masking_logbody.yaml | 8 +++---- .../testbed/integration/filtering_test.go | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/config_examples/masking_logbody.yaml b/config_examples/masking_logbody.yaml index 457145e9..668066c2 100644 --- a/config_examples/masking_logbody.yaml +++ b/config_examples/masking_logbody.yaml @@ -16,16 +16,16 @@ processors: log_statements: - context: log statements: - - replace_pattern(body, "(^|\\s)3\\s*[47](\\s*[0-9]){9}((\\s*[0-9]){4})($|\\s)", "$$1**** $$3$$5") - - replace_pattern(body, "(^|\\s)(5[1-5]([0-9]){2}|222[1-9]|22[3-9]\\d|2[3-6]\\d{2}|27[0-1]\\d|2720)(\\s*[0-9]){8}\\s*([0-9]{4})($|\\s)", "$$1**** $$5$$6") - - replace_pattern(body, "(^|\\s)4(\\s*[0-9]){8,14}\\s*(([0-9]\\s*?){4})($|\\s)", "$$1**** $$3$$5") + - replace_pattern(body, "(^|[\\s\"'])3\\s*[47](\\s*[0-9]){9}((\\s*[0-9]){4})($|[\\s\"'])", "$$1**** $$3$$5") + - replace_pattern(body, "(^|[\\s\"'])(5[1-5]([0-9]){2}|222[1-9]|22[3-9]\\d|2[3-6]\\d{2}|27[0-1]\\d|2720)(\\s*[0-9]){8}\\s*([0-9]{4})($|[\\s\"'])", "$$1**** $$5$$6") + - replace_pattern(body, "(^|[\\s\"'])4(\\s*[0-9]){8,14}\\s*(([0-9]\\s*?){4})($|[\\s\"'])", "$$1**** $$3$$5") transform/iban: error_mode: ignore trace_statements: log_statements: - context: log statements: - - replace_pattern(body, "(^|\\s)([A-Z]{2})[0-9]{2}((\\s*[A-Z0-9]){4,26})\\s*((\\s*[A-Z0-9]){4})($|\\s)", "$$1$$2 **** $$5$$7") + - replace_pattern(body, "(^|[\\s\"'])([A-Z]{2})[0-9]{2}((\\s*[A-Z0-9]){4,26})\\s*((\\s*[A-Z0-9]){4})($|[\\s\"'])", "$$1$$2 **** $$5$$7") service: diff --git a/internal/testbed/integration/filtering_test.go b/internal/testbed/integration/filtering_test.go index 4f72de9f..981ca0a7 100644 --- a/internal/testbed/integration/filtering_test.go +++ b/internal/testbed/integration/filtering_test.go @@ -161,26 +161,35 @@ func TestFilteringLogBody(t *testing.T) { "card_master_spaces2 5105 1051 0510 5100", "card_master_spaces3 2720 1051 0510 5100", "card_master_spaces4 2720 1051 0520 5100 some text after", + "2367 8901 2345 6789", "card_master_no_spaces1 2367890123456789", "card_master_no_spaces2 5105105105105100", "card_master_no_spaces3 2720105105105100", "card_master_no_spaces4 2720145107105100 testy test", + "quoted master spaces \"5105 1051 0510 5100\" here", + "quoted master '2367890123456789' there", "card_visa_spaces1 4539 1488 0343 6467", "card_visa_spaces2 4539 1488 0343 6", "card_visa_spaces3 4539 1488 0343 6467 234", "card_visa_spaces4 4539 1498 0343 6457 234 testy test", + "4539 1488 0343 6467", "card_visa_no_spaces1 4539148803436467", "card_visa_no_spaces2 4539148803436", "card_visa_no_spaces3 4539148803436467234", "card_visa_no_spaces4 4539148812336467234 asdf asdf", + "quoted visa '4539 1488 0343 6467 234' here", + "quoted visa \"4539148803436467\" there", "card_amex_spaces1 3714 496353 98431", "card_amex_spaces2 3487 344936 71000", "card_amex_spaces3 3782 822463 10005", "card_amex_spaces4 3782 822463 12305 testy test.", + "3782 822463 12305", "card_amex_no_spaces1 371449635398431", "card_amex_no_spaces2 348734493671000", "card_amex_no_spaces3 378282246310005", "card_amex_no_spaces4 378282243210005 some postfix", + "quoted amex spaces '3714 496353 98431' here", + "quoted amex \"371449635398431\" there", "safe_attribute1 371", "safe_attribute2 37810005", "safe_attribute3 987346 some postfix test", @@ -191,26 +200,35 @@ func TestFilteringLogBody(t *testing.T) { "card_master_spaces2 **** 5100", "card_master_spaces3 **** 5100", "card_master_spaces4 **** 5100 some text after", + "**** 6789", "card_master_no_spaces1 **** 6789", "card_master_no_spaces2 **** 5100", "card_master_no_spaces3 **** 5100", "card_master_no_spaces4 **** 5100 testy test", + "quoted master spaces \"**** 5100\" here", + "quoted master '**** 6789' there", "card_visa_spaces1 **** 6467", "card_visa_spaces2 **** 343 6", "card_visa_spaces3 **** 7 234", "card_visa_spaces4 **** 7 234 testy test", + "**** 6467", "card_visa_no_spaces1 **** 6467", "card_visa_no_spaces2 **** 3436", "card_visa_no_spaces3 **** 7234", "card_visa_no_spaces4 **** 7234 asdf asdf", + "quoted visa '**** 7 234' here", + "quoted visa \"**** 6467\" there", "card_amex_spaces1 **** 8431", "card_amex_spaces2 **** 1000", "card_amex_spaces3 **** 0005", "card_amex_spaces4 **** 2305 testy test.", + "**** 2305", "card_amex_no_spaces1 **** 8431", "card_amex_no_spaces2 **** 1000", "card_amex_no_spaces3 **** 0005", "card_amex_no_spaces4 **** 0005 some postfix", + "quoted amex spaces '**** 8431' here", + "quoted amex \"**** 8431\" there", "safe_attribute1 371", "safe_attribute2 37810005", "safe_attribute3 987346 some postfix test", @@ -248,6 +266,9 @@ func TestFilteringLogBody(t *testing.T) { "iban29 LU280019400644750000", "iban30 NO9386011117947", "non-iban no4444 ds", + "quoted iban spaces 'ES91 2100 0418 4502 0005 1332' here", + "quoted iban \"PL61 1090 1014 0000 0712 1981 2874\" there", + "IT60 X054 2811 1010 0000 0123 456", } logsIbanFiltered := []string{ @@ -282,6 +303,9 @@ func TestFilteringLogBody(t *testing.T) { "iban29 LU **** 0000", "iban30 NO **** 7947", "non-iban no4444 ds", + "quoted iban spaces 'ES **** 1332' here", + "quoted iban \"PL **** 2874\" there", + "IT **** 3 456", } creditCardTransformConfig := "masking_logbody.yaml"