Extend config examples with log body filtering (#357)

Signed-off-by: Moritz Wiesinger <[email protected]>
Dynatrace · Nov 12, 2024 · 2968af5 · 2968af5
1 parent 9609841
commit 2968af5
Show file tree

Hide file tree

Showing 3 changed files with 242 additions and 0 deletions.
diff --git a/.chloggen/add-log-body-cc-filtering.yaml b/.chloggen/add-log-body-cc-filtering.yaml
@@ -0,0 +1,18 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: transformprocessor
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Extend the credit card filtering example with log body filtering
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [357]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
diff --git a/config_examples/masking_logbody.yaml b/config_examples/masking_logbody.yaml
@@ -0,0 +1,36 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+exporters:
+  otlphttp:
+    endpoint: ${env:DT_ENDPOINT}
+    headers:
+      Authorization: "Api-Token ${env:API_TOKEN}"
+
+processors:
+  transform/cc:
+    error_mode: ignore
+    trace_statements:
+    log_statements:
+      - context: log
+        statements:
+          - replace_pattern(body, "(^|\\s)3\\s*[47](\\s*[0-9]){9}((\\s*[0-9]){4})($|\\s)", "$$1**** $$3$$5")
+          - replace_pattern(body, "(^|\\s)(5[1-5]([0-9]){2}|222[1-9]|22[3-9]\\d|2[3-6]\\d{2}|27[0-1]\\d|2720)(\\s*[0-9]){8}\\s*([0-9]{4})($|\\s)", "$$1**** $$5$$6")
+          - replace_pattern(body, "(^|\\s)4(\\s*[0-9]){8,14}\\s*(([0-9]\\s*?){4})($|\\s)", "$$1**** $$3$$5")
+  transform/iban:
+    error_mode: ignore
+    trace_statements:
+    log_statements:
+      - context: log
+        statements:
+          - replace_pattern(body, "(^|\\s)([A-Z]{2})[0-9]{2}((\\s*[A-Z0-9]){4,26})\\s*((\\s*[A-Z0-9]){4})($|\\s)", "$$1$$2 **** $$5$$7")
+
+
+service:
+  pipelines:
+    logs:
+      receivers: [otlp]
+      processors: [transform/iban, transform/cc]
+      exporters: [otlphttp]
diff --git a/internal/testbed/integration/filtering_test.go b/internal/testbed/integration/filtering_test.go
@@ -155,6 +155,177 @@ func TestFilteringCreditCard(t *testing.T) {
 	}
 }
 
+func TestFilteringLogBody(t *testing.T) {
+	logsCCPlain := []string{
+		"card_master_spaces1 2367 8901 2345 6789",
+		"card_master_spaces2 5105 1051 0510 5100",
+		"card_master_spaces3 2720 1051 0510 5100",
+		"card_master_spaces4 2720 1051 0520 5100 some text after",
+		"card_master_no_spaces1 2367890123456789",
+		"card_master_no_spaces2 5105105105105100",
+		"card_master_no_spaces3 2720105105105100",
+		"card_master_no_spaces4 2720145107105100 testy test",
+		"card_visa_spaces1 4539 1488 0343 6467",
+		"card_visa_spaces2 4539 1488 0343 6",
+		"card_visa_spaces3 4539 1488 0343 6467 234",
+		"card_visa_spaces4 4539 1498 0343 6457 234 testy test",
+		"card_visa_no_spaces1 4539148803436467",
+		"card_visa_no_spaces2 4539148803436",
+		"card_visa_no_spaces3 4539148803436467234",
+		"card_visa_no_spaces4 4539148812336467234 asdf asdf",
+		"card_amex_spaces1 3714 496353 98431",
+		"card_amex_spaces2 3487 344936 71000",
+		"card_amex_spaces3 3782 822463 10005",
+		"card_amex_spaces4 3782 822463 12305 testy test.",
+		"card_amex_no_spaces1 371449635398431",
+		"card_amex_no_spaces2 348734493671000",
+		"card_amex_no_spaces3 378282246310005",
+		"card_amex_no_spaces4 378282243210005 some postfix",
+		"safe_attribute1 371",
+		"safe_attribute2 37810005",
+		"safe_attribute3 987346 some postfix test",
+	}
+
+	logsCCFiltered := []string{
+		"card_master_spaces1 **** 6789",
+		"card_master_spaces2 **** 5100",
+		"card_master_spaces3 **** 5100",
+		"card_master_spaces4 **** 5100 some text after",
+		"card_master_no_spaces1 **** 6789",
+		"card_master_no_spaces2 **** 5100",
+		"card_master_no_spaces3 **** 5100",
+		"card_master_no_spaces4 **** 5100 testy test",
+		"card_visa_spaces1 **** 6467",
+		"card_visa_spaces2 **** 343 6",
+		"card_visa_spaces3 **** 7 234",
+		"card_visa_spaces4 **** 7 234 testy test",
+		"card_visa_no_spaces1 **** 6467",
+		"card_visa_no_spaces2 **** 3436",
+		"card_visa_no_spaces3 **** 7234",
+		"card_visa_no_spaces4 **** 7234 asdf asdf",
+		"card_amex_spaces1 **** 8431",
+		"card_amex_spaces2 **** 1000",
+		"card_amex_spaces3 **** 0005",
+		"card_amex_spaces4 **** 2305 testy test.",
+		"card_amex_no_spaces1 **** 8431",
+		"card_amex_no_spaces2 **** 1000",
+		"card_amex_no_spaces3 **** 0005",
+		"card_amex_no_spaces4 **** 0005 some postfix",
+		"safe_attribute1 371",
+		"safe_attribute2 37810005",
+		"safe_attribute3 987346 some postfix test",
+	}
+
+	logsIbanPlain := []string{
+		"iban1 DE89 3704 0044 0532 0130 00",
+		"iban2 FR14 2004 1010 0505 0001 3M02 606",
+		"iban3 ES91 2100 0418 4502 0005 1332",
+		"iban4 IT60 X054 2811 1010 0000 0123 456",
+		"iban5 NL91 ABNA 0417 1643 00",
+		"iban6 BE71 0961 2345 6769",
+		"iban7 AT48 3200 0000 0123 4568 testy test",
+		"iban8 SE72 8000 0810 0340 0978 3242",
+		"iban9 PL61 1090 1014 0000 0712 1981 2874",
+		"iban10 GB29 NWBK 6016 1331 9268 19",
+		"iban11 AL47 2121 1009 0000 0002 3569 8741 asdf asdf",
+		"iban12 CY17 0020 0128 0000 0012 0052 7600",
+		"iban13 KW81 CBKU 0000 0000 0000 1234 5601 01",
+		"iban14 LU28 0019 4006 4475 0000",
+		"iban15 NO93 8601 1117 947",
+		"iban16 DE89370400440532013000",
+		"iban17 FR1420041010050500013M02606 test222",
+		"iban18 ES9121000418450200051332",
+		"iban19 IT60X0542811101000000123456",
+		"iban20 NL91ABNA0417164300 text text",
+		"iban21 BE71096123456769",
+		"iban22 AT483200000001234568",
+		"iban23 SE7280000810034009783242",
+		"iban24 PL61109010140000071219812874",
+		"iban25 GB29NWBK60161331926819 postfix text",
+		"iban26 AL47212110090000000235698741",
+		"iban27 CY17002001280000001200527600",
+		"iban28 KW81CBKU0000000000001234560101",
+		"iban29 LU280019400644750000",
+		"iban30 NO9386011117947",
+		"non-iban no4444 ds",
+	}
+
+	logsIbanFiltered := []string{
+		"iban1 DE **** 30 00",
+		"iban2 FR **** 2 606",
+		"iban3 ES **** 1332",
+		"iban4 IT **** 3 456",
+		"iban5 NL **** 43 00",
+		"iban6 BE **** 6769",
+		"iban7 AT **** 4568 testy test",
+		"iban8 SE **** 3242",
+		"iban9 PL **** 2874",
+		"iban10 GB **** 68 19",
+		"iban11 AL **** 8741 asdf asdf",
+		"iban12 CY **** 7600",
+		"iban13 KW **** 01 01",
+		"iban14 LU **** 0000",
+		"iban15 NO **** 7 947",
+		"iban16 DE **** 3000",
+		"iban17 FR **** 2606 test222",
+		"iban18 ES **** 1332",
+		"iban19 IT **** 3456",
+		"iban20 NL **** 4300 text text",
+		"iban21 BE **** 6769",
+		"iban22 AT **** 4568",
+		"iban23 SE **** 3242",
+		"iban24 PL **** 2874",
+		"iban25 GB **** 6819 postfix text",
+		"iban26 AL **** 8741",
+		"iban27 CY **** 7600",
+		"iban28 KW **** 0101",
+		"iban29 LU **** 0000",
+		"iban30 NO **** 7947",
+		"non-iban no4444 ds",
+	}
+
+	creditCardTransformConfig := "masking_logbody.yaml"
+
+	tests := []struct {
+		name         string
+		dataProvider testbed.DataProvider
+		sender       SenderFunc
+		receiver     ReceiverFunc
+		validator    testbed.TestCaseValidator
+		configName   string
+	}{
+		{
+			name:         "CC log bodies transform",
+			dataProvider: NewSampleConfigsLogsDataProvider(generateLogsWithBodies(logsCCPlain)),
+			sender:       NewOTLPLogsDataSenderWrapper,
+			receiver:     testbed.NewOTLPHTTPDataReceiver,
+			validator:    NewLogsValidator(t, []plog.Logs{generateLogsWithBodies(logsCCFiltered)}),
+			configName:   creditCardTransformConfig,
+		},
+		{
+			name:         "IBAN log bodies transform",
+			dataProvider: NewSampleConfigsLogsDataProvider(generateLogsWithBodies(logsIbanPlain)),
+			sender:       NewOTLPLogsDataSenderWrapper,
+			receiver:     testbed.NewOTLPHTTPDataReceiver,
+			validator:    NewLogsValidator(t, []plog.Logs{generateLogsWithBodies(logsIbanFiltered)}),
+			configName:   creditCardTransformConfig,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			FilteringScenario(
+				t,
+				test.dataProvider,
+				test.sender,
+				test.receiver,
+				test.validator,
+				test.configName,
+			)
+		})
+	}
+}
+
 func TestFilteringDTAPITokenRedactionProcessor(t *testing.T) {
 	configName := "redaction_api_token.yaml"
 
@@ -661,3 +832,20 @@ func generateBasicLogsWithAttributes(attributes pcommon.Map) plog.Logs {
 
 	return logs
 }
+
+func generateLogsWithBodies(bodies []string) plog.Logs {
+	logs := plog.NewLogs()
+	rl := logs.ResourceLogs().AppendEmpty()
+
+	for _, body := range bodies {
+		logRecords := rl.ScopeLogs().AppendEmpty().LogRecords()
+		logRecords.EnsureCapacity(1)
+
+		record := logRecords.AppendEmpty()
+		record.SetSeverityNumber(plog.SeverityNumberInfo3)
+		record.SetSeverityText("INFO")
+		record.Body().SetStr(body)
+	}
+
+	return logs
+}