Backport/8.15/pr 675 (#695)

elastic/security/README.md

@@ -102,7 +102,9 @@ The following parameters are available:
 * `number_of_replicas` (default: 1) - The number of replicas to set per Data Stream. The same value is used for all Data Streams.
 * `bulk_indexing_clients` (default: 8) - The number of clients issuing indexing requests.
 * `bulk_size` (default: 50) - The number of documents to send per indexing request.
-* `force_merge_max_num_segments` (default: unset): An integer specifying the max amount of segments the force-merge operation should use. Only supported in `security-indexing-querying` track.
+* `force_merge_max_num_segments` (default: unset): An integer specifying the max amount of segments the force-merge operation should use. Only supported in `security-indexing-querying` track. 
+* `logs_endpoint_from_kibana` (default: false): Skip creation of endpoint templates. Used when templates are expected from kibana. 
+* `include_non_serverless_index_settings` (default: true for non-serverless clusters, false for serverless clusters): Whether to include non-serverless index settings.
 
 ### Querying parameters
 

elastic/security/pipelines/.fleet_final_pipeline-1.json

@@ -1,24 +1,17 @@
 {
-  "version": 2,
+  "version": 4,
   "_meta": {
     "managed_by": "fleet",
     "managed": true
   },
   "description": "Final pipeline for processing all incoming Fleet Agent documents.\n",
   "processors": [
-    {
-      "set": {
-        "description": "Add time when event was ingested.",
-        "field": "event.ingested",
-        "copy_from": "_ingest.timestamp"
-      }
-    },
     {
       "script": {
-        "description": "Remove sub-seconds from event.ingested to improve storage efficiency.",
+        "description": "Add time when event was ingested (and remove sub-seconds to improve storage efficiency)",
         "tag": "truncate-subseconds-event-ingested",
-        "source": "ctx.event.ingested = ctx.event.ingested.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);",
-        "ignore_failure": true
+        "ignore_failure": true,
+        "source": "if (ctx?.event == null) {\n  ctx.event = [:];\n}\n\nctx.event.ingested = metadata().now.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);"
       }
     },
     {
@@ -31,6 +24,15 @@
         "ignore_missing": true
       }
     },
+    {
+      "remove": {
+        "description": "Remove event.original unless the preserve_original_event tag is set",
+        "field": "event.original",
+        "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))",
+        "ignore_failure": true,
+        "ignore_missing": true
+      }
+    },
     {
       "set_security_user": {
         "field": "_security",

elastic/security/pipelines/logs-endpoint.events.file-8.15.1.json

@@ -0,0 +1,47 @@
+{
+  "description": "Pipeline for setting event.ingested",
+  "processors": [
+    {
+      "set": {
+        "field": "event.ingested",
+        "value": "{{ _ingest.timestamp }}",
+        "ignore_failure": true
+      }
+    },
+    {
+      "pipeline": {
+        "name": "global@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Global pipeline for all data streams"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs`"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-endpoint.integration@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `endpoint` integration"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-endpoint.events.file@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for the `endpoint.events.file` dataset"
+      }
+    }
+  ],
+  "_meta": {
+    "managed_by": "fleet",
+    "managed": true,
+    "package": {
+      "name": "endpoint"
+    }
+  }
+}

elastic/security/pipelines/logs-endpoint.events.library-8.15.1.json

@@ -0,0 +1,47 @@
+{
+  "description": "Pipeline for setting event.ingested",
+  "processors": [
+    {
+      "set": {
+        "field": "event.ingested",
+        "value": "{{ _ingest.timestamp }}",
+        "ignore_failure": true
+      }
+    },
+    {
+      "pipeline": {
+        "name": "global@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Global pipeline for all data streams"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs`"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-endpoint.integration@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `endpoint` integration"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-endpoint.events.library@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for the `endpoint.events.library` dataset"
+      }
+    }
+  ],
+  "_meta": {
+    "managed_by": "fleet",
+    "managed": true,
+    "package": {
+      "name": "endpoint"
+    }
+  }
+}

elastic/security/pipelines/logs-endpoint.events.network-8.2.0.json → elastic/security/pipelines/logs-endpoint.events.network-8.15.1.json

@@ -117,6 +117,34 @@
         "ignore_missing": true,
         "field": "dns.question.Ext_temp"
       }
+    },
+    {
+      "pipeline": {
+        "name": "global@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Global pipeline for all data streams"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs`"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-endpoint.integration@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `endpoint` integration"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-endpoint.events.network@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for the `endpoint.events.network` dataset"
+      }
     }
   ],
   "_meta": {