From b8673325e7a71bc2784332877e2cb1b9107b25e9 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 16 Dec 2024 16:52:29 +0100 Subject: [PATCH 1/4] feature: introduce a new use_synthetic_source_recovery track parameter --- elastic/logs/README.md | 1 + elastic/logs/templates/component/track-shared-logsdb-mode.json | 3 +++ elastic/security/README.md | 1 + .../security/templates/component/track-shared-logsdb-mode.json | 3 +++ tsdb/README.md | 1 + tsdb/index.json | 3 +++ 6 files changed, 12 insertions(+) diff --git a/elastic/logs/README.md b/elastic/logs/README.md index 4d5fed01..eebd4651 100644 --- a/elastic/logs/README.md +++ b/elastic/logs/README.md @@ -222,6 +222,7 @@ The following parameters are available: * `workflow-request-cache` (default: `true`) - Explicit control of request cache query parameter in searches executed in a workflow. This can be further overriden at an operation level with `request-cache` parameter. * `synthetic_source_keep` (default: unset) - Allows overriding the default synthetic source behaviour for all field types with the following values: `none` (equivalent to unset) - no source is stored, `arrays` - source stored as is only for multi-value (array) fields. * `source_mode` (default: unset) - Specifies the source mode to be used. +* `use_synthetic_source_recovery` (default: unset): If `true` recovery will take place using synthetic source instead of using `_recovery_source` ### Data Download Parameters diff --git a/elastic/logs/templates/component/track-shared-logsdb-mode.json b/elastic/logs/templates/component/track-shared-logsdb-mode.json index 8ae129fb..d6745c19 100644 --- a/elastic/logs/templates/component/track-shared-logsdb-mode.json +++ b/elastic/logs/templates/component/track-shared-logsdb-mode.json @@ -4,6 +4,9 @@ {% if index_mode %} "index": { "mode": {{ index_mode | tojson }} + {% if use_synthetic_source_recovery %} + ,"recovery.use_synthetic_source": {{use_synthetic_source_recovery | tojson}}, + {% endif %} {% if synthetic_source_keep and synthetic_source_keep != 'none' %} ,"mapping.synthetic_source_keep": {{ synthetic_source_keep | tojson }} {% endif %} diff --git a/elastic/security/README.md b/elastic/security/README.md index 0a6da1c3..e2eacb48 100644 --- a/elastic/security/README.md +++ b/elastic/security/README.md @@ -86,6 +86,7 @@ The following parameters are available: * `index_mode` (default: unset) - A parameter meant to be used internally which defines one of the available indexing modes, "standard", "logsdb" or "time_series". If not set, "standard" is used. * `synthetic_source_keep` (default: unset) - Allows overriding the default synthetic source behaviour for all field types with the following values: `none` (equivalent to unset) - no source is stored, `arrays` - source stored as is only for multi-value (array) fields. * `source_mode` (default: unset) - Specifies the source mode to be used. +* `use_synthetic_source_recovery` (default: unset): If `true` recovery will take place using synthetic source instead of using `_recovery_source` ### Data Generation Parameters diff --git a/elastic/security/templates/component/track-shared-logsdb-mode.json b/elastic/security/templates/component/track-shared-logsdb-mode.json index 570faffc..d2740341 100644 --- a/elastic/security/templates/component/track-shared-logsdb-mode.json +++ b/elastic/security/templates/component/track-shared-logsdb-mode.json @@ -4,6 +4,9 @@ {% if index_mode %} "index": { "mode": {{ index_mode | tojson }}, + {% if use_synthetic_source_recovery %} + "recovery.use_synthetic_source": {{use_synthetic_source_recovery | tojson}}, + {% endif %} {% if synthetic_source_keep and synthetic_source_keep != 'none' %} "mapping.synthetic_source_keep": {{ synthetic_source_keep | tojson}}, {% endif %} diff --git a/tsdb/README.md b/tsdb/README.md index 69600325..b8e5f24e 100644 --- a/tsdb/README.md +++ b/tsdb/README.md @@ -204,6 +204,7 @@ This track allows to overwrite the following parameters using `--track-params`: * `post_ingest_sleep_duration` (default: 30): Sleep duration in seconds. * `document_ids`: documentd IDs to use for search, get and mget apis in the `low-latency` challenge. If empty, a default set of 4 values is used. * `synthetic_source_keep` (default: unset): If specified, configures the `index.mapping.synthetic_source_keep` index setting. +* `use_synthetic_source_recovery` (default: unset): If `true` recovery will take place using synthetic source instead of using `_recovery_source` ### License diff --git a/tsdb/index.json b/tsdb/index.json index 6ff1c004..d5fe6fea 100644 --- a/tsdb/index.json +++ b/tsdb/index.json @@ -8,6 +8,9 @@ "number_of_shards": {{number_of_shards | default(1)}}, "number_of_replicas": {{number_of_replicas | default(0)}}, "requests.cache.enable": false, + {% if use_synthetic_source_recovery %} + "recovery.use_synthetic_source": {{use_synthetic_source_recovery | tojson}}, + {% endif %} {%- endif -%}{# non-serverless-index-settings-marker-end #} "codec": "{{codec | default('default')}}", "mode": "{{index_mode | default('time_series')}}", From 9447535f5edd1dd11335c214f7c6afc10c0bc8fc Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 16 Dec 2024 17:12:50 +0100 Subject: [PATCH 2/4] nit: reword parameter description --- elastic/logs/README.md | 2 +- elastic/security/README.md | 2 +- tsdb/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/elastic/logs/README.md b/elastic/logs/README.md index eebd4651..3a2aa5fa 100644 --- a/elastic/logs/README.md +++ b/elastic/logs/README.md @@ -222,7 +222,7 @@ The following parameters are available: * `workflow-request-cache` (default: `true`) - Explicit control of request cache query parameter in searches executed in a workflow. This can be further overriden at an operation level with `request-cache` parameter. * `synthetic_source_keep` (default: unset) - Allows overriding the default synthetic source behaviour for all field types with the following values: `none` (equivalent to unset) - no source is stored, `arrays` - source stored as is only for multi-value (array) fields. * `source_mode` (default: unset) - Specifies the source mode to be used. -* `use_synthetic_source_recovery` (default: unset): If `true` recovery will take place using synthetic source instead of using `_recovery_source` +* `use_synthetic_source_recovery` (default: unset): Whether synthetic recovery source will be used. ### Data Download Parameters diff --git a/elastic/security/README.md b/elastic/security/README.md index e2eacb48..1f3cc71c 100644 --- a/elastic/security/README.md +++ b/elastic/security/README.md @@ -86,7 +86,7 @@ The following parameters are available: * `index_mode` (default: unset) - A parameter meant to be used internally which defines one of the available indexing modes, "standard", "logsdb" or "time_series". If not set, "standard" is used. * `synthetic_source_keep` (default: unset) - Allows overriding the default synthetic source behaviour for all field types with the following values: `none` (equivalent to unset) - no source is stored, `arrays` - source stored as is only for multi-value (array) fields. * `source_mode` (default: unset) - Specifies the source mode to be used. -* `use_synthetic_source_recovery` (default: unset): If `true` recovery will take place using synthetic source instead of using `_recovery_source` +* `use_synthetic_source_recovery` (default: unset): Whether synthetic recovery source will be used. ### Data Generation Parameters diff --git a/tsdb/README.md b/tsdb/README.md index b8e5f24e..8492fc81 100644 --- a/tsdb/README.md +++ b/tsdb/README.md @@ -204,7 +204,7 @@ This track allows to overwrite the following parameters using `--track-params`: * `post_ingest_sleep_duration` (default: 30): Sleep duration in seconds. * `document_ids`: documentd IDs to use for search, get and mget apis in the `low-latency` challenge. If empty, a default set of 4 values is used. * `synthetic_source_keep` (default: unset): If specified, configures the `index.mapping.synthetic_source_keep` index setting. -* `use_synthetic_source_recovery` (default: unset): If `true` recovery will take place using synthetic source instead of using `_recovery_source` +* `use_synthetic_source_recovery` (default: unset): Whether synthetic recovery source will be used. ### License From ce031fe24e88a39f13d016702d9e2683a6773d12 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 17 Dec 2024 15:23:12 +0100 Subject: [PATCH 3/4] fix: remove misplaced comma --- elastic/logs/templates/component/track-shared-logsdb-mode.json | 2 +- .../security/templates/component/track-shared-logsdb-mode.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elastic/logs/templates/component/track-shared-logsdb-mode.json b/elastic/logs/templates/component/track-shared-logsdb-mode.json index d6745c19..9c640c73 100644 --- a/elastic/logs/templates/component/track-shared-logsdb-mode.json +++ b/elastic/logs/templates/component/track-shared-logsdb-mode.json @@ -5,7 +5,7 @@ "index": { "mode": {{ index_mode | tojson }} {% if use_synthetic_source_recovery %} - ,"recovery.use_synthetic_source": {{use_synthetic_source_recovery | tojson}}, + ,"recovery.use_synthetic_source": {{use_synthetic_source_recovery | tojson}} {% endif %} {% if synthetic_source_keep and synthetic_source_keep != 'none' %} ,"mapping.synthetic_source_keep": {{ synthetic_source_keep | tojson }} diff --git a/elastic/security/templates/component/track-shared-logsdb-mode.json b/elastic/security/templates/component/track-shared-logsdb-mode.json index d2740341..c2631916 100644 --- a/elastic/security/templates/component/track-shared-logsdb-mode.json +++ b/elastic/security/templates/component/track-shared-logsdb-mode.json @@ -5,7 +5,7 @@ "index": { "mode": {{ index_mode | tojson }}, {% if use_synthetic_source_recovery %} - "recovery.use_synthetic_source": {{use_synthetic_source_recovery | tojson}}, + "recovery.use_synthetic_source": {{use_synthetic_source_recovery | tojson}} {% endif %} {% if synthetic_source_keep and synthetic_source_keep != 'none' %} "mapping.synthetic_source_keep": {{ synthetic_source_keep | tojson}}, From 31010dda76776c4fa8d9096722b2539d655be97d Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 18 Dec 2024 17:18:30 +0100 Subject: [PATCH 4/4] fix: add missing comma --- .../security/templates/component/track-shared-logsdb-mode.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastic/security/templates/component/track-shared-logsdb-mode.json b/elastic/security/templates/component/track-shared-logsdb-mode.json index c2631916..d2740341 100644 --- a/elastic/security/templates/component/track-shared-logsdb-mode.json +++ b/elastic/security/templates/component/track-shared-logsdb-mode.json @@ -5,7 +5,7 @@ "index": { "mode": {{ index_mode | tojson }}, {% if use_synthetic_source_recovery %} - "recovery.use_synthetic_source": {{use_synthetic_source_recovery | tojson}} + "recovery.use_synthetic_source": {{use_synthetic_source_recovery | tojson}}, {% endif %} {% if synthetic_source_keep and synthetic_source_keep != 'none' %} "mapping.synthetic_source_keep": {{ synthetic_source_keep | tojson}},