Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add scalability queries to esql-ccs-snapshot Logs challenge #705

Merged
merged 19 commits into from
Nov 18, 2024
Merged

Add scalability queries to esql-ccs-snapshot Logs challenge #705

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
b8d09fa
Make base path configuration conditional
gbanasiak Nov 5, 2024
36a237b
Fix typo
gbanasiak Nov 5, 2024
304849d
Refactor to control local snapshot
gbanasiak Nov 5, 2024
88550c3
Rename p_skip_esql_ccs_remote_clusters to p_esql_ccs_skip_remote_clus…
gbanasiak Nov 5, 2024
13fd2f2
Add newlines for clarity
gbanasiak Nov 5, 2024
8b95300
Add track parameter to control data generation
gbanasiak Nov 5, 2024
7538bff
Remove snapshot name from task names
gbanasiak Nov 6, 2024
16ca9b2
Add scaling queries
gbanasiak Nov 6, 2024
d3656ba
Remove extra parentheses
gbanasiak Nov 6, 2024
801c8b6
Rename cluster stats operation
gbanasiak Nov 6, 2024
ff9f236
Add tojson filter
gbanasiak Nov 6, 2024
e0418c8
Add node count condition to _cluster/health
gbanasiak Nov 6, 2024
8823f2d
Check cluster health at completion
gbanasiak Nov 7, 2024
6f05f74
Fix operation syntax
gbanasiak Nov 7, 2024
0f20b69
Add local snapshot restore option
gbanasiak Nov 12, 2024
a705448
Check local cluster health later
gbanasiak Nov 12, 2024
4238e7c
Revert "Check local cluster health later"
gbanasiak Nov 12, 2024
ed80512
Add cluster health check after local snapshot restore
gbanasiak Nov 12, 2024
5c729f1
At completion cluster checks without retries
gbanasiak Nov 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 121 additions & 46 deletions elastic/logs/challenges/esql-ccs-snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,15 @@
{
"name": "esql-ccs-snapshot",
"description": "Indexes logs to the local cluster, and runs ESQL queries either locally or across remote clusters via CCS. Data replicated via snapshot/restore.",
"parameters": {
"generate-data": {{ generate_data | default('true') | tojson }}
},
"schedule": [
{% if not p_skip_esql_ccs_remote_clusters %}

{% if not p_esql_ccs_skip_remote_clusters %}
{% include "tasks/many-clusters-setup.json" %},
{% endif %}

{% include "tasks/index-setup.json" %},
{
"name": "bulk-index",
Expand All @@ -32,24 +37,20 @@
"request-timeout": 60
}
},
{% if p_skip_esql_ccs_remote_clusters %}
{
"name": "check-cluster-health-local",
"operation": {
"operation-type": "cluster-health",
"index": "_all",
"request-params": {
"wait_for_status": "{{cluster_health | default('green')}}",
"wait_for_no_relocating_shards": "true"
},
"retry-until-success": true
}
},
{% else %}{# if p_skip_esql_ccs_remote_clusters ... #}
{# mutate a copy so as not to override in other challenges. shallow copy is ok since we're changing a top-level element.' #}
{% set many_clusters_snapshot_repo_settings = p_snapshot_repo_settings.copy() %}
{# randomize base path to avoid clashes when running concurrent benchmarks #}

{# mutate a copy so as not to override in other challenges. shallow copy is ok since we're changing a top-level element.' #}
{% set many_clusters_snapshot_repo_settings = p_snapshot_repo_settings.copy() %}
{# randomize base path to avoid clashes when running concurrent benchmarks #}
{% if 'base_path' not in many_clusters_snapshot_repo_settings %}
{% set _=many_clusters_snapshot_repo_settings.update({"base_path":"many-clusters-"+((now|int)|string)}) %}
{% endif %}

{
"name": "check-cluster-health-on-local-cluster",
"operation": "check-cluster-health-on-local-cluster"
},

{% if p_esql_ccs_include_local_snapshot %}
{
"name": "register-snapshot-repository-on-local-cluster",
"operation": {
Expand All @@ -61,13 +62,38 @@
}
}
},
{
"name": "create-snapshot",
"operation": {
"operation-type": "create-snapshot",
"repository": {{ p_snapshot_repo_name | tojson }},
"snapshot": {{ p_snapshot_name | tojson }},
"wait-for-completion": false,
"body": {
"indices": {{ p_restore_data_streams | tojson }},
"ignore_unavailable": true,
"include_global_state": false,
"metadata": {{ p_snapshot_metadata | tojson }}
},
"include-in-reporting": true
}
},
{
"name": "wait-for-snapshot",
"operation": {
"operation-type": "wait-for-snapshot-create",
"snapshot": {{ p_snapshot_name | tojson }},
"repository": {{ p_snapshot_repo_name | tojson }}
}
},
{% endif %}

{% if p_esql_ccs_include_local_snapshot_restore %}
{% set _=many_clusters_snapshot_repo_settings.update({"readonly":true}) %}
{
"name": "register-snapshot-repository-on-remote-clusters",
"name": "register-snapshot-repository-on-local-cluster",
"operation": {
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "create-snapshot-repository",
"ignore-clusters": ["default"],
"operation-type": "create-snapshot-repository",
"repository": {{ p_snapshot_repo_name | tojson }},
"body": {
"type": {{ p_snapshot_repo_type | tojson }},
Expand All @@ -76,31 +102,51 @@
}
},
{
"name": "create-snapshot-{{ p_snapshot_name }}",
"name": "recover-snapshot-on-local-cluster",
"operation": {
"operation-type": "create-snapshot",
"operation-type": "restore-snapshot",
"repository": {{ p_snapshot_repo_name | tojson }},
"snapshot": {{ p_snapshot_name | tojson }},
"wait-for-completion": false,
"body": {
"indices": {{ p_restore_data_streams | tojson }},
"ignore_unavailable": true,
"include_global_state": false,
"metadata": {{ p_snapshot_metadata | tojson }}
},
"include-in-reporting": true
"rename_pattern": "(.+)",
"rename_replacement": "$1{{ p_snapshot_rename_suffix }}"
}
}
},
{
"name": "wait-for-snapshot-{{ p_snapshot_name }}",
"name": "wait-for-snapshot-recovery-on-local-cluster",
"operation": {
"operation-type": "wait-for-snapshot-create",
"snapshot": {{ p_snapshot_name | tojson }},
"repository": {{ p_snapshot_repo_name | tojson }}
"operation-type": "wait-for-recovery",
"index": {{ (p_restore_data_streams~p_snapshot_rename_suffix) | tojson }}
}
},
{
"name": "check-cluster-health-on-local-cluster-after-snapshot-restore",
"operation": "check-cluster-health-on-local-cluster"
},
{% endif %}

{% if not p_esql_ccs_skip_remote_clusters %}
{% set _=many_clusters_snapshot_repo_settings.update({"readonly":true}) %}
{
"name": "register-snapshot-repository-on-remote-clusters",
"operation": {
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "create-snapshot-repository",
"ignore-clusters": ["default"],
"repository": {{ p_snapshot_repo_name | tojson }},
"body": {
"type": {{ p_snapshot_repo_type | tojson }},
"settings": {{ many_clusters_snapshot_repo_settings | tojson(indent=2)}}
}
}
},
{
"name": "recover-snapshot-{{ p_snapshot_name }}-on-remote-clusters",
"name": "recover-snapshot-on-remote-clusters",
"operation": {
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "restore-snapshot",
Expand All @@ -118,25 +164,17 @@
}
},
{
"name": "wait-for-snapshot-recovery-{{ p_snapshot_name }}-on-all-clusters",
"name": "wait-for-snapshot-recovery-on-remote-clusters",
"operation": {
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "wait-for-recovery",
"ignore-clusters": ["default"],
"index": {{ (p_restore_data_streams~p_snapshot_rename_suffix) | tojson }}
}
},
{
"name": "check-cluster-health-all-clusters",
"operation": {
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "cluster-health",
"index": "_all",
"request-params": {
"wait_for_status": "{{wait_for_status | default('green')}}",
"wait_for_no_relocating_shards": "true"
},
"retry-until-success": true
}
"name": "check-cluster-health-on-remote-clusters",
"operation": "check-cluster-health-on-remote-clusters"
},
{
"name": "setup-bi-directional-remote-clusters",
Expand All @@ -161,7 +199,44 @@
}
}
},
{% endif %}{# if p_skip_esql_ccs_remote_clusters ... #}
{% include "tasks/many-clusters-esql-ccs-queries.json" %}
{% endif %}{# if not p_esql_ccs_skip_remote_clusters ... #}

{% include "tasks/many-clusters-esql-ccs-queries.json" %},

{# the following cluster health checks purposefully without retries #}
{
"name": "check-cluster-health-on-local-cluster-at-completion",
"operation": {
"operation-type": "cluster-health",
"index": "_all",
"request-params": {
"wait_for_status": "{{cluster_health | default('green')}}",
{% if p_esql_ccs_local_nodes_count > 0 %}
"wait_for_nodes": {{ p_esql_ccs_local_nodes_count }},
{% endif %}
"wait_for_no_relocating_shards": "true"
},
"retry-until-success": false
}
}
{% if not p_esql_ccs_skip_remote_clusters %},
{
"name": "check-cluster-health-on-remote-clusters-at-completion",
"operation": {
"operation-type": "multi-cluster-wrapper",
"base-operation-type": "cluster-health",
"ignore-clusters": ["default"],
"index": "_all",
"request-params": {
"wait_for_status": "{{wait_for_status | default('green')}}",
{% if p_esql_ccs_remote_nodes_count > 0 %}
"wait_for_nodes": {{ p_esql_ccs_remote_nodes_count }},
{% endif %}
"wait_for_no_relocating_shards": "true"
},
"retry-until-success": false
}
}
{% endif %}
]
}
Loading
Loading