From 5b6f1e2355658795e691b1bdc4e9ef9e3a62f012 Mon Sep 17 00:00:00 2001
From: Amnon Heiman <amnon@scylladb.com>
Date: Thu, 9 Jan 2025 10:37:18 +0200
Subject: [PATCH] scylla-advanced: break the rpc into per-domain rows

---
 grafana/scylla-advanced.template.json | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/grafana/scylla-advanced.template.json b/grafana/scylla-advanced.template.json
index 88fed79d7..87d696f79 100644
--- a/grafana/scylla-advanced.template.json
+++ b/grafana/scylla-advanced.template.json
@@ -730,7 +730,8 @@
                     {
                         "class": "collapsible_row_panel",
                         "collapsed": true,
-                        "title": "RPC metrics"
+                        "repeat": "domain",
+                        "title": "RPC metrics - $domain"
                     }
                 ]
             },
@@ -743,7 +744,7 @@
                         "span": 2,
                         "targets": [
                             {
-                                "expr": "$topbottom([[filter_limit]], sum(scylla_rpc_client_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]],domain)>0)",
+                                "expr": "$topbottom([[filter_limit]], sum(scylla_rpc_client_count{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", domain=\"$domain\"}) by ([[by]], domain)>0)",
                                 "intervalFactor": 1,
                                 "legendFormat": "",
                                 "metric": "",
@@ -759,7 +760,7 @@
                         "span": 2,
                         "targets": [
                             {
-                                "expr": "$topbottom([[filter_limit]], sum(rate(scylla_rpc_client_sent_messages{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])>0) by ([[by]],domain))",
+                                "expr": "$topbottom([[filter_limit]], sum(rate(scylla_rpc_client_sent_messages{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", domain=\"$domain\"}[$__rate_interval])>0) by ([[by]],domain))",
                                 "intervalFactor": 1,
                                 "legendFormat": "",
                                 "metric": "",
@@ -775,7 +776,7 @@
                         "span": 2,
                         "targets": [
                             {
-                                "expr": "$topbottom([[filter_limit]], sum(rate(scylla_rpc_client_replied{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])>0) by ([[by]],domain))",
+                                "expr": "$topbottom([[filter_limit]], sum(rate(scylla_rpc_client_replied{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", domain=\"$domain\"}[$__rate_interval])>0) by ([[by]],domain))",
                                 "intervalFactor": 1,
                                 "legendFormat": "",
                                 "metric": "",
@@ -791,7 +792,7 @@
                         "span": 2,
                         "targets": [
                             {
-                                "expr": "$topbottom([[filter_limit]], sum(rate(scylla_rpc_client_exception_received{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])>0) by ([[by]],domain))",
+                                "expr": "$topbottom([[filter_limit]], sum(rate(scylla_rpc_client_exception_received{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", domain=\"$domain\"}[$__rate_interval])>0) by ([[by]],domain))",
                                 "intervalFactor": 1,
                                 "legendFormat": "",
                                 "metric": "",
@@ -807,7 +808,7 @@
                         "span": 2,
                         "targets": [
                             {
-                                "expr": "$topbottom([[filter_limit]], sum(rate(scylla_rpc_client_timeout{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])>0) by ([[by]],domain))",
+                                "expr": "$topbottom([[filter_limit]], sum(rate(scylla_rpc_client_timeout{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", domain=\"$domain\"}[$__rate_interval])>0) by ([[by]],domain))",
                                 "intervalFactor": 1,
                                 "legendFormat": "",
                                 "metric": "",
@@ -823,7 +824,7 @@
                         "span": 2,
                         "targets": [
                             {
-                                "expr": "$topbottom([[filter_limit]], sum(scylla_rpc_client_pending{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]],domain)>0)",
+                                "expr": "$topbottom([[filter_limit]], sum(scylla_rpc_client_pending{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", domain=\"$domain\"}) by ([[by]],domain)>0)",
                                 "intervalFactor": 1,
                                 "legendFormat": "",
                                 "metric": "",
@@ -839,7 +840,7 @@
                         "span": 2,
                         "targets": [
                             {
-                                "expr": "$topbottom([[filter_limit]], sum(scylla_rpc_client_wait_reply{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]],domain)>0)",
+                                "expr": "$topbottom([[filter_limit]], sum(scylla_rpc_client_wait_reply{instance=~\"[[node]]|^$\",cluster=\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", domain=\"$domain\"}) by ([[by]],domain)>0)",
                                 "intervalFactor": 1,
                                 "legendFormat": "",
                                 "metric": "",
@@ -954,6 +955,14 @@
                     "query": "label_values(scylla_io_queue_consumption,iogroup)",
                     "sort": 1
                 },
+                {
+                    "class": "template_variable_all",
+                    "label": "domain",
+                    "name": "domain",
+                    "hide": 0,
+                    "query": "label_values(scylla_rpc_client_count,domain)",
+                    "sort": 1
+                },
                 {
                     "class": "aggregation_function",
                     "current": {