add metrics logic to perf tests (#5224)

perf-tests/suite/test_ap_reload_perf.py

@@ -29,6 +29,7 @@
     get_file_contents,
     get_first_pod_name,
     get_ingress_nginx_template_conf,
+    get_resource_metrics,
     replace_ingress_with_ap_annotations,
     wait_before_test,
     wait_for_event_increment,
@@ -71,7 +72,9 @@ def enable_prometheus_port(
 
 
 @pytest.fixture(scope="class")
-def appprotect_setup(request, kube_apis, ingress_controller_endpoint, test_namespace) -> AppProtectSetup:
+def appprotect_setup(
+    request, kube_apis, ingress_controller_prerequisites, ingress_controller_endpoint, test_namespace
+) -> AppProtectSetup:
     """
     Deploy simple application and all the AppProtect(dataguard-alarm) resources under test in one namespace.
 
@@ -106,6 +109,16 @@ def appprotect_setup(request, kube_apis, ingress_controller_endpoint, test_names
 
     def fin():
         print("Clean up:")
+        print("Collect resource usage metrics")
+        pod_metrics = get_resource_metrics(kube_apis.custom_objects, "pods", ingress_controller_prerequisites.namespace)
+        with open("ap_pod_metrics.json", "w+") as f:
+            json.dump(pod_metrics, f, ensure_ascii=False, indent=4)
+        node_metrics = get_resource_metrics(
+            kube_apis.custom_objects, "nodes", ingress_controller_prerequisites.namespace
+        )
+        with open("ap_node_metrics.json", "w+") as f:
+            json.dump(node_metrics, f, ensure_ascii=False, indent=4)
+
         delete_ap_policy(kube_apis.custom_objects, pol_name, test_namespace)
         delete_ap_logconf(kube_apis.custom_objects, log_name, test_namespace)
         delete_common_app(kube_apis, "simple", test_namespace)

perf-tests/suite/test_ingress_perf.py

@@ -13,6 +13,7 @@
     delete_secret,
     ensure_connection,
     ensure_connection_to_public_endpoint,
+    get_resource_metrics,
     wait_before_test,
     wait_until_all_pods_are_ready,
 )
@@ -34,7 +35,7 @@ def __init__(self, req_url):
 
 
 @pytest.fixture(scope="class")
-def setup(request, kube_apis, ingress_controller_endpoint, test_namespace) -> Setup:
+def setup(request, kube_apis, ingress_controller_prerequisites, ingress_controller_endpoint, test_namespace) -> Setup:
     print("------------------------- Deploy prerequisites -----------------------------------")
     secret_name = create_secret_from_yaml(kube_apis.v1, test_namespace, f"{TEST_DATA}/smoke/smoke-secret.yaml")
 
@@ -49,6 +50,16 @@ def setup(request, kube_apis, ingress_controller_endpoint, test_namespace) -> Se
 
     def fin():
         print("Clean up simple app")
+        print("Collect resource usage metrics")
+        pod_metrics = get_resource_metrics(kube_apis.custom_objects, "pods", ingress_controller_prerequisites.namespace)
+        with open("ingress_pod_metrics.json", "w+") as f:
+            json.dump(pod_metrics, f, ensure_ascii=False, indent=4)
+        node_metrics = get_resource_metrics(
+            kube_apis.custom_objects, "nodes", ingress_controller_prerequisites.namespace
+        )
+        with open("ingress_node_metrics.json", "w+") as f:
+            json.dump(node_metrics, f, ensure_ascii=False, indent=4)
+
         delete_common_app(kube_apis, "simple", test_namespace)
         delete_secret(kube_apis.v1, secret_name, test_namespace)
         with open("reload_ing.json", "w+") as f:

perf-tests/suite/test_vs_perf.py

@@ -6,14 +6,23 @@
 import pytest
 import requests
 from common import collect_prom_reload_metrics, run_perf
-from suite.utils.resources_utils import wait_before_test
+from suite.utils.resources_utils import get_resource_metrics, wait_before_test
 
 reload = []
 
 
 @pytest.fixture(scope="class")
-def collect(request, kube_apis, ingress_controller_endpoint, test_namespace) -> None:
+def collect(request, kube_apis, ingress_controller_prerequisites, ingress_controller_endpoint, test_namespace) -> None:
     def fin():
+        print("Collect resource usage metrics")
+        pod_metrics = get_resource_metrics(kube_apis.custom_objects, "pods", ingress_controller_prerequisites.namespace)
+        with open("vs_pod_metrics.json", "w+") as f:
+            json.dump(pod_metrics, f, ensure_ascii=False, indent=4)
+        node_metrics = get_resource_metrics(
+            kube_apis.custom_objects, "nodes", ingress_controller_prerequisites.namespace
+        )
+        with open("vs_node_metrics.json", "w+") as f:
+            json.dump(node_metrics, f, ensure_ascii=False, indent=4)
         with open("reload_vs.json", "w+") as f:
             json.dump(reload, f, ensure_ascii=False, indent=4)
 

tests/suite/utils/resources_utils.py

@@ -1704,3 +1704,30 @@ def get_last_log_entry(kube_apis, pod_name, namespace) -> str:
     # Our log entries end in '\n' which means the final entry when we split on a new line
     # is an empty string. Return the second to last entry instead.
     return logs.split("\n")[-2]
+
+
+def get_resource_metrics(kube_apis, plural, namespace="nginx-ingress") -> str:
+    """
+    :param kube_apis: kube apis
+    :param namespace: the namespace
+    :param plural: the plural of the resource
+    """
+    if plural == "pods":
+        metrics = kube_apis.list_namespaced_custom_object("metrics.k8s.io", "v1beta1", namespace, plural)
+        while metrics["items"] == []:
+            wait_before_test()
+            try:
+                metrics = kube_apis.list_namespaced_custom_object("metrics.k8s.io", "v1beta1", namespace, plural)
+            except ApiException as e:
+                print(f"Error: {e}")
+    elif plural == "nodes":
+        metrics = kube_apis.list_cluster_custom_object("metrics.k8s.io", "v1beta1", plural)
+        while metrics["items"] == []:
+            wait_before_test()
+            try:
+                metrics = kube_apis.list_cluster_custom_object("metrics.k8s.io", "v1beta1", plural)
+            except ApiException as e:
+                print(f"Error: {e}")
+    else:
+        return "Invalid plural specified. Please use 'pods' or 'nodes' as the plural"
+    return metrics["items"]