From 305568e6a6c2d737e442476991fe840512ddabd6 Mon Sep 17 00:00:00 2001 From: Derek Su Date: Mon, 8 Apr 2024 03:20:57 +0000 Subject: [PATCH] Expose P99 latency Expose 99th percentile. It means that 99% of the requests should be faster than given latency. Longhorn 8254 Signed-off-by: Derek Su --- fio/cmp_parse.sh | 88 +++++++++++++++++++++++++++--------------------- fio/cmp_run.sh | 6 ++-- fio/func.sh | 23 ++++++++++++- fio/parse.sh | 49 +++++++++++++++++---------- fio/run.sh | 5 ++- 5 files changed, 108 insertions(+), 63 deletions(-) diff --git a/fio/cmp_parse.sh b/fio/cmp_parse.sh index 2abcd0f..80e3f0d 100755 --- a/fio/cmp_parse.sh +++ b/fio/cmp_parse.sh @@ -17,6 +17,8 @@ if [ -z "${2}" ]; then fi METRICS="${2}" +P99_LATENCY="${3:-false}" + if [ -z "FIRST_VOL_NAME" ]; then echo Require the first volume name exit 1 @@ -49,6 +51,9 @@ parse_metrics() { for IO_TYPE in "${io_types_array[@]}"; do local output="${vol_name}-${IO_TYPE}-${METRIC}.json" local parse_func="parse_${IO_TYPE}_${METRIC}" + if [ "$P99_LATENCY" = "true" ]; then + parse_func="parse_${IO_TYPE}_${METRIC}_p99" + fi if declare -f "$parse_func" > /dev/null; then $parse_func "$output" @@ -80,8 +85,6 @@ calc_cmp_bandwidth calc_cmp_latency # Build the summary with header information -RESULT=${FIRST_VOL_NAME}_vs_${SECOND_VOL_NAME}.summary - QUICK_MODE_TEXT="Quick Mode: disabled" if [ -n "$QUICK_MODE" ]; then QUICK_MODE_TEXT="Quick Mode: enabled" @@ -92,22 +95,31 @@ if [ -n "$SIZE" ]; then SIZE_TEXT="Size: $SIZE" fi +# Determine the file name suffix and title based on P99_LATENCY +if [ "$P99_LATENCY" = "false" ]; then + FILE_SUFFIX="summary" + TITLE="FIO Benchmark Comparison Summary" +else + FILE_SUFFIX="p99_latency.summary" + TITLE="FIO Benchmark P99 Latency Comparison Summary" +fi + +RESULT="${FIRST_VOL_NAME}_vs_${SECOND_VOL_NAME}_${FILE_SUFFIX}" + SUMMARY=" -================================ -FIO Benchmark Comparsion Summary +=============================== +$TITLE For: $FIRST_VOL_NAME vs $SECOND_VOL_NAME CPU Idleness Profiling: $CPU_IDLE_PROF $SIZE_TEXT $QUICK_MODE_TEXT -================================ +=============================== " printf -v header "$CMP_FMT" \ "" $FIRST_VOL_NAME "vs" $SECOND_VOL_NAME ":" "Change" SUMMARY+=$header -#!/bin/bash - # Define a function to add metrics to the summary add_metrics_to_summary() { local metric_name="${1}" @@ -139,7 +151,7 @@ add_metrics_to_summary() { local cmp_seqwrite="${25}" local cmp_cpu_idle_pct_seqwrite="${26}" - if [ "$CPU_IDLE_PROF" = "enabled" ]; then + if [ "$CPU_IDLE_PROF" = "enabled" ] && [ "$P99_LATENCY" = "false" ]; then printf -v cxt "${metric_name} in ${metric_unit} with CPU idleness in percent (Read/Write)\n${CMP_FMT}${CMP_FMT}${CMP_FMT}\n" \ "Random:" \ "$(commaize "${first_randread}") ($(commaize "${first_cpu_idle_pct_randread}")) / $(commaize "${first_randwrite}") ($(commaize "${first_cpu_idle_pct_randwrite}"))" \ @@ -174,33 +186,35 @@ if [ "x$CPU_IDLE_PROF" = "xenabled" ]; then fi # Example usage -add_metrics_to_summary "IOPS" "ops" \ - "$FIRST_RANDREAD_IOPS" "$FIRST_CPU_IDLE_PCT_RANDREAD_IOPS" \ - "$FIRST_RANDWRITE_IOPS" "$FIRST_CPU_IDLE_PCT_RANDWRITE_IOPS" \ - "$SECOND_RANDREAD_IOPS" "$SECOND_CPU_IDLE_PCT_RANDREAD_IOPS" \ - "$SECOND_RANDWRITE_IOPS" "$SECOND_CPU_IDLE_PCT_RANDWRITE_IOPS" \ - "$CMP_RANDREAD_IOPS" "$CMP_CPU_IDLE_PCT_RANDREAD_IOPS" \ - "$CMP_RANDWRITE_IOPS" "$CMP_CPU_IDLE_PCT_RANDWRITE_IOPS" \ - "$FIRST_SEQREAD_IOPS" "$FIRST_CPU_IDLE_PCT_SEQREAD_IOPS" \ - "$FIRST_SEQWRITE_IOPS" "$FIRST_CPU_IDLE_PCT_SEQWRITE_IOPS" \ - "$SECOND_SEQREAD_IOPS" "$SECOND_CPU_IDLE_PCT_SEQREAD_IOPS" \ - "$SECOND_SEQWRITE_IOPS" "$SECOND_CPU_IDLE_PCT_SEQWRITE_IOPS" \ - "$CMP_SEQREAD_IOPS" "$CMP_CPU_IDLE_PCT_SEQREAD_IOPS" \ - "$CMP_SEQWRITE_IOPS" "$CMP_CPU_IDLE_PCT_SEQWRITE_IOPS" - -add_metrics_to_summary "Bandwidth" "KiB/sec" \ - "$FIRST_RANDREAD_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \ - "$FIRST_RANDWRITE_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \ - "$SECOND_RANDREAD_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \ - "$SECOND_RANDWRITE_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \ - "$CMP_RANDREAD_BANDWIDTH" "$CMP_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \ - "$CMP_RANDWRITE_BANDWIDTH" "$CMP_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \ - "$FIRST_SEQREAD_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \ - "$FIRST_SEQWRITE_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" \ - "$SECOND_SEQREAD_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \ - "$SECOND_SEQWRITE_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" \ - "$CMP_SEQREAD_BANDWIDTH" "$CMP_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \ - "$CMP_SEQWRITE_BANDWIDTH" "$CMP_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" +if [ "$P99_LATENCY" = "false" ]; then + add_metrics_to_summary "IOPS" "ops" \ + "$FIRST_RANDREAD_IOPS" "$FIRST_CPU_IDLE_PCT_RANDREAD_IOPS" \ + "$FIRST_RANDWRITE_IOPS" "$FIRST_CPU_IDLE_PCT_RANDWRITE_IOPS" \ + "$SECOND_RANDREAD_IOPS" "$SECOND_CPU_IDLE_PCT_RANDREAD_IOPS" \ + "$SECOND_RANDWRITE_IOPS" "$SECOND_CPU_IDLE_PCT_RANDWRITE_IOPS" \ + "$CMP_RANDREAD_IOPS" "$CMP_CPU_IDLE_PCT_RANDREAD_IOPS" \ + "$CMP_RANDWRITE_IOPS" "$CMP_CPU_IDLE_PCT_RANDWRITE_IOPS" \ + "$FIRST_SEQREAD_IOPS" "$FIRST_CPU_IDLE_PCT_SEQREAD_IOPS" \ + "$FIRST_SEQWRITE_IOPS" "$FIRST_CPU_IDLE_PCT_SEQWRITE_IOPS" \ + "$SECOND_SEQREAD_IOPS" "$SECOND_CPU_IDLE_PCT_SEQREAD_IOPS" \ + "$SECOND_SEQWRITE_IOPS" "$SECOND_CPU_IDLE_PCT_SEQWRITE_IOPS" \ + "$CMP_SEQREAD_IOPS" "$CMP_CPU_IDLE_PCT_SEQREAD_IOPS" \ + "$CMP_SEQWRITE_IOPS" "$CMP_CPU_IDLE_PCT_SEQWRITE_IOPS" + + add_metrics_to_summary "Bandwidth" "KiB/sec" \ + "$FIRST_RANDREAD_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \ + "$FIRST_RANDWRITE_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \ + "$SECOND_RANDREAD_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \ + "$SECOND_RANDWRITE_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \ + "$CMP_RANDREAD_BANDWIDTH" "$CMP_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \ + "$CMP_RANDWRITE_BANDWIDTH" "$CMP_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \ + "$FIRST_SEQREAD_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \ + "$FIRST_SEQWRITE_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" \ + "$SECOND_SEQREAD_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \ + "$SECOND_SEQWRITE_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" \ + "$CMP_SEQREAD_BANDWIDTH" "$CMP_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \ + "$CMP_SEQWRITE_BANDWIDTH" "$CMP_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" +fi add_metrics_to_summary "Latency" "ns" \ "$FIRST_RANDREAD_LATENCY" "$FIRST_CPU_IDLE_PCT_RANDREAD_LATENCY" \ @@ -218,6 +232,4 @@ add_metrics_to_summary "Latency" "ns" \ echo "$SUMMARY" > $RESULT -cat $RESULT - - +cat $RESULT \ No newline at end of file diff --git a/fio/cmp_run.sh b/fio/cmp_run.sh index 6a005a2..edf42a1 100755 --- a/fio/cmp_run.sh +++ b/fio/cmp_run.sh @@ -25,10 +25,8 @@ if [ -z "SECOND_VOL_FILE" ]; then exit 1 fi -#disable parsing in run.sh -export SKIP_PARSE=1 - $CURRENT_DIR/run.sh $FIRST_VOL_FILE $FIRST_VOL_NAME $CURRENT_DIR/run.sh $SECOND_VOL_FILE $SECOND_VOL_NAME -$CURRENT_DIR/cmp_parse.sh "$IO_TYPES" "$METRICS" +$CURRENT_DIR/cmp_parse.sh "$IO_TYPES" "$METRICS" "false" +$CURRENT_DIR/cmp_parse.sh "$IO_TYPES" "latency" "true" \ No newline at end of file diff --git a/fio/func.sh b/fio/func.sh index 7a699fd..e1d5cc0 100644 --- a/fio/func.sh +++ b/fio/func.sh @@ -75,8 +75,29 @@ parse_randwrite_latency() { CPU_IDLE_PCT_RANDWRITE_LATENCY=`cat $OUTPUT | jq '.cpu_idleness.system' | cut -f1 -d.` } +# Latency 99th percentile +parse_seqread_latency_p99() { + local OUTPUT=${1} + SEQREAD_LATENCY=`cat $OUTPUT | jq '.jobs[0].read.clat_ns.percentile["99.000000"]'| cut -f1 -d.` +} + +parse_seqwrite_latency_p99() { + local OUTPUT=${1} + SEQWRITE_LATENCY=`cat $OUTPUT | jq '.jobs[0].write.clat_ns.percentile["99.000000"]'| cut -f1 -d.` +} + +parse_randread_latency_p99() { + local OUTPUT=${1} + RANDREAD_LATENCY=`cat $OUTPUT | jq '.jobs[0].read.clat_ns.percentile["99.000000"]'| cut -f1 -d.` +} + +parse_randwrite_latency_p99() { + local OUTPUT=${1} + RANDWRITE_LATENCY=`cat $OUTPUT | jq '.jobs[0].write.clat_ns.percentile["99.000000"]'| cut -f1 -d.` +} + -FMT="%25s%25s\n" +FMT="%30s%30s\n" CMP_FMT="%20s%30s%10s%30s%10s%25s\n" commaize() { diff --git a/fio/parse.sh b/fio/parse.sh index f1a0746..a2f7ab4 100755 --- a/fio/parse.sh +++ b/fio/parse.sh @@ -17,7 +17,7 @@ append_metric() { local seqwrite="${9}" local cpu_idle_pct_seqwrite="${10}" - if [ "$CPU_IDLE_PROF" = "enabled" ]; then + if [ "$CPU_IDLE_PROF" = "enabled" ] && [ "$P99_LATENCY" = "false" ]; then # If CPU idle profiling is enabled, include it in the output printf -v cxt "%s in %s with CPU idleness in percent (Read/Write)\n$FMT$FMT\n" \ "$metric_name" "$metric_unit" \ @@ -56,6 +56,7 @@ if [ -z "${3}" ]; then fi PREFIX="${3}" +P99_LATENCY="${4:-true}" IFS=',' read -r -a io_types_array <<< "${IO_TYPES}" IFS=',' read -r -a metrics_array <<< "${METRICS}" @@ -63,35 +64,49 @@ IFS=',' read -r -a metrics_array <<< "${METRICS}" for TYPE in "${io_types_array[@]}"; do for METRIC in "${metrics_array[@]}"; do OUTPUT="${PREFIX}-${TYPE}-${METRIC}.json" - parse_${TYPE}_${METRIC} "$OUTPUT" + if [ "$P99_LATENCY" = "true" ]; then + parse_${TYPE}_${METRIC}_p99 "$OUTPUT" + else + parse_${TYPE}_${METRIC} "$OUTPUT" + fi done done # Initialize the result file name -RESULT=${PREFIX}.summary - # Build the summary with header information +if [ "$P99_LATENCY" = "false" ]; then + RESULT="${PREFIX}.summary" + TITLE="FIO Benchmark Summary" +else + RESULT="${PREFIX}_p99_latency.summary" + TITLE="FIO Benchmark P99 Latency Summary" +fi + +# Construct the SUMMARY with dynamic content SUMMARY=" -========================= -FIO Benchmark Summary +================================== +$TITLE For: $PREFIX CPU Idleness Profiling: ${CPU_IDLE_PROF:-not provided} Size: ${SIZE:-10g} Quick Mode: ${QUICK_MODE:-disabled} -========================= +================================== " # Append performance metrics to the summary -append_metric "IOPS" "ops" \ - "$RANDREAD_IOPS" "$CPU_IDLE_PCT_RANDREAD_IOPS" \ - "$RANDWRITE_IOPS" "$CPU_IDLE_PCT_RANDWRITE_IOPS" \ - "$SEQREAD_IOPS" "$CPU_IDLE_PCT_SEQREAD_IOPS" \ - "$SEQWRITE_IOPS" "$CPU_IDLE_PCT_SEQWRITE_IOPS" -append_metric "Bandwidth" "KiB/sec" \ - "$RANDREAD_BANDWIDTH" "$CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \ - "$RANDWRITE_BANDWIDTH" "$CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \ - "$SEQREAD_BANDWIDTH" "$CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \ - "$SEQWRITE_BANDWIDTH" "$CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" +if [ "$P99_LATENCY" = "false" ]; then + append_metric "IOPS" "ops" \ + "$RANDREAD_IOPS" "$CPU_IDLE_PCT_RANDREAD_IOPS" \ + "$RANDWRITE_IOPS" "$CPU_IDLE_PCT_RANDWRITE_IOPS" \ + "$SEQREAD_IOPS" "$CPU_IDLE_PCT_SEQREAD_IOPS" \ + "$SEQWRITE_IOPS" "$CPU_IDLE_PCT_SEQWRITE_IOPS" + append_metric "Bandwidth" "KiB/sec" \ + "$RANDREAD_BANDWIDTH" "$CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \ + "$RANDWRITE_BANDWIDTH" "$CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \ + "$SEQREAD_BANDWIDTH" "$CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \ + "$SEQWRITE_BANDWIDTH" "$CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" +fi + append_metric "Latency" "ns" \ "$RANDREAD_LATENCY" "$CPU_IDLE_PCT_RANDREAD_LATENCY" \ "$RANDWRITE_LATENCY" "$CPU_IDLE_PCT_RANDWRITE_LATENCY" \ diff --git a/fio/run.sh b/fio/run.sh index 66f77c6..2dfbbbc 100755 --- a/fio/run.sh +++ b/fio/run.sh @@ -42,7 +42,6 @@ for TYPE in "${io_types_array[@]}"; do done done -if [ -z "$SKIP_PARSE" ]; then - "$CURRENT_DIR/parse.sh" "$IO_TYPES" "$METRICS" "$OUTPUT" -fi +"$CURRENT_DIR/parse.sh" "$IO_TYPES" "$METRICS" "$OUTPUT" "false" +"$CURRENT_DIR/parse.sh" "$IO_TYPES" "latency" "$OUTPUT" "true"