From 4f69d2ad08845e31eccccfadefdea3f6c11996eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Garillot?= Date: Sun, 17 Mar 2024 12:17:36 -0400 Subject: [PATCH] chore: Refactor benchmarks for feature handling Helps with https://github.com/lurk-lab/ci-workflows/issues/49 - Renamed several environmental variables in various benchmarking scripts and configuration files for consistency. - Introduced a new section for performing comparative benchmarks based on respective features and architecture. - Added a new variable to the default benchmark configuration, `BENCH_OUTPUT`, set to `commit-comment`. --- .github/workflows/bench-pr-comment.yml | 2 +- .github/workflows/gpu-bench.yml | 12 ++++++------ benches/bench.env | 3 ++- benches/common/mod.rs | 7 +++---- benches/common/supernova/mod.rs | 4 ++-- benches/justfile | 26 ++++++++++++++++++++------ 6 files changed, 34 insertions(+), 20 deletions(-) diff --git a/.github/workflows/bench-pr-comment.yml b/.github/workflows/bench-pr-comment.yml index 706bd2742..7fc6e104a 100644 --- a/.github/workflows/bench-pr-comment.yml +++ b/.github/workflows/bench-pr-comment.yml @@ -20,4 +20,4 @@ jobs: with: default-runner: "self-hosted,gpu-bench" default-benches: "supernova-ci" - default-env: "ARECIBO_BENCH_OUTPUT=pr-comment ARECIBO_BENCH_NUM_CONS=16384,524288" + default-env: "BENCH_OUTPUT=pr-comment BENCH_NUM_CONS=16384,524288" diff --git a/.github/workflows/gpu-bench.yml b/.github/workflows/gpu-bench.yml index f1ba0d742..c10fdf856 100644 --- a/.github/workflows/gpu-bench.yml +++ b/.github/workflows/gpu-bench.yml @@ -38,8 +38,8 @@ jobs: cargo install criterion-table - name: Set bench output format and base SHA run: | - echo "ARECIBO_BENCH_OUTPUT=commit-comment" | tee -a $GITHUB_ENV - echo "ARECIBO_BENCH_NUM_CONS=16384,1038732" | tee -a $GITHUB_ENV + echo "BENCH_OUTPUT=commit-comment" | tee -a $GITHUB_ENV + echo "BENCH_NUM_CONS=16384,1038732" | tee -a $GITHUB_ENV echo "BASE_COMMIT=${{ github.event.merge_group.base_sha }}" | tee -a $GITHUB_ENV GPU_NAME=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader,nounits | tail -n1) echo "GPU_ID=$(echo $GPU_NAME | awk '{ print $NF }')" | tee -a $GITHUB_ENV @@ -91,8 +91,8 @@ jobs: uses: peter-evans/commit-comment@v3 with: body-path: BENCHMARKS.md - # TODO: Set `$ARECIBO_BENCH_NOISE_THRESHOLD` via `cardinalby/export-env-action` or hardcode to 1.3 - # Check for a slowdown >= `$ARECIBO_BENCH_NOISE_THRESHOLD` (fallback is 30%/1.3x). If so, open an issue but don't block merge + # TODO: Set `$BENCH_NOISE_THRESHOLD` via `cardinalby/export-env-action` or hardcode to 1.3 + # Check for a slowdown >= `$BENCH_NOISE_THRESHOLD` (fallback is 30%/1.3x). If so, open an issue but don't block merge # Since we are parsing for slowdowns, we simply add 1 to the noise threshold decimal to get the regression factor - name: Check for perf regression id: regression-check @@ -100,8 +100,8 @@ jobs: REGRESSIONS=$(grep -o '[0-9.]*x slower' BENCHMARKS.md | cut -d 'x' -f1) echo $REGRESSIONS - if [ ! -z "${{ env.ARECIBO_BENCH_NOISE_THRESHOLD}}" ]; then - REGRESSION_FACTOR=$(echo "${{ env.ARECIBO_BENCH_NOISE_THRESHOLD }}+1" | bc) + if [ ! -z "${{ env.BENCH_NOISE_THRESHOLD}}" ]; then + REGRESSION_FACTOR=$(echo "${{ env.BENCH_NOISE_THRESHOLD }}+1" | bc) else REGRESSION_FACTOR=1.3 fi diff --git a/benches/bench.env b/benches/bench.env index 9ff802b99..aadae61f7 100644 --- a/benches/bench.env +++ b/benches/bench.env @@ -1,2 +1,3 @@ # Arecibo config, used only in `justfile` by default -ARECIBO_BENCH_NUM_CONS=16384,1048576 +BENCH_NUM_CONS=16384,1048576 +BENCH_OUTPUT=commit-comment \ No newline at end of file diff --git a/benches/common/mod.rs b/benches/common/mod.rs index 3afbaf146..5893f9a53 100644 --- a/benches/common/mod.rs +++ b/benches/common/mod.rs @@ -36,13 +36,12 @@ impl BenchParams { } fn output_type_env() -> anyhow::Result { - std::env::var("ARECIBO_BENCH_OUTPUT") - .map_err(|e| anyhow!("ARECIBO_BENCH_OUTPUT env var isn't set: {e}")) + std::env::var("BENCH_OUTPUT").map_err(|e| anyhow!("BENCH_OUTPUT env var isn't set: {e}")) } pub(crate) fn noise_threshold_env() -> anyhow::Result { - std::env::var("ARECIBO_BENCH_NOISE_THRESHOLD") - .map_err(|e| anyhow!("ARECIBO_BENCH_NOISE_THRESHOLD env var isn't set: {e}")) + std::env::var("BENCH_NOISE_THRESHOLD") + .map_err(|e| anyhow!("BENCH_NOISE_THRESHOLD env var isn't set: {e}")) .and_then(|nt| { nt.parse::() .map_err(|e| anyhow!("Failed to parse noise threshold: {e}")) diff --git a/benches/common/supernova/mod.rs b/benches/common/supernova/mod.rs index 5b74c3dbe..9e4132a68 100644 --- a/benches/common/supernova/mod.rs +++ b/benches/common/supernova/mod.rs @@ -54,8 +54,8 @@ pub fn num_cons() -> Vec { } fn num_cons_env() -> anyhow::Result> { - std::env::var("ARECIBO_BENCH_NUM_CONS") - .map_err(|e| anyhow!("ARECIBO_BENCH_NUM_CONS env var not set: {e}")) + std::env::var("BENCH_NUM_CONS") + .map_err(|e| anyhow!("BENCH_NUM_CONS env var not set: {e}")) .and_then(|rc| { let vec: anyhow::Result> = rc .split(',') diff --git a/benches/justfile b/benches/justfile index 6d7061897..557c2770a 100644 --- a/benches/justfile +++ b/benches/justfile @@ -9,8 +9,10 @@ commit := `git rev-parse HEAD` # Run CPU benchmarks bench +benches: #!/bin/sh + [ $(uname -m) = "x86_64" ] && FEATURES="asm" || FEATURES="default" + for bench in {{benches}}; do - cargo criterion --bench $bench + cargo criterion --bench $bench --features $FEATURES done # Run CUDA benchmarks on GPU @@ -22,6 +24,7 @@ gpu-bench +benches: export CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | sed 's/\.//g') export EC_GPU_CUDA_NVCC_ARGS="--fatbin --gpu-architecture=sm_$CUDA_ARCH --generate-code=arch=compute_$CUDA_ARCH,code=sm_$CUDA_ARCH" export EC_GPU_FRAMEWORK="cuda" + [ $(uname -m) = "x86_64" ] && FEATURES="cuda, asm" || FEATURES="cuda" for bench in {{benches}}; do cargo criterion --bench $bench --features "cuda" @@ -31,12 +34,23 @@ gpu-bench +benches: gpu-bench-ci +benches: #!/bin/sh printenv PATH - if [ $(uname -m) = "x86_64" ]; then - FEATURES="cuda,asm" - else - FEATURES="cuda" + [ $(uname -m) = "x86_64" ] && FEATURES="cuda, asm" || FEATURES="cuda" + + for bench in {{benches}}; do + cargo criterion --bench $bench --features $FEATURES --message-format=json > "$bench-{{commit}}".json + done + +comparative-bench +benches: + #!/bin/sh + # Initialize FEATURES based on architecture + [ $(uname -m) = "x86_64" ] && FEATURES="asm" || FEATURES="" + # Append cuda to FEATURES if nvcc is found + if which nvcc > /dev/null; then + FEATURES="${FEATURES:+$FEATURES,}cuda" fi + # add default if FEATURES is empty + FEATURES="${FEATURES:-default}" for bench in {{benches}}; do cargo criterion --bench $bench --features $FEATURES --message-format=json > "$bench-{{commit}}".json - done \ No newline at end of file + done