From bd0de34bd8860d8681e46865d99fa9985976b60f Mon Sep 17 00:00:00 2001 From: David Rubin Date: Thu, 16 Jan 2025 16:37:58 -0800 Subject: [PATCH] ci: cache build objects in between internal runs --- .github/actions/setup/action.yml | 18 +++ .github/workflows/check.yml | 191 ++++++++++++++++--------------- scripts/kcov_test.sh | 13 ++- 3 files changed, 125 insertions(+), 97 deletions(-) create mode 100644 .github/actions/setup/action.yml diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 000000000..eb95f9db3 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,18 @@ +name: setup + +inputs: + name: + required: true + type: string + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + name: ${{ inputs.name }} + + - run: tar xf ${{ inputs.name }}.tar + shell: bash diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5a9245e2e..370cd8409 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -10,23 +10,19 @@ jobs: lint: runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - name: setup-zig - uses: mlugg/setup-zig@v1 + - uses: mlugg/setup-zig@v1 with: version: 0.13.0 - name: lint - run: | - zig fmt --check src/ build.zig + run: zig fmt --check src/ build.zig check_style: runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.10 uses: actions/setup-python@v3 @@ -39,59 +35,77 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v3 - - - name: setup-zig - uses: mlugg/setup-zig@v1 - with: - version: 0.13.0 - - # GitHub's CI runners for macos seem to alternate between using a x86 and an ARM machine - # which is annoying to handle and results in them having invalid hardware info configurations - # on the virtual machine. This leads to incorrect feature detection and the CI is flaky. - # - # If we at some point setup a self-hosted MacOS runner, we could work around the issue and - # enable full testing on MacOS! - - name: build - run: | - zig build -Dtarget=x86_64-linux -Dcpu=x86_64_v4 - zig build -Dtarget=aarch64-macos -Dcpu=apple_m3 + - uses: actions/checkout@v4 + + - uses: mlugg/setup-zig@v1 + with: + version: 0.13.0 + + - name: build + run: | + zig build test -Denable-tsan=true -Dno-run -p output/tsan + zig build test -Denable-tsan=false -Dno-run -p output/no-tsan/ + zig build test -Denable-tsan=true -Dblockstore=hashmap -Dfilter="ledger" -Dno-run -p output/hashmap + + # In order to not loose the execution permissions of the binaries, + # we tar everything together before uploading + - name: bundle + run: tar -cvf zig-build.tar output/ + + - uses: actions/upload-artifact@v4 + with: + name: zig-build + path: zig-build.tar + + # GitHub's CI runners for macos seem to alternate between using a x86 and an ARM machine + # which is annoying to handle and results in them having invalid hardware info configurations + # on the virtual machine. This leads to incorrect feature detection and the CI is flaky. + # + # If we at some point setup a self-hosted MacOS runner, we could work around the issue and + # enable full testing on MacOS! + build-macos: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: mlugg/setup-zig@v1 + with: + version: 0.13.0 + + - name: build + run: zig build -Dtarget=aarch64-macos -Dcpu=apple_m3 test: strategy: matrix: os: [ubuntu-latest] runs-on: ${{matrix.os}} + needs: build timeout-minutes: 60 steps: - - name: checkout - uses: actions/checkout@v2 - - - name: setup-zig - uses: mlugg/setup-zig@v1 + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup with: - version: 0.13.0 + name: zig-build - name: test - run: | - zig build test -Denable-tsan=true - zig build test -Denable-tsan=true -Dblockstore=hashmap -Dfilter="ledger" + run: output/tsan/bin/test + + - name: test-hashmap + run: output/hashmap/bin/test kcov_test: strategy: matrix: os: [ubuntu-latest] runs-on: ${{matrix.os}} + needs: build timeout-minutes: 60 steps: - - name: checkout - uses: actions/checkout@v2 - - - name: setup-zig - uses: mlugg/setup-zig@v1 + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup with: - version: 0.13.0 + name: zig-build - name: Set up dependencies run: sudo apt-get update @@ -108,30 +122,43 @@ jobs: sudo ln libbfd-2.42-system.so libbfd-2.38-system.so || echo libbfd not found - name: run kcov - run: | - bash scripts/kcov_test.sh + run: bash scripts/kcov_test.sh output/no-tsan/bin/test - name: print coverage report - run: | - python scripts/parse_kcov.py kcov-output/test/coverage.json + run: python scripts/parse_kcov.py kcov-output/test/coverage.json + + build-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: mlugg/setup-zig@v1 + with: + version: 0.13.0 + + - name: build + run: zig build sig fuzz -Doptimize=ReleaseSafe -Dno-run + + - name: bundle + run: tar -cvf zig-build-release.tar zig-out/ + + - uses: actions/upload-artifact@v4 + with: + name: zig-build-release + path: zig-build-release.tar gossip: strategy: matrix: os: [ubuntu-latest] runs-on: ${{matrix.os}} + needs: build-release timeout-minutes: 60 steps: - - name: checkout - uses: actions/checkout@v2 - - - name: setup-zig - uses: mlugg/setup-zig@v1 + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup with: - version: 0.13.0 - - - name: build release - run: zig build -Doptimize=ReleaseSafe + name: zig-build-release - name: run gossip run: bash scripts/gossip_test.sh 120 # in seconds @@ -142,17 +169,12 @@ jobs: os: [ubuntu-latest] runs-on: ${{matrix.os}} timeout-minutes: 60 + needs: build-release steps: - - name: checkout - uses: actions/checkout@v2 - - - name: setup zig - uses: mlugg/setup-zig@v1 + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup with: - version: 0.13.0 - - - name: build - run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz + name: zig-build-release - name: run run: ./zig-out/bin/fuzz gossip_service 19 10000 @@ -162,21 +184,13 @@ jobs: matrix: os: [ubuntu-latest] runs-on: ${{matrix.os}} + needs: build-release timeout-minutes: 60 steps: - - name: checkout - uses: actions/checkout@v2 - + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup with: - submodules: recursive - - - name: setup zig - uses: mlugg/setup-zig@v1 - with: - version: 0.13.0 - - - name: build - run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz + name: zig-build-release - name: run run: ./zig-out/bin/fuzz gossip_table 19 100000 @@ -186,18 +200,13 @@ jobs: matrix: os: [ubuntu-latest] runs-on: ${{matrix.os}} + needs: build-release timeout-minutes: 60 steps: - - name: checkout - uses: actions/checkout@v2 - - - name: setup zig - uses: mlugg/setup-zig@v1 + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup with: - version: 0.13.0 - - - name: build - run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz + name: zig-build-release - name: run run: ./zig-out/bin/fuzz allocators 19 10000 @@ -207,18 +216,14 @@ jobs: matrix: os: [ubuntu-latest] runs-on: ${{matrix.os}} + needs: build-release timeout-minutes: 60 steps: - - name: checkout - uses: actions/checkout@v2 - with: - submodules: recursive - - name: setup zig - uses: mlugg/setup-zig@v1 + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup with: - version: 0.13.0 - - name: build - run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz + name: zig-build-release + - name: run run: ./zig-out/bin/fuzz ledger 19 10000 diff --git a/scripts/kcov_test.sh b/scripts/kcov_test.sh index 7778a1bd6..04a8fe0ee 100755 --- a/scripts/kcov_test.sh +++ b/scripts/kcov_test.sh @@ -13,21 +13,26 @@ # export PATH=$PATH:/path/to/kcov/build/src # ``` -set -euxo pipefail +set -exo pipefail echo "=> Cleaning up" rm -rf kcov-output mkdir kcov-output -echo "=> Building Sig" -zig build +if [ -z "$1" ]; then + echo "=> Building Sig" + zig build + test_bin="./zig-out/bin/test" +else + test_bin="$1" +fi echo "=> Running kcov on tests" kcov \ --include-pattern=src/ \ --exclude-pattern=$HOME/.cache \ kcov-output \ - ./zig-out/bin/test + $test_bin echo "=> Opening kcov-output/index.html" open kcov-output/index.html || echo "=> Failed to open kcov-output/index.html"