ci: cache build objects in between internal runs

Syndica · Jan 17, 2025 · bd0de34 · bd0de34
1 parent 63544d0
commit bd0de34
Show file tree

Hide file tree

Showing 3 changed files with 125 additions and 97 deletions.
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
@@ -0,0 +1,18 @@
+name: setup
+
+inputs:
+  name:
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/download-artifact@v4
+      with:
+          name: ${{ inputs.name }}
+
+    - run: tar xf ${{ inputs.name }}.tar
+      shell: bash
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -10,23 +10,19 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - name: checkout
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - name: setup-zig
-        uses: mlugg/setup-zig@v1
+      - uses: mlugg/setup-zig@v1
         with:
           version: 0.13.0
 
       - name: lint
-        run: |
-          zig fmt --check src/ build.zig
+        run: zig fmt --check src/ build.zig
 
   check_style:
     runs-on: ubuntu-latest
     steps:
-      - name: checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python 3.10
         uses: actions/setup-python@v3
@@ -39,59 +35,77 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-        - name: checkout
-          uses: actions/checkout@v3
-
-        - name: setup-zig
-          uses: mlugg/setup-zig@v1
-          with:
-            version: 0.13.0
-
-        # GitHub's CI runners for macos seem to alternate between using a x86 and an ARM machine
-        # which is annoying to handle and results in them having invalid hardware info configurations
-        # on the virtual machine. This leads to incorrect feature detection and the CI is flaky.
-        # 
-        # If we at some point setup a self-hosted MacOS runner, we could work around the issue and
-        # enable full testing on MacOS!
-        - name: build
-          run: |
-            zig build -Dtarget=x86_64-linux -Dcpu=x86_64_v4
-            zig build -Dtarget=aarch64-macos -Dcpu=apple_m3 
+      - uses: actions/checkout@v4
+
+      - uses: mlugg/setup-zig@v1
+        with:
+          version: 0.13.0
+
+      - name: build
+        run: |
+          zig build test -Denable-tsan=true  -Dno-run -p output/tsan
+          zig build test -Denable-tsan=false -Dno-run -p output/no-tsan/
+          zig build test -Denable-tsan=true -Dblockstore=hashmap -Dfilter="ledger" -Dno-run -p output/hashmap
+      
+      # In order to not loose the execution permissions of the binaries, 
+      # we tar everything together before uploading
+      - name: bundle
+        run: tar -cvf zig-build.tar output/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: zig-build
+          path: zig-build.tar
+
+  # GitHub's CI runners for macos seem to alternate between using a x86 and an ARM machine
+  # which is annoying to handle and results in them having invalid hardware info configurations
+  # on the virtual machine. This leads to incorrect feature detection and the CI is flaky.
+  # 
+  # If we at some point setup a self-hosted MacOS runner, we could work around the issue and
+  # enable full testing on MacOS!
+  build-macos:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: mlugg/setup-zig@v1
+        with:
+          version: 0.13.0
+
+      - name: build
+        run: zig build -Dtarget=aarch64-macos -Dcpu=apple_m3
 
   test:
     strategy:
       matrix:
         os: [ubuntu-latest]
     runs-on: ${{matrix.os}}
+    needs: build
     timeout-minutes: 60
     steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
-      - name: setup-zig
-        uses: mlugg/setup-zig@v1
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
         with:
-          version: 0.13.0
+          name: zig-build
 
       - name: test
-        run: |
-          zig build test -Denable-tsan=true
-          zig build test -Denable-tsan=true -Dblockstore=hashmap -Dfilter="ledger"
+        run: output/tsan/bin/test
+
+      - name: test-hashmap
+        run: output/hashmap/bin/test
 
   kcov_test:
     strategy:
       matrix:
         os: [ubuntu-latest]
     runs-on: ${{matrix.os}}
+    needs: build
     timeout-minutes: 60
     steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
-      - name: setup-zig
-        uses: mlugg/setup-zig@v1
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
         with:
-          version: 0.13.0
+          name: zig-build
 
       - name: Set up dependencies
         run: sudo apt-get update
@@ -108,30 +122,43 @@ jobs:
           sudo ln libbfd-2.42-system.so libbfd-2.38-system.so || echo libbfd not found
 
       - name: run kcov
-        run: |
-          bash scripts/kcov_test.sh
+        run: bash scripts/kcov_test.sh output/no-tsan/bin/test
 
       - name: print coverage report
-        run: |
-          python scripts/parse_kcov.py kcov-output/test/coverage.json
+        run: python scripts/parse_kcov.py kcov-output/test/coverage.json
+
+  build-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: mlugg/setup-zig@v1
+        with:
+          version: 0.13.0
+
+      - name: build
+        run: zig build sig fuzz -Doptimize=ReleaseSafe -Dno-run
+
+      - name: bundle
+        run: tar -cvf zig-build-release.tar zig-out/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: zig-build-release
+          path: zig-build-release.tar
 
   gossip:
     strategy:
       matrix:
         os: [ubuntu-latest]
     runs-on: ${{matrix.os}}
+    needs: build-release
     timeout-minutes: 60
     steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
-      - name: setup-zig
-        uses: mlugg/setup-zig@v1
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
         with:
-          version: 0.13.0
-
-      - name: build release
-        run: zig build -Doptimize=ReleaseSafe
+          name: zig-build-release 
 
       - name: run gossip
         run: bash scripts/gossip_test.sh 120 # in seconds
@@ -142,17 +169,12 @@ jobs:
         os: [ubuntu-latest]
     runs-on: ${{matrix.os}}
     timeout-minutes: 60
+    needs: build-release
     steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
-      - name: setup zig
-        uses: mlugg/setup-zig@v1
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
         with:
-          version: 0.13.0
-
-      - name: build
-        run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz
+          name: zig-build-release 
 
       - name: run
         run: ./zig-out/bin/fuzz gossip_service 19 10000
@@ -162,21 +184,13 @@ jobs:
       matrix:
         os: [ubuntu-latest]
     runs-on: ${{matrix.os}}
+    needs: build-release
     timeout-minutes: 60
     steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
         with:
-          submodules: recursive
-
-      - name: setup zig
-        uses: mlugg/setup-zig@v1
-        with:
-          version: 0.13.0
-
-      - name: build
-        run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz
+          name: zig-build-release 
 
       - name: run
         run: ./zig-out/bin/fuzz gossip_table 19 100000
@@ -186,18 +200,13 @@ jobs:
       matrix:
         os: [ubuntu-latest]
     runs-on: ${{matrix.os}}
+    needs: build-release
     timeout-minutes: 60
     steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
-      - name: setup zig
-        uses: mlugg/setup-zig@v1
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
         with:
-          version: 0.13.0
-
-      - name: build
-        run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz
+          name: zig-build-release 
 
       - name: run
         run: ./zig-out/bin/fuzz allocators 19 10000
@@ -207,18 +216,14 @@ jobs:
       matrix:
         os: [ubuntu-latest]
     runs-on: ${{matrix.os}}
+    needs: build-release
     timeout-minutes: 60
     steps:
-      - name: checkout
-        uses: actions/checkout@v2
-        with:
-          submodules: recursive
-      - name: setup zig
-        uses: mlugg/setup-zig@v1
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
         with:
-          version: 0.13.0
-      - name: build
-        run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz
+          name: zig-build-release 
+
       - name: run
         run: ./zig-out/bin/fuzz ledger 19 10000
 

diff --git a/scripts/kcov_test.sh b/scripts/kcov_test.sh
@@ -13,21 +13,26 @@
 #   export PATH=$PATH:/path/to/kcov/build/src
 #   ```
 
-set -euxo pipefail
+set -exo pipefail
 
 echo "=> Cleaning up" 
 rm -rf kcov-output 
 mkdir kcov-output 
 
-echo "=> Building Sig" 
-zig build 
+if [ -z "$1" ]; then
+    echo "=> Building Sig" 
+    zig build 
+    test_bin="./zig-out/bin/test"
+else
+    test_bin="$1"
+fi
 
 echo "=> Running kcov on tests" 
 kcov \
     --include-pattern=src/ \
     --exclude-pattern=$HOME/.cache \
     kcov-output \
-    ./zig-out/bin/test
+    $test_bin
 
 echo "=> Opening kcov-output/index.html" 
 open kcov-output/index.html || echo "=> Failed to open kcov-output/index.html"