ci: cache build objects in between internal runs

Syndica · Jan 17, 2025 · 88aabd7 · 88aabd7
1 parent 63544d0
commit 88aabd7
Show file tree

Hide file tree

Showing 4 changed files with 225 additions and 158 deletions.
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
@@ -0,0 +1,22 @@
+name: setup
+
+inputs:
+  name:
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v4
+
+    - uses: mlugg/setup-zig@v1
+      with:
+        version: 0.13.0
+
+    - uses: actions/download-artifact@v4
+      with:
+          name: ${{ inputs.name }}
+          path: ${{ inputs.name }}.tar
+
+    - run: tar xf ${{ inputs.name }}.tar
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -10,8 +10,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - name: checkout
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: setup-zig
         uses: mlugg/setup-zig@v1
@@ -25,8 +24,7 @@ jobs:
   check_style:
     runs-on: ubuntu-latest
     steps:
-      - name: checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python 3.10
         uses: actions/setup-python@v3
@@ -39,188 +37,205 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-        - name: checkout
-          uses: actions/checkout@v3
-
-        - name: setup-zig
-          uses: mlugg/setup-zig@v1
-          with:
-            version: 0.13.0
-
-        # GitHub's CI runners for macos seem to alternate between using a x86 and an ARM machine
-        # which is annoying to handle and results in them having invalid hardware info configurations
-        # on the virtual machine. This leads to incorrect feature detection and the CI is flaky.
-        # 
-        # If we at some point setup a self-hosted MacOS runner, we could work around the issue and
-        # enable full testing on MacOS!
-        - name: build
-          run: |
-            zig build -Dtarget=x86_64-linux -Dcpu=x86_64_v4
-            zig build -Dtarget=aarch64-macos -Dcpu=apple_m3 
-
-  test:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-    runs-on: ${{matrix.os}}
-    timeout-minutes: 60
-    steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
-      - name: setup-zig
-        uses: mlugg/setup-zig@v1
-        with:
-          version: 0.13.0
-
-      - name: test
-        run: |
-          zig build test -Denable-tsan=true
-          zig build test -Denable-tsan=true -Dblockstore=hashmap -Dfilter="ledger"
-
-  kcov_test:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-    runs-on: ${{matrix.os}}
-    timeout-minutes: 60
-    steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
+      - uses: actions/checkout@v4
+
       - name: setup-zig
         uses: mlugg/setup-zig@v1
         with:
           version: 0.13.0
-
-      - name: Set up dependencies
-        run: sudo apt-get update
-
-      - name: install kcov
-        run: |
-          wget https://github.com/SimonKagstrom/kcov/releases/download/v42/kcov-amd64.tar.gz
-          sudo tar xf kcov-amd64.tar.gz -C /
 
-      - name: fix kcov dependencies
+      - name: build
         run: |
-          cd /usr/lib/x86_64-linux-gnu/
-          sudo ln libopcodes-2.42-system.so libopcodes-2.38-system.so || echo libopcodes not found
-          sudo ln libbfd-2.42-system.so libbfd-2.38-system.so || echo libbfd not found
+          zig build -Denable-tsan -p output/
+          zig build test -Denable-tsan=true -Dblockstore=hashmap -Dfilter="ledger" -Dno-run -p output/tsan
+      
+      # in order to not loose the execution permissions of the binaries, we tar everything together 
+      # before uploading
+      - name: bundle
+        run: tar -cvf zig-build.tar output/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: zig-build
+          path: zig-build.tar
+
+  # # GitHub's CI runners for macos seem to alternate between using a x86 and an ARM machine
+  # # which is annoying to handle and results in them having invalid hardware info configurations
+  # # on the virtual machine. This leads to incorrect feature detection and the CI is flaky.
+  # # 
+  # # If we at some point setup a self-hosted MacOS runner, we could work around the issue and
+  # # enable full testing on MacOS!
+  # build-macos:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
 
-      - name: run kcov
-        run: |
-          bash scripts/kcov_test.sh
+  #     - name: setup-zig
+  #       uses: mlugg/setup-zig@v1
+  #       with:
+  #         version: 0.13.0
 
-      - name: print coverage report
-        run: |
-          python scripts/parse_kcov.py kcov-output/test/coverage.json
+  #     - name: build
+  #       run: zig build -Dtarget=aarch64-macos -Dcpu=apple_m3
 
-  gossip:
+  test:
     strategy:
       matrix:
         os: [ubuntu-latest]
     runs-on: ${{matrix.os}}
+    needs: build
     timeout-minutes: 60
     steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
-      - name: setup-zig
-        uses: mlugg/setup-zig@v1
+      - uses: actions/checkout@v4
+      - uses: .github/actions/setup
         with:
-          version: 0.13.0
+          name: zig-build
 
-      - name: build release
-        run: zig build -Doptimize=ReleaseSafe
+      - name: test
+        run: |
+          output/bin/test
+          output/tsan/bin/test
 
-      - name: run gossip
-        run: bash scripts/gossip_test.sh 120 # in seconds
+  # kcov_test:
+  #   strategy:
+  #     matrix:
+  #       os: [ubuntu-latest]
+  #   runs-on: ${{matrix.os}}
+  #   needs: build
+  #   timeout-minutes: 60
+  #   steps:
+  #     - uses: actions/checkout@v4
 
-  gossip_service_fuzz:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-    runs-on: ${{matrix.os}}
-    timeout-minutes: 60
-    steps:
-      - name: checkout
-        uses: actions/checkout@v2
+  #     - name: setup-zig
+  #       uses: mlugg/setup-zig@v1
+  #       with:
+  #         version: 0.13.0
 
-      - name: setup zig
-        uses: mlugg/setup-zig@v1
-        with:
-          version: 0.13.0
+  #     - uses: actions/download-artifact@v4
+  #       with:
+  #         name: zig-build
+  #         path: output/
 
-      - name: build
-        run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz
+  #     - name: Set up dependencies
+  #       run: sudo apt-get update
 
-      - name: run
-        run: ./zig-out/bin/fuzz gossip_service 19 10000
+  #     - name: install kcov
+  #       run: |
+  #         wget https://github.com/SimonKagstrom/kcov/releases/download/v42/kcov-amd64.tar.gz
+  #         sudo tar xf kcov-amd64.tar.gz -C /
+
+  #     - name: fix kcov dependencies
+  #       run: |
+  #         cd /usr/lib/x86_64-linux-gnu/
+  #         sudo ln libopcodes-2.42-system.so libopcodes-2.38-system.so || echo libopcodes not found
+  #         sudo ln libbfd-2.42-system.so libbfd-2.38-system.so || echo libbfd not found
+
+  #     - name: run kcov
+  #       run: |
+  #         bash scripts/kcov_test.sh output/zig-out/bin/test
+
+  #     - name: print coverage report
+  #       run: |
+  #         python scripts/parse_kcov.py kcov-output/test/coverage.json
+
+  # build-release:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+
+  #     - name: setup-zig
+  #       uses: mlugg/setup-zig@v1
+  #       with:
+  #         version: 0.13.0
+
+  #     - name: build
+  #       run: |
+  #         zig build sig fuzz -Doptimize=ReleaseSafe -Dno-run
+
+  #     - uses: actions/upload-artifact@v4
+  #       with:
+  #         name: zig-build-release
+  #         path: zig-out/
 
-  gossip_table_fuzz:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-    runs-on: ${{matrix.os}}
-    timeout-minutes: 60
-    steps:
-      - name: checkout
-        uses: actions/checkout@v2
+  # gossip:
+  #   strategy:
+  #     matrix:
+  #       os: [ubuntu-latest]
+  #   runs-on: ${{matrix.os}}
+  #   needs: build-release
+  #   timeout-minutes: 60
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: ./.github/workflows/setup.yml
+  #       with:
+  #         name: zig-build-release 
 
-        with:
-          submodules: recursive
+  #     - name: run gossip
+  #       run: bash scripts/gossip_test.sh 120 # in seconds
 
-      - name: setup zig
-        uses: mlugg/setup-zig@v1
-        with:
-          version: 0.13.0
+  # gossip_service_fuzz:
+  #   strategy:
+  #     matrix:
+  #       os: [ubuntu-latest]
+  #   runs-on: ${{matrix.os}}
+  #   timeout-minutes: 60
+  #   needs: build-release
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: ./.github/workflows/setup.yml
+  #       with:
+  #         name: zig-build-release 
 
-      - name: build
-        run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz
+  #     - name: run
+  #       run: ./zig-out/bin/fuzz gossip_service 19 10000
 
-      - name: run
-        run: ./zig-out/bin/fuzz gossip_table 19 100000
+  # gossip_table_fuzz:
+  #   strategy:
+  #     matrix:
+  #       os: [ubuntu-latest]
+  #   runs-on: ${{matrix.os}}
+  #   needs: build-release
+  #   timeout-minutes: 60
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: ./.github/workflows/setup.yml
+  #       with:
+  #         name: zig-build-release 
 
-  allocators_fuzz:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-    runs-on: ${{matrix.os}}
-    timeout-minutes: 60
-    steps:
-      - name: checkout
-        uses: actions/checkout@v2
+  #     - name: run
+  #       run: ./zig-out/bin/fuzz gossip_table 19 100000
 
-      - name: setup zig
-        uses: mlugg/setup-zig@v1
-        with:
-          version: 0.13.0
+  # allocators_fuzz:
+  #   strategy:
+  #     matrix:
+  #       os: [ubuntu-latest]
+  #   runs-on: ${{matrix.os}}
+  #   needs: build-release
+  #   timeout-minutes: 60
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: ./.github/workflows/setup.yml
+  #       with:
+  #         name: zig-build-release 
 
-      - name: build
-        run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz
+  #     - name: run
+  #       run: ./zig-out/bin/fuzz allocators 19 10000
 
-      - name: run
-        run: ./zig-out/bin/fuzz allocators 19 10000
+  # ledger_fuzz:
+  #   strategy:
+  #     matrix:
+  #       os: [ubuntu-latest]
+  #   runs-on: ${{matrix.os}}
+  #   needs: build-release
+  #   timeout-minutes: 60
+  #   steps:
+      # - uses: actions/checkout@v4
+      # - uses: ./.github/workflows/setup.yml
+      #   with:
+      #     name: zig-build-release 
 
-  ledger_fuzz:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-    runs-on: ${{matrix.os}}
-    timeout-minutes: 60
-    steps:
-      - name: checkout
-        uses: actions/checkout@v2
-        with:
-          submodules: recursive
-      - name: setup zig
-        uses: mlugg/setup-zig@v1
-        with:
-          version: 0.13.0
-      - name: build
-        run: zig build -Doptimize=ReleaseSafe -Dno-run fuzz
-      - name: run
-        run: ./zig-out/bin/fuzz ledger 19 10000
+  #     - name: run
+  #       run: ./zig-out/bin/fuzz ledger 19 10000
 
   # benchmarks:
   #   if: ${{ github.ref != 'refs/heads/main' }}