Various CI improvements (#519)

alexklibisz · May 28, 2023 · cba5b9c · cba5b9c
1 parent 466483b
commit cba5b9c
Show file tree

Hide file tree

Showing 5 changed files with 89 additions and 96 deletions.
diff --git a/.github/scripts/delete-snapshot-releases.sh b/.github/scripts/delete-snapshot-releases.sh
@@ -1,24 +1,18 @@
 #!/bin/bash
 set -e
 
-# Delete snapshots from closed PRs.
-PRNUMS=$(hub pr list -s closed --format='%I ' --limit=10 --sort=updated)
+limit=100
+closed_prs=$(gh pr list --state closed --json number --jq '.[].number' --limit $limit)
+pr_release_tags=$(gh release list --limit $limit | grep -E "PR[0-9]+-SNAPSHOT" | awk '{print $3}')
 
-for N in $PRNUMS;
+# Loop over the closed PRs and delete any releases that correspond to a closed PR.
+for pr in $closed_prs
 do
-  TAGS=$(hub release | grep "PR$N-" || true)
-  for T in $TAGS;
+  echo "Checking PR $pr"
+  for tag in $(echo "$pr_release_tags" | grep "PR$pr-SNAPSHOT")
   do
-    echo "Deleting $T"
-    gh release delete "$T" --yes
-    git push --delete origin "$T"
+    echo "Deleting release $tag"
+    gh release delete "$tag" --yes
+    git push --delete origin "$tag"
   done
 done
-
-# Delete snapshots from master.
-TAGS=$(hub release | grep "MAIN[0-9]*-SNAPSHOT" || true)
-for T in $TAGS;
-do
-  gh release delete "$T" --yes
-  git push --delete origin "$T"
-done
diff --git a/.github/workflows/ci.yml → .github/workflows/ci.yaml b/.github/workflows/ci.yml → .github/workflows/ci.yaml
@@ -4,8 +4,6 @@ on:
   pull_request:
     branches:
       - main
-  schedule:
-    - cron: "0 0 * * 1"
 
 concurrency:
   group: ${{ github.ref }}
@@ -26,81 +24,86 @@ jobs:
     name: "Test JVM Code"
     runs-on:
       - ubuntu-22.04
-    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/cache@v3
-        with:
-          key: ${{ github.workflow }}.${{ github.job }}.r${{ github.run_number }}
-          restore-keys: |
-            ${{ github.workflow }}.${{ github.job }}
-          path: |
-            **/target/**/*
+        timeout-minutes: 1
       - uses: actions/setup-java@v3
+        timeout-minutes: 1
         with:
           distribution: 'adopt'
           java-version: 19
           cache: 'sbt'
       - uses: arduino/setup-task@v1
+        timeout-minutes: 1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Increase MMAP Limits
+        timeout-minutes: 1
         run: sudo sysctl -w vm.max_map_count=262144
       - name: Compile
+        timeout-minutes: 5
         run: task jvmCompile
       - name: Assemble
+        timeout-minutes: 1
         run: task jvmAssemble
       - name: Run Unit Tests
-        run: task jvmUnitTestQuick
+        timeout-minutes: 1
+        run: task jvmUnitTest
       - name: Run Cluster
+        timeout-minutes: 5
         run: task dockerRunTestingCluster
       - name: Run Integration Tests
-        run: task jvmIntegrationTestQuick
+        timeout-minutes: 20
+        run: task jvmIntegrationTest
       - name: Cluster Logs
+        timeout-minutes: 1
         if: always()
         run: task dockerLogTestingCluster
       - name: Stop Cluster
+        timeout-minutes: 1
         if: always()
         run: task dockerStopTestingCluster
 
   test-python:
-    name: Test Python Code
+    name: Test Python Client Code
     runs-on:
       - ubuntu-22.04
-    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/cache@v3
-        with:
-          key: ${{ github.workflow }}.${{ github.job }}.r${{ github.run_number }}
-          restore-keys: |
-            ${{ github.workflow }}.${{ github.job }}
-          path: |
-            **/target/**/*
+        timeout-minutes: 1
       - uses: actions/setup-java@v3
+        timeout-minutes: 1
         with:
           distribution: 'adopt'
           java-version: 19
           cache: 'sbt'
       - uses: actions/setup-python@v4
+        timeout-minutes: 1
         with:
           python-version: '3.7.15'
           cache: 'pip'
       - uses: arduino/setup-task@v1
+        timeout-minutes: 1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Increase MMAP Limits
+        timeout-minutes: 1
         run: sudo sysctl -w vm.max_map_count=262144
       - name: Docs
+        timeout-minutes: 1
         run: task pyDocs
       - name: Run Cluster
+        timeout-minutes: 5
         run: task dockerRunTestingCluster
       - name: Test
+        timeout-minutes: 1
         run: task pyTest
       - name: Cluster Logs
+        timeout-minutes: 1
         if: always()
         run: task dockerLogTestingCluster
       - name: Stop Cluster
+        timeout-minutes: 1
         if: always()
         run: task dockerStopTestingCluster
 
@@ -112,50 +115,56 @@ jobs:
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/cache@v3
-        with:
-          key: ${{ github.workflow }}.${{ github.job }}.r${{ github.run_number }}
-          restore-keys: |
-            ${{ github.workflow }}.${{ github.job }}
-          path: |
-            **/target/**/*
+        timeout-minutes: 1
       - uses: actions/setup-java@v3
+        timeout-minutes: 1
         with:
           distribution: 'adopt'
           java-version: 19
           cache: 'sbt'
       - uses: arduino/setup-task@v1
+        timeout-minutes: 1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/setup-python@v4
+        timeout-minutes: 1
         with:
           python-version: '3.6'
           cache: 'pip'
       - name: Increase MMAP Limits
+        timeout-minutes: 1
         run: sudo sysctl -w vm.max_map_count=262144
       - name: Initialize Submodule
+        timeout-minutes: 1
         run: task annbCreateSubmodule
       - name: Install Dependencies
+        timeout-minutes: 1
         run: task annbInstallRequirements
       - name: Run Cluster
+        timeout-minutes: 5
         run: task dockerRunTestingCluster
       - name: Test
+        timeout-minutes: 5
         run: task annbTest
       - name: Cluster Logs
+        timeout-minutes: 1
         if: always()
         run: task dockerLogTestingCluster
       - name: Stop Cluster
+        timeout-minutes: 1
         if: always()
         run: task dockerStopTestingCluster
 
-  build-jekyll-site:
+  test-jekyll-site:
     name: Test Jekyll Site
     runs-on:
       - ubuntu-22.04
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@v3
+        timeout-minutes: 1
       - uses: arduino/setup-task@v1
+        timeout-minutes: 1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Compile Jekyll Site
@@ -169,49 +178,43 @@ jobs:
     name: Publish Snapshots
     runs-on:
       - ubuntu-22.04
-    timeout-minutes: 10
-    needs: [show-github-context, test-jvm, test-python, test-benchmarks]
+    needs:
+      - test-jvm
+      - test-python
+      - test-benchmarks
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v3
+        timeout-minutes: 1
         with:
           fetch-depth: 0 # Needed for git-based changelog.
-      - uses: actions/cache@v3
-        with:
-          key: ${{ github.workflow }}.${{ github.job }}.r${{ github.run_number }}
-          restore-keys: |
-            ${{ github.workflow }}.${{ github.job }}
-          path: |
-            **/target/**/*
       - name: Setup Release Credentials
+        timeout-minutes: 1
         env:
           PYPIRC_B64: ${{ secrets.PYPIRC_B64 }}
         run: ./.github/scripts/setup-env.sh
       - uses: actions/setup-java@v3
+        timeout-minutes: 1
         with:
           distribution: 'adopt'
           java-version: 19
           cache: 'sbt'
       - uses: actions/setup-python@v4
+        timeout-minutes: 1
         with:
           python-version: '3.7.15'
           cache: 'pip'
       - uses: arduino/setup-task@v1
+        timeout-minutes: 1
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Setup Setuptools
+      - name: Install Setuptools
+        timeout-minutes: 1
         run: python3 -m pip install setuptools
       - name: Publish to PyPi
+        timeout-minutes: 1
         run: task pyPublishSnapshot VERSION=$(cat version)-dev${{ github.run_number }}
-        if: github.event_name == 'pull_request'
-      - name: Publish Plugin from PR
-        if: github.event_name == 'pull_request'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Publish Plugin to Github
+        timeout-minutes: 5
         run: task jvmPublishSnapshot VERSION=$(cat version)-PR${{ github.event.pull_request.number }}-SNAPSHOT
-      - name: Publish Plugin from Main
-        if: github.event_name == 'push'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          ./.github/scripts/delete-snapshot-releases.sh
-          task jvmPublishSnapshot VERSION=$(cat version)-MAIN${{ github.run_number }}-SNAPSHOT
diff --git a/.github/workflows/cleanup-snapshots.yaml b/.github/workflows/cleanup-snapshots.yaml
@@ -0,0 +1,18 @@
+name: "Cleanup Snapshots"
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  cleanup-snapshots:
+    name: Cleanup Snapshots
+    runs-on:
+      - ubuntu-22.04
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v3
+      - run: ./.github/scripts/delete-snapshot-releases.sh
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -54,13 +54,6 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0 # Needed for git-based changelog.
-      - uses: actions/cache@v3
-        with:
-          key: ${{ github.workflow }}.${{ github.job }}.r${{ github.run_number }}
-          restore-keys: |
-            ${{ github.workflow }}.${{ github.job }}
-          path: |
-            **/target/**/*
       - uses: actions/setup-python@v4
         with:
           python-version: '3.7.15'