migrate image builders to ubicloud

.github/workflows/docker-image.yml

@@ -33,70 +33,89 @@ concurrency:
 
 permissions: write-all
 
+# build:
+
 jobs:
   build:
-    runs-on: ubicloud
-    if:
-      (github.event_name != 'workflow_dispatch') || (github.event.inputs &&
-      !github.event.inputs.ee)
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Read EE repo commit hash
-        run: |
-          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
-
-      - uses: actions/checkout@v4
-        with:
-          repository: windmill-labs/windmill-ee-private
-          path: ./windmill-ee-private
-          ref: ${{ env.ee_repo_ref }}
-          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
-          fetch-depth: 0
-
-      # - name: Set up Docker Buildx
-      #   uses: docker/setup-buildx-action@v2
-      - uses: depot/setup-action@v1
-
-      - name: Login to registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Substitute EE code (EE logic is behind feature flag)
-        run: |
-          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
-
-      - name: Docker meta
-        id: meta-public
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          flavor: |
-            latest=false
-          tags: |
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-
-      - name: Build and push publicly
-        uses: depot/build-push-action@v1
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          push: true
-          build-args: |
-            features=embedding,parquet,openidconnect,jemalloc,deno_core,license,http_trigger,zip,oauth2,dind,php,mysql,mssql,bigquery,oracledb,websocket,python,smtp,csharp,static_frontend,rust
-          tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
-            ${{ steps.meta-public.outputs.tags }}
-          labels: |
-            ${{ steps.meta-public.outputs.labels }}
-            org.opencontainers.image.licenses=AGPLv3
+    uses: ./.github/workflows/docker-multi-platform.yml
+    with:
+      registry: docker.io
+      registry_username: ubicloud
+      image_name: "ubicloud/windmill-test"
+      fetch_depth: 0
+      push_image: true
+      build_runner_cpus: 32
+      metadata_flavor: |
+        latest=false
+      metadata_labels: |
+        org.opencontainers.image.licenses=AGPLv3
+      build_args: |
+        features=embedding,parquet,openidconnect,jemalloc,deno_core,license,http_trigger,zip,oauth2,dind,php,mysql,mssql,bigquery,oracledb,websocket,python,smtp,csharp,static_frontend,rust
+      checkout_ee: true
+    secrets: inherit # that makes secrets available to the called workflow
+
+    # runs-on: ubicloud
+    # if:
+    #   (github.event_name != 'workflow_dispatch') || (github.event.inputs &&
+    #   !github.event.inputs.ee)
+    # steps:
+    #   - uses: actions/checkout@v4
+    #     with:
+    #       fetch-depth: 0
+
+    #   - name: Read EE repo commit hash
+    #     run: |
+    #       echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
+
+    #   - uses: actions/checkout@v4
+    #     with:
+    #       repository: windmill-labs/windmill-ee-private
+    #       path: ./windmill-ee-private
+    #       ref: ${{ env.ee_repo_ref }}
+    #       token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
+    #       fetch-depth: 0
+
+    #   # - name: Set up Docker Buildx
+    #   #   uses: docker/setup-buildx-action@v2
+    #   - uses: depot/setup-action@v1
+
+    #   - name: Login to registry
+    #     uses: docker/login-action@v3
+    #     with:
+    #       registry: ${{ env.REGISTRY }}
+    #       username: ${{ github.actor }}
+    #       password: ${{ secrets.GITHUB_TOKEN }}
+
+    #   - name: Substitute EE code (EE logic is behind feature flag)
+    #     run: |
+    #       ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
+
+    #   - name: Docker meta
+    #     id: meta-public
+    #     uses: docker/metadata-action@v5
+    #     with:
+    #       images: |
+    #         ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+    #       flavor: |
+    #         latest=false
+    #       tags: |
+    #         type=semver,pattern={{version}}
+    #         type=semver,pattern={{major}}.{{minor}}
+
+    #   - name: Build and push publicly
+    #     uses: depot/build-push-action@v1
+    #     with:
+    #       context: .
+    #       platforms: linux/amd64,linux/arm64
+    #       push: true
+    #       build-args: |
+    #         features=embedding,parquet,openidconnect,jemalloc,deno_core,license,http_trigger,zip,oauth2,dind,php,mysql,mssql,bigquery,oracledb,websocket,python,smtp,csharp,static_frontend,rust
+    #       tags: |
+    #         ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
+    #         ${{ steps.meta-public.outputs.tags }}
+    #       labels: |
+    #         ${{ steps.meta-public.outputs.labels }}
+    #         org.opencontainers.image.licenses=AGPLv3
 
   build_ee:
     runs-on: ubicloud

.github/workflows/multi-build-arch.yml

@@ -0,0 +1,174 @@
+name: Build multi-platform Docker image
+
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      registry_username:
+        required: true
+        type: string
+      image_name:
+        required: true
+        type: string
+      metadata_flavor:
+        default: |
+          latest=auto
+        type: string
+      metadata_tags:
+        type: string
+        default: |
+          type=schedule
+          type=semver,pattern={{version}}
+          type=semver,pattern={{major}}.{{minor}}
+          type=semver,pattern={{major}}
+          type=ref,event=branch
+          type=ref,event=pr
+          type=sha
+      metadata_labels:
+        type: string
+        default: |
+          org.opencontainers.image.licenses=Windmill-Enterprise-License
+      build_args:
+        type: string
+        default: ""
+      push_image:
+        default: false
+        type: boolean
+      checkout_ee:
+        default: false
+        type: boolean
+      fetch_depth:
+        default: 1
+        type: number
+      after_checkout_command:
+        default: ""
+        type: string
+      build_runner_cpus:
+        default: 4
+        type: number
+
+jobs:
+  docker:
+    name: Docker ${{ matrix.platform }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubicloud-standard-${{ inputs.build_runner_cpus }}
+            platform: linux/amd64
+          - runner: ubicloud-standard-${{ inputs.build_runner_cpus }}-arm
+            platform: linux/arm64
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: ${{ inputs.fetch_depth }}
+
+      - name: Read EE repo commit hash
+        run: |
+          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v4
+        if: ${{ inputs.checkout_ee }}
+        with:
+          repository: windmill-labs/windmill-ee-private
+          path: ./windmill-ee-private
+          ref: ${{ env.ee_repo_ref }}
+          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
+          fetch-depth: ${{ inputs.fetch_depth }}
+
+      - name: Run command after checkout
+        if: ${{ inputs.after_checkout_command }}
+        run: ${{ inputs.after_checkout_command }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ inputs.registry }}/${{ inputs.image_name }}
+          tags: ${{ inputs.metadata_tags }}
+          labels: ${{ inputs.metadata_labels }}
+
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ inputs.registry }}
+          username: ${{ inputs.registry_username }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: ${{ inputs.build_args }}
+          cache-from: type=gha,scope=ubicloud-${{ matrix.runner }}
+          cache-to: type=gha,mode=max,scope=ubicloud-${{ matrix.runner }}
+          outputs:
+            type=image,name=${{ inputs.registry }}/${{ inputs.image_name
+            }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        if: ${{ inputs.push_image }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.runner }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    if: ${{ inputs.push_image }}
+    runs-on: ubicloud
+    needs:
+      - docker
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          pattern: digests-*
+          merge-multiple: true
+          path: /tmp/digests
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ inputs.registry }}/${{ inputs.image_name }}
+          tags: ${{ inputs.metadata_tags }}
+          labels: ${{ inputs.metadata_labels }}
+
+      - name: Login to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ inputs.registry }}
+          username: ${{ inputs.registry_username }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ inputs.registry }}/${{ inputs.image_name }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ inputs.registry }}/${{ inputs.image_name }}:${{ steps.meta.outputs.version }}