From 8a848616e62cf73e7387ac8177ead6e59aa550c6 Mon Sep 17 00:00:00 2001 From: avalonche Date: Wed, 11 Dec 2024 11:20:49 +1100 Subject: [PATCH 1/4] Docker build time in CI --- .github/workflows/release.yml | 33 ++++++---------- Dockerfile | 73 ++++++++++++++++++++++++++--------- 2 files changed, 67 insertions(+), 39 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 15dda34..3ebd930 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ permissions: jobs: docker-image: name: Publish Docker Image - runs-on: warp-ubuntu-latest-x64-16x + runs-on: ubuntu-latest-xlarge steps: - name: Checkout sources @@ -37,36 +37,27 @@ jobs: type=pep440,pattern={{major}}.{{minor}} type=raw,value=latest,enable=${{ !contains(env.RELEASE_VERSION, '-') }} - # https://github.com/WarpBuilds/rust-cache - - name: Run WarpBuilds/rust-cache - uses: WarpBuilds/rust-cache@v2 - with: - cache-on-failure: true - - # https://github.com/Mozilla-Actions/sccache-action - - name: Setup sccache-action - uses: mozilla-actions/sccache-action@v0.0.5 - - - name: Set env vars - run: | - echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV - echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.FLASHBOTS_DOCKERHUB_USERNAME }} password: ${{ secrets.FLASHBOTS_DOCKERHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: + cache-from: | + type=gha,scope=amd64 + type=gha,scope=arm64 + cache-to: | + type=gha,mode=max,scope=amd64 + type=gha,mode=max,scope=arm64 context: . push: true build-args: | @@ -79,7 +70,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout sources - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Create release id: create_release diff --git a/Dockerfile b/Dockerfile index c2b4446..10b34d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,64 @@ -FROM lukemathwalker/cargo-chef:latest AS chef +# +# Base container (with sccache and cargo-chef) +# +# - https://github.com/mozilla/sccache +# - https://github.com/LukeMathWalker/cargo-chef +# +# Based on https://depot.dev/blog/rust-dockerfile-best-practices +# +FROM rust:1.82 as base + +ARG FEATURES + +RUN cargo install sccache --version ^0.8 +RUN cargo install cargo-chef --version ^0.1 + +RUN apt-get update \ + && apt-get install -y clang libclang-dev + +ENV CARGO_HOME=/usr/local/cargo +ENV RUSTC_WRAPPER=sccache +ENV SCCACHE_DIR=/sccache + +# +# Planner container (running "cargo chef prepare") +# +FROM base AS planner WORKDIR /app -# Prepare build plan -FROM chef AS planner -COPY ./Cargo.toml ./Cargo.lock ./ -COPY ./src ./src -RUN cargo chef prepare +COPY . . -# Build application -FROM chef AS builder +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \ + cargo chef prepare --recipe-path recipe.json -# Install system dependencies -RUN apt-get update && \ - apt-get install -y openssl libclang-dev libssl3 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* +# +# Builder container (running "cargo chef cook" and "cargo build --release") +# +FROM base as builder +WORKDIR /app +# Default binary filename +ARG ROLLUP_BOOST_BIN="rollup-boost" +COPY --from=planner /app/recipe.json recipe.json + +RUN --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \ + cargo chef cook --release --recipe-path recipe.json -COPY --from=planner /app/recipe.json . -RUN cargo chef cook --release COPY . . -RUN cargo build --release -FROM debian:bullseye-slim AS final -COPY --from=builder /app/target/release/rollup-boost /usr/local/bin/ +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \ + cargo build --release --features="$FEATURES" --package=${ROLLUP_BOOST_BIN} + +# +# Runtime container +# +FROM gcr.io/distroless/cc-debian12 +WORKDIR /app + +ARG ROLLUP_BOOST_BIN="rollup-boost" +COPY --from=builder /app/target/release/${ROLLUP_BOOST_BIN} /usr/local/bin/ ENTRYPOINT ["/usr/local/bin/rollup-boost"] \ No newline at end of file From 8525d647862f5ae65143c0cd20682dddc352426f Mon Sep 17 00:00:00 2001 From: avalonche Date: Wed, 11 Dec 2024 16:37:22 +1100 Subject: [PATCH 2/4] testing --- .github/workflows/release.yml | 39 +++++++++++++++++++++++++---------- Dockerfile | 2 +- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ebd930..5058dc6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,8 +12,16 @@ permissions: jobs: docker-image: name: Publish Docker Image - runs-on: ubuntu-latest-xlarge - + strategy: + matrix: + config: + - target: x86_64-unknown-linux-gnu + runner: warp-ubuntu-latest-x64-16x + platform: linux/amd64 + - target: aarch64-unknown-linux-gnu + runner: warp-ubuntu-latest-arm64-16x + platform: linux/arm64 + runs-on: ${{ matrix.config.runner }} steps: - name: Checkout sources uses: actions/checkout@v2 @@ -50,21 +58,30 @@ jobs: password: ${{ secrets.FLASHBOTS_DOCKERHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: - cache-from: | - type=gha,scope=amd64 - type=gha,scope=arm64 - cache-to: | - type=gha,mode=max,scope=amd64 - type=gha,mode=max,scope=arm64 + cache-from: type=gha + cache-to: type=gha,mode=max context: . - push: true build-args: | VERSION=${{ env.RELEASE_VERSION }} - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.config.platform }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ github.repository }},push-by-digest=true,name-canonical=true,push=true + + - name: Export Digest + run: | + digest="${{ steps.build.outputs.digest }}" + [ "$digest" ] || exit 1 + mkdir -p /tmp/digests + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload Digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.config.platform }} + path: /tmp/digests/* github-release: runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index 10b34d2..1da923b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ FROM rust:1.82 as base ARG FEATURES -RUN cargo install sccache --version ^0.8 +RUN cargo install sccache --version ^0.9 RUN cargo install cargo-chef --version ^0.1 RUN apt-get update \ From cc08ee6378c800cc6e1582aa4b9147d30136325d Mon Sep 17 00:00:00 2001 From: avalonche Date: Wed, 11 Dec 2024 20:08:05 +1100 Subject: [PATCH 3/4] merge tags --- .github/workflows/release.yml | 81 ++++++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5058dc6..529a660 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,25 +9,29 @@ on: permissions: contents: write +env: + REGISTRY_IMAGE: flashbots/rollup-boost + jobs: - docker-image: + build: name: Publish Docker Image strategy: matrix: config: - - target: x86_64-unknown-linux-gnu + - platform: linux/amd64 runner: warp-ubuntu-latest-x64-16x - platform: linux/amd64 - - target: aarch64-unknown-linux-gnu + - platform: linux/arm64 runner: warp-ubuntu-latest-arm64-16x - platform: linux/arm64 runs-on: ${{ matrix.config.runner }} steps: - name: Checkout sources uses: actions/checkout@v2 - - name: Get tag version - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Set env + run: | + platform=${{ matrix.config.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Print version run: | @@ -38,12 +42,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: flashbots/rollup-boost - tags: | - type=sha - type=pep440,pattern={{version}} - type=pep440,pattern={{major}}.{{minor}} - type=raw,value=latest,enable=${{ !contains(env.RELEASE_VERSION, '-') }} + images: ${{ env.REGISTRY_IMAGE }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -66,22 +65,64 @@ jobs: build-args: | VERSION=${{ env.RELEASE_VERSION }} platforms: ${{ matrix.config.platform }} - tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ github.repository }},push-by-digest=true,name-canonical=true,push=true + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - - name: Export Digest + - name: Export digest run: | - digest="${{ steps.build.outputs.digest }}" - [ "$digest" ] || exit 1 mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" touch "/tmp/digests/${digest#sha256:}" - - name: Upload Digest + - name: Upload digest uses: actions/upload-artifact@v4 with: - name: digests-${{ matrix.config.platform }} + name: digests-${{ env.PLATFORM_PAIR }} path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: ubuntu-latest + needs: + - build + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.FLASHBOTS_DOCKERHUB_USERNAME }} + password: ${{ secrets.FLASHBOTS_DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY_IMAGE }} + tags: | + type=sha + type=pep440,pattern={{version}} + type=pep440,pattern={{major}}.{{minor}} + type=raw,value=latest,enable=${{ !contains(env.RELEASE_VERSION, '-') }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} github-release: runs-on: ubuntu-latest From 9f24457c9a29c8abb0d76d6a2c876becbdcf34a7 Mon Sep 17 00:00:00 2001 From: avalonche Date: Wed, 11 Dec 2024 21:30:03 +1100 Subject: [PATCH 4/4] debug --- .github/workflows/release.yml | 1 + Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 529a660..28b1d64 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,6 +57,7 @@ jobs: password: ${{ secrets.FLASHBOTS_DOCKERHUB_TOKEN }} - name: Build and push + id: build uses: docker/build-push-action@v6 with: cache-from: type=gha diff --git a/Dockerfile b/Dockerfile index 1da923b..cf63059 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ # # Based on https://depot.dev/blog/rust-dockerfile-best-practices # -FROM rust:1.82 as base +FROM rust:1.82 AS base ARG FEATURES @@ -36,7 +36,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ # # Builder container (running "cargo chef cook" and "cargo build --release") # -FROM base as builder +FROM base AS builder WORKDIR /app # Default binary filename ARG ROLLUP_BOOST_BIN="rollup-boost"