From 2c30dc34022f55455c3848524ed0199d87ec5c12 Mon Sep 17 00:00:00 2001 From: Paul Abel <128620221+pdabelf5@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:27:44 +0000 Subject: [PATCH] refactor base image generation on main CI flow (#5187) --- .github/data/matrix-regression.json | 8 +- .github/data/matrix-smoke.json | 102 ++++++++++++---- .github/workflows/build-base-images.yml | 7 +- .github/workflows/ci.yml | 155 ++++++++++++++++-------- 4 files changed, 188 insertions(+), 84 deletions(-) diff --git a/.github/data/matrix-regression.json b/.github/data/matrix-regression.json index f6b8782825..ab3c909788 100644 --- a/.github/data/matrix-regression.json +++ b/.github/data/matrix-regression.json @@ -3,11 +3,15 @@ "images": [ { "label": "regression", - "image": "debian" + "image": "debian", + "type": "oss", + "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" }, { "label": "regression", - "image": "debian-plus" + "image": "debian-plus", + "type": "plus", + "platforms": "linux/arm64, linux/amd64" } ] } diff --git a/.github/data/matrix-smoke.json b/.github/data/matrix-smoke.json index dec0f1c3f8..332475089b 100644 --- a/.github/data/matrix-smoke.json +++ b/.github/data/matrix-smoke.json @@ -3,122 +3,176 @@ { "label": "ingresses 1/2", "image": "debian", - "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'" + "type": "oss", + "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'", + "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" }, { "label": "ingresses 2/2", "image": "debian", - "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'" + "type": "oss", + "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'", + "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" }, { "label": "VSR", "image": "alpine", - "marker": "vsr" + "type": "oss", + "marker": "vsr", + "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" }, { "label": "policies 1/2", "image": "alpine", - "marker": "'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'" + "type": "oss", + "marker": "'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'", + "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" }, { "label": "policies 2/2", "image": "alpine", - "marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls'" + "type": "oss", + "marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls'", + "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" }, { "label": "VS 1/3", "image": "debian", - "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'" + "type": "oss", + "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'", + "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" }, { "label": "VS 2/3", "image": "debian", - "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'" + "type": "oss", + "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'", + "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" }, { "label": "VS 3/3", "image": "debian", - "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'" + "type": "oss", + "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'", + "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" }, { "label": "TS", "image": "ubi", - "marker": "ts" + "type": "oss", + "marker": "ts", + "platforms": "linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" }, { "label": "VS 1/3", "image": "debian-plus", - "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'" + "type": "plus", + "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'", + "platforms": "linux/arm64, linux/amd64" }, { "label": "VS 2/3", "image": "debian-plus", - "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'" + "type": "plus", + "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'", + "platforms": "linux/arm64, linux/amd64" }, { "label": "VS 3/3", "image": "debian-plus", - "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'" + "type": "plus", + "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'", + "platforms": "linux/arm64, linux/amd64" }, { "label": "TS", "image": "debian-plus", - "marker": "ts" + "type": "plus", + "marker": "ts", + "platforms": "linux/arm64, linux/amd64" }, { "label": "ingresses 1/2", "image": "alpine-plus", - "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'" + "type": "plus", + "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'", + "platforms": "linux/arm64, linux/amd64" }, { "label": "ingresses 2/2", "image": "alpine-plus", - "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'" + "type": "plus", + "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'", + "platforms": "linux/arm64, linux/amd64" }, { "label": "VSR", "image": "alpine-plus", - "marker": "vsr" + "type": "plus", + "marker": "vsr", + "platforms": "linux/arm64, linux/amd64" }, { "label": "policies 1/2", "image": "ubi-plus", - "marker": "'policies and not policies_ac and not policies_jwt and not policies_mtls'" + "type": "plus", + "marker": "'policies and not policies_ac and not policies_jwt and not policies_mtls'", + "platforms": "linux/arm64, linux/amd64, linux/s390x" }, { "label": "policies 2/2", "image": "ubi-plus", - "marker": "'policies_ac or policies_jwt or policies_mtls'" + "type": "plus", + "marker": "'policies_ac or policies_jwt or policies_mtls'", + "platforms": "linux/arm64, linux/amd64, linux/s390x" }, { "label": "AP_WAF 1/4", "image": "ubi-9-plus-nap", - "marker": "appprotect_waf_policies_allow" + "type": "plus", + "nap_modules": "waf", + "marker": "appprotect_waf_policies_allow", + "platforms": "linux/amd64" }, { "label": "AP_WAF 2/4", "image": "alpine-plus-nap-fips", - "marker": "'appprotect_waf_policies and not appprotect_waf_policies_allow'" + "type": "plus", + "nap_modules": "waf", + "marker": "'appprotect_waf_policies and not appprotect_waf_policies_allow'", + "platforms": "linux/amd64" }, { "label": "AP_WAF 3/4", "image": "debian-plus-nap", - "marker": "appprotect_waf_policies_grpc" + "type": "plus", + "nap_modules": "waf", + "marker": "appprotect_waf_policies_grpc", + "platforms": "linux/amd64" }, { "label": "AP_WAF 4/4", "image": "debian-plus-nap", - "marker": "'appprotect_watch or appprotect_batch or appprotect_integration'" + "type": "plus", + "nap_modules": "waf", + "marker": "'appprotect_watch or appprotect_batch or appprotect_integration'", + "platforms": "linux/amd64" }, { "label": "AP_DOS", "image": "debian-plus-nap", - "marker": "'dos and not dos_learning'" + "type": "plus", + "nap_modules": "dos", + "marker": "'dos and not dos_learning'", + "platforms": "linux/amd64" }, { "label": "AP_DOS_Learning", "image": "ubi-8-plus-nap", - "marker": "dos_learning" + "type": "plus", + "nap_modules": "dos", + "marker": "dos_learning", + "platforms": "linux/amd64" } ], "k8s": [] diff --git a/.github/workflows/build-base-images.yml b/.github/workflows/build-base-images.yml index bf5c84e061..66ad132805 100644 --- a/.github/workflows/build-base-images.yml +++ b/.github/workflows/build-base-images.yml @@ -127,7 +127,7 @@ jobs: - name: Setup QEMU uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 with: - platforms: arm,arm64,ppc64le,s390x + platforms: arm64,s390x - name: Authenticate to Google Cloud id: auth @@ -204,11 +204,6 @@ jobs: - name: Docker Buildx uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0 - - name: Setup QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 - with: - platforms: arm,arm64,ppc64le,s390x - - name: Authenticate to Google Cloud id: auth uses: google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c # v2.1.2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1ce17812c..41ad29cbbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -234,63 +234,19 @@ jobs: key: nginx-ingress-${{ needs.checks.outputs.go_code_md5 }} if: ${{ needs.checks.outputs.binary_cache_hit != 'true' }} - rebuild-base-images: - name: Rebuild NIC Base images - runs-on: ubuntu-22.04 - needs: checks - permissions: - contents: read - id-token: write - steps: - - name: Checkout Repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: Docker Buildx - uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0 - if: ${{ needs.checks.outputs.forked_workflow == 'false' }} - - - name: Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c # v2.1.2 - with: - token_format: access_token - workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }} - service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }} - if: ${{ needs.checks.outputs.forked_workflow == 'false' }} - - - name: Login to GCR - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 - with: - registry: gcr.io - username: oauth2accesstoken - password: ${{ steps.auth.outputs.access_token }} - if: ${{ needs.checks.outputs.forked_workflow == 'false' }} - - - name: Check if base images exist - id: base_exists - run: | - if docker manifest inspect gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/oss:${{ needs.checks.outputs.docker_md5 }}-debian; then - echo "exists=0" >> $GITHUB_OUTPUT - else - echo "exists=1" >> $GITHUB_OUTPUT - fi - if: ${{ needs.checks.outputs.forked_workflow == 'false' }} - - - name: Rebuild base images - uses: ./.github/workflows/build-base-images.yml - if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }} - helm-tests: name: Helm Tests runs-on: ubuntu-22.04 - needs: [checks, binaries, rebuild-base-images] + needs: [checks, binaries] strategy: matrix: include: - image: debian type: oss + platforms: "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" - image: debian-plus type: plus + platforms: "linux/arm64, linux/amd64" permissions: contents: read id-token: write @@ -307,6 +263,12 @@ jobs: - name: Docker Buildx uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0 + - name: Setup QEMU + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 + with: + platforms: ${{ matrix.platforms }} + if: ${{ needs.checks.outputs.forked_workflow == 'false' }} + - name: Authenticate to Google Cloud id: auth uses: google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c # v2.1.2 @@ -324,6 +286,36 @@ jobs: password: ${{ steps.auth.outputs.access_token }} if: ${{ needs.checks.outputs.forked_workflow == 'false' }} + - name: Check if base images exist + id: base_exists + run: | + if docker manifest inspect gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.image }}; then + echo "exists=0" >> $GITHUB_OUTPUT + else + echo "exists=1" >> $GITHUB_OUTPUT + fi + if: ${{ needs.checks.outputs.forked_workflow == 'false' }} + + - name: Build Base Container + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 + with: + file: build/Dockerfile + context: "." + cache-from: type=gha,scope=${{ matrix.image }} + cache-to: type=gha,scope=${{ matrix.image }},mode=max + target: common + tags: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.image }} + platforms: ${{ matrix.platforms }} + pull: true + push: true + build-args: | + BUILD_OS=${{ matrix.image }} + IC_VERSION=${{ needs.checks.outputs.ic_version }} + secrets: | + ${{ matrix.type == 'plus' && format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) || '' }} + ${{ matrix.type == 'plus' && format('"nginx-repo.key={0}"', secrets.NGINX_KEY) || '' }} + if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }} + - name: Build Docker Image ${{ matrix.image }} uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 with: @@ -336,11 +328,11 @@ jobs: load: true build-args: | BUILD_OS=${{ matrix.image }} - PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ contains(matrix.image, 'plus') && 'plus' || 'oss' }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.image }} + PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.image }} IC_VERSION=CI secrets: | - ${{ contains(matrix.type, 'plus') && format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) || '' }} - ${{ contains(matrix.type, 'plus') && format('"nginx-repo.key={0}"', secrets.NGINX_KEY) || '' }} + ${{ matrix.type == 'plus' && format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) || '' }} + ${{ matrix.type == 'plus' && format('"nginx-repo.key={0}"', secrets.NGINX_KEY) || '' }} - name: Deploy Kubernetes id: k8s @@ -389,7 +381,7 @@ jobs: setup-matrix: name: Setup Matrix for Smoke Tests runs-on: ubuntu-22.04 - needs: [binaries, checks, rebuild-base-images] + needs: [binaries, checks] permissions: contents: read id-token: write @@ -427,7 +419,7 @@ jobs: password: ${{ steps.auth.outputs.access_token }} if: ${{ needs.checks.outputs.forked_workflow == 'false' }} - - name: Check if image exists + - name: Check if test image exists id: check-image run: | docker manifest inspect "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}" @@ -461,6 +453,65 @@ jobs: - name: Checkout Repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c # v2.1.2 + with: + token_format: access_token + workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }} + service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }} + if: ${{ needs.checks.outputs.forked_workflow == 'false' }} + + - name: Login to GCR + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: gcr.io + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + if: ${{ needs.checks.outputs.forked_workflow == 'false' }} + + - name: Check if base images exist + id: base_exists + run: | + if docker manifest inspect gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }}; then + echo "exists=0" >> $GITHUB_OUTPUT + else + echo "exists=1" >> $GITHUB_OUTPUT + fi + if: ${{ needs.checks.outputs.forked_workflow == 'false' }} + + - name: Docker Buildx + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0 + if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }} + + - name: Setup QEMU + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 + with: + platforms: ${{ matrix.platforms }} + if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }} + + - name: Build Base Container + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 + with: + file: build/Dockerfile + context: "." + cache-from: type=gha,scope=${{ matrix.images.image }} + cache-to: type=gha,scope=${{ matrix.images.image }},mode=max + target: common + tags: gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ matrix.images.type }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.images.image }} + platforms: ${{ matrix.images.platforms }} + pull: true + push: true + build-args: | + BUILD_OS=${{ matrix.images.image }} + IC_VERSION=${{ needs.checks.outputs.ic_version }} + NAP_MODULES=${{ matrix.images.nap_modules }} + secrets: | + ${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.crt={0}"', secrets.NGINX_AP_CRT) || format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) }} + ${{ contains(matrix.images.image, 'nap') && format('"nginx-repo.key={0}"', secrets.NGINX_AP_KEY) || format('"nginx-repo.key={0}"', secrets.NGINX_KEY) }} + ${{ contains(matrix.images.image, 'ubi') && format('"rhel_license={0}"', secrets.RHEL_LICENSE) || '' }} + if: ${{ needs.checks.outputs.forked_workflow == 'false' && steps.base_exists.outputs.exists != 0 }} + - name: Run Smoke Tests id: smoke-tests uses: ./.github/actions/smoke-tests