From ecd46ec903c26e5de55040611794465d615de8c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20M=C3=B6ller?= Date: Thu, 21 Nov 2024 11:59:59 +0100 Subject: [PATCH] chore: setup release to reuse CTF from components workflow (#1077) #### What this PR does / why we need it This makes sure that the CTF is only built once in the release flow (during the components build) and then reused across the release. For this it needs to be built with the correct version during release (e.g. v0.18.0-rc.1) which we now pass via `EFFECTIVE_VERSION` into the Makefiles in the repo. This allows us to get rid of the `make CTF_TYPE=directory ctf` step in the goreleaser configuration so we now no longer need to build the ctf twice. #### Which issue(s) this PR fixes Part of the release candidate transparency rework in https://github.com/open-component-model/ocm/issues/995 as it allows us to prepare the release to only use the binaries from the CTFs. --------- Co-authored-by: Hilmar Falkenberg --- .github/config/goreleaser.yaml | 1 - .github/workflows/components.yaml | 42 ++++++++++++++--- .github/workflows/release.yaml | 47 ++++++++++++++++++- Makefile | 8 +++- components/demoplugin/Makefile | 10 ++-- components/ecrplugin/Makefile | 10 ++-- components/helmdemo/Makefile | 8 +++- components/helminstaller/Makefile | 12 +++-- .../helminstaller/component-constructor.yaml | 2 +- components/ocmcli/Makefile | 10 ++-- components/subchartsdemo/Makefile | 8 +++- examples/make/Makefile | 4 +- 12 files changed, 130 insertions(+), 32 deletions(-) diff --git a/.github/config/goreleaser.yaml b/.github/config/goreleaser.yaml index 5f8e6ebec..b42711776 100644 --- a/.github/config/goreleaser.yaml +++ b/.github/config/goreleaser.yaml @@ -8,7 +8,6 @@ release: before: hooks: - go mod tidy - - make CTF_TYPE=directory ctf builds: - <<: &build_defaults diff --git a/.github/workflows/components.yaml b/.github/workflows/components.yaml index ff7f6b9af..f0e796887 100644 --- a/.github/workflows/components.yaml +++ b/.github/workflows/components.yaml @@ -3,6 +3,22 @@ name: Components on: pull_request: workflow_call: + inputs: + effective-version: + type: string + required: false + description: "The version to use for the build" + default: "" + upload-ctf: + type: boolean + required: false + description: "Whether to upload the final CTF" + default: false + ref: + type: string + description: "The ref to use for the component build, defaults to the ref where the workflow was triggered from" + required: false + default: "" push: branches: - main @@ -12,8 +28,11 @@ permissions: pull-requests: read env: + REF: ${{ inputs.ref == '' && github.ref || inputs.ref }} CTF_TYPE: directory components: '["ocmcli", "helminstaller", "helmdemo", "subchartsdemo", "ecrplugin"]' + IMAGE_PLATFORMS: 'linux/amd64 linux/arm64' + PLATFORMS: 'windows/amd64 darwin/arm64 darwin/amd64 linux/amd64 linux/arm64' jobs: define-matrix: @@ -40,6 +59,8 @@ jobs: uses: TooMuch4U/actions-clean@v2.2 - name: Checkout uses: actions/checkout@v4 + with: + ref: ${{ env.REF }} - name: Setup Go uses: actions/setup-go@v5 with: @@ -68,7 +89,14 @@ jobs: - name: CTF run: | cd components/${{ matrix.component }} - PATH=$PATH:$(go env GOPATH)/bin CTF_TYPE=${{ env.CTF_TYPE }} make ctf descriptor describe + + PATH=$PATH:$(go env GOPATH)/bin \ + CTF_TYPE=${{ env.CTF_TYPE }} \ + EFFECTIVE_VERSION=${{ inputs.effective-version }} \ + PLATFORMS="${{ env.PLATFORMS }}" \ + IMAGE_PLATFORMS="${{ env.IMAGE_PLATFORMS }}" \ + make \ + ctf descriptor describe - name: Upload CTF uses: actions/upload-artifact@v4 with: @@ -89,6 +117,8 @@ jobs: uses: TooMuch4U/actions-clean@v2.2 - name: Checkout uses: actions/checkout@v4 + with: + ref: ${{ env.REF }} - name: Download CTFs uses: actions/download-artifact@v4 with: @@ -96,8 +126,7 @@ jobs: path: gen/downloaded-ctfs - name: Move CTFs into correct directory for aggregation run: | - IFS=" " read -a COMPONENTS <<< "${{ env.components }}" - for i in "${COMPONENTS[@]}"; do + for i in ${{ env.components }}; do mkdir -p ${{ github.workspace }}/gen/${i} mv ${{ github.workspace }}/gen/downloaded-ctfs/ctf-component-${i} ${{ github.workspace }}/gen/${i}/ctf ls -R ${{ github.workspace }}/gen/${i} @@ -130,14 +159,13 @@ jobs: ${{ github.workspace }}/gen/ctf done - name: Upload aggregated CTF - # TODO This is currently permanently disabled, - # until we integrate it with the release build, in which it would be reused - if: false + # only upload the artifact if we are not on a PR + if: inputs.upload-ctf uses: actions/upload-artifact@v4 with: if-no-files-found: error overwrite: true - retention-days: 60 + retention-days: 30 name: ctf-aggregated path: gen/ctf - name: Delete old CTFs that lead up to aggregation diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 507ac7cbc..aaa885225 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -81,18 +81,22 @@ jobs: components: name: Component CTF Builds uses: ./.github/workflows/components.yaml - needs: check + needs: [check,release-version] + with: + effective-version: ${{ needs.release-version.outputs.version_no_prefix }} + upload-ctf: true + ref: ${{ github.ref }} permissions: contents: read pull-requests: read - release: needs: # run check before actual release to make sure we succeed # they will be skipped from the needs check - check - release-version + - components name: Release Build runs-on: large_runner permissions: @@ -124,6 +128,45 @@ jobs: - name: Setup Cosign uses: sigstore/cosign-installer@v3.7.0 + - name: Download CTF + uses: actions/download-artifact@v4 + with: + pattern: 'ctf-aggregated' + path: gen/downloaded-ctfs + - name: Move CTF into correct directory to be recognized by the release process + run: | + mv \ + ${{ github.workspace }}/gen/downloaded-ctfs/ctf-aggregated \ + ${{ github.workspace }}/gen/ctf + + # TODO: Remove Go setup once binaries no longer need to be built by goreleaser. + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: '${{ github.workspace }}/go.mod' + check-latest: false + cache: false + - name: Get go environment for use with cache + run: | + echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV + echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV + # This step will only reuse the go mod and build cache from main made during the Build, + # see push_ocm.yaml => "ocm-cli-latest" Job + # This means it never caches by itself and PRs cannot cause cache pollution / thrashing + # This is because we have huge storage requirements for our cache because of the mass of dependencies + - name: Restore / Reuse Cache from central build + id: cache-golang-restore + uses: actions/cache/restore@v4 # Only Restore, not build another cache (too big) + with: + path: | + ${{ env.go_cache }} + ${{ env.go_modcache }} + key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }} + restore-keys: | + ${{ env.cache_name }}-${{ runner.os }}-go- + env: + cache_name: ocm-cli-latest-go-cache # needs to be the same key in the end as in the build step + - name: Setup git config run: | git config user.name "GitHub Actions Bot" diff --git a/Makefile b/Makefile index 405434ff3..726de3d3f 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,13 @@ REPO_ROOT := $(shell dirname $(realpath $(l GITHUBORG ?= open-component-model OCMREPO ?= ghcr.io/$(GITHUBORG)/ocm VERSION := $(shell go run api/version/generate/release_generate.go print-rc-version $(CANDIDATE)) -EFFECTIVE_VERSION := $(VERSION)+$(shell git rev-parse HEAD) +COMMIT = $(shell git rev-parse --verify HEAD) +# if EFFECTIVE_VERSION is not set, set it to VERSION+HEAD +# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string +ifeq ($(EFFECTIVE_VERSION),) +EFFECTIVE_VERSION := $(VERSION)+$(COMMIT) +endif GIT_TREE_STATE := $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty) -COMMIT := $(shell git rev-parse --verify HEAD) CONTROLLER_TOOLS_VERSION ?= v0.14.0 CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen diff --git a/components/demoplugin/Makefile b/components/demoplugin/Makefile index 43b8b5803..ce83a088b 100644 --- a/components/demoplugin/Makefile +++ b/components/demoplugin/Makefile @@ -3,13 +3,17 @@ PROVIDER ?= ocm.software GITHUBORG ?= open-component-model COMPONENT = $(PROVIDER)/plugins/$(NAME) OCMREPO ?= ghcr.io/$(GITHUBORG)/ocm -PLATFORMS = linux/amd64 linux/arm64 +PLATFORMS ?= linux/amd64 linux/arm64 CTF_TYPE ?= directory REPO_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))/../.. VERSION = $(shell go run ../../api/version/generate/release_generate.go print-rc-version $(CANDIDATE)) -COMMIT = $(shell git rev-parse HEAD) -EFFECTIVE_VERSION = $(VERSION)+$(COMMIT) +COMMIT = $(shell git rev-parse --verify HEAD) +# if EFFECTIVE_VERSION is not set, set it to VERSION+COMMIT +# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string +ifeq ($(EFFECTIVE_VERSION),) +EFFECTIVE_VERSION := $(VERSION)+$(COMMIT) +endif GIT_TREE_STATE := $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty) CMDSRCS=$(shell find $(REPO_ROOT)/cmds/$(NAME) -type f) diff --git a/components/ecrplugin/Makefile b/components/ecrplugin/Makefile index ff96ab485..548be8751 100644 --- a/components/ecrplugin/Makefile +++ b/components/ecrplugin/Makefile @@ -3,14 +3,18 @@ PROVIDER ?= ocm.software GITHUBORG ?= open-component-model COMPONENT = $(PROVIDER)/plugins/$(NAME) OCMREPO ?= ghcr.io/$(GITHUBORG)/ocm -PLATFORMS = linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 +PLATFORMS ?= linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 CTF_TYPE ?= directory REPO_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))/../.. VERSION = $(shell go run ../../api/version/generate/release_generate.go print-rc-version $(CANDIDATE)) -COMMIT = $(shell git rev-parse HEAD) -EFFECTIVE_VERSION = $(VERSION)+$(COMMIT) +COMMIT = $(shell git rev-parse --verify HEAD) +# if EFFECTIVE_VERSION is not set, set it to VERSION+COMMIT +# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string +ifeq ($(EFFECTIVE_VERSION),) +EFFECTIVE_VERSION := $(VERSION)+$(COMMIT) +endif GIT_TREE_STATE := $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty) CMDSRCS=$(shell find $(REPO_ROOT)/cmds/$(NAME) -type f) diff --git a/components/helmdemo/Makefile b/components/helmdemo/Makefile index 4ceba662f..351a5626b 100644 --- a/components/helmdemo/Makefile +++ b/components/helmdemo/Makefile @@ -9,8 +9,12 @@ HELMINSTCOMP = $(PROVIDER)/toi/installers/helminstaller REPO_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))/../.. VERSION = $(shell go run ../../api/version/generate/release_generate.go print-rc-version $(CANDIDATE)) -COMMIT = $(shell git rev-parse HEAD) -EFFECTIVE_VERSION = $(VERSION)-$(COMMIT) +COMMIT = $(shell git rev-parse --verify HEAD) +# if EFFECTIVE_VERSION is not set, set it to VERSION-COMMIT +# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string +ifeq ($(EFFECTIVE_VERSION),) +EFFECTIVE_VERSION := $(VERSION)+$(COMMIT) +endif HELMINSTVERSION ?= $(VERSION) CREDS ?= diff --git a/components/helminstaller/Makefile b/components/helminstaller/Makefile index ed3d368da..761c0fd44 100644 --- a/components/helminstaller/Makefile +++ b/components/helminstaller/Makefile @@ -5,13 +5,17 @@ IMAGE := $(NAME) COMPONENT := $(PROVIDER)/toi/installers/$(NAME) OCMREPO ?= ghcr.io/$(GITHUBORG)/ocm MULTI ?= true -PLATFORMS ?= linux/amd64 linux/arm64 +IMAGE_PLATFORMS ?= linux/amd64 linux/arm64 CTF_TYPE ?= directory REPO_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))/../.. VERSION := $(shell go run ../../api/version/generate/release_generate.go print-rc-version $(CANDIDATE)) COMMIT := $(shell git rev-parse --verify HEAD) -EFFECTIVE_VERSION := $(VERSION)-$(COMMIT) +# if EFFECTIVE_VERSION is not set, set it to VERSION-COMMIT +# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string +ifeq ($(EFFECTIVE_VERSION),) +EFFECTIVE_VERSION := $(VERSION)+$(COMMIT) +endif GIT_TREE_STATE := $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty) PLATFORM := $(shell go env GOOS)/$(shell go env GOARCH) CACHE_DIR := $(shell go env GOCACHE) @@ -64,7 +68,7 @@ $(GEN)/ctf: $(OCM_BIN) $(GEN)/.exists $(GEN)/image.$(NAME)$(FLAGSUF) component-c PROVIDER="$(PROVIDER)" \ COMMIT="$(COMMIT)" \ GEN="$(GEN)" \ - PLATFORMS="$(PLATFORMS)" \ + IMAGE_PLATFORMS="$(IMAGE_PLATFORMS)" \ MULTI="$(MULTI)" \ IMAGE="$(IMAGE):$(VERSION)" \ component-constructor.yaml @@ -94,7 +98,7 @@ push-image: multi: $(GEN)/image.$(NAME).multi $(GEN)/image.$(NAME).multi: $(GEN)/.exists Dockerfile $(CMDSRCS) $(OCMSRCS) - for i in $(PLATFORMS); do \ + for i in $(IMAGE_PLATFORMS); do \ tag=$$(echo $$i | sed -e s:/:-:g); \ echo building platform $$i; \ docker buildx build --load -t $(IMAGE):$(VERSION)-$$tag --platform $$i --file Dockerfile $(REPO_ROOT) \ diff --git a/components/helminstaller/component-constructor.yaml b/components/helminstaller/component-constructor.yaml index a2223653a..c78b2c88c 100644 --- a/components/helminstaller/component-constructor.yaml +++ b/components/helminstaller/component-constructor.yaml @@ -21,5 +21,5 @@ components: input: type: (( bool(values.MULTI) ? "dockermulti" :"docker" )) repository: (( index(values.IMAGE, ":") >= 0 ? substr(values.IMAGE,0,index(values.IMAGE,":")) :values.IMAGE )) - variants: (( bool(values.MULTI) ? map[split(" ", values.PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ )) + variants: (( bool(values.MULTI) ? map[split(" ", values.IMAGE_PLATFORMS)|v|-> values.IMAGE "-" replace(v,"/","-")] :~~ )) path: (( !bool(values.MULTI) ? values.IMAGE :~~ )) \ No newline at end of file diff --git a/components/ocmcli/Makefile b/components/ocmcli/Makefile index e972f9103..b9ca7a137 100644 --- a/components/ocmcli/Makefile +++ b/components/ocmcli/Makefile @@ -7,14 +7,18 @@ COMPONENT = $(PROVIDER)/$(NAME) OCMREPO ?= ghcr.io/$(GITHUBORG)/ocm MULTI ?= true IMAGE_PLATFORMS ?= linux/amd64 linux/arm64 -PLATFORMS = $(IMAGE_PLATFORMS) darwin/arm64 darwin/amd64 windows/amd64 +PLATFORMS ?= $(IMAGE_PLATFORMS) darwin/arm64 darwin/amd64 windows/amd64 CTF_TYPE ?= directory REPO_ROOT := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))../.. GIT_TREE_STATE = $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty) VERSION = $(shell go run ../../api/version/generate/release_generate.go print-rc-version $(CANDIDATE)) -COMMIT = $(shell git rev-parse HEAD) -EFFECTIVE_VERSION = $(VERSION)+$(COMMIT) +COMMIT = $(shell git rev-parse --verify HEAD) +# if EFFECTIVE_VERSION is not set, set it to VERSION+COMMIT +# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string +ifeq ($(EFFECTIVE_VERSION),) +EFFECTIVE_VERSION := $(VERSION)+$(COMMIT) +endif PLATFORM_OS := $(shell go env GOOS) PLATFORM_ARCH := $(shell go env GOARCH) diff --git a/components/subchartsdemo/Makefile b/components/subchartsdemo/Makefile index 93ae7b583..7f67b5a7c 100644 --- a/components/subchartsdemo/Makefile +++ b/components/subchartsdemo/Makefile @@ -14,8 +14,12 @@ PODINFO_CHART_VERSION = 6.3.5 REPO_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))/../.. GIT_TREE_STATE = $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty) VERSION = $(shell go run $(REPO_ROOT)/api/version/generate/release_generate.go print-rc-version $(CANDIDATE)) -COMMIT = $(shell git rev-parse HEAD) -EFFECTIVE_VERSION = $(VERSION)-$(COMMIT) +COMMIT = $(shell git rev-parse --verify HEAD) +# if EFFECTIVE_VERSION is not set, set it to VERSION-COMMIT +# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string +ifeq ($(EFFECTIVE_VERSION),) +EFFECTIVE_VERSION := $(VERSION)+$(COMMIT) +endif HELMINSTVERSION ?= $(VERSION) CREDS ?= diff --git a/examples/make/Makefile b/examples/make/Makefile index 98fa2abac..24b19f968 100644 --- a/examples/make/Makefile +++ b/examples/make/Makefile @@ -3,8 +3,8 @@ OCI_REPO := ghcr.io/mandelsoft/cnudie REPO_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) VERSION = $(shell cat $(REPO_ROOT)/VERSION) -COMMIT = $(shell git rev-parse HEAD) -EFFECTIVE_VERSION = $(VERSION)-$(COMMIT) +COMMIT = $(shell git rev-parse --verify HEAD) +EFFECTIVE_VERSION = $(VERSION)+$(COMMIT) .PHONY: ctf ctf: ca ## Create CTF from component archive