From f8774faf94ad85a0a5cb968909a5342824c0508a Mon Sep 17 00:00:00 2001 From: Leandro Mendes Date: Wed, 18 Dec 2024 13:03:13 +0100 Subject: [PATCH] refactor(RELEASE-1041): move and rename iib pipeline this PR moves the IIB internal pipeline and tasks to the release-service-catalog and renames it to update-fbc-catalog. also adds a workspace to save files that are used by the task and tests. Signed-off-by: Leandro Mendes --- .../update-fbc-catalog-pipeline.yaml | 1 + .../resources/update-fbc-catalog-task.yaml | 1 + .../internal/update-fbc-catalog/README.md | 16 + .../update-fbc-catalog-pipeline.yaml | 86 +++++ .../update-fbc-catalog-task/README.md | 14 + .../update-fbc-catalog-task/tests/mocks.sh | 139 ++++++++ .../tests/pre-apply-task-hook.sh | 24 ++ .../tests/test-update-fbc-catalog-error.yaml | 98 ++++++ .../tests/test-update-fbc-catalog-hotfix.yaml | 105 ++++++ .../tests/test-update-fbc-catalog-prod.yaml | 103 ++++++ .../test-update-fbc-catalog-staged-index.yaml | 105 ++++++ .../test-update-fbc-catalog-timeout.yaml | 57 +++ .../update-fbc-catalog-task.yaml | 324 ++++++++++++++++++ tasks/managed/add-fbc-contribution/README.md | 3 + .../add-fbc-contribution.yaml | 4 +- 15 files changed, 1078 insertions(+), 2 deletions(-) create mode 120000 internal/resources/update-fbc-catalog-pipeline.yaml create mode 120000 internal/resources/update-fbc-catalog-task.yaml create mode 100644 pipelines/internal/update-fbc-catalog/README.md create mode 100644 pipelines/internal/update-fbc-catalog/update-fbc-catalog-pipeline.yaml create mode 100644 tasks/internal/update-fbc-catalog-task/README.md create mode 100755 tasks/internal/update-fbc-catalog-task/tests/mocks.sh create mode 100755 tasks/internal/update-fbc-catalog-task/tests/pre-apply-task-hook.sh create mode 100644 tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-error.yaml create mode 100644 tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-hotfix.yaml create mode 100644 tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-prod.yaml create mode 100644 tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-staged-index.yaml create mode 100644 tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-timeout.yaml create mode 100644 tasks/internal/update-fbc-catalog-task/update-fbc-catalog-task.yaml diff --git a/internal/resources/update-fbc-catalog-pipeline.yaml b/internal/resources/update-fbc-catalog-pipeline.yaml new file mode 120000 index 000000000..24e755aed --- /dev/null +++ b/internal/resources/update-fbc-catalog-pipeline.yaml @@ -0,0 +1 @@ +../../pipelines/internal/update-fbc-catalog/update-fbc-catalog-pipeline.yaml \ No newline at end of file diff --git a/internal/resources/update-fbc-catalog-task.yaml b/internal/resources/update-fbc-catalog-task.yaml new file mode 120000 index 000000000..c95c799b0 --- /dev/null +++ b/internal/resources/update-fbc-catalog-task.yaml @@ -0,0 +1 @@ +../../tasks/internal/update-fbc-catalog-task/update-fbc-catalog-task.yaml \ No newline at end of file diff --git a/pipelines/internal/update-fbc-catalog/README.md b/pipelines/internal/update-fbc-catalog/README.md new file mode 100644 index 000000000..842fa94d8 --- /dev/null +++ b/pipelines/internal/update-fbc-catalog/README.md @@ -0,0 +1,16 @@ +# update-fbc-catalog pipeline + +Tekton pipeline add/update FBC fragments to the FBC catalog by interacting with IIB service for File Based Catalogs + +## Parameters + +| Name | Description | Optional | Default value | +|-------------------------|-----------------------------------------------------------------------------|----------|---------------------| +| iibServiceAccountSecret | Secret containing the credentials for IIB service | yes | iib-service-account | +| fbcFragment | FBC fragment built by HACBS | no | - | +| fromIndex | Index image (catalog of catalogs) the FBC fragment will be added to | no | - | +| buildTags | List of additional tags the internal index image copy should be tagged with | yes | '[]' | +| addArches | List of arches the index image should be built for | yes | '[]' | +| hotfix | Whether this build is a hotfix build | yes | false | +| stagedIndex | Whether this build is a staged index build | yes | false | +| buildTimeoutSeconds | IIB Build Service timeout seconds | no | - | diff --git a/pipelines/internal/update-fbc-catalog/update-fbc-catalog-pipeline.yaml b/pipelines/internal/update-fbc-catalog/update-fbc-catalog-pipeline.yaml new file mode 100644 index 000000000..8bf5f67ff --- /dev/null +++ b/pipelines/internal/update-fbc-catalog/update-fbc-catalog-pipeline.yaml @@ -0,0 +1,86 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: update-fbc-catalog + labels: + app.kubernetes.io/version: "1.0.0" + annotations: + tekton.dev/pipelines.minVersion: "0.12.1" + tekton.dev/tags: fbc +spec: + description: >- + Tekton pipeline add/update FBC fragments to the FBC catalog by interacting with IIB service for File Based Catalogs + params: + - name: iibServiceAccountSecret + type: string + description: Secret containing the credentials for IIB service + default: iib-service-account + - name: fbcFragment + type: string + description: FBC fragment built by HACBS + - name: fromIndex + type: string + description: >- + Index image (catalog of catalogs) the FBC fragment will be added to + - name: targetIndex + type: string + description: >- + Target index is the pullspec the FBC catalog will be pushed to + - name: buildTags + type: string + default: '[]' + description: >- + List of additional tags the internal index image copy should be + tagged with + - name: addArches + type: string + default: '[]' + description: List of arches the index image should be built for + - name: hotfix + type: string + default: "false" + description: Whether this build is a hotfix build + - name: stagedIndex + type: string + default: "false" + description: Whether this build is a staged index build + - name: buildTimeoutSeconds + type: string + description: IIB Build Service timeout seconds + tasks: + - name: update-fbc-catalog-task + taskRef: + name: update-fbc-catalog-task + params: + - name: iibServiceAccountSecret + value: $(params.iibServiceAccountSecret) + - name: fbcFragment + value: $(params.fbcFragment) + - name: fromIndex + value: $(params.fromIndex) + - name: targetIndex + value: $(params.targetIndex) + - name: buildTags + value: $(params.buildTags) + - name: addArches + value: $(params.addArches) + - name: hotfix + value: $(params.hotfix) + - name: stagedIndex + value: $(params.stagedIndex) + - name: buildTimeoutSeconds + value: $(params.buildTimeoutSeconds) + results: + - name: jsonBuildInfo + value: $(tasks.update-fbc-catalog-task.results.jsonBuildInfo) + - name: buildState + value: $(tasks.update-fbc-catalog-task.results.buildState) + - name: genericResult + value: $(tasks.update-fbc-catalog-task.results.genericResult) + - name: indexImageDigests + value: $(tasks.update-fbc-catalog-task.results.indexImageDigests) + - name: iibLog + value: $(tasks.update-fbc-catalog-task.results.iibLog) + - name: exitCode + value: $(tasks.update-fbc-catalog-task.results.exitCode) diff --git a/tasks/internal/update-fbc-catalog-task/README.md b/tasks/internal/update-fbc-catalog-task/README.md new file mode 100644 index 000000000..2ae75c6c6 --- /dev/null +++ b/tasks/internal/update-fbc-catalog-task/README.md @@ -0,0 +1,14 @@ +# update-fbc-catalog task + +Tekton task to submit a IIB build request to add/update a fbc-fragment to an index image + +| Name | Description | Optional | Default value | +| ----------------------- | ---------------------------------------------------------------------------- | -------- | ------------- | +| fbcFragment | FBC fragment built by HACBS | No | - | +| fromIndex | Index image (catalog of catalogs) the FBC fragment will be added to | No | - | +| buildTags | List of additional tags the internal index image copy should be tagged with. | No | - | +| addArches | List of arches the index image should be built for. | No | - | +| buildTimeoutSeconds | Timeout seconds to receive the build state | Yes | "300" | +| iibServiceAccountSecret | Secret with IIB credentials to be used | No | - | +| hotfix | Whether this build is a hotfix build | Yes | "false" | +| stagedIndex | Whether this build is for a staged index build | Yes | "false" | diff --git a/tasks/internal/update-fbc-catalog-task/tests/mocks.sh b/tasks/internal/update-fbc-catalog-task/tests/mocks.sh new file mode 100755 index 000000000..45750005f --- /dev/null +++ b/tasks/internal/update-fbc-catalog-task/tests/mocks.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +set -x + +# seed for the build status +yq -o json <<< ' +items: +- id: 1 + distribution_scope: "stage" + fbc_fragment: "registry.io/image0@sha256:0000" + internal_index_image_copy: "registry-proxy-stage.engineering.redhat.com/rh-osbs-stage/iib:1" + index_image_resolved: "registry-proxy-stage.engineering.redhat.com/rh-osbs-stage/iib@sha256:0000" + logs: + url: "https://fakeiib.host/api/v1/builds/1/logs" + request_type: "fbc-operations" + state: "in_progress" + state_reason: "The request was initiated" + state_history: + - state: "in_progress" + state_reason: "The request was initiated" + user: "iib@kerberos"' > /tmp/build-seed + +buildSeed=$(cat /tmp/build-seed) +buildJson=$(jq -cr '.items[0]' <<< "${buildSeed}") + +export buildSeed buildJson calls + +function mock_build_progress() { + + state_reason[1]="Resolving the fbc fragment" + state_reason[2]="Resolving the container images" + state_reason[3]="The FBC fragment was successfully added in the index image" + + encoded_script="$2" + calls="$1" + mock_error="$3" + + build="$(base64 -d <<< "$encoded_script")" + if [ -n "$mock_error" ]; then + build=$(jq -rc '.state |= "failed"' <<< "$build") + build=$(jq -rc '.state_reason |= "IIB Mocked Error"' <<< "${build}") + jq -rc --argjson progress "{ \"state\": \"failed\", \"state_reason\": \"IIB Mocked Error\" }" '.state_history |= [$progress] + .' <<< "${build}" + exit + fi + + if [ "$calls" -gt "${#state_reason[@]}" ]; then + jq -cr . <<< "${build}" + elif [ "$calls" -eq "${#state_reason[@]}" ]; then + build=$(jq -rc '.state |= "complete"' <<< "$build") + build=$(jq -rc '.state_reason |= "The FBC fragment was successfully added in the index image"' <<< "${build}") + jq -rc --argjson progress "{ \"state\": \"complete\", \"state_reason\": \"${state_reason[$calls]}\" }" '.state_history |= [$progress] + .' <<< "${build}" + if [[ "$(context.taskRun.name)" =~ test-update-fbc-catalog-retry-outdated* ]]; then + build=$(jq -rc '.items[0].retry = "outdated"' <<< "${build}") + fi + exit + else + jq -rc --argjson progress "{ \"state\": \"in_progress\", \"state_reason\": \"${state_reason[$calls]}\" }" '.state_history |= [$progress] + .' <<< "${build}" + fi +} + +function curl() { + params="$*" + + if [[ "$params" =~ "--negotiate -u: https://pyxis.engineering.redhat.com/v1/repositories/registry/quay.io/repository/repo/image -o"* ]]; then + tempfile="$5" + echo -e '{ "fbc_opt_in": true }' > "$tempfile" + + elif [[ "$params" =~ "-s https://fakeiib.host/builds?user=iib@kerberos&from_index=quay.io/scoheb/fbc-index-testing:"* ]]; then + if [[ "$(context.taskRun.name)" =~ "test-update-fbc-catalog-retry-in-progress"* ]]; then + echo -en "${buildSeed}" + elif [[ "$(context.taskRun.name)" =~ test-update-fbc-catalog-retry* ]]; then + build=$(jq -rc '.items[0].state = "complete"' <<< "$buildSeed") + build=$(jq -rc '.items[0].state_reason = "The FBC fragment was successfully added in the index image"' <<< "${build}") + + if [[ "$(context.taskRun.name)" =~ test-update-fbc-catalog-retry-outdated* ]]; then + build=$(jq -rc '.items[0].retry = "outdated"' <<< "${build}") + fi + fi + + elif [[ "$params" == "-s https://fakeiib.host/builds/1" ]]; then + echo "$*" >> mock_build_progress_calls + if [[ "$(context.taskRun.name)" =~ "test-update-fbc-catalog-error"* ]]; then + mock_error="true" + fi + + mock_build_progress "$(awk 'END{ print NR }' mock_build_progress_calls)" "$(base64 <<< "${buildJson}")" "$mock_error" | tee build_json + export -n buildJson + buildJson=$(cat build_json) + export buildJson + + elif [[ "$params" == "-s https://fakeiib.host/api/v1/builds/1/logs" ]]; then + echo "Logs are for weaks" + + elif [[ "$params" =~ "-u : --negotiate -s -X POST -H Content-Type: application/json -d@".*" --insecure https://fakeiib.host/builds/fbc-operations" ]]; then + echo -en "${buildJson}" | jq -cr + + else + echo "" + + fi +} + +function opm() { + echo '{ "schema": "olm.bundle", "image": "quay.io/repo/image@sha256:abcd1234"}' +} + +function base64() { + echo "decrypted-keytab" +} + +function kinit() { + echo "Ok" +} + +function skopeo() { + today="$(date --iso-8601="seconds")" + yesterday="$(date --date="yesterday" --iso-8601="seconds")" + tomorrow="$(date --date="tomorrow" --iso-8601="seconds")" + + shift + if [[ "$*" == "--raw docker://registry-proxy-stage.engineering.redhat.com/rh-osbs-stage/iib:1" ]]; then + echo '{"manifests": [ { "mediaType": "application/vnd.docker.distribution.manifest.v2+json", "digest": "sha256:000" }]}' + fi + + if [[ "$*" == "--config docker://registry-proxy-stage.engineering.redhat.com/rh-osbs-stage/iib@sha256:0000" ]]; then + echo '{"created": "'"${today}"'"}' + fi + + if [[ "$*" == "--config docker://quay.io/fbc/catalog:complete" ]]; then + echo '{"created": "'"${yesterday}"'"}' + fi + + if [[ "$*" == "--config docker://quay.io/fbc/catalog:outdated" ]]; then + echo '{"created": "'"${tomorrow}"'"}' + fi +} + +# the watch_build_state can't reach some mocks by default, so exporting them fixes it. +export -f curl +export -f mock_build_progress diff --git a/tasks/internal/update-fbc-catalog-task/tests/pre-apply-task-hook.sh b/tasks/internal/update-fbc-catalog-task/tests/pre-apply-task-hook.sh new file mode 100755 index 000000000..4e5f74572 --- /dev/null +++ b/tasks/internal/update-fbc-catalog-task/tests/pre-apply-task-hook.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Install the CRDs so we can create/get them +.github/scripts/install_crds.sh + +# Add RBAC so that the SA executing the tests can retrieve CRs +kubectl apply -f .github/resources/crd_rbac.yaml + +# create required secrets +kubectl create secret generic iib-service-account-secret \ + --from-literal=principal="iib@kerberos" \ + --from-literal=keytab="something" +kubectl create secret generic iib-services-config \ + --from-literal=krb5.conf="" \ + --from-literal=url="https://fakeiib.host" + +kubectl create secret generic iib-overwrite-fromimage-credentials \ + --from-literal=username="bot+user" \ + --from-literal=token="token" +# Add mocks to the beginning of task step script +TASK_PATH="$1" +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +yq -i '.spec.steps[0].script = load_str("'$SCRIPT_DIR'/mocks.sh") + .spec.steps[0].script' "$TASK_PATH" +yq -i '.spec.steps[1].script = load_str("'$SCRIPT_DIR'/mocks.sh") + .spec.steps[1].script' "$TASK_PATH" diff --git a/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-error.yaml b/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-error.yaml new file mode 100644 index 000000000..9e9922642 --- /dev/null +++ b/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-error.yaml @@ -0,0 +1,98 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: test-update-fbc-catalog-error +spec: + description: Tests a production FBC release, but with a failure scenario due to an error returned by IIB + tasks: + - name: run-task + taskRef: + name: update-fbc-catalog-task + params: + - name: fbcFragment + value: "registry.io/image0@sha256:0000" + - name: fromIndex + value: "quay.io/scoheb/fbc-index-testing:error" + - name: targetIndex + value: "quay.io/fbc/catalog:test" + - name: buildTags + value: "[]" + - name: addArches + value: "[]" + - name: iibServiceAccountSecret + value: "iib-service-account-secret" + - name: check-result + params: + - name: jsonBuildInfo + value: $(tasks.run-task.results.jsonBuildInfo) + - name: buildState + value: $(tasks.run-task.results.buildState) + - name: genericResult + value: $(tasks.run-task.results.genericResult) + - name: indexImageDigests + value: $(tasks.run-task.results.indexImageDigests) + - name: iibLog + value: $(tasks.run-task.results.iibLog) + - name: exitCode + value: $(tasks.run-task.results.exitCode) + taskSpec: + params: + - name: jsonBuildInfo + type: string + - name: buildState + type: string + - name: genericResult + type: string + - name: indexImageDigests + type: string + - name: iibLog + type: string + - name: exitCode + type: string + steps: + - name: check-result + image: quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f + script: | + #!/bin/bash + set -x + + # the jsonBuild mockes has 10 keys on it + keyLength=$(jq '. | length' <<< '$(params.jsonBuildInfo)') + if [ "$keyLength" -ne 10 ]; then + echo "The task did not save a valid json in jsonBuildInfo result" + exit 1 + fi + + # shellcheck disable=SC2046 # (warning): Quote this to prevent word splitting + # reason: wrapping the tekton parameters in the following line with double quotes is adding unwanted + # quotes to the contained json, breaking jq. + state="$(jq -cr '.state' <<< $(params.buildState))" + if [ "$state" != "failed" ]; then + echo "The task did not save a completed IIB build in buildState result" + exit 1 + fi + + # shellcheck disable=SC2046 # (warning): Quote this to prevent word splitting + # reason: wrapping the tekton parameters in the following line with double quotes is adding unwanted + # quotes to the contained json, breaking jq. + genericResult=$(jq -r \ + '. |[.fbc_opt_in, .overwrite_fromindex_image, .publish_index_image,.sign_index_image] |@csv' \ + <<< $(params.genericResult)) + if [ "$genericResult" != '"true","true","true","true"' ]; then + echo "The stask did not save the correct values in genericResult result" + exit 1 + fi + + iibLog=$(awk '{match($0, /https.*/); print(substr($0, RSTART)) }' <<< "$(params.iibLog)") + if [ "$iibLog" != "https://fakeiib.host/api/v1/builds/1/logs" ]; then + echo "The task did not save a valid iib log url in the iibLog result" + exit 1 + fi + + if [ "$(params.exitCode)" != "1" ]; then + echo "The task did not finish with a fail exit code" + exit 1 + fi + runAfter: + - run-task diff --git a/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-hotfix.yaml b/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-hotfix.yaml new file mode 100644 index 000000000..a5ef25017 --- /dev/null +++ b/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-hotfix.yaml @@ -0,0 +1,105 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: test-update-fbc-catalog-hotfix +spec: + description: Tests a hotfix FBC release + tasks: + - name: run-task + taskRef: + name: update-fbc-catalog-task + params: + - name: fbcFragment + value: "registry.io/image0@sha256:0000" + - name: fromIndex + value: "quay.io/scoheb/fbc-index-testing:latest" + - name: targetIndex + value: "quay.io/fbc/catalog:test" + - name: buildTags + value: "[]" + - name: addArches + value: "[]" + - name: iibServiceAccountSecret + value: "iib-service-account-secret" + - name: hotfix + value: true + - name: check-result + params: + - name: jsonBuildInfo + value: $(tasks.run-task.results.jsonBuildInfo) + - name: buildState + value: $(tasks.run-task.results.buildState) + - name: genericResult + value: $(tasks.run-task.results.genericResult) + - name: indexImageDigests + value: $(tasks.run-task.results.indexImageDigests) + - name: iibLog + value: $(tasks.run-task.results.iibLog) + - name: exitCode + value: $(tasks.run-task.results.exitCode) + taskSpec: + params: + - name: jsonBuildInfo + type: string + - name: buildState + type: string + - name: genericResult + type: string + - name: indexImageDigests + type: string + - name: iibLog + type: string + - name: exitCode + type: string + steps: + - name: check-result + image: quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f + script: | + #!/bin/bash + set -x + + # the jsonBuild mockes has 10 keys on it + keyLength=$(jq '. | length' <<< '$(params.jsonBuildInfo)') + if [ "$keyLength" -ne 10 ]; then + echo "The task did not save a valid json in jsonBuildInfo result" + exit 1 + fi + + # shellcheck disable=SC2046 # (warning): Quote this to prevent word splitting + # reason: wrapping the tekton parameters in the following line with double quotes is adding unwanted + # quotes to the contained json, breaking jq. + state="$(jq -cr '.state' <<< $(params.buildState))" + if [ "$state" != "complete" ]; then + echo "The task did not save a completed IIB build in buildState result" + exit 1 + fi + + if [ "$(params.indexImageDigests)" != "sha256:000" ]; then + echo "The task did not save a valid digest image in the indexImageDigests result" + exit 1 + fi + + iibLog=$(awk '{match($0, /https.*/); print(substr($0, RSTART)) }' <<< "$(params.iibLog)") + if [ "$iibLog" != "https://fakeiib.host/api/v1/builds/1/logs" ]; then + echo "The task did not save a valid iib log url in the iibLog result" + exit 1 + fi + + # shellcheck disable=SC2046 # (warning): Quote this to prevent word splitting + # reason: wrapping the tekton parameters in the following line with double quotes is adding unwanted + # quotes to the contained json, breaking jq. + genericResult=$(jq -r \ + '. |[.fbc_opt_in, .overwrite_fromindex_image, .publish_index_image, .sign_index_image] |@csv' \ + <<< $(params.genericResult)) + if [ "$genericResult" != '"true","false","true","true"' ]; then + echo "The stask did not save the correct values in genericResult result" + exit 1 + fi + + if [ "$(params.exitCode)" != "0" ]; then + echo "The task did not finish with a successful exit code" + exit 1 + fi + runAfter: + - run-task diff --git a/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-prod.yaml b/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-prod.yaml new file mode 100644 index 000000000..6866bc58e --- /dev/null +++ b/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-prod.yaml @@ -0,0 +1,103 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: test-update-fbc-catalog-prod +spec: + description: Tests a production FBC release + tasks: + - name: run-task + taskRef: + name: update-fbc-catalog-task + params: + - name: fbcFragment + value: "registry.io/image0@sha256:0000" + - name: fromIndex + value: "quay.io/scoheb/fbc-index-testing:latest" + - name: targetIndex + value: "quay.io/fbc/catalog:test" + - name: buildTags + value: "[]" + - name: addArches + value: "[]" + - name: iibServiceAccountSecret + value: "iib-service-account-secret" + - name: check-result + params: + - name: jsonBuildInfo + value: $(tasks.run-task.results.jsonBuildInfo) + - name: buildState + value: $(tasks.run-task.results.buildState) + - name: genericResult + value: $(tasks.run-task.results.genericResult) + - name: indexImageDigests + value: $(tasks.run-task.results.indexImageDigests) + - name: iibLog + value: $(tasks.run-task.results.iibLog) + - name: exitCode + value: $(tasks.run-task.results.exitCode) + taskSpec: + params: + - name: jsonBuildInfo + type: string + - name: buildState + type: string + - name: genericResult + type: string + - name: indexImageDigests + type: string + - name: iibLog + type: string + - name: exitCode + type: string + steps: + - name: check-result + image: quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f + script: | + #!/bin/bash + set -x + + # the jsonBuild mockes has 10 keys on it + keyLength=$(jq '. | length' <<< '$(params.jsonBuildInfo)') + if [ "$keyLength" -ne 10 ]; then + echo "The task did not save a valid json in jsonBuildInfo result" + exit 1 + fi + + # shellcheck disable=SC2046 # (warning): Quote this to prevent word splitting + # reason: wrapping the tekton parameters in the following line with double quotes is adding unwanted + # quotes to the contained json, breaking jq. + state="$(jq -cr '.state' <<< $(params.buildState))" + if [ "$state" != "complete" ]; then + echo "The task did not save a completed IIB build in buildState result" + exit 1 + fi + + # shellcheck disable=SC2046 # (warning): Quote this to prevent word splitting + # reason: wrapping the tekton parameters in the following line with double quotes is adding unwanted + # quotes to the contained json, breaking jq. + genericResult=$(jq -r \ + '. |[.fbc_opt_in, .overwrite_fromindex_image, .publish_index_image,.sign_index_image] |@csv' \ + <<< $(params.genericResult)) + if [ "$genericResult" != '"true","true","true","true"' ]; then + echo "The stask did not save the correct values in genericResult result" + exit 1 + fi + + if [ "$(params.indexImageDigests)" != "sha256:000" ]; then + echo "The task did not save a valid digest image in the indexImageDigests result" + exit 1 + fi + + iibLog=$(awk '{match($0, /https.*/); print(substr($0, RSTART)) }' <<< "$(params.iibLog)") + if [ "$iibLog" != "https://fakeiib.host/api/v1/builds/1/logs" ]; then + echo "The task did not save a valid iib log url in the iibLog result" + exit 1 + fi + + if [ "$(params.exitCode)" != "0" ]; then + echo "The task did not finish with a successful exit code" + exit 1 + fi + runAfter: + - run-task diff --git a/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-staged-index.yaml b/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-staged-index.yaml new file mode 100644 index 000000000..fa3b1c85b --- /dev/null +++ b/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-staged-index.yaml @@ -0,0 +1,105 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: test-update-fbc-catalog-staged-index +spec: + description: Tests a stage FBC release + tasks: + - name: run-task + taskRef: + name: update-fbc-catalog-task + params: + - name: fbcFragment + value: "registry.io/image0@sha256:0000" + - name: fromIndex + value: "quay.io/scoheb/fbc-index-testing:latest" + - name: targetIndex + value: "quay.io/fbc/catalog:test" + - name: buildTags + value: "[]" + - name: addArches + value: "[]" + - name: iibServiceAccountSecret + value: "iib-service-account-secret" + - name: stagedIndex + value: true + - name: check-result + params: + - name: jsonBuildInfo + value: $(tasks.run-task.results.jsonBuildInfo) + - name: buildState + value: $(tasks.run-task.results.buildState) + - name: genericResult + value: $(tasks.run-task.results.genericResult) + - name: indexImageDigests + value: $(tasks.run-task.results.indexImageDigests) + - name: iibLog + value: $(tasks.run-task.results.iibLog) + - name: exitCode + value: $(tasks.run-task.results.exitCode) + taskSpec: + params: + - name: jsonBuildInfo + type: string + - name: buildState + type: string + - name: genericResult + type: string + - name: indexImageDigests + type: string + - name: iibLog + type: string + - name: exitCode + type: string + steps: + - name: check-result + image: quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f + script: | + #!/bin/bash + set -x + + # the jsonBuild mockes has 10 keys on it + keyLength=$(jq '. | length' <<< '$(params.jsonBuildInfo)') + if [ "$keyLength" -ne 10 ]; then + echo "The task did not save a valid json in jsonBuildInfo result" + exit 1 + fi + + # shellcheck disable=SC2046 # (warning): Quote this to prevent word splitting + # reason: wrapping the tekton parameters in the following line with double quotes is adding unwanted + # quotes to the contained json, breaking jq. + state="$(jq -cr '.state' <<< $(params.buildState))" + if [ "$state" != "complete" ]; then + echo "The task did not save a completed IIB build in buildState result" + exit 1 + fi + + # shellcheck disable=SC2046 # (warning): Quote this to prevent word splitting + # reason: wrapping the tekton parameters in the following line with double quotes is adding unwanted + # quotes to the contained json, breaking jq. + genericResult=$(jq -r \ + '. |[.fbc_opt_in, .overwrite_fromindex_image, .publish_index_image,.sign_index_image] |@csv' \ + <<< $(params.genericResult)) + if [ "$genericResult" != '"true","false","false","false"' ]; then + echo "The stask did not save the correct values in genericResult result" + exit 1 + fi + + if [ "$(params.indexImageDigests)" != "sha256:000" ]; then + echo "The task did not save a valid digest image in the indexImageDigests result" + exit 1 + fi + + iibLog=$(awk '{match($0, /https.*/); print(substr($0, RSTART)) }' <<< "$(params.iibLog)") + if [ "$iibLog" != "https://fakeiib.host/api/v1/builds/1/logs" ]; then + echo "The task did not save a valid iib log url in the iibLog result" + exit 1 + fi + + if [ "$(params.exitCode)" != "0" ]; then + echo "The task did not finish with a successful exit code" + exit 1 + fi + runAfter: + - run-task diff --git a/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-timeout.yaml b/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-timeout.yaml new file mode 100644 index 000000000..fa1b22192 --- /dev/null +++ b/tasks/internal/update-fbc-catalog-task/tests/test-update-fbc-catalog-timeout.yaml @@ -0,0 +1,57 @@ +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: test-update-fbc-catalog-timeout +spec: + description: Tests a timed out FBC release + tasks: + - name: run-task + taskRef: + name: update-fbc-catalog-task + params: + - name: fbcFragment + value: "registry.io/image0@sha256:0000" + - name: fromIndex + value: "quay.io/scoheb/fbc-index-testing:timeout" + - name: targetIndex + value: "quay.io/fbc/catalog:test" + - name: buildTags + value: "[]" + - name: addArches + value: "[]" + - name: iibServiceAccountSecret + value: "iib-service-account-secret" + - name: buildTimeoutSeconds + value: 1 + - name: check-result + params: + - name: iibLog + value: $(tasks.run-task.results.iibLog) + - name: exitCode + value: $(tasks.run-task.results.exitCode) + taskSpec: + params: + - name: iibLog + type: string + - name: exitCode + type: string + steps: + - name: check-result + image: quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f + script: | + #!/bin/bash + set -x + + iibLog=$(awk '{match($0, /https.*/); print(substr($0, RSTART)) }' <<< "$(params.iibLog)") + if [ "$iibLog" != "https://fakeiib.host/api/v1/builds/1/logs" ]; then + echo "The task did not save a valid iib log url in the iibLog result" + exit 1 + fi + + if [ "$(params.exitCode)" != "124" ]; then + echo "The task did not finish with a timeout exit code" + exit 1 + fi + runAfter: + - run-task diff --git a/tasks/internal/update-fbc-catalog-task/update-fbc-catalog-task.yaml b/tasks/internal/update-fbc-catalog-task/update-fbc-catalog-task.yaml new file mode 100644 index 000000000..cbc7645ba --- /dev/null +++ b/tasks/internal/update-fbc-catalog-task/update-fbc-catalog-task.yaml @@ -0,0 +1,324 @@ +--- +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: update-fbc-catalog-task + labels: + app.kubernetes.io/version: "1.0.0" + annotations: + tekton.dev/pipelines.minVersion: "0.12.1" + tekton.dev/tags: release +spec: + description: >- + Tekton task to submit a IIB build request to add/update a fbc-fragment to an index image + params: + - name: fbcFragment + type: string + description: FBC fragment built by HACBS + - name: fromIndex + type: string + description: >- + Index image (catalog of catalogs) the FBC fragment will be added to + - name: targetIndex + type: string + description: >- + Target index is the pullspec the FBC catalog will be pushed to + - name: buildTags + type: string + description: >- + List of additional tags the internal index image copy should be + tagged with. + - name: addArches + type: string + description: List of arches the index image should be built for. + - name: buildTimeoutSeconds + type: string + default: "300" + description: Timeout seconds to receive the build state + - name: iibServiceAccountSecret + type: string + description: Secret with IIB credentials to be used + - name: hotfix + type: string + default: "false" + description: Whether this build is a hotfix build + - name: stagedIndex + type: string + default: "false" + description: Whether this build is for a staged index build + results: + - name: jsonBuildInfo + description: JSON build information for the requested build + - name: buildState + description: IIB Service build state + - name: genericResult + description: Set the genericResult if FBC Fragment is Opt-In and should be published + - name: indexImageDigests + description: The digests for each arch for the manifest list of the index image + - name: iibLog + description: The link to the log from the IIB request + - name: exitCode + description: The exit code from the task + steps: + - name: update-fbc-catalog-prepare-and-call-iib-step + image: >- + quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f + env: + - name: IIB_SERVICE_URL + valueFrom: + secretKeyRef: + name: iib-services-config + key: url + - name: IIB_OVERWRITE_FROM_INDEX_USERNAME + valueFrom: + secretKeyRef: + name: iib-overwrite-fromimage-credentials + key: username + - name: IIB_OVERWRITE_FROM_INDEX_TOKEN + valueFrom: + secretKeyRef: + name: iib-overwrite-fromimage-credentials + key: token + - name: KRB5_CONF_CONTENT + valueFrom: + secretKeyRef: + name: iib-services-config + key: krb5.conf + script: | + #!/usr/bin/env bash + + isFBCOptIn() { + TMPFILE=$(mktemp) + PYXIS_URL="https://pyxis.engineering.redhat.com/v1" + + IFS="/" read -r REGISTRY REPO IMAGE <<< "${1}" + IFS=":" read -r IMAGE TAG <<< "${IMAGE}" + + FETCH_URL="${PYXIS_URL}/repositories/registry/${REGISTRY}/repository/${REPO}/${IMAGE}/tag/${TAG}" + + # strips the last "/tag" in case $TAG is not set + [ -z "${TAG}" ] && FETCH_URL=${FETCH_URL%/tag*} + + curl --negotiate -u: "${FETCH_URL}" -o "${TMPFILE}" + + # prints "false" in case .fbc_opt_in entry is missing + jq -e -r '.fbc_opt_in //false' "${TMPFILE}" && rm -f "${TMPFILE}" + } + + # checks if there is any previous build for the same fbc_fragment. + # in case multiple builds are found, returns only the last one. + check_previous_build() { + user="${1}" + from_index="${2}" + fbc_fragment="${3}" + + # fetch only builds in progress or completed + build=$(curl -s "${IIB_SERVICE_URL}/builds?user=${user}&from_index=${from_index}" | \ + jq --arg fbc_fragment "${fbc_fragment}" \ + '[.items[] |select(.fbc_fragment==$fbc_fragment and .state!="failed")][0] // empty') + + if [ "$(jq -r '.state' <<< "${build}")" = "complete" ]; then + indexImageResolved="$(jq -r '.index_image_resolved' <<< "${build}")" + newCatalogCreatedDate="$(date --date "$(skopeo inspect --config "docker://${indexImageResolved}" | \ + jq -r .created)" "+%s")" + upstreamCatalogCreatedDate="$(date --date "$(skopeo inspect --config "docker://$(params.targetIndex)" | \ + jq -r .created)" "+%s")" + # checks if the index_image_resolved in the previous completed build is newer + # than the upstream catalog index. + # in case the new catalog index is older than the upstream, a new build is + # required to assure the catalog integrity. + if [ "${newCatalogCreatedDate}" -gt "${upstreamCatalogCreatedDate}" ]; then + echo "${build}" + exit + fi + fi + } + + # performs kerberos authentication. + base64 -d /mnt/service-account-secret/keytab > "/tmp/keytab" + + KRB5_TEMP_CONF=$(mktemp) + KRB5_PRINCIPAL=$(cat /mnt/service-account-secret/principal) + + echo "${KRB5_CONF_CONTENT}" > "${KRB5_TEMP_CONF}" + export KRB5_CONFIG="${KRB5_TEMP_CONF}" + export KRB5_TRACE=/dev/stderr + + kinit -V "${KRB5_PRINCIPAL}" -k -t "/tmp/keytab" + + set -x + # check if this fbc fragment is opt-in + echo "Fetching the image bundle from $(params.fbcFragment)..." + PULL_SPEC_LIST=$(opm render "$(params.fbcFragment)" | jq -r \ + 'select(.schema == "olm.bundle") | "\(.image)" | split("@")[0]' |uniq) + + fbcOptIn="true" + for PULL_SPEC in ${PULL_SPEC_LIST}; do + # make sure they query is done using the internal name instead of the public + PULL_SPEC="${PULL_SPEC//registry.redhat.io/registry.access.redhat.com}" + echo "Attempting to fetch from ${FETCH_URL} to check if fragment is \`fbc_opt_in==true\`..." + if [ "$(isFBCOptIn "${PULL_SPEC}")" = "false" ]; then + fbcOptIn="false" + break + fi + done + mustOverwriteFromIndexImage="${fbcOptIn}" + mustPublishIndexImage="${fbcOptIn}" + mustSignIndexImage="${fbcOptIn}" + + if [ "$(params.hotfix)" == "true" ]; then + echo "Hotfix build" + mustOverwriteFromIndexImage="false" + mustSignIndexImage="true" + mustPublishIndexImage="true" + elif [ "$(params.stagedIndex)" == "true" ]; then + echo "Staged Index build" + mustOverwriteFromIndexImage="false" + mustSignIndexImage="false" + mustPublishIndexImage="false" + fi + + echo "Fragment has \`fbc_opt_in==${fbcOptIn}\`" + echo " \`mustOverwriteFromIndexImage==${mustOverwriteFromIndexImage}\`" + echo " \`mustPublishIndexImage==${mustPublishIndexImage}\`" + echo " \`mustSignIndexImage==${mustSignIndexImage}\`" + + # these results will be used by add-fbc-contribution to control + # signing and publishing of the built fragment + jq -n -c \ + --arg fbc_opt_in "${fbcOptIn}" \ + --arg overwrite_fromindex_image "${mustOverwriteFromIndexImage}" \ + --arg publish_index_image "${mustPublishIndexImage}" \ + --arg sign_index_image "${mustSignIndexImage}" \ + '{ + "fbc_opt_in": $fbc_opt_in, + "overwrite_fromindex_image": $overwrite_fromindex_image, + "publish_index_image": $publish_index_image, + "sign_index_image": $sign_index_image + } | tostring' | tee "$(results.genericResult.path)" + + # if it finds a build which is completed or in progress, it should exit this step and jump to + # the next step `s-wait-for-build-state` which will watch the build until it is completed. + build=$(check_previous_build "${KRB5_PRINCIPAL}" "$(params.fromIndex)" "$(params.fbcFragment)") + if [ -n "${build}" ]; then + echo "=== A previous build for this fragment was found ===" + echo "${build}" |tee "$(results.jsonBuildInfo.path)" + exit 0 + fi + + # adds the json request parameters to a file to be used as input data + # for curl and preventing shell expansion. + json_input="/tmp/$$.tmp" + json_raw_input="/tmp/$$_raw.tmp" + + cat > "$json_raw_input" < "${json_input}" + + echo "Calling IIB endpoint" > "$(results.buildState.path)" + # adds image to the index. + curl -u : --negotiate -s -X POST -H "Content-Type: application/json" -d@"${json_input}" --insecure \ + "${IIB_SERVICE_URL}/builds/fbc-operations" |tee "$(results.jsonBuildInfo.path)" + + # checks if the previous call returned an error. + ! jq -e -r ".error | select( . != null )" "$(results.jsonBuildInfo.path)" + volumeMounts: + - name: service-account-secret + mountPath: /mnt/service-account-secret + - name: update-fbc-catalog-wait-for-iib-build-step + image: >- + quay.io/konflux-ci/release-service-utils:e633d51cd41d73e4b3310face21bb980af7a662f + env: + - name: IIB_SERVICE_URL + valueFrom: + secretKeyRef: + name: iib-services-config + key: url + script: | + #!/usr/bin/env bash + # shellcheck disable=SC2317 # shellcheck calls all the commands in the function unreachable + set -x + + watch_build_state() { + build_id="$(jq -r ".id" "$(results.jsonBuildInfo.path)")" + state="" + while true; do + # + # fetching build information. + build_info=$(curl -s "${IIB_SERVICE_URL}/builds/${build_id}") + # get state from the build information. + state="$(jq -r ".state" <<< "${build_info}")" + # remove the history as it breaks the results build up + jq -r 'del(.state_history)' <<< "${build_info}" | jq -c . > "$(results.jsonBuildInfo.path)" + url="$(jq -r ".logs.url" <<< "${build_info}")" + echo IIB log url is: "${url}" > "$(results.iibLog.path)" + case ${state} in + "complete") break ;; + "failed") break ;; + *) echo -en "."; sleep 30; continue ;; + esac + done + echo + jq -cr '{ "state": .state, "state_reason": .state_reason }' "$(results.jsonBuildInfo.path)" | jq -Rc \ + | tee "$(results.buildState.path)" + test "${state}" = "complete" && exit 0 || exit 1 + } + + echo -en "waiting for build state to exit..." + # adding timeout here due to the Task timeout not accepting $(params.buildTimeoutSeconds) + # as parameter. + export -f watch_build_state + timeout "$(params.buildTimeoutSeconds)" bash -c watch_build_state + BUILDEXIT=$? + + # it should continue only if the IIB build status is complete + if [ ${BUILDEXIT} -eq 0 ]; then + echo -n 0 > "$(results.exitCode.path)" + + # get the manifest digests + indexImageCopy=$(jq -cr .internal_index_image_copy < "$(results.jsonBuildInfo.path)") + # Use this to obtain the manifest digests for each arch in manifest list + indexImageDigestsRaw=$(skopeo inspect --raw "docker://${indexImageCopy}") + # according the IIB team, + # "all index images will always be multi-arch with a manifest list" + # + indexImageDigests=$(echo "${indexImageDigestsRaw}" | \ + jq -r \ + '.manifests[]? | select(.mediaType=="application/vnd.docker.distribution.manifest.v2+json") | .digest') + echo -n "${indexImageDigests}" > "$(results.indexImageDigests.path)" + if [ -z "${indexImageDigests}" ] ; then + echo "Index image produced is not multi-arch with a manifest list" + echo -n 1 > "$(results.exitCode.path)" + fi + else + if [ ${BUILDEXIT} -eq 124 ]; then + echo "Timeout while waiting for the build to finish" + echo "Build timeout" > "$(results.buildState.path)" + fi + echo -n "" > "$(results.indexImageDigests.path)" + echo -n "$BUILDEXIT" > "$(results.exitCode.path)" + fi + # We don't put the log in a result because tekton results are too limited for what we can put + # to be useful, but still print it for debugging + curl -s "$(awk '{print $NF}' < "$(results.iibLog.path)")" + + exit 0 + volumes: + - name: service-account-secret + secret: + secretName: $(params.iibServiceAccountSecret) + defaultMode: + 0400 diff --git a/tasks/managed/add-fbc-contribution/README.md b/tasks/managed/add-fbc-contribution/README.md index b05f63e24..1ad9df341 100644 --- a/tasks/managed/add-fbc-contribution/README.md +++ b/tasks/managed/add-fbc-contribution/README.md @@ -13,6 +13,9 @@ Task to create a internalrequest to add fbc contributions to index images | targetIndex | targetIndex value updated by update-ocp-tag task | No | - | | resultsDirPath | Path to results directory in the data workspace | No | - | +## Changes in 3.4.3 +- Changes the internal request pipeline "request" parameter from `iib` to `update-fbc-catalog` + ## Changes in 3.4.2 * Improve clarity of log statements when fbc_opt_in is not set to True. diff --git a/tasks/managed/add-fbc-contribution/add-fbc-contribution.yaml b/tasks/managed/add-fbc-contribution/add-fbc-contribution.yaml index cdf7c0c8e..848fc1215 100644 --- a/tasks/managed/add-fbc-contribution/add-fbc-contribution.yaml +++ b/tasks/managed/add-fbc-contribution/add-fbc-contribution.yaml @@ -4,7 +4,7 @@ kind: Task metadata: name: add-fbc-contribution labels: - app.kubernetes.io/version: "3.4.2" + app.kubernetes.io/version: "3.4.3" annotations: tekton.dev/pipelines.minVersion: "0.12.1" tekton.dev/tags: release @@ -131,7 +131,7 @@ spec: # The internal-request script will create the InternalRequest and wait until it finishes to get its status # If it fails (Failed, Rejected or Timed out) the script will exit and display the reason. echo "Creating InternalRequest to add FBC contribution to index image:" - internal-request -r "iib" \ + internal-request -r "update-fbc-catalog" \ -p fromIndex="$(params.fromIndex)" \ -p targetIndex="${target_index}" \ -p fbcFragment="${fbc_fragment}" \