diff --git a/.github/workflows/e2e-1.19.yaml b/.github/workflows/e2e-1.19.yaml new file mode 100644 index 00000000..84a20729 --- /dev/null +++ b/.github/workflows/e2e-1.19.yaml @@ -0,0 +1,203 @@ +name: E2E-1.19 + +on: + push: + branches: + - master + - release-* + pull_request: {} + workflow_dispatch: {} + +env: + # Common versions + GO_VERSION: '1.17' + KIND_IMAGE: 'kindest/node:v1.19.16' + KIND_CLUSTER_NAME: 'ci-testing' + +jobs: + + rollout: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + node_image: ${{ env.KIND_IMAGE }} + cluster_name: ${{ env.KIND_CLUSTER_NAME }} + config: ./test/kind-conf.yaml + - name: Build image + run: | + export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}" + docker build --pull --no-cache . -t $IMAGE + kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; } + - name: Install Kruise + run: | + set -ex + kubectl cluster-info + make helm + helm repo add openkruise https://openkruise.github.io/charts/ + helm repo update + helm install kruise openkruise/kruise + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + echo "Wait for kruise-manager ready successfully" + else + echo "Timeout to wait for kruise-manager ready" + exit 1 + fi + - name: Install Kruise Rollout + run: | + set -ex + kubectl cluster-info + IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + set -e + if [ "$PODS" -eq "1" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + kubectl get node -o yaml + kubectl get all -n kruise-rollout -o yaml + set -e + if [ "$PODS" -eq "1" ]; then + echo "Wait for kruise-rollout ready successfully" + else + echo "Timeout to wait for kruise-rollout ready" + exit 1 + fi + - name: Run E2E Tests + run: | + export KUBECONFIG=/home/runner/.kube/config + make ginkgo + set +e + ./bin/ginkgo -timeout 60m -v --focus='\[rollouts\] (Rollout)' test/e2e + retVal=$? + # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout + restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') + if [ "${restartCount}" -eq "0" ];then + echo "Kruise-rollout has not restarted" + else + kubectl get pod -n kruise-rollout --no-headers + echo "Kruise-rollout has restarted, abort!!!" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout + exit 1 + fi + exit $retVal + + batchRelease: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + node_image: ${{ env.KIND_IMAGE }} + cluster_name: ${{ env.KIND_CLUSTER_NAME }} + config: ./test/kind-conf.yaml + - name: Build image + run: | + export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}" + docker build --pull --no-cache . -t $IMAGE + kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; } + - name: Install Kruise + run: | + set -ex + kubectl cluster-info + make helm + helm repo add openkruise https://openkruise.github.io/charts/ + helm repo update + helm install kruise openkruise/kruise + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + echo "Wait for kruise-manager ready successfully" + else + echo "Timeout to wait for kruise-manager ready" + exit 1 + fi + - name: Install Kruise Rollout + run: | + set -ex + kubectl cluster-info + IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + set -e + if [ "$PODS" -eq "1" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + kubectl get node -o yaml + kubectl get all -n kruise-rollout -o yaml + set -e + if [ "$PODS" -eq "1" ]; then + echo "Wait for kruise-rollout ready successfully" + else + echo "Timeout to wait for kruise-rollout ready" + exit 1 + fi + - name: Run E2E Tests + run: | + export KUBECONFIG=/home/runner/.kube/config + make ginkgo + set +e + ./bin/ginkgo -timeout 60m -v --focus='\[rollouts\] (BatchRelease)' test/e2e + retVal=$? + # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout + restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') + if [ "${restartCount}" -eq "0" ];then + echo "Kruise-rollout has not restarted" + else + kubectl get pod -n kruise-rollout --no-headers + echo "Kruise-rollout has restarted, abort!!!" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout + exit 1 + fi + exit $retVal diff --git a/.github/workflows/e2e-1.23.yaml b/.github/workflows/e2e-1.23.yaml new file mode 100644 index 00000000..ab70915d --- /dev/null +++ b/.github/workflows/e2e-1.23.yaml @@ -0,0 +1,203 @@ +name: E2E-1.23 + +on: + push: + branches: + - master + - release-* + pull_request: {} + workflow_dispatch: {} + +env: + # Common versions + GO_VERSION: '1.17' + KIND_IMAGE: 'kindest/node:v1.23.3' + KIND_CLUSTER_NAME: 'ci-testing' + +jobs: + + rollout: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + node_image: ${{ env.KIND_IMAGE }} + cluster_name: ${{ env.KIND_CLUSTER_NAME }} + config: ./test/kind-conf.yaml + - name: Build image + run: | + export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}" + docker build --pull --no-cache . -t $IMAGE + kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; } + - name: Install Kruise + run: | + set -ex + kubectl cluster-info + make helm + helm repo add openkruise https://openkruise.github.io/charts/ + helm repo update + helm install kruise openkruise/kruise + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + echo "Wait for kruise-manager ready successfully" + else + echo "Timeout to wait for kruise-manager ready" + exit 1 + fi + - name: Install Kruise Rollout + run: | + set -ex + kubectl cluster-info + IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + set -e + if [ "$PODS" -eq "1" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + kubectl get node -o yaml + kubectl get all -n kruise-rollout -o yaml + set -e + if [ "$PODS" -eq "1" ]; then + echo "Wait for kruise-rollout ready successfully" + else + echo "Timeout to wait for kruise-rollout ready" + exit 1 + fi + - name: Run E2E Tests + run: | + export KUBECONFIG=/home/runner/.kube/config + make ginkgo + set +e + ./bin/ginkgo -timeout 60m -v --focus='\[rollouts\] (Rollout)' test/e2e + retVal=$? + # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout + restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') + if [ "${restartCount}" -eq "0" ];then + echo "Kruise-rollout has not restarted" + else + kubectl get pod -n kruise-rollout --no-headers + echo "Kruise-rollout has restarted, abort!!!" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout + exit 1 + fi + exit $retVal + + batchRelease: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + node_image: ${{ env.KIND_IMAGE }} + cluster_name: ${{ env.KIND_CLUSTER_NAME }} + config: ./test/kind-conf.yaml + - name: Build image + run: | + export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}" + docker build --pull --no-cache . -t $IMAGE + kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; } + - name: Install Kruise + run: | + set -ex + kubectl cluster-info + make helm + helm repo add openkruise https://openkruise.github.io/charts/ + helm repo update + helm install kruise openkruise/kruise + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + echo "Wait for kruise-manager ready successfully" + else + echo "Timeout to wait for kruise-manager ready" + exit 1 + fi + - name: Install Kruise Rollout + run: | + set -ex + kubectl cluster-info + IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + set -e + if [ "$PODS" -eq "1" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + kubectl get node -o yaml + kubectl get all -n kruise-rollout -o yaml + set -e + if [ "$PODS" -eq "1" ]; then + echo "Wait for kruise-rollout ready successfully" + else + echo "Timeout to wait for kruise-rollout ready" + exit 1 + fi + - name: Run E2E Tests + run: | + export KUBECONFIG=/home/runner/.kube/config + make ginkgo + set +e + ./bin/ginkgo -timeout 60m -v --focus='\[rollouts\] (BatchRelease)' test/e2e + retVal=$? + # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout + restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') + if [ "${restartCount}" -eq "0" ];then + echo "Kruise-rollout has not restarted" + else + kubectl get pod -n kruise-rollout --no-headers + echo "Kruise-rollout has restarted, abort!!!" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout + exit 1 + fi + exit $retVal diff --git a/.golangci.yml b/.golangci.yml index d7bdb8d5..a4eac5e0 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -69,14 +69,10 @@ linters: disable-all: true enable: # TODO Enforce the below linters later - - deadcode - gofmt - - govet - goimports - ineffassign - misspell - - vet - - unconvert issues: exclude: # staticcheck diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..17ab16aa --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Change Log + +## v0.1.0 + +### Kruise-Rollout-Controller +- Support Canary Publishing + Nginx Ingress + Workload(CloneSet, Deployment) +- Support for Batch Release(e.g. 20%, 40%, 60%, 80, 100%) for workload(CloneSet) + +### Documents +- Introduction, Installation, Basic Usage diff --git a/Makefile b/Makefile index 9c84adb6..38999fd3 100644 --- a/Makefile +++ b/Makefile @@ -97,6 +97,14 @@ KUSTOMIZE = $(shell pwd)/bin/kustomize kustomize: ## Download kustomize locally if necessary. $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) +GINKGO = $(shell pwd)/bin/ginkgo +ginkgo: ## Download ginkgo locally if necessary. + $(call go-get-tool,$(GINKGO),github.com/onsi/ginkgo/ginkgo@v1.16.4) + +HELM = $(shell pwd)/bin/helm +helm: ## Download helm locally if necessary. + $(call go-get-tool,$(HELM),helm.sh/helm/v3@v3.8.1) + # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) define go-get-tool @@ -106,7 +114,7 @@ TMP_DIR=$$(mktemp -d) ;\ cd $$TMP_DIR ;\ go mod init tmp ;\ echo "Downloading $(2)" ;\ -GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\ +GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\ rm -rf $$TMP_DIR ;\ } endef diff --git a/api/v1alpha1/batchrelease_plan_types.go b/api/v1alpha1/batchrelease_plan_types.go index 64e20b45..fe820d16 100644 --- a/api/v1alpha1/batchrelease_plan_types.go +++ b/api/v1alpha1/batchrelease_plan_types.go @@ -97,7 +97,7 @@ type BatchReleaseCanaryStatus struct { // BatchReadyTime is the ready timestamp of the current batch or the last batch. // This field is updated once a batch ready, and the batches[x].pausedSeconds // relies on this field to calculate the real-time duration. - BatchReadyTime metav1.Time `json:"lastBatchReadyTime,omitempty"` + BatchReadyTime *metav1.Time `json:"batchReadyTime,omitempty"` // UpdatedReplicas is the number of upgraded Pods. UpdatedReplicas int32 `json:"updatedReplicas,omitempty"` // UpdatedReadyReplicas is the number upgraded Pods that have a Ready Condition. diff --git a/api/v1alpha1/rollout_types.go b/api/v1alpha1/rollout_types.go index a382b901..6a5edfad 100644 --- a/api/v1alpha1/rollout_types.go +++ b/api/v1alpha1/rollout_types.go @@ -36,9 +36,6 @@ type RolloutSpec struct { } type ObjectRef struct { - // workloadRef, revisionRef - // default is workloadRef - Type ObjectRefType `json:"type,omitempty"` // WorkloadRef contains enough information to let you identify a workload for Rollout // Batch release of the bypass WorkloadRef *WorkloadRef `json:"workloadRef,omitempty"` @@ -75,9 +72,6 @@ type RolloutStrategy struct { // Paused indicates that the Rollout is paused. // Default value is false Paused bool `json:"paused,omitempty"` - // canary, BlueGreenPlan - // Default value is canary - Type RolloutStrategyType `json:"type,omitempty"` // +optional Canary *CanaryStrategy `json:"canary,omitempty"` // +optional @@ -96,9 +90,9 @@ type CanaryStrategy struct { // Steps define the order of phases to execute release in batches(20%, 40%, 60%, 80%, 100%) // +optional Steps []CanaryStep `json:"steps,omitempty"` - // TrafficRouting hosts all the supported service meshes supported to enable more fine-grained traffic routing + // TrafficRoutings hosts all the supported service meshes supported to enable more fine-grained traffic routing // todo current only support one - TrafficRouting []*TrafficRouting `json:"trafficRouting,omitempty"` + TrafficRoutings []*TrafficRouting `json:"trafficRoutings,omitempty"` // MetricsAnalysis *MetricsAnalysisBackground `json:"metricsAnalysis,omitempty"` } @@ -125,11 +119,10 @@ type RolloutPause struct { // TrafficRouting hosts all the different configuration for supported service meshes to enable more fine-grained traffic routing type TrafficRouting struct { // Service holds the name of a service which selects pods with stable version and don't select any pods with canary version. - // +optional Service string `json:"service"` // Optional duration in seconds the traffic provider(e.g. nginx ingress controller) consumes the service, ingress configuration changes gracefully. GracePeriodSeconds int32 `json:"gracePeriodSeconds,omitempty"` - // nginx, alb etc. + // nginx, alb, istio etc. Type string `json:"type"` // Ingress holds Ingress specific configuration to route traffic, e.g. Nginx, Alb. Ingress *IngressTrafficRouting `json:"ingress,omitempty"` @@ -150,7 +143,7 @@ type RolloutStatus struct { ObservedGeneration int64 `json:"observedGeneration,omitempty"` // CanaryRevision the hash of the canary pod template // +optional - CanaryRevision string `json:"canaryRevision,omitempty"` + //CanaryRevision string `json:"canaryRevision,omitempty"` // StableRevision indicates the revision pods that has successfully rolled out StableRevision string `json:"stableRevision,omitempty"` // Conditions a list of conditions a rollout can have. @@ -217,9 +210,12 @@ type CanaryStatus struct { RolloutHash string `json:"rolloutHash,omitempty"` // CanaryService holds the name of a service which selects pods with canary version and don't select any pods with stable version. CanaryService string `json:"canaryService"` - // CanaryRevision the hash of the current pod template + // CanaryRevision is calculated by rollout based on podTemplateHash, and the internal logic flow uses + // It may be different from rs podTemplateHash in different k8s versions, so it cannot be used as service selector label // +optional CanaryRevision string `json:"canaryRevision"` + // pod template hash is used as service selector label + PodTemplateHash string `json:"podTemplateHash"` // CanaryReplicas the numbers of canary revision pods CanaryReplicas int32 `json:"canaryReplicas"` // CanaryReadyReplicas the numbers of ready canary revision pods @@ -274,6 +270,7 @@ const ( // +kubebuilder:printcolumn:name="CANARY_STEP",type="integer",JSONPath=".status.canaryStatus.currentStepIndex",description="The rollout canary status step" // +kubebuilder:printcolumn:name="CANARY_STATE",type="string",JSONPath=".status.canaryStatus.currentStepState",description="The rollout canary status step state" // +kubebuilder:printcolumn:name="MESSAGE",type="string",JSONPath=".status.message",description="The rollout canary status message" +// +kubebuilder:printcolumn:name="AGE",type=date,JSONPath=".metadata.creationTimestamp" // Rollout is the Schema for the rollouts API type Rollout struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 5fa24359..a227eeff 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -56,7 +56,10 @@ func (in *BatchRelease) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BatchReleaseCanaryStatus) DeepCopyInto(out *BatchReleaseCanaryStatus) { *out = *in - in.BatchReadyTime.DeepCopyInto(&out.BatchReadyTime) + if in.BatchReadyTime != nil { + in, out := &in.BatchReadyTime, &out.BatchReadyTime + *out = (*in).DeepCopy() + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchReleaseCanaryStatus. @@ -196,8 +199,8 @@ func (in *CanaryStrategy) DeepCopyInto(out *CanaryStrategy) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.TrafficRouting != nil { - in, out := &in.TrafficRouting, &out.TrafficRouting + if in.TrafficRoutings != nil { + in, out := &in.TrafficRoutings, &out.TrafficRoutings *out = make([]*TrafficRouting, len(*in)) for i := range *in { if (*in)[i] != nil { diff --git a/config/crd/bases/rollouts.kruise.io_batchreleases.yaml b/config/crd/bases/rollouts.kruise.io_batchreleases.yaml index 74dd6472..86e19774 100644 --- a/config/crd/bases/rollouts.kruise.io_batchreleases.yaml +++ b/config/crd/bases/rollouts.kruise.io_batchreleases.yaml @@ -102,9 +102,6 @@ spec: description: TargetRef contains the GVK and name of the workload that we need to upgrade to. properties: - type: - description: workloadRef, revisionRef default is workloadRef - type: string workloadRef: description: WorkloadRef contains enough information to let you identify a workload for Rollout Batch release of the bypass @@ -135,6 +132,13 @@ spec: canaryStatus: description: CanaryStatus describes the state of the canary rollout. properties: + batchReadyTime: + description: BatchReadyTime is the ready timestamp of the current + batch or the last batch. This field is updated once a batch + ready, and the batches[x].pausedSeconds relies on this field + to calculate the real-time duration. + format: date-time + type: string batchState: description: CurrentBatchState indicates the release state of the current batch. @@ -144,13 +148,6 @@ spec: it starts from 0 format: int32 type: integer - lastBatchReadyTime: - description: BatchReadyTime is the ready timestamp of the current - batch or the last batch. This field is updated once a batch - ready, and the batches[x].pausedSeconds relies on this field - to calculate the real-time duration. - format: date-time - type: string updatedReadyReplicas: description: UpdatedReadyReplicas is the number upgraded Pods that have a Ready Condition. diff --git a/config/crd/bases/rollouts.kruise.io_rollouts.yaml b/config/crd/bases/rollouts.kruise.io_rollouts.yaml index 56c05e9b..9819a100 100644 --- a/config/crd/bases/rollouts.kruise.io_rollouts.yaml +++ b/config/crd/bases/rollouts.kruise.io_rollouts.yaml @@ -33,6 +33,9 @@ spec: jsonPath: .status.message name: MESSAGE type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date name: v1alpha1 schema: openAPIV3Schema: @@ -58,9 +61,6 @@ spec: Important: Run "make" to regenerate code after modifying this file ObjectRef indicates workload' properties: - type: - description: workloadRef, revisionRef default is workloadRef - type: string workloadRef: description: WorkloadRef contains enough information to let you identify a workload for Rollout Batch release of the bypass @@ -118,8 +118,8 @@ spec: type: integer type: object type: array - trafficRouting: - description: TrafficRouting hosts all the supported service + trafficRoutings: + description: TrafficRoutings hosts all the supported service meshes supported to enable more fine-grained traffic routing todo current only support one items: @@ -150,9 +150,10 @@ spec: any pods with canary version. type: string type: - description: nginx, alb etc. + description: nginx, alb, istio etc. type: string required: + - service - type type: object type: array @@ -161,9 +162,6 @@ spec: description: Paused indicates that the Rollout is paused. Default value is false type: boolean - type: - description: canary, BlueGreenPlan Default value is canary - type: string type: object required: - objectRef @@ -172,9 +170,6 @@ spec: status: description: RolloutStatus defines the observed state of Rollout properties: - canaryRevision: - description: CanaryRevision the hash of the canary pod template - type: string canaryStatus: description: Canary describes the state of the canary rollout properties: @@ -188,7 +183,10 @@ spec: format: int32 type: integer canaryRevision: - description: CanaryRevision the hash of the current pod template + description: CanaryRevision is calculated by rollout based on + podTemplateHash, and the internal logic flow uses It may be + different from rs podTemplateHash in different k8s versions, + so it cannot be used as service selector label type: string canaryService: description: CanaryService holds the name of a service which selects @@ -214,6 +212,9 @@ spec: observed for this Rollout ref workload generation. format: int64 type: integer + podTemplateHash: + description: pod template hash is used as service selector label + type: string rolloutHash: description: RolloutHash from rollout.spec object type: string @@ -222,6 +223,7 @@ spec: - canaryReplicas - canaryService - currentStepState + - podTemplateHash type: object conditions: description: Conditions a list of conditions a rollout can have. @@ -272,8 +274,9 @@ spec: Phase is the rollout phase. type: string stableRevision: - description: StableRevision indicates the revision pods that has successfully - rolled out + description: CanaryRevision the hash of the canary pod template CanaryRevision + string `json:"canaryRevision,omitempty"` StableRevision indicates + the revision pods that has successfully rolled out type: string type: object type: object diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md index 32938bd2..0587ec03 100644 --- a/docs/getting_started/installation.md +++ b/docs/getting_started/installation.md @@ -1,7 +1,7 @@ # Installation ## Requirements -- Install Kubernetes Cluster, requires **Kubernetes version >= 1.16**. +- Install Kubernetes Cluster, requires **Kubernetes version >= 1.19**. - (Optional, If Use CloneSet) Helm installation of OpenKruise, **Since v1.1.0**, Reference [Install OpenKruise](https://openkruise.io/docs/installation). ## Install with helm diff --git a/docs/getting_started/introduction.md b/docs/getting_started/introduction.md index 2c07ad74..2a9713c2 100644 --- a/docs/getting_started/introduction.md +++ b/docs/getting_started/introduction.md @@ -5,7 +5,7 @@ Kruise Rollouts is **a Bypass component which provides advanced deployment capab Kruise Rollout integrates with ingress controllers and service meshes, leveraging their traffic shaping abilities to gradually shift traffic to the new version during an update. In addition, the business Pods metrics analysis can be used during rollout to determine whether the release will continue or be suspended. -![arch](../images/rollout-arch.jpg) +![arch](../images/rollout-arch.png) ## Why is Kruise Rollout? The native Kubernetes Deployment Object supports the **RollingUpdate** strategy which provides a basic set of safety guarantees(maxUnavailable, maxSurge) during an update. However the rolling update strategy faces many limitations: diff --git a/docs/images/rollout-arch.jpg b/docs/images/rollout-arch.jpg deleted file mode 100644 index 1b246f14..00000000 Binary files a/docs/images/rollout-arch.jpg and /dev/null differ diff --git a/docs/images/rollout-arch.png b/docs/images/rollout-arch.png new file mode 100644 index 00000000..aff5fd67 Binary files /dev/null and b/docs/images/rollout-arch.png differ diff --git a/docs/tutorials/basic_usage.md b/docs/tutorials/basic_usage.md index c6d989bc..c6a75bb8 100644 --- a/docs/tutorials/basic_usage.md +++ b/docs/tutorials/basic_usage.md @@ -105,7 +105,7 @@ spec: pause: {} # optional, The first step of released replicas. If not set, the default is to use 'weight', as shown above is 5%. replicas: 20% - trafficRouting: + trafficRoutings: # echoserver service name - service: echoserver # nginx ingress diff --git a/pkg/controller/batchrelease/batchrelease_controller_test.go b/pkg/controller/batchrelease/batchrelease_controller_test.go index 3e90d6d1..3e752a45 100644 --- a/pkg/controller/batchrelease/batchrelease_controller_test.go +++ b/pkg/controller/batchrelease/batchrelease_controller_test.go @@ -371,7 +371,7 @@ func TestReconcile_CloneSet(t *testing.T) { Name: "Progressing, stage=0->1, Input-State=BatchReady, Output-State=Upgrade", GetRelease: func() client.Object { release := setState(releaseClone, v1alpha1.ReadyBatchState) - release.Status.CanaryStatus.BatchReadyTime = *getOldTime() + release.Status.CanaryStatus.BatchReadyTime = getOldTime() stableTemplate := stableClone.Spec.Template.DeepCopy() canaryTemplate := stableClone.Spec.Template.DeepCopy() stableTemplate.Spec.Containers = containers("v1") @@ -395,7 +395,8 @@ func TestReconcile_CloneSet(t *testing.T) { Name: "Progressing, stage=0->1, Input-State=BatchReady, Output-State=BatchReady", GetRelease: func() client.Object { release := setState(releaseClone, v1alpha1.ReadyBatchState) - release.Status.CanaryStatus.BatchReadyTime = metav1.Now() + now := metav1.Now() + release.Status.CanaryStatus.BatchReadyTime = &now stableTemplate := stableClone.Spec.Template.DeepCopy() canaryTemplate := stableClone.Spec.Template.DeepCopy() stableTemplate.Spec.Containers = containers("v1") @@ -418,7 +419,8 @@ func TestReconcile_CloneSet(t *testing.T) { Name: "Special Case: Scaling, Input-State=BatchReady, Output-State=Upgrade", GetRelease: func() client.Object { release := setState(releaseClone, v1alpha1.ReadyBatchState) - release.Status.CanaryStatus.BatchReadyTime = metav1.Now() + now := metav1.Now() + release.Status.CanaryStatus.BatchReadyTime = &now stableTemplate := stableClone.Spec.Template.DeepCopy() canaryTemplate := stableClone.Spec.Template.DeepCopy() stableTemplate.Spec.Containers = containers("v1") @@ -442,7 +444,8 @@ func TestReconcile_CloneSet(t *testing.T) { Name: `Special Case: RollBack, Input-Phase=Progressing, Output-Phase=Abort`, GetRelease: func() client.Object { release := setState(releaseClone, v1alpha1.ReadyBatchState) - release.Status.CanaryStatus.BatchReadyTime = metav1.Now() + now := metav1.Now() + release.Status.CanaryStatus.BatchReadyTime = &now stableTemplate := stableClone.Spec.Template.DeepCopy() canaryTemplate := stableClone.Spec.Template.DeepCopy() stableTemplate.Spec.Containers = containers("v1") @@ -469,7 +472,8 @@ func TestReconcile_CloneSet(t *testing.T) { Name: `Special Case: Deletion, Input-Phase=Progressing, Output-Phase=Terminating`, GetRelease: func() client.Object { release := setState(releaseClone, v1alpha1.ReadyBatchState) - release.Status.CanaryStatus.BatchReadyTime = metav1.Now() + now := metav1.Now() + release.Status.CanaryStatus.BatchReadyTime = &now stableTemplate := stableClone.Spec.Template.DeepCopy() canaryTemplate := stableClone.Spec.Template.DeepCopy() stableTemplate.Spec.Containers = containers("v1") @@ -494,7 +498,8 @@ func TestReconcile_CloneSet(t *testing.T) { Name: `Special Case: Continuous Release, Input-Phase=Progressing, Output-Phase=Initial`, GetRelease: func() client.Object { release := setState(releaseClone, v1alpha1.ReadyBatchState) - release.Status.CanaryStatus.BatchReadyTime = metav1.Now() + now := metav1.Now() + release.Status.CanaryStatus.BatchReadyTime = &now stableTemplate := stableClone.Spec.Template.DeepCopy() canaryTemplate := stableClone.Spec.Template.DeepCopy() stableTemplate.Spec.Containers = containers("v1") @@ -663,7 +668,7 @@ func TestReconcile_Deployment(t *testing.T) { Name: "Progressing, stage=0->1, Input-State=BatchReady, Output-State=Upgrade", GetRelease: func() client.Object { release := setState(releaseDeploy, v1alpha1.ReadyBatchState) - release.Status.CanaryStatus.BatchReadyTime = *getOldTime() + release.Status.CanaryStatus.BatchReadyTime = getOldTime() return release }, GetDeployments: func() []client.Object { @@ -681,7 +686,8 @@ func TestReconcile_Deployment(t *testing.T) { Name: "Progressing, stage=0->1, Input-State=BatchReady, Output-State=BatchReady", GetRelease: func() client.Object { release := setState(releaseDeploy, v1alpha1.ReadyBatchState) - release.Status.CanaryStatus.BatchReadyTime = metav1.Now() + now := metav1.Now() + release.Status.CanaryStatus.BatchReadyTime = &now return release }, GetDeployments: func() []client.Object { @@ -698,7 +704,8 @@ func TestReconcile_Deployment(t *testing.T) { Name: "Special Case: Scaling, Input-State=BatchReady, Output-State=Upgrade", GetRelease: func() client.Object { release := setState(releaseDeploy, v1alpha1.ReadyBatchState) - release.Status.CanaryStatus.BatchReadyTime = metav1.Now() + now := metav1.Now() + release.Status.CanaryStatus.BatchReadyTime = &now return release }, GetDeployments: func() []client.Object { @@ -716,7 +723,8 @@ func TestReconcile_Deployment(t *testing.T) { Name: `Special Case: RollBack, Input-Phase=Progressing, Output-Phase=Abort`, GetRelease: func() client.Object { release := setState(releaseDeploy, v1alpha1.ReadyBatchState) - release.Status.CanaryStatus.BatchReadyTime = metav1.Now() + now := metav1.Now() + release.Status.CanaryStatus.BatchReadyTime = &now stableTemplate := stableDeploy.Spec.Template.DeepCopy() canaryTemplate := stableDeploy.Spec.Template.DeepCopy() stableTemplate.Spec.Containers = containers("v1") @@ -739,7 +747,8 @@ func TestReconcile_Deployment(t *testing.T) { Name: `Special Case: Deletion, Input-Phase=Progressing, Output-Phase=Terminating`, GetRelease: func() client.Object { release := setState(releaseDeploy, v1alpha1.ReadyBatchState) - release.Status.CanaryStatus.BatchReadyTime = metav1.Now() + now := metav1.Now() + release.Status.CanaryStatus.BatchReadyTime = &now stableTemplate := stableDeploy.Spec.Template.DeepCopy() canaryTemplate := stableDeploy.Spec.Template.DeepCopy() stableTemplate.Spec.Containers = containers("v1") @@ -764,7 +773,8 @@ func TestReconcile_Deployment(t *testing.T) { Name: `Special Case: Continuous Release, Input-Phase=Progressing, Output-Phase=Initial`, GetRelease: func() client.Object { release := setState(releaseDeploy, v1alpha1.ReadyBatchState) - release.Status.CanaryStatus.BatchReadyTime = metav1.Now() + now := metav1.Now() + release.Status.CanaryStatus.BatchReadyTime = &now stableTemplate := stableDeploy.Spec.Template.DeepCopy() canaryTemplate := stableDeploy.Spec.Template.DeepCopy() stableTemplate.Spec.Containers = containers("v1") @@ -840,7 +850,7 @@ func setPhase(release *v1alpha1.BatchRelease, phase v1alpha1.RolloutPhase) *v1al case v1alpha1.RolloutPhaseInitial, v1alpha1.RolloutPhaseHealthy: default: r.Status.ObservedWorkloadReplicas = 100 - r.Status.ObservedReleasePlanHash = hashReleasePlanBatches(&release.Spec.ReleasePlan) + r.Status.ObservedReleasePlanHash = util.HashReleasePlanBatches(&release.Spec.ReleasePlan) } return r } @@ -850,7 +860,7 @@ func setState(release *v1alpha1.BatchRelease, state v1alpha1.BatchReleaseBatchSt r.Status.Phase = v1alpha1.RolloutPhaseProgressing r.Status.CanaryStatus.CurrentBatchState = state r.Status.ObservedWorkloadReplicas = 100 - r.Status.ObservedReleasePlanHash = hashReleasePlanBatches(&release.Spec.ReleasePlan) + r.Status.ObservedReleasePlanHash = util.HashReleasePlanBatches(&release.Spec.ReleasePlan) return r } diff --git a/pkg/controller/batchrelease/batchrelease_plan_executor.go b/pkg/controller/batchrelease/batchrelease_plan_executor.go index 864532f0..d3582b62 100644 --- a/pkg/controller/batchrelease/batchrelease_plan_executor.go +++ b/pkg/controller/batchrelease/batchrelease_plan_executor.go @@ -205,7 +205,8 @@ func (r *Executor) progressBatches(workloadController workloads.WorkloadControll setCondition(status, "Progressing", v1.ConditionFalse, "VerifyBatchFailed", err.Error()) case verified: result = reconcile.Result{RequeueAfter: DefaultDuration} - status.CanaryStatus.BatchReadyTime = metav1.Now() + now := metav1.Now() + status.CanaryStatus.BatchReadyTime = &now status.CanaryStatus.CurrentBatchState = v1alpha1.ReadyBatchState default: status.CanaryStatus.CurrentBatchState = v1alpha1.UpgradingBatchState diff --git a/pkg/controller/batchrelease/batchrelease_special_cases_handler.go b/pkg/controller/batchrelease/batchrelease_special_cases_handler.go index ca198336..bd950232 100644 --- a/pkg/controller/batchrelease/batchrelease_special_cases_handler.go +++ b/pkg/controller/batchrelease/batchrelease_special_cases_handler.go @@ -21,6 +21,7 @@ import ( "github.com/openkruise/rollouts/api/v1alpha1" "github.com/openkruise/rollouts/pkg/controller/batchrelease/workloads" + "github.com/openkruise/rollouts/pkg/util" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/klog/v2" @@ -170,10 +171,9 @@ func refreshStatus(release *v1alpha1.BatchRelease, newStatus *v1alpha1.BatchRele newStatus.CanaryStatus.UpdatedReplicas = workloadInfo.Status.UpdatedReplicas newStatus.CanaryStatus.UpdatedReadyReplicas = workloadInfo.Status.UpdatedReadyReplicas } - planHash := hashReleasePlanBatches(&release.Spec.ReleasePlan) - if newStatus.ObservedReleasePlanHash != planHash { - newStatus.ObservedReleasePlanHash = planHash - } + } + if len(newStatus.ObservedReleasePlanHash) == 0 { + newStatus.ObservedReleasePlanHash = util.HashReleasePlanBatches(&release.Spec.ReleasePlan) } } @@ -182,7 +182,7 @@ func isPlanTerminating(release *v1alpha1.BatchRelease, status *v1alpha1.BatchRel } func isPlanChanged(plan *v1alpha1.ReleasePlan, status *v1alpha1.BatchReleaseStatus) bool { - return status.ObservedReleasePlanHash != hashReleasePlanBatches(plan) && status.Phase == v1alpha1.RolloutPhaseProgressing + return status.ObservedReleasePlanHash != util.HashReleasePlanBatches(plan) && status.Phase == v1alpha1.RolloutPhaseProgressing } func isPlanUnhealthy(plan *v1alpha1.ReleasePlan, status *v1alpha1.BatchReleaseStatus) bool { diff --git a/pkg/controller/batchrelease/batchrelease_util.go b/pkg/controller/batchrelease/batchrelease_util.go index 4c818d1e..467fa883 100644 --- a/pkg/controller/batchrelease/batchrelease_util.go +++ b/pkg/controller/batchrelease/batchrelease_util.go @@ -17,11 +17,8 @@ limitations under the License. package batchrelease import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" @@ -39,12 +36,6 @@ func HasTerminatingCondition(status v1alpha1.BatchReleaseStatus) bool { return false } -func hashReleasePlanBatches(releasePlan *v1alpha1.ReleasePlan) string { - by, _ := json.Marshal(releasePlan.Batches) - md5Hash := sha256.Sum256(by) - return hex.EncodeToString(md5Hash[:]) -} - func initializeStatusIfNeeds(status *v1alpha1.BatchReleaseStatus) { if len(status.Phase) == 0 { resetStatus(status) @@ -76,13 +67,14 @@ func signalRecalculate(release *v1alpha1.BatchRelease, newStatus *v1alpha1.Batch currentBatch := int32(0) if release.Spec.ReleasePlan.BatchPartition != nil { // ensure current batch upper bound - currentBatch = integer.Int32Min(*release.Spec.ReleasePlan.BatchPartition, currentBatch) + currentBatch = integer.Int32Min(*release.Spec.ReleasePlan.BatchPartition, int32(len(release.Spec.ReleasePlan.Batches)-1)) } klog.Infof("BatchRelease(%v) canary batch changed from %v to %v when the release plan changed", client.ObjectKeyFromObject(release), newStatus.CanaryStatus.CurrentBatch, currentBatch) newStatus.CanaryStatus.CurrentBatch = currentBatch newStatus.CanaryStatus.CurrentBatchState = v1alpha1.UpgradingBatchState + newStatus.ObservedReleasePlanHash = util.HashReleasePlanBatches(&release.Spec.ReleasePlan) } func resetStatus(status *v1alpha1.BatchReleaseStatus) { @@ -101,11 +93,12 @@ func setCondition(status *v1alpha1.BatchReleaseStatus, condType v1alpha1.Rollout if len(status.Conditions) == 0 { status.Conditions = append(status.Conditions, v1alpha1.RolloutCondition{ - Type: condType, - Status: condStatus, - Reason: reason, - Message: message, - LastUpdateTime: metav1.Now(), + Type: condType, + Status: condStatus, + Reason: reason, + Message: message, + LastUpdateTime: metav1.Now(), + LastTransitionTime: metav1.Now(), }) return } diff --git a/pkg/controller/rollout/batchrelease/batchrelease.go b/pkg/controller/rollout/batchrelease/batchrelease.go index dc0a6b2c..99de7446 100644 --- a/pkg/controller/rollout/batchrelease/batchrelease.go +++ b/pkg/controller/rollout/batchrelease/batchrelease.go @@ -21,8 +21,9 @@ import rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" // BatchRelease is not the actual controller of the BatchRelease controller, // but rather the ability to interact with the BatchRelease controller through the BatchRelease CRD to achieve a batch release type BatchRelease interface { - // Verify will create batchRelease or update batchRelease steps configuration - Verify(index int32) error + // Verify will create batchRelease or update batchRelease steps configuration and + // return whether the batchRelease configuration is consistent with the rollout step + Verify(index int32) (bool, error) // 1. Promote release workload in step(index), 1<=index<=len(step) // 2. Promote will resume stable workload if the last batch(index=-1) is finished diff --git a/pkg/controller/rollout/batchrelease/inner_batchrelease.go b/pkg/controller/rollout/batchrelease/inner_batchrelease.go index 98e39403..d81fa4e2 100644 --- a/pkg/controller/rollout/batchrelease/inner_batchrelease.go +++ b/pkg/controller/rollout/batchrelease/inner_batchrelease.go @@ -59,7 +59,7 @@ func NewInnerBatchController(c client.Client, rollout *rolloutv1alpha1.Rollout) return r } -func (r *innerBatchRelease) Verify(index int32) error { +func (r *innerBatchRelease) Verify(index int32) (bool, error) { index = index - 1 batch := &rolloutv1alpha1.BatchRelease{} err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch) @@ -68,42 +68,42 @@ func (r *innerBatchRelease) Verify(index int32) error { br := createBatchRelease(r.rollout, r.batchName) if err = r.Create(context.TODO(), br); err != nil && !errors.IsAlreadyExists(err) { klog.Errorf("rollout(%s/%s) create BatchRelease failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return err + return false, err } data := util.DumpJSON(br) klog.Infof("rollout(%s/%s) create BatchRelease(%s) success", r.rollout.Namespace, r.rollout.Name, data) - return nil + return false, nil } else if err != nil { klog.Errorf("rollout(%s/%s) fetch BatchRelease failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return err + return false, err } // check whether batchRelease configuration is the latest newBr := createBatchRelease(r.rollout, r.batchName) if reflect.DeepEqual(batch.Spec.ReleasePlan.Batches, newBr.Spec.ReleasePlan.Batches) { - klog.Infof("rollout(%s/%s) batchRelease is initialize done", r.rollout.Namespace, r.rollout.Name) - return nil + klog.Infof("rollout(%s/%s) batchRelease(generation:%d) configuration is the latest", r.rollout.Namespace, r.rollout.Name, batch.Generation) + return true, nil } // update batchRelease to the latest version - if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch); err != nil { + if err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if err = r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch); err != nil { klog.Errorf("error getting updated BatchRelease(%s/%s) from client", batch.Namespace, batch.Name) return err } batch.Spec.ReleasePlan.Batches = newBr.Spec.ReleasePlan.Batches batch.Spec.ReleasePlan.BatchPartition = utilpointer.Int32Ptr(index) - if err := r.Client.Update(context.TODO(), batch); err != nil { + if err = r.Client.Update(context.TODO(), batch); err != nil { return err } return nil }); err != nil { klog.Errorf("rollout(%s/%s) update batchRelease configuration failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) - return err + return false, err } data := util.DumpJSON(batch) klog.Infof("rollout(%s/%s) update batchRelease configuration(%s) to the latest", r.rollout.Namespace, r.rollout.Name, data) - return nil + return false, nil } func (r *innerBatchRelease) FetchBatchRelease() (*rolloutv1alpha1.BatchRelease, error) { @@ -161,7 +161,7 @@ func (r *innerBatchRelease) resumeStableWorkload(checkReady bool) (bool, error) return false, err } // default partition.IntVal=0 - if !obj.Spec.UpdateStrategy.Paused && obj.Spec.UpdateStrategy.Partition.IntVal == 0 { + if !obj.Spec.UpdateStrategy.Paused && obj.Spec.UpdateStrategy.Partition.IntVal == 0 && obj.Spec.UpdateStrategy.Partition.Type == intstr.Int { return true, nil } @@ -276,7 +276,6 @@ func createBatchRelease(rollout *rolloutv1alpha1.Rollout, batchName string) *rol }, Spec: rolloutv1alpha1.BatchReleaseSpec{ TargetRef: rolloutv1alpha1.ObjectRef{ - Type: rolloutv1alpha1.WorkloadRefType, WorkloadRef: &rolloutv1alpha1.WorkloadRef{ APIVersion: rollout.Spec.ObjectRef.WorkloadRef.APIVersion, Kind: rollout.Spec.ObjectRef.WorkloadRef.Kind, diff --git a/pkg/controller/rollout/canary.go b/pkg/controller/rollout/canary.go index 348eb14b..605fb7ba 100644 --- a/pkg/controller/rollout/canary.go +++ b/pkg/controller/rollout/canary.go @@ -26,7 +26,6 @@ import ( "github.com/openkruise/rollouts/pkg/util" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" @@ -43,42 +42,41 @@ func (r *rolloutContext) runCanary() error { canaryStatus.CurrentStepIndex = 1 canaryStatus.RolloutHash = r.rollout.Annotations[util.RolloutHashAnnotation] } - if r.rollout.Spec.Strategy.Canary.TrafficRouting[0].GracePeriodSeconds <= 0 { - r.rollout.Spec.Strategy.Canary.TrafficRouting[0].GracePeriodSeconds = defaultGracePeriodSeconds - } + // update canary status canaryStatus.CanaryReplicas = r.workload.CanaryReplicas canaryStatus.CanaryReadyReplicas = r.workload.CanaryReadyReplicas switch canaryStatus.CurrentStepState { case rolloutv1alpha1.CanaryStepStateUpgrade: klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, rolloutv1alpha1.CanaryStepStateUpgrade) - done, err := r.doCanaryUpgrade() - if err != nil { - return err - } else if done { - canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateTrafficRouting + // If the last step is 100%, there is no need to execute the canary process at this time + if r.rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1].Weight == 100 { + klog.Infof("rollout(%s/%s) last step is 100%, there is no need to execute the canary process at this time, and set state=%s", + r.rollout.Namespace, r.rollout.Name, canaryStatus.CurrentStepIndex-1, canaryStatus.CurrentStepIndex, rolloutv1alpha1.CanaryStepStateCompleted) canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name, - canaryStatus.CurrentStepIndex, rolloutv1alpha1.CanaryStepStateUpgrade, canaryStatus.CurrentStepState) + canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateCompleted + } else { + done, err := r.doCanaryUpgrade() + if err != nil { + return err + } else if done { + canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateTrafficRouting + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name, + canaryStatus.CurrentStepIndex, rolloutv1alpha1.CanaryStepStateUpgrade, canaryStatus.CurrentStepState) + } } + case rolloutv1alpha1.CanaryStepStateTrafficRouting: klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, rolloutv1alpha1.CanaryStepStateTrafficRouting) done, err := r.doCanaryTrafficRouting() if err != nil { return err } else if done { - // if 100% completed - if len(r.rollout.Spec.Strategy.Canary.Steps) == int(canaryStatus.CurrentStepIndex) && - r.rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1].Weight == 100 { - klog.Infof("rollout(%s/%s) canary run all steps, and completed", r.rollout.Namespace, r.rollout.Name, canaryStatus.CurrentStepIndex-1, canaryStatus.CurrentStepIndex) - canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateCompleted - } else { - canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} - canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateMetricsAnalysis - klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name, - canaryStatus.CurrentStepIndex, rolloutv1alpha1.CanaryStepStateTrafficRouting, canaryStatus.CurrentStepState) - } + canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} + canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateMetricsAnalysis + klog.Infof("rollout(%s/%s) step(%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name, + canaryStatus.CurrentStepIndex, rolloutv1alpha1.CanaryStepStateTrafficRouting, canaryStatus.CurrentStepState) } expectedTime := time.Now().Add(time.Duration(defaultGracePeriodSeconds) * time.Second) r.recheckTime = &expectedTime @@ -115,7 +113,7 @@ func (r *rolloutContext) runCanary() error { canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateUpgrade klog.Infof("rollout(%s/%s) canary step from(%d) -> to(%d)", r.rollout.Namespace, r.rollout.Name, canaryStatus.CurrentStepIndex-1, canaryStatus.CurrentStepIndex) } else { - klog.Infof("rollout(%s/%s) canary run all steps, and completed", r.rollout.Namespace, r.rollout.Name, canaryStatus.CurrentStepIndex-1, canaryStatus.CurrentStepIndex) + klog.Infof("rollout(%s/%s) canary run all steps, and completed", r.rollout.Namespace, r.rollout.Name) canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} canaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateCompleted } @@ -142,18 +140,26 @@ func (r *rolloutContext) doCanaryUpgrade() (bool, error) { return false, nil }*/ + // verify whether batchRelease configuration is the latest steps := len(r.rollout.Spec.Strategy.Canary.Steps) - // canary release + canaryStatus := r.newStatus.CanaryStatus + isLatest, err := r.batchControl.Verify(canaryStatus.CurrentStepIndex) + if err != nil { + return false, err + } else if !isLatest { + return false, nil + } + + // fetch batchRelease batch, err := r.batchControl.FetchBatchRelease() - if errors.IsNotFound(err) { - // the first step, and create batch release crd - return false, r.batchControl.Verify(r.newStatus.CanaryStatus.CurrentStepIndex) - } else if err != nil { + if err != nil { return false, err - } else if batch.Generation != batch.Status.ObservedGeneration { + } else if batch.Status.ObservedReleasePlanHash != util.HashReleasePlanBatches(&batch.Spec.ReleasePlan) || + batch.Generation != batch.Status.ObservedGeneration { + klog.Infof("rollout(%s/%s) batchReleasePlan is not consistent, and wait a moment", r.rollout.Namespace, r.rollout.Name) return false, nil } - canaryStatus := r.newStatus.CanaryStatus + batchData := util.DumpJSON(batch.Status) cond := util.GetRolloutCondition(*r.newStatus, rolloutv1alpha1.RolloutConditionProgressing) cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and upgrade workload new versions", canaryStatus.CurrentStepIndex, steps) r.newStatus.Message = cond.Message @@ -165,14 +171,15 @@ func (r *rolloutContext) doCanaryUpgrade() (bool, error) { } // check whether batchRelease is ready - if batch.Status.CanaryStatus.CurrentBatchState != rolloutv1alpha1.ReadyBatchState { - klog.Infof("rollout(%s/%s) workload(%s) batch(%d) ReadyReplicas(%d) state(%s), and wait a moment", - r.rollout.Namespace, r.rollout.Name, r.workload.Name, canaryStatus.CurrentStepIndex, batch.Status.CanaryStatus.UpdatedReadyReplicas, batch.Status.CanaryStatus.CurrentBatchState) + if batch.Status.CanaryStatus.CurrentBatchState != rolloutv1alpha1.ReadyBatchState || + batch.Status.CanaryStatus.CurrentBatch+1 != canaryStatus.CurrentStepIndex { + klog.Infof("rollout(%s/%s) batch(%s) state(%s), and wait a moment", + r.rollout.Namespace, r.rollout.Name, batchData, batch.Status.CanaryStatus.CurrentBatchState) return false, nil } r.recorder.Eventf(r.rollout, corev1.EventTypeNormal, "Progressing", fmt.Sprintf("upgrade step(%d) canary pods with new versions done", canaryStatus.CurrentStepIndex)) - klog.Infof("rollout(%s/%s) workload(%s) batch(%d) availableReplicas(%d) state(%s), and continue", - r.rollout.Namespace, r.rollout.Name, r.workload.Name, canaryStatus.CurrentStepIndex, batch.Status.CanaryStatus.UpdatedReadyReplicas, batch.Status.CanaryStatus.CurrentBatchState) + klog.Infof("rollout(%s/%s) batch(%s) state(%s), and success", + r.rollout.Namespace, r.rollout.Name, batchData, batch.Status.CanaryStatus.CurrentBatchState) return true, nil } @@ -194,7 +201,7 @@ func (r *rolloutContext) doCanaryPaused() (bool, error) { // need manual confirmation if currentStep.Pause.Duration == nil { klog.Infof("rollout(%s/%s) don't set pause duration, and need manual confirmation", r.rollout.Namespace, r.rollout.Name) - cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and you need manually confirm(kube-cli approve) to enter the next step", canaryStatus.CurrentStepIndex, steps) + cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and you need manually confirm to enter the next step", canaryStatus.CurrentStepIndex, steps) r.newStatus.Message = cond.Message return false, nil } @@ -218,9 +225,6 @@ func (r *rolloutContext) doCanaryFinalising() (bool, error) { if r.newStatus.CanaryStatus == nil { return true, nil } - if r.rollout.Spec.Strategy.Canary.TrafficRouting[0].GracePeriodSeconds <= 0 { - r.rollout.Spec.Strategy.Canary.TrafficRouting[0].GracePeriodSeconds = defaultGracePeriodSeconds - } // 1. rollout progressing complete, allow workload paused=false in webhook err := r.removeRolloutStateInWorkload() if err != nil { diff --git a/pkg/controller/rollout/context.go b/pkg/controller/rollout/context.go index a23af63a..9d2598dc 100644 --- a/pkg/controller/rollout/context.go +++ b/pkg/controller/rollout/context.go @@ -22,6 +22,7 @@ import ( rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" "github.com/openkruise/rollouts/pkg/controller/rollout/batchrelease" "github.com/openkruise/rollouts/pkg/util" + apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" @@ -74,5 +75,5 @@ func (r *rolloutContext) finalising() (bool, error) { } func (r *rolloutContext) podRevisionLabelKey() string { - return util.RsPodRevisionLabelKey + return apps.DefaultDeploymentUniqueLabelKey } diff --git a/pkg/controller/rollout/progressing.go b/pkg/controller/rollout/progressing.go index 5c811786..066a1af8 100644 --- a/pkg/controller/rollout/progressing.go +++ b/pkg/controller/rollout/progressing.go @@ -40,8 +40,18 @@ var defaultGracePeriodSeconds int32 = 3 func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *rolloutv1alpha1.Rollout) (*time.Time, error) { cond := util.GetRolloutCondition(rollout.Status, rolloutv1alpha1.RolloutConditionProgressing) klog.Infof("reconcile rollout(%s/%s) progressing action", rollout.Namespace, rollout.Name) + workload, err := r.Finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef) + if err != nil { + klog.Errorf("rollout(%s/%s) get workload failed: %s", rollout.Namespace, rollout.Name, err.Error()) + return nil, err + } else if workload == nil { + klog.Errorf("rollout(%s/%s) workload Not Found", rollout.Namespace, rollout.Name) + return nil, nil + } else if !workload.IsStatusConsistent { + klog.Infof("rollout(%s/%s) workload status isn't consistent, then wait a moment", rollout.Namespace, rollout.Name) + return nil, nil + } - var err error var recheckTime *time.Time newStatus := rollout.Status.DeepCopy() switch cond.Reason { @@ -63,25 +73,21 @@ func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *rolloutv1alpha1 } case rolloutv1alpha1.ProgressingReasonInRolling: - // paused rollout progress - if rollout.Spec.Strategy.Paused { - klog.Infof("rollout(%s/%s) is Progressing, but paused", rollout.Namespace, rollout.Name) - progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonPaused, "Rollout has been paused, you can resume it by kube-cli") - // rollout canceled, indicates rollback(v1 -> v2 -> v1) - } else if newStatus.StableRevision == newStatus.CanaryRevision && newStatus.CanaryRevision != newStatus.CanaryStatus.CanaryRevision { - workload, _ := r.Finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef) - if workload != nil { - // rollback, mark stable revision - newStatus.CanaryStatus.CanaryRevision = workload.CurrentPodTemplateHash - } + // rollout canceled, indicates rollback(v1 -> v2 -> v1) + if workload.IsInRollback { + newStatus.CanaryStatus.CanaryRevision = workload.CanaryRevision r.Recorder.Eventf(rollout, corev1.EventTypeNormal, "Progressing", "workload has been rollback, then rollout is canceled") klog.Infof("rollout(%s/%s) workload has been rollback, then rollout canceled", rollout.Namespace, rollout.Name) - progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonCancelling, "The workload has been rolled back and the rollout process has been cancelled") + progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonCancelling, "The workload has been rolled back and the rollout process will be cancelled") + // paused rollout progress + } else if rollout.Spec.Strategy.Paused { + klog.Infof("rollout(%s/%s) is Progressing, but paused", rollout.Namespace, rollout.Name) + progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonPaused, "Rollout has been paused, you can resume it by kube-cli") // In case of continuous publishing(v1 -> v2 -> v3), then restart publishing - } else if newStatus.CanaryStatus.CanaryRevision != "" && newStatus.CanaryRevision != newStatus.CanaryStatus.CanaryRevision { + } else if newStatus.CanaryStatus.CanaryRevision != "" && workload.CanaryRevision != newStatus.CanaryStatus.CanaryRevision { r.Recorder.Eventf(rollout, corev1.EventTypeNormal, "Progressing", "workload continuous publishing canaryRevision, then restart publishing") klog.Infof("rollout(%s/%s) workload continuous publishing canaryRevision from(%s) -> to(%s), then restart publishing", - rollout.Namespace, rollout.Name, newStatus.CanaryStatus.CanaryRevision, newStatus.CanaryRevision) + rollout.Namespace, rollout.Name, newStatus.CanaryStatus.CanaryRevision, workload.CanaryRevision) done, err := r.doProgressingReset(rollout, newStatus) if err != nil { klog.Errorf("rollout(%s/%s) doProgressingReset failed: %s", rollout.Namespace, rollout.Name, err.Error()) @@ -103,12 +109,6 @@ func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *rolloutv1alpha1 klog.Errorf("rollout(%s/%s) reCalculate Canary StepIndex failed: %s", rollout.Namespace, rollout.Name, err.Error()) return nil, err } - // update batchRelease to the latest version - err = batchControl.Verify(newStepIndex) - if err != nil { - klog.Errorf("rollout(%s/%s) canary step configuration change, but update batchRelease crd failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return nil, err - } // canary step configuration change causes current step index change newStatus.CanaryStatus.CurrentStepIndex = newStepIndex newStatus.CanaryStatus.CurrentStepState = rolloutv1alpha1.CanaryStepStateUpgrade @@ -123,6 +123,7 @@ func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *rolloutv1alpha1 klog.Infof("rollout(%s/%s) progressing rolling done", rollout.Namespace, rollout.Name) progressingStateTransition(newStatus, corev1.ConditionTrue, rolloutv1alpha1.ProgressingReasonFinalising, "Rollout has been completed and some closing work is being done") } else { // rollout is in rolling + newStatus.CanaryStatus.PodTemplateHash = workload.PodTemplateHash recheckTime, err = r.doProgressingInRolling(rollout, newStatus) if err != nil { return nil, err @@ -142,7 +143,14 @@ func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *rolloutv1alpha1 } case rolloutv1alpha1.ProgressingReasonPaused: - if !rollout.Spec.Strategy.Paused { + // rollout canceled, indicates rollback(v1 -> v2 -> v1) + if workload.IsInRollback { + newStatus.CanaryStatus.CanaryRevision = workload.CanaryRevision + r.Recorder.Eventf(rollout, corev1.EventTypeNormal, "Progressing", "workload has been rollback, then rollout is canceled") + klog.Infof("rollout(%s/%s) workload has been rollback, then rollout canceled", rollout.Namespace, rollout.Name) + progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonCancelling, "The workload has been rolled back and the rollout process will be cancelled") + // from paused to inRolling + } else if !rollout.Spec.Strategy.Paused { klog.Infof("rollout(%s/%s) is Progressing, but paused", rollout.Namespace, rollout.Name) progressingStateTransition(newStatus, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonInRolling, "") } @@ -187,10 +195,7 @@ func progressingStateTransition(status *rolloutv1alpha1.RolloutStatus, condStatu func (r *RolloutReconciler) doProgressingInitializing(rollout *rolloutv1alpha1.Rollout, newStatus *rolloutv1alpha1.RolloutStatus) (bool, string, error) { // canary release - if rollout.Spec.Strategy.Type == "" || rollout.Spec.Strategy.Type == rolloutv1alpha1.RolloutStrategyCanary { - return r.verifyCanaryStrategy(rollout, newStatus) - } - return true, "", nil + return r.verifyCanaryStrategy(rollout, newStatus) } func (r *RolloutReconciler) doProgressingInRolling(rollout *rolloutv1alpha1.Rollout, newStatus *rolloutv1alpha1.RolloutStatus) (*time.Time, error) { @@ -230,7 +235,7 @@ func (r *RolloutReconciler) doProgressingReset(rollout *rolloutv1alpha1.Rollout, recorder: r.Recorder, } - if rolloutCon.rollout.Spec.Strategy.Canary.TrafficRouting != nil { + if rolloutCon.rollout.Spec.Strategy.Canary.TrafficRoutings != nil { // 1. remove stable service podRevision selector done, err := rolloutCon.restoreStableService() if err != nil || !done { @@ -257,8 +262,8 @@ func (r *RolloutReconciler) doProgressingReset(rollout *rolloutv1alpha1.Rollout, func (r *RolloutReconciler) verifyCanaryStrategy(rollout *rolloutv1alpha1.Rollout, newStatus *rolloutv1alpha1.RolloutStatus) (bool, string, error) { canary := rollout.Spec.Strategy.Canary // Traffic routing - if canary.TrafficRouting != nil { - if ok, msg, err := r.verifyTrafficRouting(rollout.Namespace, canary.TrafficRouting[0]); !ok { + if canary.TrafficRoutings != nil { + if ok, msg, err := r.verifyTrafficRouting(rollout.Namespace, canary.TrafficRoutings[0]); !ok { return ok, msg, err } } diff --git a/pkg/controller/rollout/progressing_test.go b/pkg/controller/rollout/progressing_test.go index 64a029a9..623d6a7a 100644 --- a/pkg/controller/rollout/progressing_test.go +++ b/pkg/controller/rollout/progressing_test.go @@ -32,16 +32,16 @@ import ( func TestReCalculateCanaryStepIndex(t *testing.T) { cases := []struct { name string - getObj func() *apps.Deployment + getObj func() (*apps.Deployment, *apps.ReplicaSet) getRollout func() *rolloutv1alpha1.Rollout getBatchRelease func() *rolloutv1alpha1.BatchRelease expectStepIndex int32 }{ { name: "steps changed v1", - getObj: func() *apps.Deployment { + getObj: func() (*apps.Deployment, *apps.ReplicaSet) { obj := deploymentDemo.DeepCopy() - return obj + return obj, rsDemo.DeepCopy() }, getRollout: func() *rolloutv1alpha1.Rollout { obj := rolloutDemo.DeepCopy() @@ -78,9 +78,9 @@ func TestReCalculateCanaryStepIndex(t *testing.T) { }, { name: "steps changed v2", - getObj: func() *apps.Deployment { + getObj: func() (*apps.Deployment, *apps.ReplicaSet) { obj := deploymentDemo.DeepCopy() - return obj + return obj, rsDemo.DeepCopy() }, getRollout: func() *rolloutv1alpha1.Rollout { obj := rolloutDemo.DeepCopy() @@ -117,9 +117,9 @@ func TestReCalculateCanaryStepIndex(t *testing.T) { }, { name: "steps changed v3", - getObj: func() *apps.Deployment { + getObj: func() (*apps.Deployment, *apps.ReplicaSet) { obj := deploymentDemo.DeepCopy() - return obj + return obj, rsDemo.DeepCopy() }, getRollout: func() *rolloutv1alpha1.Rollout { obj := rolloutDemo.DeepCopy() @@ -156,9 +156,9 @@ func TestReCalculateCanaryStepIndex(t *testing.T) { }, { name: "steps changed v4", - getObj: func() *apps.Deployment { + getObj: func() (*apps.Deployment, *apps.ReplicaSet) { obj := deploymentDemo.DeepCopy() - return obj + return obj, rsDemo.DeepCopy() }, getRollout: func() *rolloutv1alpha1.Rollout { obj := rolloutDemo.DeepCopy() @@ -193,13 +193,59 @@ func TestReCalculateCanaryStepIndex(t *testing.T) { }, expectStepIndex: 2, }, + { + name: "steps changed v5", + getObj: func() (*apps.Deployment, *apps.ReplicaSet) { + obj := deploymentDemo.DeepCopy() + return obj, rsDemo.DeepCopy() + }, + getRollout: func() *rolloutv1alpha1.Rollout { + obj := rolloutDemo.DeepCopy() + obj.Spec.Strategy.Canary.Steps = []rolloutv1alpha1.CanaryStep{ + { + Weight: 2, + Replicas: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "10%", + }, + }, + { + Weight: 3, + Replicas: &intstr.IntOrString{ + Type: intstr.String, + StrVal: "10%", + }, + }, + } + return obj + }, + getBatchRelease: func() *rolloutv1alpha1.BatchRelease { + obj := batchDemo.DeepCopy() + obj.Spec.ReleasePlan.Batches = []rolloutv1alpha1.ReleaseBatch{ + { + CanaryReplicas: intstr.FromString("10%"), + }, + { + CanaryReplicas: intstr.FromString("20%"), + }, + { + CanaryReplicas: intstr.FromString("30%"), + }, + } + obj.Spec.ReleasePlan.BatchPartition = utilpointer.Int32(0) + return obj + }, + expectStepIndex: 1, + }, } for _, cs := range cases { t.Run(cs.name, func(t *testing.T) { client := fake.NewClientBuilder().WithScheme(scheme).Build() client.Create(context.TODO(), cs.getBatchRelease()) - client.Create(context.TODO(), cs.getObj()) + dep, rs := cs.getObj() + client.Create(context.TODO(), dep) + client.Create(context.TODO(), rs) client.Create(context.TODO(), cs.getRollout()) reconciler := &RolloutReconciler{ diff --git a/pkg/controller/rollout/rollout_controller.go b/pkg/controller/rollout/rollout_controller.go index bcf11120..1fcc6854 100644 --- a/pkg/controller/rollout/rollout_controller.go +++ b/pkg/controller/rollout/rollout_controller.go @@ -87,9 +87,11 @@ func (r *RolloutReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, err } // update rollout status - err = r.updateRolloutStatus(rollout) + done, err := r.updateRolloutStatus(rollout) if err != nil { return ctrl.Result{}, err + } else if !done { + return ctrl.Result{}, nil } var recheckTime *time.Time diff --git a/pkg/controller/rollout/rollout_controller_test.go b/pkg/controller/rollout/rollout_controller_test.go index 450ed0be..e0831946 100644 --- a/pkg/controller/rollout/rollout_controller_test.go +++ b/pkg/controller/rollout/rollout_controller_test.go @@ -22,6 +22,7 @@ import ( apps "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" utilpointer "k8s.io/utils/pointer" ) @@ -36,7 +37,6 @@ var ( }, Spec: rolloutv1alpha1.RolloutSpec{ ObjectRef: rolloutv1alpha1.ObjectRef{ - Type: rolloutv1alpha1.WorkloadRefType, WorkloadRef: &rolloutv1alpha1.WorkloadRef{ APIVersion: "apps/v1", Kind: "Deployment", @@ -55,11 +55,43 @@ var ( Kind: "Deployment", }, ObjectMeta: metav1.ObjectMeta{ - Name: "echoserver", - Labels: map[string]string{}, + Name: "echoserver", + Labels: map[string]string{}, + Generation: 1, + UID: types.UID("606132e0-85ef-460a-8cf5-cd8f915a8cc3"), }, Spec: apps.DeploymentSpec{ Replicas: utilpointer.Int32(100), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "echoserver", + }, + }, + }, + Status: apps.DeploymentStatus{ + ObservedGeneration: 1, + }, + } + + rsDemo = &apps.ReplicaSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "ReplicaSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "echoserver-xxx", + Labels: map[string]string{ + "app": "echoserver", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "echoserver", + UID: types.UID("606132e0-85ef-460a-8cf5-cd8f915a8cc3"), + Controller: utilpointer.BoolPtr(true), + }, + }, }, } @@ -70,7 +102,6 @@ var ( }, Spec: rolloutv1alpha1.BatchReleaseSpec{ TargetRef: rolloutv1alpha1.ObjectRef{ - Type: rolloutv1alpha1.WorkloadRefType, WorkloadRef: &rolloutv1alpha1.WorkloadRef{ APIVersion: "apps/v1", Kind: "Deployment", diff --git a/pkg/controller/rollout/status.go b/pkg/controller/rollout/status.go index 5d693bc4..6b83c10d 100644 --- a/pkg/controller/rollout/status.go +++ b/pkg/controller/rollout/status.go @@ -33,9 +33,22 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func (r *RolloutReconciler) updateRolloutStatus(rollout *rolloutv1alpha1.Rollout) error { +func (r *RolloutReconciler) updateRolloutStatus(rollout *rolloutv1alpha1.Rollout) (done bool, err error) { newStatus := *rollout.Status.DeepCopy() newStatus.ObservedGeneration = rollout.GetGeneration() + defer func() { + err = r.updateRolloutStatusInternal(rollout, newStatus) + if err != nil { + klog.Errorf("update rollout(%s/%s) status failed: %s", rollout.Namespace, rollout.Name, err.Error()) + return + } + err = r.calculateRolloutHash(rollout) + if err != nil { + return + } + rollout.Status = newStatus + }() + // delete rollout CRD if !rollout.DeletionTimestamp.IsZero() && newStatus.Phase != rolloutv1alpha1.RolloutPhaseTerminating { newStatus.Phase = rolloutv1alpha1.RolloutPhaseTerminating @@ -48,29 +61,36 @@ func (r *RolloutReconciler) updateRolloutStatus(rollout *rolloutv1alpha1.Rollout workload, err := r.Finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef) if err != nil { klog.Errorf("rollout(%s/%s) get workload failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return err - } else if workload == nil && rollout.DeletionTimestamp.IsZero() { - resetStatus(&newStatus) - klog.Infof("rollout(%s/%s) workload not found, and reset status be Initial", rollout.Namespace, rollout.Name) - } else if workload != nil { - newStatus.StableRevision = workload.StableRevision - newStatus.CanaryRevision = workload.CanaryRevision - // update workload generation to canaryStatus.ObservedWorkloadGeneration - // rollout is a target ref bypass, so there needs to be a field to identify the rollout execution process or results, - // which version of deployment is targeted, ObservedWorkloadGeneration that is to compare with the workload generation - if newStatus.CanaryStatus != nil && newStatus.CanaryStatus.CanaryRevision != "" && - newStatus.CanaryStatus.CanaryRevision == workload.CanaryRevision { - newStatus.CanaryStatus.ObservedWorkloadGeneration = workload.Generation + return + } else if workload == nil { + if rollout.DeletionTimestamp.IsZero() { + resetStatus(&newStatus) + klog.Infof("rollout(%s/%s) workload not found, and reset status be Initial", rollout.Namespace, rollout.Name) } + done = true + return + } + + // workload status is not consistent + if !workload.IsStatusConsistent { + klog.Infof("rollout(%s/%s) workload status isn't consistent, then wait a moment", rollout.Namespace, rollout.Name) + done = false + return + } + newStatus.StableRevision = workload.StableRevision + // update workload generation to canaryStatus.ObservedWorkloadGeneration + // rollout is a target ref bypass, so there needs to be a field to identify the rollout execution process or results, + // which version of deployment is targeted, ObservedWorkloadGeneration that is to compare with the workload generation + if newStatus.CanaryStatus != nil && newStatus.CanaryStatus.CanaryRevision != "" && + newStatus.CanaryStatus.CanaryRevision == workload.CanaryRevision { + newStatus.CanaryStatus.ObservedWorkloadGeneration = workload.Generation } switch newStatus.Phase { case rolloutv1alpha1.RolloutPhaseInitial: - if workload != nil { - klog.Infof("rollout(%s/%s) status phase from(%s) -> to(%s)", rollout.Namespace, rollout.Name, rolloutv1alpha1.RolloutPhaseInitial, rolloutv1alpha1.RolloutPhaseHealthy) - newStatus.Phase = rolloutv1alpha1.RolloutPhaseHealthy - newStatus.Message = "rollout is healthy" - } + klog.Infof("rollout(%s/%s) status phase from(%s) -> to(%s)", rollout.Namespace, rollout.Name, rolloutv1alpha1.RolloutPhaseInitial, rolloutv1alpha1.RolloutPhaseHealthy) + newStatus.Phase = rolloutv1alpha1.RolloutPhaseHealthy + newStatus.Message = "rollout is healthy" case rolloutv1alpha1.RolloutPhaseHealthy: // from healthy to progressing if workload.InRolloutProgressing { @@ -85,17 +105,8 @@ func (r *RolloutReconciler) updateRolloutStatus(rollout *rolloutv1alpha1.Rollout newStatus.Phase = rolloutv1alpha1.RolloutPhaseHealthy } } - err = r.updateRolloutStatusInternal(rollout, newStatus) - if err != nil { - klog.Errorf("update rollout(%s/%s) status failed: %s", rollout.Namespace, rollout.Name, err.Error()) - return err - } - err = r.calculateRolloutHash(rollout) - if err != nil { - return err - } - rollout.Status = newStatus - return nil + done = true + return } func (r *RolloutReconciler) updateRolloutStatusInternal(rollout *rolloutv1alpha1.Rollout, newStatus rolloutv1alpha1.RolloutStatus) error { @@ -124,9 +135,8 @@ func (r *RolloutReconciler) updateRolloutStatusInternal(rollout *rolloutv1alpha1 // ResetStatus resets the status of the rollout to start from beginning func resetStatus(status *rolloutv1alpha1.RolloutStatus) { - status.CanaryRevision = "" status.StableRevision = "" - util.RemoveRolloutCondition(status, rolloutv1alpha1.RolloutConditionProgressing) + //util.RemoveRolloutCondition(status, rolloutv1alpha1.RolloutConditionProgressing) status.Phase = rolloutv1alpha1.RolloutPhaseInitial status.Message = "workload not found" } diff --git a/pkg/controller/rollout/trafficrouting.go b/pkg/controller/rollout/trafficrouting.go index 809156e6..e2ceaa02 100644 --- a/pkg/controller/rollout/trafficrouting.go +++ b/pkg/controller/rollout/trafficrouting.go @@ -36,12 +36,20 @@ import ( ) func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) { - if r.rollout.Spec.Strategy.Canary.TrafficRouting == nil { + if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { return true, nil } + + if r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds <= 0 { + r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds = defaultGracePeriodSeconds + } canaryStatus := r.newStatus.CanaryStatus + if r.newStatus.StableRevision == "" || canaryStatus.PodTemplateHash == "" { + klog.Warningf("rollout(%s/%s) stableRevision or podTemplateHash can't be empty, and wait a moment", r.rollout.Namespace, r.rollout.Name) + return false, nil + } //fetch stable service - sName := r.rollout.Spec.Strategy.Canary.TrafficRouting[0].Service + sName := r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service r.stableService = &corev1.Service{} err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: sName}, r.stableService) if err != nil { @@ -71,8 +79,8 @@ func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) { // update service selector // update service selector specific revision pods - if r.canaryService.Spec.Selector[r.podRevisionLabelKey()] != canaryStatus.CanaryRevision { - body := fmt.Sprintf(`{"spec":{"selector":{"%s":"%s"}}}`, r.podRevisionLabelKey(), canaryStatus.CanaryRevision) + if r.canaryService.Spec.Selector[r.podRevisionLabelKey()] != canaryStatus.PodTemplateHash { + body := fmt.Sprintf(`{"spec":{"selector":{"%s":"%s"}}}`, r.podRevisionLabelKey(), canaryStatus.PodTemplateHash) if err = r.Patch(context.TODO(), r.canaryService, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil { klog.Errorf("rollout(%s/%s) patch canary service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.canaryService.Name, err.Error()) return false, err @@ -80,7 +88,7 @@ func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) { // update canary service time, and wait 3 seconds, just to be safe canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()} klog.Infof("add rollout(%s/%s) canary service(%s) selector(%s=%s) success", - r.rollout.Namespace, r.rollout.Name, r.canaryService.Name, r.podRevisionLabelKey(), canaryStatus.CanaryRevision) + r.rollout.Namespace, r.rollout.Name, r.canaryService.Name, r.podRevisionLabelKey(), canaryStatus.PodTemplateHash) } if r.stableService.Spec.Selector[r.podRevisionLabelKey()] != r.newStatus.StableRevision { body := fmt.Sprintf(`{"spec":{"selector":{"%s":"%s"}}}`, r.podRevisionLabelKey(), r.newStatus.StableRevision) @@ -96,7 +104,7 @@ func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) { } // After restore stable service configuration, give the ingress provider 3 seconds to take effect - if verifyTime := canaryStatus.LastUpdateTime.Add(time.Second * time.Duration(r.rollout.Spec.Strategy.Canary.TrafficRouting[0].GracePeriodSeconds)); verifyTime.After(time.Now()) { + if verifyTime := canaryStatus.LastUpdateTime.Add(time.Second * time.Duration(r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds)); verifyTime.After(time.Now()) { klog.Infof("update rollout(%s/%s) stable service(%s) done, and wait 3 seconds", r.rollout.Namespace, r.rollout.Name, r.stableService.Name) return false, nil } @@ -121,16 +129,20 @@ func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) { r.recorder.Eventf(r.rollout, corev1.EventTypeNormal, "Progressing", fmt.Sprintf("traffic route weight(%d) done", desiredWeight)) return false, trController.SetRoutes(desiredWeight) } - klog.Infof("rollout(%s/%s) do step(%d) trafficRouting(%d%) success", r.rollout.Namespace, r.rollout.Name, r.newStatus.CanaryStatus.CurrentStepIndex, desiredWeight) + klog.Infof("rollout(%s/%s) do step(%d) trafficRouting(%d) success", r.rollout.Namespace, r.rollout.Name, r.newStatus.CanaryStatus.CurrentStepIndex, desiredWeight) return true, nil } func (r *rolloutContext) restoreStableService() (bool, error) { - if r.rollout.Spec.Strategy.Canary.TrafficRouting == nil { + if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { return true, nil } + + if r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds <= 0 { + r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds = defaultGracePeriodSeconds + } //fetch stable service - sName := r.rollout.Spec.Strategy.Canary.TrafficRouting[0].Service + sName := r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service r.stableService = &corev1.Service{} err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: sName}, r.stableService) if err != nil { @@ -157,7 +169,7 @@ func (r *rolloutContext) restoreStableService() (bool, error) { } // After restore stable service configuration, give the ingress provider 3 seconds to take effect if r.newStatus.CanaryStatus.LastUpdateTime != nil { - if verifyTime := r.newStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(r.rollout.Spec.Strategy.Canary.TrafficRouting[0].GracePeriodSeconds)); verifyTime.After(time.Now()) { + if verifyTime := r.newStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds)); verifyTime.After(time.Now()) { klog.Infof("restore rollout(%s/%s) stable service(%s) done, and wait a moment", r.rollout.Namespace, r.rollout.Name, r.stableService.Name) return false, nil } @@ -167,9 +179,13 @@ func (r *rolloutContext) restoreStableService() (bool, error) { } func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) { - if r.rollout.Spec.Strategy.Canary.TrafficRouting == nil { + if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { return true, nil } + + if r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds <= 0 { + r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds = defaultGracePeriodSeconds + } if r.newStatus.CanaryStatus == nil { r.newStatus.CanaryStatus = &rolloutv1alpha1.CanaryStatus{} } @@ -194,7 +210,7 @@ func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) { // After do TrafficRouting configuration, give the ingress provider 3 seconds to take effect if r.newStatus.CanaryStatus.LastUpdateTime != nil { - if verifyTime := r.newStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(r.rollout.Spec.Strategy.Canary.TrafficRouting[0].GracePeriodSeconds)); verifyTime.After(time.Now()) { + if verifyTime := r.newStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds)); verifyTime.After(time.Now()) { klog.Infof("rollout(%s/%s) doFinalisingTrafficRouting done, and wait a moment", r.rollout.Namespace, r.rollout.Name) return false, nil } @@ -225,7 +241,7 @@ func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) { func (r *rolloutContext) newTrafficRoutingController(roCtx *rolloutContext) (trafficrouting.Controller, error) { canary := roCtx.rollout.Spec.Strategy.Canary - switch canary.TrafficRouting[0].Type { + switch canary.TrafficRoutings[0].Type { case "nginx": gvk := schema.GroupVersionKind{Group: rolloutv1alpha1.GroupVersion.Group, Version: rolloutv1alpha1.GroupVersion.Version, Kind: "Rollout"} return nginx.NewNginxTrafficRouting(r.Client, r.newStatus, nginx.Config{ @@ -233,12 +249,12 @@ func (r *rolloutContext) newTrafficRoutingController(roCtx *rolloutContext) (tra RolloutNs: r.rollout.Namespace, CanaryService: r.canaryService, StableService: r.stableService, - TrafficConf: r.rollout.Spec.Strategy.Canary.TrafficRouting[0].Ingress, + TrafficConf: r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].Ingress, OwnerRef: *metav1.NewControllerRef(r.rollout, gvk), }) } - return nil, fmt.Errorf("TrafficRouting(%s) not support", canary.TrafficRouting[0].Type) + return nil, fmt.Errorf("TrafficRouting(%s) not support", canary.TrafficRoutings[0].Type) } func (r *rolloutContext) createCanaryService() error { @@ -266,7 +282,7 @@ func (r *rolloutContext) createCanaryService() error { r.canaryService.Spec.IPFamilyPolicy = nil r.canaryService.Spec.IPFamilies = nil r.canaryService.Spec.LoadBalancerIP = "" - r.canaryService.Spec.Selector[r.podRevisionLabelKey()] = r.newStatus.CanaryStatus.CanaryRevision + r.canaryService.Spec.Selector[r.podRevisionLabelKey()] = r.newStatus.CanaryStatus.PodTemplateHash err := r.Create(context.TODO(), r.canaryService) if err != nil && !errors.IsAlreadyExists(err) { klog.Errorf("create rollout(%s/%s) canary service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.canaryService.Name, err.Error()) diff --git a/pkg/util/controller_finder.go b/pkg/util/controller_finder.go index e8117779..a95cec44 100644 --- a/pkg/util/controller_finder.go +++ b/pkg/util/controller_finder.go @@ -43,17 +43,23 @@ type Workload struct { StableRevision string // canary revision CanaryRevision string + // pod template hash is used as service selector hash + PodTemplateHash string // canary replicas CanaryReplicas int32 // canary ready replicas CanaryReadyReplicas int32 - // spec.pod.template hash - CurrentPodTemplateHash string + // Is it in rollback phase + IsInRollback bool // indicate whether the workload can enter the rollout process // 1. workload.Spec.Paused = true // 2. the Deployment is not in a stable version (only one version of RS) InRolloutProgressing bool + + // whether the status consistent with the spec + // workload.generation == status.observedGeneration + IsStatusConsistent bool } // ControllerFinderFunc is a function type that maps a pod to a list of @@ -112,15 +118,19 @@ func (r *ControllerFinder) getKruiseCloneSet(namespace string, ref *rolloutv1alp } return nil, err } + if cloneSet.Generation != cloneSet.Status.ObservedGeneration { + return &Workload{IsStatusConsistent: false}, nil + } workload := &Workload{ - StableRevision: cloneSet.Status.CurrentRevision[strings.LastIndex(cloneSet.Status.CurrentRevision, "-")+1:], - CanaryRevision: cloneSet.Status.UpdateRevision[strings.LastIndex(cloneSet.Status.UpdateRevision, "-")+1:], - CanaryReplicas: cloneSet.Status.UpdatedReplicas, - CanaryReadyReplicas: cloneSet.Status.UpdatedReadyReplicas, - ObjectMeta: cloneSet.ObjectMeta, - TypeMeta: cloneSet.TypeMeta, - Replicas: *cloneSet.Spec.Replicas, - CurrentPodTemplateHash: cloneSet.Status.UpdateRevision, + StableRevision: cloneSet.Status.CurrentRevision[strings.LastIndex(cloneSet.Status.CurrentRevision, "-")+1:], + CanaryRevision: cloneSet.Status.UpdateRevision[strings.LastIndex(cloneSet.Status.UpdateRevision, "-")+1:], + CanaryReplicas: cloneSet.Status.UpdatedReplicas, + CanaryReadyReplicas: cloneSet.Status.UpdatedReadyReplicas, + ObjectMeta: cloneSet.ObjectMeta, + TypeMeta: cloneSet.TypeMeta, + Replicas: *cloneSet.Spec.Replicas, + PodTemplateHash: cloneSet.Status.UpdateRevision[strings.LastIndex(cloneSet.Status.UpdateRevision, "-")+1:], + IsStatusConsistent: true, } // not in rollout progressing if _, ok = workload.Annotations[InRolloutProgressingAnnotation]; !ok { @@ -128,6 +138,10 @@ func (r *ControllerFinder) getKruiseCloneSet(namespace string, ref *rolloutv1alp } // in rollout progressing workload.InRolloutProgressing = true + // Is it in rollback phase + if cloneSet.Status.CurrentRevision == cloneSet.Status.UpdateRevision && cloneSet.Status.UpdatedReplicas != cloneSet.Status.Replicas { + workload.IsInRollback = true + } return workload, nil } @@ -147,21 +161,23 @@ func (r *ControllerFinder) getDeployment(namespace string, ref *rolloutv1alpha1. } return nil, err } - workload := &Workload{ - ObjectMeta: stable.ObjectMeta, - TypeMeta: stable.TypeMeta, - Replicas: *stable.Spec.Replicas, + if stable.Generation != stable.Status.ObservedGeneration { + return &Workload{IsStatusConsistent: false}, nil } // stable replicaSet - stableRs, err := r.GetDeploymentStableRs(stable) + stableRs, err := r.getDeploymentStableRs(stable) if err != nil || stableRs == nil { - return workload, err + return &Workload{IsStatusConsistent: false}, err + } + + workload := &Workload{ + ObjectMeta: stable.ObjectMeta, + TypeMeta: stable.TypeMeta, + Replicas: *stable.Spec.Replicas, + IsStatusConsistent: true, + StableRevision: stableRs.Labels[apps.DefaultDeploymentUniqueLabelKey], + CanaryRevision: ComputeHash(&stable.Spec.Template, nil), } - // stable revision - workload.StableRevision = stableRs.Labels[RsPodRevisionLabelKey] - // canary revision - workload.CanaryRevision = ComputeHash(&stable.Spec.Template, nil) - workload.CurrentPodTemplateHash = workload.CanaryRevision // not in rollout progressing if _, ok = workload.Annotations[InRolloutProgressingAnnotation]; !ok { return workload, nil @@ -171,24 +187,24 @@ func (r *ControllerFinder) getDeployment(namespace string, ref *rolloutv1alpha1. workload.InRolloutProgressing = true // workload is continuous release, indicates rollback(v1 -> v2 -> v1) // delete auto-generated labels - delete(stableRs.Spec.Template.Labels, RsPodRevisionLabelKey) + delete(stableRs.Spec.Template.Labels, apps.DefaultDeploymentUniqueLabelKey) if EqualIgnoreHash(&stableRs.Spec.Template, &stable.Spec.Template) { - workload.CanaryRevision = workload.StableRevision + workload.IsInRollback = true return workload, nil } - // canary workload status + // canary deployment canary, err := r.getLatestCanaryDeployment(stable) - if err != nil { - return nil, err - } else if canary != nil { - workload.CanaryReplicas = canary.Status.Replicas - workload.CanaryReadyReplicas = canary.Status.ReadyReplicas - canaryRs, err := r.GetDeploymentStableRs(canary) - if err != nil || canaryRs == nil { - return workload, err - } + if err != nil || canary == nil { + return workload, err + } + workload.CanaryReplicas = canary.Status.Replicas + workload.CanaryReadyReplicas = canary.Status.ReadyReplicas + canaryRs, err := r.getDeploymentStableRs(canary) + if err != nil || canaryRs == nil { + return workload, err } + workload.PodTemplateHash = canaryRs.Labels[apps.DefaultDeploymentUniqueLabelKey] return workload, err } @@ -204,10 +220,16 @@ func (r *ControllerFinder) getLatestCanaryDeployment(stable *apps.Deployment) (* sort.Slice(canaryList.Items, func(i, j int) bool { return canaryList.Items[j].CreationTimestamp.Before(&canaryList.Items[i].CreationTimestamp) }) - return &canaryList.Items[0], nil + for i := range canaryList.Items { + obj := &canaryList.Items[i] + if obj.DeletionTimestamp.IsZero() { + return obj, nil + } + } + return nil, nil } -func (r *ControllerFinder) getReplicaSetsForDeployment(obj *apps.Deployment) ([]apps.ReplicaSet, error) { +func (r *ControllerFinder) GetReplicaSetsForDeployment(obj *apps.Deployment) ([]apps.ReplicaSet, error) { // List ReplicaSets owned by this Deployment rsList := &apps.ReplicaSetList{} selector, err := metav1.LabelSelectorAsSelector(obj.Spec.Selector) @@ -219,6 +241,7 @@ func (r *ControllerFinder) getReplicaSetsForDeployment(obj *apps.Deployment) ([] if err != nil { return nil, err } + rss := make([]apps.ReplicaSet, 0) for i := range rsList.Items { rs := rsList.Items[i] @@ -234,14 +257,18 @@ func (r *ControllerFinder) getReplicaSetsForDeployment(obj *apps.Deployment) ([] return rss, nil } -func (r *ControllerFinder) GetDeploymentStableRs(obj *apps.Deployment) (*apps.ReplicaSet, error) { - rss, err := r.getReplicaSetsForDeployment(obj) +func (r *ControllerFinder) getDeploymentStableRs(obj *apps.Deployment) (*apps.ReplicaSet, error) { + rss, err := r.GetReplicaSetsForDeployment(obj) if err != nil { return nil, err } - if len(rss) != 1 { + if len(rss) == 0 { return nil, nil } + // get oldest rs + sort.Slice(rss, func(i, j int) bool { + return rss[i].CreationTimestamp.Before(&rss[j].CreationTimestamp) + }) return &rss[0], nil } diff --git a/pkg/util/workloads_utils.go b/pkg/util/workloads_utils.go index 9b15b4c4..3981b8dd 100644 --- a/pkg/util/workloads_utils.go +++ b/pkg/util/workloads_utils.go @@ -18,7 +18,9 @@ package util import ( "context" + "crypto/sha256" "encoding/binary" + "encoding/hex" "encoding/json" "fmt" "hash" @@ -41,8 +43,6 @@ import ( ) const ( - // workload pod revision label - RsPodRevisionLabelKey = "pod-template-hash" CanaryDeploymentLabel = "rollouts.kruise.io/canary-deployment" BatchReleaseControlAnnotation = "batchrelease.rollouts.kruise.io/control-info" StashCloneSetPartition = "batchrelease.rollouts.kruise.io/stash-partition" @@ -137,6 +137,12 @@ func EqualIgnoreHash(template1, template2 *v1.PodTemplateSpec) bool { return apiequality.Semantic.DeepEqual(t1Copy, t2Copy) } +func HashReleasePlanBatches(releasePlan *v1alpha1.ReleasePlan) string { + by, _ := json.Marshal(releasePlan.Batches) + md5Hash := sha256.Sum256(by) + return hex.EncodeToString(md5Hash[:]) +} + type FinalizerOpType string const ( @@ -148,7 +154,7 @@ func UpdateFinalizer(c client.Client, object client.Object, op FinalizerOpType, switch op { case AddFinalizerOpType, RemoveFinalizerOpType: default: - panic(fmt.Sprintf("UpdateFinalizer Func 'op' parameter must be 'Add' or 'Remove'")) + panic("UpdateFinalizer Func 'op' parameter must be 'Add' or 'Remove'") } key := client.ObjectKeyFromObject(object) diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler.go b/pkg/webhook/rollout/validating/rollout_create_update_handler.go index d4eab9e3..ebc57602 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler.go @@ -92,14 +92,9 @@ func (h *RolloutCreateUpdateHandler) validateRolloutUpdate(oldObj, newObj *appsv if !reflect.DeepEqual(oldObj.Spec.ObjectRef, newObj.Spec.ObjectRef) { return field.ErrorList{field.Forbidden(field.NewPath("Spec.ObjectRef"), "Rollout 'ObjectRef' field is immutable")} } - if oldObj.Spec.Strategy.Type != newObj.Spec.Strategy.Type { - return field.ErrorList{field.Forbidden(field.NewPath("Spec.Strategy"), "Rollout 'Strategy.type' field is immutable")} - } // canary strategy - if oldObj.Spec.Strategy.Type != appsv1alpha1.RolloutStrategyBlueGreen { - if !reflect.DeepEqual(oldObj.Spec.Strategy.Canary.TrafficRouting, newObj.Spec.Strategy.Canary.TrafficRouting) { - return field.ErrorList{field.Forbidden(field.NewPath("Spec.Strategy.Canary.TrafficRouting"), "Rollout 'Strategy.Canary.TrafficRouting' field is immutable")} - } + if !reflect.DeepEqual(oldObj.Spec.Strategy.Canary.TrafficRoutings, newObj.Spec.Strategy.Canary.TrafficRoutings) { + return field.ErrorList{field.Forbidden(field.NewPath("Spec.Strategy.Canary.TrafficRoutings"), "Rollout 'Strategy.Canary.TrafficRoutings' field is immutable")} } } @@ -147,24 +142,14 @@ func validateRolloutSpec(rollout *appsv1alpha1.Rollout, fldPath *field.Path) fie } func validateRolloutSpecObjectRef(objectRef *appsv1alpha1.ObjectRef, fldPath *field.Path) field.ErrorList { - switch objectRef.Type { - case "", appsv1alpha1.WorkloadRefType: - if objectRef.WorkloadRef == nil || (objectRef.WorkloadRef.Kind != "Deployment" && objectRef.WorkloadRef.Kind != "CloneSet") { - return field.ErrorList{field.Invalid(fldPath.Child("WorkloadRef"), objectRef.WorkloadRef, "WorkloadRef only support 'Deployments', 'CloneSet'")} - } - default: - return field.ErrorList{field.Invalid(fldPath.Child("Type"), objectRef.Type, "ObjectRef only support 'workloadRef' type")} + if objectRef.WorkloadRef == nil || (objectRef.WorkloadRef.Kind != "Deployment" && objectRef.WorkloadRef.Kind != "CloneSet") { + return field.ErrorList{field.Invalid(fldPath.Child("WorkloadRef"), objectRef.WorkloadRef, "WorkloadRef only support 'Deployments', 'CloneSet'")} } return nil } func validateRolloutSpecStrategy(strategy *appsv1alpha1.RolloutStrategy, fldPath *field.Path) field.ErrorList { - switch strategy.Type { - case "", appsv1alpha1.RolloutStrategyCanary: - return validateRolloutSpecCanaryStrategy(strategy.Canary, fldPath.Child("Canary")) - default: - return field.ErrorList{field.Invalid(fldPath.Child("Type"), strategy.Type, "Strategy type only support 'canary'")} - } + return validateRolloutSpecCanaryStrategy(strategy.Canary, fldPath.Child("Canary")) } func validateRolloutSpecCanaryStrategy(canary *appsv1alpha1.CanaryStrategy, fldPath *field.Path) field.ErrorList { @@ -173,7 +158,10 @@ func validateRolloutSpecCanaryStrategy(canary *appsv1alpha1.CanaryStrategy, fldP } errList := validateRolloutSpecCanarySteps(canary.Steps, fldPath.Child("Steps")) - for _, traffic := range canary.TrafficRouting { + if len(canary.TrafficRoutings) > 1 { + errList = append(errList, field.Invalid(fldPath, canary.TrafficRoutings, "Rollout currently only support single TrafficRouting.")) + } + for _, traffic := range canary.TrafficRoutings { errList = append(errList, validateRolloutSpecCanaryTraffic(traffic, fldPath.Child("TrafficRouting"))...) } return errList @@ -181,7 +169,7 @@ func validateRolloutSpecCanaryStrategy(canary *appsv1alpha1.CanaryStrategy, fldP func validateRolloutSpecCanaryTraffic(traffic *appsv1alpha1.TrafficRouting, fldPath *field.Path) field.ErrorList { if traffic == nil { - return field.ErrorList{field.Invalid(fldPath, nil, "Canary.TrafficRouting cannot be empty")} + return field.ErrorList{field.Invalid(fldPath, nil, "Canary.TrafficRoutings cannot be empty")} } errList := field.ErrorList{} diff --git a/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go b/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go index add5ff2e..0c8b97bc 100644 --- a/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go +++ b/pkg/webhook/rollout/validating/rollout_create_update_handler_test.go @@ -29,7 +29,6 @@ var ( }, Spec: appsv1alpha1.RolloutSpec{ ObjectRef: appsv1alpha1.ObjectRef{ - Type: appsv1alpha1.WorkloadRefType, WorkloadRef: &appsv1alpha1.WorkloadRef{ APIVersion: apps.SchemeGroupVersion.String(), Kind: "Deployment", @@ -37,7 +36,6 @@ var ( }, }, Strategy: appsv1alpha1.RolloutStrategy{ - Type: appsv1alpha1.RolloutStrategyCanary, Canary: &appsv1alpha1.CanaryStrategy{ Steps: []appsv1alpha1.CanaryStep{ { @@ -58,7 +56,7 @@ var ( Weight: 100, }, }, - TrafficRouting: []*appsv1alpha1.TrafficRouting{ + TrafficRoutings: []*appsv1alpha1.TrafficRouting{ { Type: "nginx", Service: "service-demo", @@ -126,7 +124,7 @@ func TestRolloutValidateCreate(t *testing.T) { Succeed: false, GetObject: func() []client.Object { object := rollout.DeepCopy() - object.Spec.Strategy.Canary.TrafficRouting[0].Service = "" + object.Spec.Strategy.Canary.TrafficRoutings[0].Service = "" return []client.Object{object} }, }, @@ -135,7 +133,7 @@ func TestRolloutValidateCreate(t *testing.T) { Succeed: false, GetObject: func() []client.Object { object := rollout.DeepCopy() - object.Spec.Strategy.Canary.TrafficRouting[0].Ingress.Name = "" + object.Spec.Strategy.Canary.TrafficRoutings[0].Ingress.Name = "" return []client.Object{object} }, }, @@ -234,30 +232,12 @@ func TestRolloutValidateCreate(t *testing.T) { // return []client.Object{object} // }, //}, - { - Name: "Wrong objectRef type", - Succeed: false, - GetObject: func() []client.Object { - object := rollout.DeepCopy() - object.Spec.ObjectRef.Type = "Whatever" - return []client.Object{object} - }, - }, - { - Name: "Wrong strategy type", - Succeed: false, - GetObject: func() []client.Object { - object := rollout.DeepCopy() - object.Spec.Strategy.Type = "Whatever" - return []client.Object{object} - }, - }, { Name: "Wrong Traffic type", Succeed: false, GetObject: func() []client.Object { object := rollout.DeepCopy() - object.Spec.Strategy.Canary.TrafficRouting[0].Type = "Whatever" + object.Spec.Strategy.Canary.TrafficRoutings[0].Type = "Whatever" return []client.Object{object} }, }, @@ -390,7 +370,7 @@ func TestRolloutValidateUpdate(t *testing.T) { GetNewObject: func() client.Object { object := rollout.DeepCopy() object.Status.Phase = appsv1alpha1.RolloutPhaseTerminating - object.Spec.Strategy.Canary.TrafficRouting[0].Type = "alb" + object.Spec.Strategy.Canary.TrafficRoutings[0].Type = "alb" return object }, }, diff --git a/pkg/webhook/workload/mutating/workload_update_handler.go b/pkg/webhook/workload/mutating/workload_update_handler.go index fb289f9f..8b394ea5 100644 --- a/pkg/webhook/workload/mutating/workload_update_handler.go +++ b/pkg/webhook/workload/mutating/workload_update_handler.go @@ -148,10 +148,11 @@ func (h *WorkloadHandler) handlerDeployment(newObj, oldObj *apps.Deployment) (ch return } // 4. the deployment must be in a stable version (only one version of rs) - stableRs, err := h.Finder.GetDeploymentStableRs(newObj) + rss, err := h.Finder.GetReplicaSetsForDeployment(newObj) if err != nil { return - } else if stableRs == nil { + } else if len(rss) != 1 { + klog.Warningf("deployment(%s/%s) contains len(%d) replicaSet, can't in rollout progressing", newObj.Namespace, newObj.Name, len(rss)) return } // 5. have matched rollout crd @@ -184,8 +185,7 @@ func (h *WorkloadHandler) fetchMatchedRollout(obj client.Object) (*appsv1alpha1. } for i := range rolloutList.Items { rollout := &rolloutList.Items[i] - if !rollout.DeletionTimestamp.IsZero() || rollout.Spec.ObjectRef.Type == appsv1alpha1.RevisionRefType || - rollout.Spec.ObjectRef.WorkloadRef == nil { + if !rollout.DeletionTimestamp.IsZero() || rollout.Spec.ObjectRef.WorkloadRef == nil { continue } ref := rollout.Spec.ObjectRef.WorkloadRef diff --git a/pkg/webhook/workload/mutating/workload_update_handler_test.go b/pkg/webhook/workload/mutating/workload_update_handler_test.go index 885b3b3d..5d5f632b 100644 --- a/pkg/webhook/workload/mutating/workload_update_handler_test.go +++ b/pkg/webhook/workload/mutating/workload_update_handler_test.go @@ -159,7 +159,6 @@ var ( }, Spec: appsv1alpha1.RolloutSpec{ ObjectRef: appsv1alpha1.ObjectRef{ - Type: appsv1alpha1.WorkloadRefType, WorkloadRef: &appsv1alpha1.WorkloadRef{ APIVersion: "apps/v1", Kind: "Deployment", diff --git a/scripts/deploy_kind.sh b/scripts/deploy_kind.sh new file mode 100755 index 00000000..741bd623 --- /dev/null +++ b/scripts/deploy_kind.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +if [ -z "$IMG" ]; then + echo "no found IMG env" + exit 1 +fi + +set -e + +make kustomize +KUSTOMIZE=$(pwd)/bin/kustomize +(cd config/manager && "${KUSTOMIZE}" edit set image controller="${IMG}") +"${KUSTOMIZE}" build config/default | sed -e 's/imagePullPolicy: Always/imagePullPolicy: IfNotPresent/g' > /tmp/rollout-kustomization.yaml +echo -e "resources:\n- manager.yaml" > config/manager/kustomization.yaml +kubectl apply -f /tmp/rollout-kustomization.yaml diff --git a/temp b/temp deleted file mode 100644 index e69de29b..00000000 diff --git a/test/e2e/batchrelease_test.go b/test/e2e/batchrelease_test.go index a7e92595..712c0cf1 100644 --- a/test/e2e/batchrelease_test.go +++ b/test/e2e/batchrelease_test.go @@ -678,7 +678,7 @@ var _ = SIGDescribe("BatchRelease", func() { }, 15*time.Minute, 5*time.Second).Should(BeTrue()) }) - It("Rollback V1->V2->V1: Percentage, 100%, Succeeded", func() { + /*It("Rollback V1->V2->V1: Percentage, 100%, Succeeded", func() { release := &rolloutsv1alpha1.BatchRelease{} Expect(ReadYamlToObject("./test_data/batchrelease/cloneset_percentage_100.yaml", release)).ToNot(HaveOccurred()) CreateObject(release) @@ -695,7 +695,8 @@ var _ = SIGDescribe("BatchRelease", func() { stableRevision := GetUpdateRevision(cloneset) cloneset.Spec.UpdateStrategy.Paused = true - cloneset.Spec.Replicas = pointer.Int32Ptr(10) + // todo + //cloneset.Spec.Replicas = pointer.Int32Ptr(10) cloneset.Spec.Template.Spec.Containers[0].Image = images.GetE2EImage(images.FailedImage) UpdateCloneSet(cloneset) @@ -708,6 +709,8 @@ var _ = SIGDescribe("BatchRelease", func() { for i := 0; i < 30; i++ { fetchedRelease := &rolloutsv1alpha1.BatchRelease{} Expect(GetObject(release.Namespace, release.Name, fetchedRelease)).NotTo(HaveOccurred()) + Expect(fetchedRelease.Status.CanaryStatus.UpdatedReplicas).Should(Equal(int32(1))) + Expect(fetchedRelease.Status.CanaryStatus.UpdatedReadyReplicas).Should(Equal(int32(0))) Expect(fetchedRelease.Status.CanaryStatus.CurrentBatch).Should(Equal(int32(0))) time.Sleep(time.Second) } @@ -723,7 +726,7 @@ var _ = SIGDescribe("BatchRelease", func() { Expect(GetObject(release.Namespace, release.Name, clone)).NotTo(HaveOccurred()) return clone.Status.Phase }, 15*time.Minute, 5*time.Second).Should(Equal(rolloutsv1alpha1.RolloutPhaseCancelled)) - }) + })*/ }) KruiseDescribe("Deployment BatchRelease Checker", func() { @@ -1286,7 +1289,7 @@ var _ = SIGDescribe("BatchRelease", func() { stableRevisionV1 := workloads.ComputeHash(&deployment.Spec.Template, deployment.Status.CollisionCount) deployment.Spec.Paused = true - deployment.Spec.Replicas = pointer.Int32Ptr(10) + //deployment.Spec.Replicas = pointer.Int32Ptr(10) deployment.Spec.Template.Spec.Containers[0].Image = images.GetE2EImage(images.FailedImage) UpdateDeployment(deployment) @@ -1299,6 +1302,8 @@ var _ = SIGDescribe("BatchRelease", func() { for i := 0; i < 30; i++ { fetchedRelease := &rolloutsv1alpha1.BatchRelease{} Expect(GetObject(release.Namespace, release.Name, fetchedRelease)).NotTo(HaveOccurred()) + Expect(fetchedRelease.Status.CanaryStatus.UpdatedReplicas).Should(Equal(int32(1))) + Expect(fetchedRelease.Status.CanaryStatus.UpdatedReadyReplicas).Should(Equal(int32(0))) Expect(fetchedRelease.Status.CanaryStatus.CurrentBatch).Should(Equal(int32(0))) time.Sleep(time.Second) } diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index ef3904e1..1f778d91 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "sort" + "strings" "time" . "github.com/onsi/ginkgo" @@ -47,6 +48,24 @@ const ( var _ = SIGDescribe("Rollout", func() { var namespace string + DumpAllResources := func() { + rollout := &rolloutsv1alpha1.RolloutList{} + k8sClient.List(context.TODO(), rollout, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(rollout)) + batch := &rolloutsv1alpha1.BatchReleaseList{} + k8sClient.List(context.TODO(), batch, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(batch)) + deploy := &apps.DeploymentList{} + k8sClient.List(context.TODO(), deploy, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(deploy)) + rs := &apps.ReplicaSetList{} + k8sClient.List(context.TODO(), rs, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(rs)) + cloneSet := &appsv1alpha1.CloneSetList{} + k8sClient.List(context.TODO(), cloneSet, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(cloneSet)) + } + CreateObject := func(object client.Object, options ...client.CreateOption) { object.SetNamespace(namespace) Expect(k8sClient.Create(context.TODO(), object)).NotTo(HaveOccurred()) @@ -103,15 +122,18 @@ var _ = SIGDescribe("Rollout", func() { } ResumeRolloutCanary := func(name string) { - Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + Eventually(func() bool { clone := &rolloutsv1alpha1.Rollout{} - err := GetObject(name, clone) - if err != nil { - return err + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + if clone.Status.CanaryStatus.CurrentStepState != rolloutsv1alpha1.CanaryStepStatePaused { + fmt.Println("resume rollout success, and CurrentStepState", util.DumpJSON(clone.Status)) + return true } - clone.Status.CanaryStatus.CurrentStepState = rolloutsv1alpha1.CanaryStepStateReady - return k8sClient.Status().Update(context.TODO(), clone) - })).NotTo(HaveOccurred()) + + body := fmt.Sprintf(`{"status":{"canaryStatus":{"currentStepState":"%s"}}}`, rolloutsv1alpha1.CanaryStepStateReady) + Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred()) + return false + }, 10*time.Second, time.Second).Should(BeTrue()) } WaitDeploymentAllPodsReady := func(deployment *apps.Deployment) { @@ -142,14 +164,19 @@ var _ = SIGDescribe("Rollout", func() { } WaitRolloutCanaryStepPaused := func(name string, stepIndex int32) { + start := time.Now() Eventually(func() bool { + if start.Add(time.Minute * 5).Before(time.Now()) { + DumpAllResources() + Expect(true).Should(BeFalse()) + } clone := &rolloutsv1alpha1.Rollout{} Expect(GetObject(name, clone)).NotTo(HaveOccurred()) if clone.Status.CanaryStatus == nil { return false } return clone.Status.CanaryStatus.CurrentStepIndex == stepIndex && clone.Status.CanaryStatus.CurrentStepState == rolloutsv1alpha1.CanaryStepStatePaused - }, 5*time.Minute, time.Second).Should(BeTrue()) + }, 10*time.Minute, time.Second).Should(BeTrue()) } WaitRolloutStatusPhase := func(name string, phase rolloutsv1alpha1.RolloutPhase) { @@ -157,7 +184,15 @@ var _ = SIGDescribe("Rollout", func() { clone := &rolloutsv1alpha1.Rollout{} Expect(GetObject(name, clone)).NotTo(HaveOccurred()) return clone.Status.Phase == phase - }, 5*time.Minute, time.Second).Should(BeTrue()) + }, 10*time.Minute, time.Second).Should(BeTrue()) + } + + WaitRolloutWorkloadGenration := func(name string, generation int64) { + Eventually(func() bool { + clone := &rolloutsv1alpha1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + return clone.Status.CanaryStatus.ObservedWorkloadGeneration == generation + }, time.Minute, time.Second).Should(BeTrue()) } WaitRolloutNotFound := func(name string) { @@ -296,6 +331,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout canary ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(2)") WaitRolloutCanaryStepPaused(rollout.Name, 2) // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) @@ -306,6 +342,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout canary ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(3)") WaitRolloutCanaryStepPaused(rollout.Name, 3) // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) @@ -316,6 +353,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout canary ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(4)") WaitRolloutCanaryStepPaused(rollout.Name, 4) // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) @@ -326,6 +364,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout canary ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(5)") WaitRolloutCanaryStepPaused(rollout.Name, 5) // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) @@ -345,7 +384,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService := &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // deployment @@ -365,10 +404,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage 20%,60% Succeeded", func() { + finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") rollout := &rolloutsv1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) @@ -400,12 +440,16 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) CreateObject(workload) WaitDeploymentAllPodsReady(workload) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).ShouldNot(Equal("")) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -425,32 +469,15 @@ var _ = SIGDescribe("Rollout", func() { // wait step 1 complete WaitRolloutCanaryStepPaused(rollout.Name, 1) - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) - Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryRevision).ShouldNot(Equal("")) - canaryRevision := rollout.Status.CanaryRevision - Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) - Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 2)) - Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 2)) // check stable, canary service & ingress - // stable service - Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal(stableRevision)) - //canary service - cService := &v1.Service{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal(canaryRevision)) - // canary ingress - cIngress := &netv1.Ingress{} - Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) - Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", rollout.Spec.Strategy.Canary.Steps[0].Weight))) // canary deployment cWorkload, err := GetCanaryDeployment(workload) Expect(err).NotTo(HaveOccurred()) + crss, err := finder.GetReplicaSetsForDeployment(cWorkload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(crss)).Should(BeNumerically("==", 1)) + canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 2)) Expect(cWorkload.Status.ReadyReplicas).Should(BeNumerically("==", 2)) for _, env := range cWorkload.Spec.Template.Spec.Containers[0].Env { @@ -458,10 +485,32 @@ var _ = SIGDescribe("Rollout", func() { Expect(env.Value).Should(Equal("version2")) } } + // stable service + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) + //canary service + cService := &v1.Service{} + Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // canary ingress + cIngress := &netv1.Ingress{} + Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) + Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", rollout.Spec.Strategy.Canary.Steps[0].Weight))) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 2)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision)) // resume rollout canary ResumeRolloutCanary(rollout.Name) - By("check rollout canary status success, resume rollout, and wait rollout canary complete") + By("resume rollout, and wait next step(2)") WaitRolloutCanaryStepPaused(rollout.Name, 2) // check stable, canary service & ingress @@ -492,7 +541,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService = &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // deployment @@ -506,6 +555,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(env.Value).Should(Equal("version2")) } } + rss, err = finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + Expect(rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) + // check progressing succeed Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) @@ -513,7 +567,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) // scale up replicas 5 -> 6 workload.Spec.Replicas = utilpointer.Int32(6) @@ -523,10 +577,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage 20%, and rollback(v1)", func() { + finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") rollout := &rolloutsv1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) @@ -546,18 +601,22 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) CreateObject(workload) WaitDeploymentAllPodsReady(workload) - apps, err := GetPodsOfDeployment(workload) + pods, err := GetPodsOfDeployment(workload) Expect(err).NotTo(HaveOccurred()) appNames := make(map[string]struct{}) - for _, app := range apps { + for _, app := range pods { appNames[app.Name] = struct{}{} } + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).ShouldNot(Equal("")) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -582,10 +641,10 @@ var _ = SIGDescribe("Rollout", func() { workload.Spec.Template.Spec.Containers[0].Env = newEnvs UpdateDeployment(workload) By("Rollback deployment env NODE_NAME from(version2) -> to(version1)") - time.Sleep(time.Second * 2) WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) klog.Infof("rollout(%s) completed, and check", namespace) + time.Sleep(time.Second * 10) // check service & ingress & deployment // ingress @@ -594,7 +653,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService := &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // deployment @@ -609,23 +668,25 @@ var _ = SIGDescribe("Rollout", func() { } } // deployment pods not changed - capps, err := GetPodsOfDeployment(workload) + cpods, err := GetPodsOfDeployment(workload) Expect(err).NotTo(HaveOccurred()) cappNames := make(map[string]struct{}) - for _, app := range capps { - cappNames[app.Name] = struct{}{} + for _, pod := range cpods { + cappNames[pod.Name] = struct{}{} } Expect(cappNames).Should(Equal(appNames)) // check progressing canceled + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonCanceled)) Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) Expect(string(cond.Status)).Should(Equal("False")) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage 20%,40% and continuous release v3", func() { + finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") rollout := &rolloutsv1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) @@ -645,12 +706,16 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) CreateObject(workload) WaitDeploymentAllPodsReady(workload) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).ShouldNot(Equal("")) - stableRevision := rollout.Status.StableRevision + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -669,12 +734,19 @@ var _ = SIGDescribe("Rollout", func() { By("check deployment status & paused success") // wait step 1 complete WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // canary deployment + cWorkload, err := GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + crss, err := finder.GetReplicaSetsForDeployment(cWorkload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(crss)).Should(BeNumerically("==", 1)) + canaryRevisionV1 := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryRevision).ShouldNot(Equal("")) - canaryRevisionV1 := rollout.Status.CanaryRevision + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevisionV1)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) // resume rollout canary @@ -688,34 +760,38 @@ var _ = SIGDescribe("Rollout", func() { UpdateDeployment(workload) By("Update deployment env NODE_NAME from(version2) -> to(version3)") time.Sleep(time.Second * 2) - // wait step 0 complete WaitRolloutCanaryStepPaused(rollout.Name, 1) + + cWorkload, err = GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + crss, err = finder.GetReplicaSetsForDeployment(cWorkload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(crss)).Should(BeNumerically("==", 1)) + canaryRevisionV2 := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryRevision).ShouldNot(Equal("")) - Expect(rollout.Status.CanaryRevision).ShouldNot(Equal(canaryRevisionV1)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevisionV2)) Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 1)) - canaryRevisionV2 := rollout.Status.CanaryRevision Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) // check stable, canary service & ingress // stable service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal(stableRevision)) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal(canaryRevisionV2)) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevisionV2)) // canary ingress cIngress := &netv1.Ingress{} Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", rollout.Spec.Strategy.Canary.Steps[0].Weight))) // canary deployment - cWorkload, err := GetCanaryDeployment(workload) + cWorkload, err = GetCanaryDeployment(workload) Expect(err).NotTo(HaveOccurred()) Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 1)) Expect(cWorkload.Status.ReadyReplicas).Should(BeNumerically("==", 1)) @@ -738,7 +814,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService = &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // deployment @@ -759,10 +835,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(rollout.Status.StableRevision).Should(Equal(canaryRevisionV2)) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage 20%,40% and scale up replicas from(5) -> to(10)", func() { + finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") rollout := &rolloutsv1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) @@ -820,21 +897,29 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Spec.Paused).Should(BeTrue()) By("check deployment status & paused success") - // wait step 2 complete WaitRolloutCanaryStepPaused(rollout.Name, 2) + + // canary deployment + cWorkload, err := GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + crss, err := finder.GetReplicaSetsForDeployment(cWorkload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(crss)).Should(BeNumerically("==", 1)) + canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) - canaryRevision := rollout.Status.CanaryStatus.CanaryRevision + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision)) // scale up replicas, 5 -> 10 workload.Spec.Replicas = utilpointer.Int32(10) UpdateDeployment(workload) time.Sleep(time.Second * 3) - cWorkload, _ := GetCanaryDeployment(workload) + cWorkload, _ = GetCanaryDeployment(workload) WaitDeploymentAllPodsReady(cWorkload) + time.Sleep(time.Second * 5) // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) @@ -843,7 +928,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 4)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 4)) Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision)) // resume rollout canary ResumeRolloutCanary(rollout.Name) @@ -858,7 +943,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService := &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // deployment @@ -879,10 +964,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage 20%,40%, and scale down replicas from(10) -> to(5)", func() { + finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") rollout := &rolloutsv1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) @@ -939,14 +1025,20 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Spec.Paused).Should(BeTrue()) By("check deployment status & paused success") - // wait step 3 complete WaitRolloutCanaryStepPaused(rollout.Name, 3) + + // canary deployment + cWorkload, err := GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + crss, err := finder.GetReplicaSetsForDeployment(cWorkload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(crss)).Should(BeNumerically("==", 1)) + canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 3)) - canaryRevision := rollout.Status.CanaryStatus.CanaryRevision // scale up replicas, 10 -> 5 workload.Spec.Replicas = utilpointer.Int32(5) @@ -961,7 +1053,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 6)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 6)) Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) - Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(canaryRevision)) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision)) // resume rollout canary ResumeRolloutCanary(rollout.Name) @@ -976,7 +1068,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService := &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // deployment @@ -997,10 +1089,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage 20%,40%, paused and resume", func() { + finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") rollout := &rolloutsv1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) @@ -1049,11 +1142,16 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) CreateObject(workload) WaitDeploymentAllPodsReady(workload) + // deployment + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).ShouldNot(Equal("")) + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -1098,7 +1196,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService := &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // deployment @@ -1118,10 +1216,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage 20%,40%, but delete rollout crd", func() { + finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") rollout := &rolloutsv1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) @@ -1160,11 +1259,16 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) CreateObject(workload) WaitDeploymentAllPodsReady(workload) + // deployment + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).ShouldNot(Equal("")) + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -1190,7 +1294,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService := &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // deployment @@ -1207,6 +1311,7 @@ var _ = SIGDescribe("Rollout", func() { }) It("V1->V2: Percentage 20% v2 failed image, and v3 succeed image", func() { + finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") rollout := &rolloutsv1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) @@ -1245,11 +1350,16 @@ var _ = SIGDescribe("Rollout", func() { Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred()) CreateObject(workload) WaitDeploymentAllPodsReady(workload) + // deployment + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).ShouldNot(Equal("")) + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) By("check rollout status & paused success") // v1 -> v2, start rollout action @@ -1290,7 +1400,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService := &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // deployment @@ -1310,10 +1420,11 @@ var _ = SIGDescribe("Rollout", func() { Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage, 20%,40%,60%,80%,100%, steps changed v1", func() { + finder := util.NewControllerFinder(k8sClient) By("Creating Rollout...") rollout := &rolloutsv1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) @@ -1333,9 +1444,10 @@ var _ = SIGDescribe("Rollout", func() { workload.Spec.Replicas = utilpointer.Int32(10) CreateObject(workload) WaitDeploymentAllPodsReady(workload) - // check rollout status - Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) - stableRevision := rollout.Status.StableRevision + rss, err := finder.GetReplicaSetsForDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(rss)).Should(BeNumerically("==", 1)) + stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // v1 -> v2, start rollout action newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) @@ -1364,31 +1476,40 @@ var _ = SIGDescribe("Rollout", func() { }, } rollout = UpdateRollout(rollout) - time.Sleep(time.Second * 3) + By("update rollout configuration, and wait rollout next step(2)") // wait step 1 complete WaitRolloutCanaryStepPaused(rollout.Name, 2) + batch := &rolloutsv1alpha1.BatchRelease{} + Expect(GetObject(rollout.Name, batch)).NotTo(HaveOccurred()) + + // canary deployment + cWorkload, err := GetCanaryDeployment(workload) + Expect(err).NotTo(HaveOccurred()) + crss, err := finder.GetReplicaSetsForDeployment(cWorkload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(crss)).Should(BeNumerically("==", 1)) + canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey] // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryRevision).ShouldNot(Equal("")) - canaryRevision := rollout.Status.CanaryRevision + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision)) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) // check stable, canary service & ingress // stable service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal(stableRevision)) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal(canaryRevision)) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) // canary ingress cIngress := &netv1.Ingress{} Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true")) Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", rollout.Spec.Strategy.Canary.Steps[1].Weight))) // canary deployment - cWorkload, err := GetCanaryDeployment(workload) + cWorkload, err = GetCanaryDeployment(workload) Expect(err).NotTo(HaveOccurred()) Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 3)) Expect(cWorkload.Status.ReadyReplicas).Should(BeNumerically("==", 3)) @@ -1406,7 +1527,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService = &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // deployment @@ -1426,13 +1547,12 @@ var _ = SIGDescribe("Rollout", func() { Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) }) }) KruiseDescribe("CloneSet rollout canary nginx", func() { - - It("V1->V2: Percentage, 20%,40%,60%,80%,100% Succeeded", func() { + It("V1->V2: Percentage, 20%,60% Succeeded", func() { By("Creating Rollout...") rollout := &rolloutsv1alpha1.Rollout{} Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) @@ -1472,7 +1592,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) stableRevision := rollout.Status.StableRevision By("check rollout status & paused success") @@ -1488,7 +1608,6 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 4)) Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) By("check cloneSet status & paused success") @@ -1496,18 +1615,19 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - canaryRevision := rollout.Status.CanaryRevision + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) // check stable, canary service & ingress // stable service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal(stableRevision)) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal(canaryRevision)) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision)) // canary ingress cIngress := &netv1.Ingress{} Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) @@ -1516,7 +1636,7 @@ var _ = SIGDescribe("Rollout", func() { // resume rollout canary ResumeRolloutCanary(rollout.Name) - By("check rollout canary status success, resume rollout, and wait rollout canary complete") + By("resume rollout, and wait next step(2)") WaitRolloutCanaryStepPaused(rollout.Name, 2) // check stable, canary service & ingress @@ -1528,7 +1648,6 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) - Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 2)) Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) // resume rollout @@ -1544,7 +1663,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService = &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // cloneset @@ -1568,7 +1687,7 @@ var _ = SIGDescribe("Rollout", func() { cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) // scale up replicas 5 -> 6 @@ -1579,7 +1698,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) }) It("V1->V2: Percentage, 20%, and rollback(v1)", func() { @@ -1608,11 +1727,12 @@ var _ = SIGDescribe("Rollout", func() { CreateObject(workload) WaitCloneSetAllPodsReady(workload) + // check rollout status // check rollout status Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) stableRevision := rollout.Status.StableRevision By("check rollout status & paused success") @@ -1622,13 +1742,13 @@ var _ = SIGDescribe("Rollout", func() { workload.Spec.Template.Spec.Containers[0].Env = newEnvs UpdateCloneSet(workload) By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") - time.Sleep(time.Second * 10) + // wait step 1 complete + time.Sleep(time.Second * 20) // check workload status & paused Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 0)) - Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 4)) Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) By("check cloneSet status & paused success") @@ -1636,11 +1756,16 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateUpgrade)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + time.Sleep(time.Second * 15) + // rollback -> v1 newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:latest" @@ -1666,7 +1791,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService := &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // cloneset @@ -1714,7 +1839,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) - Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision)) + Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) stableRevision := rollout.Status.StableRevision By("check rollout status & paused success") @@ -1730,7 +1855,6 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) - Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 4)) Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) By("check cloneSet status & paused success") @@ -1739,14 +1863,14 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - canaryRevisionV1 := rollout.Status.CanaryRevision + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevisionV1 := rollout.Status.CanaryStatus.PodTemplateHash Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) // resume rollout canary ResumeRolloutCanary(rollout.Name) - By("check rollout canary status success, resume rollout, and wait rollout canary complete") time.Sleep(time.Second * 15) // v1 -> v2 -> v3, continuous release @@ -1765,18 +1889,19 @@ var _ = SIGDescribe("Rollout", func() { Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1)) Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 1)) Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) - Expect(rollout.Status.CanaryRevision).ShouldNot(Equal(canaryRevisionV1)) - Expect(rollout.Status.CanaryRevision).Should(Equal(workload.Status.UpdateRevision)) - canaryRevisionV2 := rollout.Status.CanaryRevision + Expect(rollout.Status.CanaryStatus.CanaryRevision).ShouldNot(Equal(canaryRevisionV1)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevisionV2 := rollout.Status.CanaryStatus.PodTemplateHash Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) // check stable, canary service & ingress // stable service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal(stableRevision)) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision)) //canary service cService := &v1.Service{} Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred()) - Expect(cService.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal(canaryRevisionV2)) + Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevisionV2)) // canary ingress cIngress := &netv1.Ingress{} Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred()) @@ -1787,6 +1912,7 @@ var _ = SIGDescribe("Rollout", func() { ResumeRolloutCanary(rollout.Name) By("check rollout canary status success, resume rollout, and wait rollout canary complete") WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitCloneSetAllPodsReady(workload) By("rollout completed, and check") // check service & ingress & deployment @@ -1796,7 +1922,7 @@ var _ = SIGDescribe("Rollout", func() { Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred()) // service Expect(GetObject(service.Name, service)).NotTo(HaveOccurred()) - Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal("")) + Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal("")) cService = &v1.Service{} Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred()) // cloneset @@ -1820,9 +1946,111 @@ var _ = SIGDescribe("Rollout", func() { cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) - Expect(rollout.Status.CanaryStatus.ObservedWorkloadGeneration).Should(BeNumerically("==", workload.Generation)) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) Expect(rollout.Status.StableRevision).Should(Equal(canaryRevisionV2)) }) + + It("V1->V2: Percentage, 20%,40%,60%,80%,100%, no traffic, Succeeded", func() { + By("Creating Rollout...") + rollout := &rolloutsv1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + rollout.Spec.Strategy.Canary.TrafficRoutings = nil + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + workload.Spec.UpdateStrategy.Type = appsv1alpha1.InPlaceOnlyCloneSetUpdateStrategyType + workload.Spec.UpdateStrategy.MaxUnavailable = &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + } + workload.Spec.UpdateStrategy.MaxSurge = nil + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.StableRevision).Should(Equal(workload.Status.CurrentRevision[strings.LastIndex(workload.Status.CurrentRevision, "-")+1:])) + stableRevision := rollout.Status.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + //newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:1.10.2" + UpdateCloneSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + WaitRolloutCanaryStepPaused(rollout.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CanaryRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:])) + canaryRevision := rollout.Status.CanaryStatus.PodTemplateHash + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation])) + + // resume rollout + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitCloneSetAllPodsReady(workload) + By("rollout completed, and check") + + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 0)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + Expect(workload.Status.CurrentRevision).Should(ContainSubstring(canaryRevision)) + Expect(workload.Status.UpdateRevision).Should(ContainSubstring(canaryRevision)) + time.Sleep(time.Second * 3) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) + Expect(rollout.Status.StableRevision).Should(Equal(canaryRevision)) + + // scale up replicas 5 -> 6 + workload.Spec.Replicas = utilpointer.Int32(6) + UpdateCloneSet(workload) + By("Update cloneSet replicas from(5) -> to(6)") + time.Sleep(time.Second * 2) + + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + WaitRolloutWorkloadGenration(rollout.Name, workload.Generation) + }) }) }) diff --git a/test/e2e/test_data/batchrelease/cloneset.yaml b/test/e2e/test_data/batchrelease/cloneset.yaml index da657a2c..1d187df0 100644 --- a/test/e2e/test_data/batchrelease/cloneset.yaml +++ b/test/e2e/test_data/batchrelease/cloneset.yaml @@ -6,6 +6,9 @@ metadata: name: sample spec: replicas: 5 + updateStrategy: + maxUnavailable: 0 + maxSurge: 1 selector: matchLabels: app: busybox diff --git a/test/e2e/test_data/batchrelease/deployment.yaml b/test/e2e/test_data/batchrelease/deployment.yaml index edf9a8db..fbd137aa 100644 --- a/test/e2e/test_data/batchrelease/deployment.yaml +++ b/test/e2e/test_data/batchrelease/deployment.yaml @@ -6,6 +6,11 @@ metadata: app: busybox spec: replicas: 5 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 selector: matchLabels: app: busybox diff --git a/test/e2e/test_data/rollout/cloneset.yaml b/test/e2e/test_data/rollout/cloneset.yaml index 032d4e20..fa48094a 100644 --- a/test/e2e/test_data/rollout/cloneset.yaml +++ b/test/e2e/test_data/rollout/cloneset.yaml @@ -20,10 +20,12 @@ spec: containers: - name: echoserver image: cilium/echoserver:latest - imagePullPolicy: IfNotPresent + # imagePullPolicy: IfNotPresent ports: - containerPort: 8080 env: + - name: PORT + value: '8080' - name: POD_NAME valueFrom: fieldRef: diff --git a/test/e2e/test_data/rollout/deployment.yaml b/test/e2e/test_data/rollout/deployment.yaml index bfdc2396..68e1fea0 100644 --- a/test/e2e/test_data/rollout/deployment.yaml +++ b/test/e2e/test_data/rollout/deployment.yaml @@ -5,7 +5,12 @@ metadata: labels: app: echoserver spec: - replicas: 1 + replicas: 5 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 selector: matchLabels: app: echoserver @@ -17,43 +22,24 @@ spec: containers: - name: echoserver image: cilium/echoserver:latest + # imagePullPolicy: IfNotPresent ports: - containerPort: 8080 env: - name: PORT value: '8080' ---- -apiVersion: v1 -kind: Service -metadata: - name: echoserver - labels: - app: echoserver -spec: - ports: - - port: 80 - targetPort: 8080 - protocol: TCP - name: http - selector: - app: echoserver ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: echoserver - annotations: - kubernetes.io/ingress.class: nginx -spec: - rules: - - host: echoserver.example.com - http: - paths: - - backend: - service: - name: echoserver - port: - number: 80 - path: /apis/echo - pathType: Exact + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + value: version1 diff --git a/test/e2e/test_data/rollout/rollout_canary_base.yaml b/test/e2e/test_data/rollout/rollout_canary_base.yaml index dd46464d..f230dfeb 100644 --- a/test/e2e/test_data/rollout/rollout_canary_base.yaml +++ b/test/e2e/test_data/rollout/rollout_canary_base.yaml @@ -22,7 +22,7 @@ spec: - weight: 80 pause: {duration: 10} - weight: 100 - trafficRouting: + trafficRoutings: - service: echoserver type: nginx ingress: diff --git a/test/kind-conf.yaml b/test/kind-conf.yaml new file mode 100644 index 00000000..bb854bff --- /dev/null +++ b/test/kind-conf.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker + - role: worker +featureGates: + EphemeralContainers: true