fix: make image pre pull failure block the kubernetes update

If the wrong version of the kubernetes components is used it won't bring down any pods as the upgrade will stop on the pre-pull phase. Signed-off-by: Artem Chernyshev <[email protected]>
siderolabs · Jul 19, 2024 · 23fb0c1 · 23fb0c1
1 parent b8db949
commit 23fb0c1
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 31 deletions.
diff --git a/cmd/integration-test/pkg/tests/kubernetes.go b/cmd/integration-test/pkg/tests/kubernetes.go
@@ -276,19 +276,8 @@ func AssertKubernetesUpgradeIsRevertible(testCtx context.Context, st state.State
 			assert.Equal(currentKubernetesVersion, r.TypedSpec().Value.LastUpgradeVersion, resourceDetails(r))
 		})
 
-		// verify that the upgrade took one API server down
-		rtestutils.AssertResources(ctx, t, st, []resource.ID{clusterName}, func(r *omni.KubernetesStatus, assert *assert.Assertions) {
-			var notReadyAPIServers int
-
-			for _, nodePods := range r.TypedSpec().Value.StaticPods {
-				for _, pod := range nodePods.StaticPods {
-					if pod.App == "kube-apiserver" && !pod.Ready {
-						notReadyAPIServers++
-					}
-				}
-			}
-
-			assert.Equal(1, notReadyAPIServers)
+		rtestutils.AssertResources(ctx, t, st, []resource.ID{clusterName}, func(r *omni.ImagePullStatus, assert *assert.Assertions) {
+			assert.Contains(r.TypedSpec().Value.LastProcessedError, "-bad")
 		})
 
 		t.Log("revert an upgrade")

diff --git a/internal/backend/runtime/omni/controllers/omni/internal/task/image/pulltask.go b/internal/backend/runtime/omni/controllers/omni/internal/task/image/pulltask.go
@@ -77,25 +77,31 @@ func (p PullTaskSpec) RunTask(ctx context.Context, _ *zap.Logger, pullStatusCh P
 		currentNode = imageList.Node
 
 		for _, img := range imageList.Images {
+			currentError = nil
 			currentImage = img
-			currentNum++
-
-			if nodeImageInfoMap.shouldPull(imageList.Node, img) {
-				currentError = p.pullImage(ctx, clusterID, imageList.Node, img)
-				if currentError != nil {
-					errs = multierror.Append(errs, currentError)
-				}
-
-				if !channel.SendWithContext(ctx, pullStatusCh, PullStatus{
-					Request:    p.request,
-					Node:       currentNode,
-					Image:      currentImage,
-					CurrentNum: currentNum,
-					TotalNum:   totalNum,
-					Error:      currentError,
-				}) {
-					return errs
-				}
+
+			if !nodeImageInfoMap.shouldPull(imageList.Node, img) {
+				currentNum++
+
+				continue
+			}
+
+			currentError = p.pullImage(ctx, clusterID, imageList.Node, img)
+			if currentError == nil {
+				currentNum++
+			} else {
+				errs = multierror.Append(errs, currentError)
+			}
+
+			if !channel.SendWithContext(ctx, pullStatusCh, PullStatus{
+				Request:    p.request,
+				Node:       currentNode,
+				Image:      currentImage,
+				CurrentNum: currentNum,
+				TotalNum:   totalNum,
+				Error:      currentError,
+			}) {
+				return errs
 			}
 		}
 	}