e2e: check workload health for discovered apps

e2e/deployers/discoveredapps.go

@@ -55,8 +55,9 @@ func (d DiscoveredApps) Deploy(w workloads.Workload) error {
 		return err
 	}
 
-	// TODO: modify it based on shyam's comment
-	// return waitDeploymentReady(util.Ctx.C1.CtrlClient, namespace, "busybox")
+	if err = WaitWorkloadHealth(util.Ctx.C1.CtrlClient, namespace, w); err != nil {
+		return err
+	}
 
 	util.Ctx.Log.Info(name + " is deployed")
 

e2e/deployers/retry.go

@@ -8,7 +8,9 @@ import (
 	"time"
 
 	"github.com/ramendr/ramen/e2e/util"
+	"github.com/ramendr/ramen/e2e/workloads"
 	subscriptionv1 "open-cluster-management.io/multicloud-operators-subscription/pkg/apis/apps/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const FiveSecondsDuration = 5 * time.Second
@@ -40,28 +42,26 @@ func waitSubscriptionPhase(namespace, name string, phase subscriptionv1.Subscrip
 	}
 }
 
-// func waitDeploymentReady(client client.Client, namespace, name string) error {
-// 	time.Sleep(FiveSecondsDuration)
+func WaitWorkloadHealth(client client.Client, namespace string, w workloads.Workload) error {
+	time.Sleep(FiveSecondsDuration)
 
-// 	startTime := time.Now()
+	startTime := time.Now()
 
-// 	for {
-// 		deploy, err := getDeployment(client, namespace, name)
-// 		if err != nil {
-// 			return err
-// 		}
+	for {
+		err := w.Health(client, namespace)
+		if err == nil {
+			util.Ctx.Log.Info(fmt.Sprintf("workload %s is ready", w.GetName()))
 
-// 		if deploy.Status.Replicas == deploy.Status.ReadyReplicas {
-// 			util.Ctx.Log.Info(fmt.Sprintf("deployment %s is ready", name))
+			return nil
+		}
 
-// 			return nil
-// 		}
+		if time.Since(startTime) > time.Second*time.Duration(util.Timeout) {
+			util.Ctx.Log.Info(err.Error())
 
-// 		if time.Since(startTime) > time.Second*time.Duration(util.Timeout) {
-// 			return fmt.Errorf(fmt.Sprintf("deployment %s is not ready yet before timeout of %v",
-// 				name, util.Timeout))
-// 		}
+			return fmt.Errorf(fmt.Sprintf("workload %s is not ready yet before timeout of %v",
+				w.GetName(), util.Timeout))
+		}
 
-// 		time.Sleep(time.Second * time.Duration(util.TimeInterval))
-// 	}
-// }
+		time.Sleep(time.Second * time.Duration(util.TimeInterval))
+	}
+}

e2e/dractions/actionsdiscoveredapps.go

@@ -109,6 +109,18 @@ func failoverRelocateDiscoveredApps(w workloads.Workload, d deployers.Deployer,
 		return err
 	}
 
+	drPolicyName := util.DefaultDRPolicyName
+
+	drpolicy, err := util.GetDRPolicy(client, drPolicyName)
+	if err != nil {
+		return err
+	}
+
+	targetCluster, err := getTargetCluster(client, namespace, drpcName, drpolicy)
+	if err != nil {
+		return err
+	}
+
 	if err := waitAndUpdateDRPC(client, namespace, drpcName, action); err != nil {
 		return err
 	}
@@ -128,5 +140,11 @@ func failoverRelocateDiscoveredApps(w workloads.Workload, d deployers.Deployer,
 		return err
 	}
 
-	return waitDRPCReady(client, namespace, name)
+	if err = waitDRPCReady(client, namespace, name); err != nil {
+		return err
+	}
+
+	drClient := getDRClusterClient(targetCluster, drpolicy)
+
+	return deployers.WaitWorkloadHealth(drClient, namespaceInDrCluster, w)
 }

e2e/dractions/retry.go

@@ -147,19 +147,18 @@ func getCurrentCluster(client client.Client, namespace string, placementName str
 	}
 
 	clusterName := placementDecision.Status.Decisions[0].ClusterName
-	util.Ctx.Log.Info(placementName + " placementdecision clusterName: " + clusterName)
 
 	return clusterName, nil
 }
 
 // return dr cluster client
-// func getDRClusterClient(clusterName string, drpolicy *ramen.DRPolicy) client.Client {
-// 	if clusterName == drpolicy.Spec.DRClusters[0] {
-// 		return util.Ctx.C1.CtrlClient
-// 	}
+func getDRClusterClient(clusterName string, drpolicy *ramen.DRPolicy) client.Client {
+	if clusterName == drpolicy.Spec.DRClusters[0] {
+		return util.Ctx.C1.CtrlClient
+	}
 
-// 	return util.Ctx.C2.CtrlClient
-// }
+	return util.Ctx.C2.CtrlClient
+}
 
 func getTargetCluster(client client.Client, namespace, placementName string, drpolicy *ramen.DRPolicy) (string, error) {
 	currentCluster, err := getCurrentCluster(client, namespace, placementName)

e2e/workloads/deployment.go

@@ -3,7 +3,15 @@
 
 package workloads
 
-import "github.com/ramendr/ramen/e2e/util"
+import (
+	"context"
+	"fmt"
+
+	"github.com/ramendr/ramen/e2e/util"
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
 
 type Deployment struct {
 	Path     string
@@ -63,7 +71,30 @@ func (w Deployment) GetResources() error {
 	return nil
 }
 
-func (w Deployment) Health() error {
-	// Check the workload health on a targetCluster
+// Check the workload health deployed in a cluster namespace
+func (w Deployment) Health(client client.Client, namespace string) error {
+	deploy, err := getDeployment(client, namespace, w.GetAppName())
+	if err != nil {
+		return err
+	}
+
+	if deploy.Status.Replicas == deploy.Status.ReadyReplicas {
+		util.Ctx.Log.Info(fmt.Sprintf("deployment %s is ready", w.GetAppName()))
+
+		return nil
+	}
+
 	return nil
 }
+
+func getDeployment(client client.Client, namespace, name string) (*appsv1.Deployment, error) {
+	deploy := &appsv1.Deployment{}
+	key := types.NamespacedName{Name: name, Namespace: namespace}
+
+	err := client.Get(context.Background(), key, deploy)
+	if err != nil {
+		return nil, err
+	}
+
+	return deploy, nil
+}

e2e/workloads/workload.go

@@ -3,6 +3,8 @@
 
 package workloads
 
+import "sigs.k8s.io/controller-runtime/pkg/client"
+
 type Workload interface {
 	Kustomize() string // Can differ based on the workload, hence part of the Workload interface
 	// GetResources() error // Get the actual workload resources
@@ -13,4 +15,6 @@ type Workload interface {
 	// GetRepoURL() string // Possibly all this is part of Workload than each implementation of the interfaces?
 	GetPath() string
 	GetRevision() string
+
+	Health(client client.Client, namespace string) error
 }