Skip to content

Commit

Permalink
check workload health for discovered apps
Browse files Browse the repository at this point in the history
Signed-off-by: jacklu <[email protected]>
  • Loading branch information
jacklu authored and ShyamsundarR committed Aug 26, 2024
1 parent eff856f commit 09f1c04
Show file tree
Hide file tree
Showing 6 changed files with 85 additions and 32 deletions.
5 changes: 3 additions & 2 deletions e2e/deployers/discoveredapps.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,9 @@ func (d DiscoveredApps) Deploy(w workloads.Workload) error {
return err
}

// TODO: modify it based on shyam's comment
// return waitDeploymentReady(util.Ctx.C1.CtrlClient, namespace, "busybox")
if err = WaitWorkloadHealth(util.Ctx.C1.CtrlClient, namespace, w); err != nil {
return err
}

util.Ctx.Log.Info(name + " is deployed")

Expand Down
38 changes: 19 additions & 19 deletions e2e/deployers/retry.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ import (
"time"

"github.com/ramendr/ramen/e2e/util"
"github.com/ramendr/ramen/e2e/workloads"
subscriptionv1 "open-cluster-management.io/multicloud-operators-subscription/pkg/apis/apps/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)

const FiveSecondsDuration = 5 * time.Second
Expand Down Expand Up @@ -40,28 +42,26 @@ func waitSubscriptionPhase(namespace, name string, phase subscriptionv1.Subscrip
}
}

// func waitDeploymentReady(client client.Client, namespace, name string) error {
// time.Sleep(FiveSecondsDuration)
func WaitWorkloadHealth(client client.Client, namespace string, w workloads.Workload) error {
time.Sleep(FiveSecondsDuration)

// startTime := time.Now()
startTime := time.Now()

// for {
// deploy, err := getDeployment(client, namespace, name)
// if err != nil {
// return err
// }
for {
err := w.Health(client, namespace)
if err == nil {
util.Ctx.Log.Info(fmt.Sprintf("workload %s is ready", w.GetName()))

// if deploy.Status.Replicas == deploy.Status.ReadyReplicas {
// util.Ctx.Log.Info(fmt.Sprintf("deployment %s is ready", name))
return nil
}

// return nil
// }
if time.Since(startTime) > time.Second*time.Duration(util.Timeout) {
util.Ctx.Log.Info(err.Error())

// if time.Since(startTime) > time.Second*time.Duration(util.Timeout) {
// return fmt.Errorf(fmt.Sprintf("deployment %s is not ready yet before timeout of %v",
// name, util.Timeout))
// }
return fmt.Errorf(fmt.Sprintf("workload %s is not ready yet before timeout of %v",
w.GetName(), util.Timeout))
}

// time.Sleep(time.Second * time.Duration(util.TimeInterval))
// }
// }
time.Sleep(time.Second * time.Duration(util.TimeInterval))
}
}
20 changes: 19 additions & 1 deletion e2e/dractions/actionsdiscoveredapps.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,18 @@ func failoverRelocateDiscoveredApps(w workloads.Workload, d deployers.Deployer,
return err
}

drPolicyName := util.DefaultDRPolicyName

drpolicy, err := util.GetDRPolicy(client, drPolicyName)
if err != nil {
return err
}

targetCluster, err := getTargetCluster(client, namespace, drpcName, drpolicy)
if err != nil {
return err
}

if err := waitAndUpdateDRPC(client, namespace, drpcName, action); err != nil {
return err
}
Expand All @@ -128,5 +140,11 @@ func failoverRelocateDiscoveredApps(w workloads.Workload, d deployers.Deployer,
return err
}

return waitDRPCReady(client, namespace, name)
if err = waitDRPCReady(client, namespace, name); err != nil {
return err
}

drClient := getDRClusterClient(targetCluster, drpolicy)

return deployers.WaitWorkloadHealth(drClient, namespaceInDrCluster, w)
}
13 changes: 6 additions & 7 deletions e2e/dractions/retry.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,19 +147,18 @@ func getCurrentCluster(client client.Client, namespace string, placementName str
}

clusterName := placementDecision.Status.Decisions[0].ClusterName
util.Ctx.Log.Info(placementName + " placementdecision clusterName: " + clusterName)

return clusterName, nil
}

// return dr cluster client
// func getDRClusterClient(clusterName string, drpolicy *ramen.DRPolicy) client.Client {
// if clusterName == drpolicy.Spec.DRClusters[0] {
// return util.Ctx.C1.CtrlClient
// }
func getDRClusterClient(clusterName string, drpolicy *ramen.DRPolicy) client.Client {
if clusterName == drpolicy.Spec.DRClusters[0] {
return util.Ctx.C1.CtrlClient
}

// return util.Ctx.C2.CtrlClient
// }
return util.Ctx.C2.CtrlClient
}

func getTargetCluster(client client.Client, namespace, placementName string, drpolicy *ramen.DRPolicy) (string, error) {
currentCluster, err := getCurrentCluster(client, namespace, placementName)
Expand Down
37 changes: 34 additions & 3 deletions e2e/workloads/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,15 @@

package workloads

import "github.com/ramendr/ramen/e2e/util"
import (
"context"
"fmt"

"github.com/ramendr/ramen/e2e/util"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)

type Deployment struct {
Path string
Expand Down Expand Up @@ -63,7 +71,30 @@ func (w Deployment) GetResources() error {
return nil
}

func (w Deployment) Health() error {
// Check the workload health on a targetCluster
// Check the workload health deployed in a cluster namespace
func (w Deployment) Health(client client.Client, namespace string) error {
deploy, err := getDeployment(client, namespace, w.GetAppName())
if err != nil {
return err
}

if deploy.Status.Replicas == deploy.Status.ReadyReplicas {
util.Ctx.Log.Info(fmt.Sprintf("deployment %s is ready", w.GetAppName()))

return nil
}

return nil
}

func getDeployment(client client.Client, namespace, name string) (*appsv1.Deployment, error) {
deploy := &appsv1.Deployment{}
key := types.NamespacedName{Name: name, Namespace: namespace}

err := client.Get(context.Background(), key, deploy)
if err != nil {
return nil, err
}

return deploy, nil
}
4 changes: 4 additions & 0 deletions e2e/workloads/workload.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

package workloads

import "sigs.k8s.io/controller-runtime/pkg/client"

type Workload interface {
Kustomize() string // Can differ based on the workload, hence part of the Workload interface
// GetResources() error // Get the actual workload resources
Expand All @@ -13,4 +15,6 @@ type Workload interface {
// GetRepoURL() string // Possibly all this is part of Workload than each implementation of the interfaces?
GetPath() string
GetRevision() string

Health(client client.Client, namespace string) error
}

0 comments on commit 09f1c04

Please sign in to comment.