Skip to content

Commit

Permalink
e2e: enable discovered apps (#1509)
Browse files Browse the repository at this point in the history
* e2e: enable discovered apps

Signed-off-by: jacklu <[email protected]>

* use kubectl -k to deploy and undeploy discovered apps

Signed-off-by: jacklu <[email protected]>

---------

Signed-off-by: jacklu <[email protected]>
Co-authored-by: jacklu <[email protected]>
  • Loading branch information
par97 and jacklu authored Aug 21, 2024
1 parent 4f51905 commit eff856f
Show file tree
Hide file tree
Showing 14 changed files with 673 additions and 156 deletions.
20 changes: 12 additions & 8 deletions e2e/deployers/applicationset.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,22 @@ func (a ApplicationSet) Deploy(w workloads.Workload) error {

util.Ctx.Log.Info("enter Deploy " + name)

err := createManagedClusterSetBinding(McsbName, namespace)
err := CreateManagedClusterSetBinding(McsbName, namespace)
if err != nil {
return err
}

err = createPlacement(name, namespace)
err = CreatePlacement(name, namespace)
if err != nil {
return err
}

err = createPlacementDecisionConfigMap(name, namespace)
err = CreatePlacementDecisionConfigMap(name, namespace)
if err != nil {
return err
}

err = createApplicationSet(a, w)
err = CreateApplicationSet(a, w)
if err != nil {
return err
}
Expand All @@ -45,17 +45,17 @@ func (a ApplicationSet) Undeploy(w workloads.Workload) error {

util.Ctx.Log.Info("enter Undeploy " + name)

err := deleteApplicationSet(a, w)
err := DeleteApplicationSet(a, w)
if err != nil {
return err
}

err = deleteConfigMap(name, namespace)
err = DeleteConfigMap(name, namespace)
if err != nil {
return err
}

err = deletePlacement(name, namespace)
err = DeletePlacement(name, namespace)
if err != nil {
return err
}
Expand All @@ -68,7 +68,7 @@ func (a ApplicationSet) Undeploy(w workloads.Workload) error {
}

if lastAppset {
err = deleteManagedClusterSetBinding(McsbName, namespace)
err = DeleteManagedClusterSetBinding(McsbName, namespace)
if err != nil {
return err
}
Expand All @@ -80,3 +80,7 @@ func (a ApplicationSet) Undeploy(w workloads.Workload) error {
func (a ApplicationSet) GetName() string {
return "Appset"
}

func (a ApplicationSet) IsWorkloadSupported(w workloads.Workload) bool {
return true
}
109 changes: 96 additions & 13 deletions e2e/deployers/crud.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,20 @@ package deployers

import (
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"strings"

"github.com/ramendr/ramen/e2e/util"
"github.com/ramendr/ramen/e2e/workloads"
"gopkg.in/yaml.v2"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"

"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/yaml"

argocdv1alpha1hack "github.com/ramendr/ramen/e2e/argocd"
corev1 "k8s.io/api/core/v1"
Expand All @@ -30,7 +34,7 @@ const (
ClusterSetName = "default"
)

func createManagedClusterSetBinding(name, namespace string) error {
func CreateManagedClusterSetBinding(name, namespace string) error {
labels := make(map[string]string)
labels[AppLabelKey] = namespace
mcsb := &ocmv1b2.ManagedClusterSetBinding{
Expand All @@ -54,7 +58,7 @@ func createManagedClusterSetBinding(name, namespace string) error {
return nil
}

func deleteManagedClusterSetBinding(name, namespace string) error {
func DeleteManagedClusterSetBinding(name, namespace string) error {
mcsb := &ocmv1b2.ManagedClusterSetBinding{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Expand All @@ -74,7 +78,7 @@ func deleteManagedClusterSetBinding(name, namespace string) error {
return nil
}

func createPlacement(name, namespace string) error {
func CreatePlacement(name, namespace string) error {
labels := make(map[string]string)
labels[AppLabelKey] = name
clusterSet := []string{ClusterSetName}
Expand Down Expand Up @@ -104,7 +108,7 @@ func createPlacement(name, namespace string) error {
return nil
}

func deletePlacement(name, namespace string) error {
func DeletePlacement(name, namespace string) error {
placement := &ocmv1b1.Placement{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Expand All @@ -124,7 +128,7 @@ func deletePlacement(name, namespace string) error {
return nil
}

func createSubscription(s Subscription, w workloads.Workload) error {
func CreateSubscription(s Subscription, w workloads.Workload) error {
name := GetCombinedName(s, w)
namespace := name

Expand Down Expand Up @@ -180,7 +184,7 @@ func createSubscription(s Subscription, w workloads.Workload) error {
return nil
}

func deleteSubscription(s Subscription, w workloads.Workload) error {
func DeleteSubscription(s Subscription, w workloads.Workload) error {
name := GetCombinedName(s, w)
namespace := name

Expand All @@ -207,19 +211,33 @@ func GetCombinedName(d Deployer, w workloads.Workload) string {
return strings.ToLower(d.GetName() + "-" + w.GetName() + "-" + w.GetAppName())
}

func getSubscription(ctrlClient client.Client, namespace, name string) (*subscriptionv1.Subscription, error) {
func GetNamespace(d Deployer, w workloads.Workload) string {
_, isAppSet := d.(*ApplicationSet)
if isAppSet {
// appset need be deployed in argocd ns
return util.ArgocdNamespace
}

if _, isDiscoveredApps := d.(*DiscoveredApps); isDiscoveredApps {
return util.RamenOpsNs
}

return GetCombinedName(d, w)
}

func getSubscription(client client.Client, namespace, name string) (*subscriptionv1.Subscription, error) {
subscription := &subscriptionv1.Subscription{}
key := types.NamespacedName{Name: name, Namespace: namespace}

err := ctrlClient.Get(context.Background(), key, subscription)
err := client.Get(context.Background(), key, subscription)
if err != nil {
return nil, err
}

return subscription, nil
}

func createPlacementDecisionConfigMap(cmName string, cmNamespace string) error {
func CreatePlacementDecisionConfigMap(cmName string, cmNamespace string) error {
object := metav1.ObjectMeta{Name: cmName, Namespace: cmNamespace}

data := map[string]string{
Expand All @@ -243,7 +261,7 @@ func createPlacementDecisionConfigMap(cmName string, cmNamespace string) error {
return nil
}

func deleteConfigMap(cmName string, cmNamespace string) error {
func DeleteConfigMap(cmName string, cmNamespace string) error {
object := metav1.ObjectMeta{Name: cmName, Namespace: cmNamespace}

configMap := &corev1.ConfigMap{
Expand All @@ -263,7 +281,7 @@ func deleteConfigMap(cmName string, cmNamespace string) error {
}

// nolint:funlen
func createApplicationSet(a ApplicationSet, w workloads.Workload) error {
func CreateApplicationSet(a ApplicationSet, w workloads.Workload) error {
var requeueSeconds int64 = 180

name := GetCombinedName(a, w)
Expand Down Expand Up @@ -341,7 +359,7 @@ func createApplicationSet(a ApplicationSet, w workloads.Workload) error {
return nil
}

func deleteApplicationSet(a ApplicationSet, w workloads.Workload) error {
func DeleteApplicationSet(a ApplicationSet, w workloads.Workload) error {
name := GetCombinedName(a, w)
namespace := util.ArgocdNamespace

Expand Down Expand Up @@ -378,3 +396,68 @@ func isLastAppsetInArgocdNs(namespace string) (bool, error) {

return len(appsetList.Items) == 1, nil
}

func DeleteDiscoveredApps(w workloads.Workload, namespace, cluster string) error {
tempDir, err := os.MkdirTemp("", "ramen-")
if err != nil {
return err
}

// Clean up by removing the temporary directory when done
defer os.RemoveAll(tempDir)

if err = CreateKustomizationFile(w, tempDir); err != nil {
return err
}

cmd := exec.Command("kubectl", "delete", "-k", tempDir, "-n", namespace,
"--context", cluster, "--timeout=5m", "--ignore-not-found=true")

// Run the command and capture the output
if out, err := cmd.Output(); err != nil {
util.Ctx.Log.Info(string(out))

return err
}

return nil
}

type CombinedData map[string]interface{}

func CreateKustomizationFile(w workloads.Workload, dir string) error {
yamlData := `resources:
- ` + util.GetGitURL() + `/` + w.GetPath() + `?ref=` + w.GetRevision()

var yamlContent CombinedData

err := yaml.Unmarshal([]byte(yamlData), &yamlContent)
if err != nil {
return err
}

patch := w.Kustomize()

var jsonContent CombinedData

err = json.Unmarshal([]byte(patch), &jsonContent)
if err != nil {
return err
}

// Merge JSON content into YAML content
for key, value := range jsonContent {
yamlContent[key] = value
}

// Convert the combined content back to YAML
combinedYAML, err := yaml.Marshal(&yamlContent)
if err != nil {
return err
}

// Write the combined content to a new YAML file
outputFile := dir + "/kustomization.yaml"

return os.WriteFile(outputFile, combinedYAML, os.ModePerm)
}
2 changes: 2 additions & 0 deletions e2e/deployers/deployer.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import "github.com/ramendr/ramen/e2e/workloads"
type Deployer interface {
Deploy(workloads.Workload) error
Undeploy(workloads.Workload) error

IsWorkloadSupported(workloads.Workload) bool
// Scale(Workload) for adding/removing PVCs; in Deployer even though scaling is a Workload interface
// as we can Kustomize the Workload and change the deployer to perform the right action
// Resize(Workload) for changing PVC(s) size
Expand Down
110 changes: 110 additions & 0 deletions e2e/deployers/discoveredapps.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// SPDX-FileCopyrightText: The RamenDR authors
// SPDX-License-Identifier: Apache-2.0

package deployers

import (
"os"
"os/exec"

"github.com/ramendr/ramen/e2e/util"
"github.com/ramendr/ramen/e2e/workloads"
)

type DiscoveredApps struct{}

func (d DiscoveredApps) GetName() string {
return "Disapp"
}

func (d DiscoveredApps) Deploy(w workloads.Workload) error {
name := GetCombinedName(d, w)
namespace := name

util.Ctx.Log.Info("enter Deploy " + name)

// create namespace in both dr clusters
if err := util.CreateNamespaceAndAddAnnotation(namespace); err != nil {
return err
}

tempDir, err := os.MkdirTemp("", "ramen-")
if err != nil {
return err
}

// Clean up by removing the temporary directory when done
defer os.RemoveAll(tempDir)

if err = CreateKustomizationFile(w, tempDir); err != nil {
return err
}

drpolicy, err := util.GetDRPolicy(util.Ctx.Hub.CtrlClient, util.DefaultDRPolicyName)
if err != nil {
return err
}

cmd := exec.Command("kubectl", "apply", "-k", tempDir, "-n", namespace,
"--context", drpolicy.Spec.DRClusters[0], "--timeout=5m")

// Run the command and capture the output
if out, err := cmd.Output(); err != nil {
util.Ctx.Log.Info(string(out))

return err
}

// TODO: modify it based on shyam's comment
// return waitDeploymentReady(util.Ctx.C1.CtrlClient, namespace, "busybox")

util.Ctx.Log.Info(name + " is deployed")

return nil
}

func (d DiscoveredApps) Undeploy(w workloads.Workload) error {
name := GetCombinedName(d, w)
namespace := name // this namespace is in dr clusters

util.Ctx.Log.Info("enter Undeploy " + name)

drpolicy, err := util.GetDRPolicy(util.Ctx.Hub.CtrlClient, util.DefaultDRPolicyName)
if err != nil {
return err
}

util.Ctx.Log.Info("starts to delete discovered apps on " + drpolicy.Spec.DRClusters[0])

// delete app on both clusters
if err := DeleteDiscoveredApps(w, namespace, drpolicy.Spec.DRClusters[0]); err != nil {
return err
}

util.Ctx.Log.Info("starts to delete discovered apps on " + drpolicy.Spec.DRClusters[1])

if err := DeleteDiscoveredApps(w, namespace, drpolicy.Spec.DRClusters[1]); err != nil {
return err
}

util.Ctx.Log.Info("starts to delete namespace " + namespace + " on " + drpolicy.Spec.DRClusters[0])

// delete namespace on both clusters
if err := util.DeleteNamespace(util.Ctx.C1.CtrlClient, namespace); err != nil {
return err
}

util.Ctx.Log.Info("starts to delete namespace " + namespace + " on " + drpolicy.Spec.DRClusters[1])

if err := util.DeleteNamespace(util.Ctx.C2.CtrlClient, namespace); err != nil {
return err
}

util.Ctx.Log.Info(name + " is undeployed")

return nil
}

func (d DiscoveredApps) IsWorkloadSupported(w workloads.Workload) bool {
return w.GetName() != "Deploy-cephfs"
}
Loading

0 comments on commit eff856f

Please sign in to comment.