diff --git a/internal/controller/drclusterconfig_controller.go b/internal/controller/drclusterconfig_controller.go index 7e0c465d7e..364269ed89 100644 --- a/internal/controller/drclusterconfig_controller.go +++ b/internal/controller/drclusterconfig_controller.go @@ -9,6 +9,8 @@ import ( "slices" "time" + volrep "github.com/csi-addons/kubernetes-csi-addons/apis/replication.storage/v1alpha1" + snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" clusterv1alpha1 "github.com/open-cluster-management-io/api/cluster/v1alpha1" "golang.org/x/time/rate" storagev1 "k8s.io/api/storage/v1" @@ -38,7 +40,9 @@ const ( maxReconcileBackoff = 5 * time.Minute // Prefixes for various ClusterClaims - ccSCPrefix = "storage.class" + ccSCPrefix = "storage.class" + ccVSCPrefix = "snapshot.class" + ccVRCPrefix = "replication.class" ) // DRClusterConfigReconciler reconciles a DRClusterConfig object @@ -173,17 +177,13 @@ func (r *DRClusterConfigReconciler) processCreateOrUpdate( return ctrl.Result{Requeue: true}, fmt.Errorf("failed to add finalizer for DRClusterConfig resource, %w", err) } - allSurvivors := []string{} - - survivors, err := r.createSCClusterClaims(ctx, log) + allSurvivors, err := r.CreateClassClaims(ctx, log) if err != nil { log.Info("Reconcile error", "error", err) return ctrl.Result{Requeue: true}, err } - allSurvivors = append(allSurvivors, survivors...) - if err := r.pruneClusterClaims(ctx, log, allSurvivors); err != nil { log.Info("Reconcile error", "error", err) @@ -193,7 +193,35 @@ func (r *DRClusterConfigReconciler) processCreateOrUpdate( return ctrl.Result{}, nil } -// createSCClusterClaims lists all StorageClasses and creates ClusterClaims for them +// CreateClassClaims creates cluster claims for various storage related classes of interest +func (r *DRClusterConfigReconciler) CreateClassClaims(ctx context.Context, log logr.Logger) ([]string, error) { + allSurvivors := []string{} + + survivors, err := r.createSCClusterClaims(ctx, log) + if err != nil { + return nil, err + } + + allSurvivors = append(allSurvivors, survivors...) + + survivors, err = r.createVSCClusterClaims(ctx, log) + if err != nil { + return nil, err + } + + allSurvivors = append(allSurvivors, survivors...) + + survivors, err = r.createVRCClusterClaims(ctx, log) + if err != nil { + return nil, err + } + + allSurvivors = append(allSurvivors, survivors...) + + return allSurvivors, nil +} + +// createSCClusterClaims lists StorageClasses and creates ClusterClaims for ones marked for ramen func (r *DRClusterConfigReconciler) createSCClusterClaims( ctx context.Context, log logr.Logger, ) ([]string, error) { @@ -205,7 +233,6 @@ func (r *DRClusterConfigReconciler) createSCClusterClaims( } for i := range sClasses.Items { - // TODO: If something is labeled later is there an Update event? if !util.HasLabel(&sClasses.Items[i], StorageIDLabel) { continue } @@ -220,6 +247,58 @@ func (r *DRClusterConfigReconciler) createSCClusterClaims( return claims, nil } +// createVSCClusterClaims lists VolumeSnapshotClasses and creates ClusterClaims for ones marked for ramen +func (r *DRClusterConfigReconciler) createVSCClusterClaims( + ctx context.Context, log logr.Logger, +) ([]string, error) { + claims := []string{} + + vsClasses := &snapv1.VolumeSnapshotClassList{} + if err := r.Client.List(ctx, vsClasses); err != nil { + return nil, fmt.Errorf("failed to list VolumeSnapshotClasses, %w", err) + } + + for i := range vsClasses.Items { + if !util.HasLabel(&vsClasses.Items[i], StorageIDLabel) { + continue + } + + if err := r.ensureClusterClaim(ctx, log, ccVSCPrefix, vsClasses.Items[i].GetName()); err != nil { + return nil, err + } + + claims = append(claims, claimName(ccVSCPrefix, vsClasses.Items[i].GetName())) + } + + return claims, nil +} + +// createVRCClusterClaims lists VolumeReplicationClasses and creates ClusterClaims for ones marked for ramen +func (r *DRClusterConfigReconciler) createVRCClusterClaims( + ctx context.Context, log logr.Logger, +) ([]string, error) { + claims := []string{} + + vrClasses := &volrep.VolumeReplicationClassList{} + if err := r.Client.List(ctx, vrClasses); err != nil { + return nil, fmt.Errorf("failed to list VolumeReplicationClasses, %w", err) + } + + for i := range vrClasses.Items { + if !util.HasLabel(&vrClasses.Items[i], VolumeReplicationIDLabel) { + continue + } + + if err := r.ensureClusterClaim(ctx, log, ccVRCPrefix, vrClasses.Items[i].GetName()); err != nil { + return nil, err + } + + claims = append(claims, claimName(ccVRCPrefix, vrClasses.Items[i].GetName())) + } + + return claims, nil +} + // ensureClusterClaim is a generic ClusterClaim creation function, that creates a claim named "prefix.name", with // the passed in name as the ClusterClaim spec.Value func (r *DRClusterConfigReconciler) ensureClusterClaim( @@ -296,5 +375,7 @@ func (r *DRClusterConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { RateLimiter: rateLimiter, }).For(&ramen.DRClusterConfig{}). Watches(&storagev1.StorageClass{}, drccMapFn, drccPredFn). + Watches(&snapv1.VolumeSnapshotClass{}, drccMapFn, drccPredFn). + Watches(&volrep.VolumeReplicationClass{}, drccMapFn, drccPredFn). Complete(r) } diff --git a/internal/controller/drclusterconfig_controller_test.go b/internal/controller/drclusterconfig_controller_test.go index b2ae8dd6c1..5939d4b07d 100644 --- a/internal/controller/drclusterconfig_controller_test.go +++ b/internal/controller/drclusterconfig_controller_test.go @@ -10,6 +10,8 @@ import ( "path/filepath" "time" + volrep "github.com/csi-addons/kubernetes-csi-addons/apis/replication.storage/v1alpha1" + snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" clusterv1alpha1 "github.com/open-cluster-management-io/api/cluster/v1alpha1" @@ -43,7 +45,6 @@ func ensureClaimCount(apiReader client.Reader, count int) { }, timeout, interval).Should(BeTrue()) } -//nolint:unparam func ensureClusterClaim(apiReader client.Reader, class, name string) { Eventually(func() error { ccName := types.NamespacedName{ @@ -67,14 +68,17 @@ func ensureClusterClaim(apiReader client.Reader, class, name string) { var _ = Describe("DRClusterConfig-ClusterClaimsTests", Ordered, func() { var ( - ctx context.Context - cancel context.CancelFunc - cfg *rest.Config - testEnv *envtest.Environment - k8sClient client.Client - apiReader client.Reader - drCConfig *ramen.DRClusterConfig - baseSC, sc1, sc2 *storagev1.StorageClass + ctx context.Context + cancel context.CancelFunc + cfg *rest.Config + testEnv *envtest.Environment + k8sClient client.Client + apiReader client.Reader + drCConfig *ramen.DRClusterConfig + baseSC, sc1, sc2 *storagev1.StorageClass + baseVSC, vsc1, vsc2 *snapv1.VolumeSnapshotClass + baseVRC, vrc1, vrc2 *volrep.VolumeReplicationClass + claimCount int ) BeforeAll(func() { @@ -156,7 +160,7 @@ var _ = Describe("DRClusterConfig-ClusterClaimsTests", Ordered, func() { } Expect(k8sClient.Create(context.TODO(), drCConfig)).To(Succeed()) - By("Defining a basic StroageClass") + By("Defining basic Classes") baseSC = &storagev1.StorageClass{ ObjectMeta: metav1.ObjectMeta{ @@ -167,6 +171,29 @@ var _ = Describe("DRClusterConfig-ClusterClaimsTests", Ordered, func() { }, Provisioner: "fake.ramen.com", } + + baseVSC = &snapv1.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baseVSC", + Labels: map[string]string{ + ramencontrollers.StorageIDLabel: "fake", + }, + }, + Driver: "fake.ramen.com", + DeletionPolicy: snapv1.VolumeSnapshotContentDelete, + } + + baseVRC = &volrep.VolumeReplicationClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baseVRC", + Labels: map[string]string{ + ramencontrollers.VolumeReplicationIDLabel: "fake", + }, + }, + Spec: volrep.VolumeReplicationClassSpec{ + Provisioner: "fake.ramen.com", + }, + } }) AfterAll(func() { @@ -189,7 +216,7 @@ var _ = Describe("DRClusterConfig-ClusterClaimsTests", Ordered, func() { Expect(err).NotTo(HaveOccurred()) }) - Describe("ClusterClaims-StorageClasses", Ordered, func() { + Describe("ClusterClaims", Ordered, func() { Context("Given DRClusterConfig resource", func() { When("there is a StorageClass created with required labels", func() { It("creates a ClusterClaim", func() { @@ -199,8 +226,9 @@ var _ = Describe("DRClusterConfig-ClusterClaimsTests", Ordered, func() { sc1.Name = "sc1" Expect(k8sClient.Create(context.TODO(), sc1)).To(Succeed()) + claimCount++ ensureClusterClaim(apiReader, "storage.class", "sc1") - ensureClaimCount(apiReader, 1) + ensureClaimCount(apiReader, claimCount) }) }) When("a StorageClass with required labels is deleted", func() { @@ -209,7 +237,8 @@ var _ = Describe("DRClusterConfig-ClusterClaimsTests", Ordered, func() { Expect(k8sClient.Delete(context.TODO(), sc1)).To(Succeed()) - ensureClaimCount(apiReader, 0) + claimCount-- + ensureClaimCount(apiReader, claimCount) }) }) When("there are multiple StorageClass created with required labels", func() { @@ -224,9 +253,10 @@ var _ = Describe("DRClusterConfig-ClusterClaimsTests", Ordered, func() { sc2.Name = "sc2" Expect(k8sClient.Create(context.TODO(), sc2)).To(Succeed()) + claimCount += 2 ensureClusterClaim(apiReader, "storage.class", "sc1") ensureClusterClaim(apiReader, "storage.class", "sc2") - ensureClaimCount(apiReader, 2) + ensureClaimCount(apiReader, claimCount) }) }) When("a StorageClass label is deleted", func() { @@ -236,10 +266,117 @@ var _ = Describe("DRClusterConfig-ClusterClaimsTests", Ordered, func() { sc1.Labels = map[string]string{} Expect(k8sClient.Update(context.TODO(), sc1)).To(Succeed()) - ensureClaimCount(apiReader, 1) + claimCount-- + ensureClaimCount(apiReader, claimCount) ensureClusterClaim(apiReader, "storage.class", "sc2") }) }) }) + When("there is a SnapshotCLass created with required labels", func() { + It("creates a ClusterClaim", func() { + By("creating a SnapshotClass") + + vsc1 = baseVSC.DeepCopy() + vsc1.Name = "vsc1" + Expect(k8sClient.Create(context.TODO(), vsc1)).To(Succeed()) + + claimCount++ + ensureClusterClaim(apiReader, "snapshot.class", "vsc1") + ensureClaimCount(apiReader, claimCount) + }) + }) + When("a SnapshotClass with required labels is deleted", func() { + It("deletes the associated ClusterClaim", func() { + By("deleting a SnapshotClass") + + Expect(k8sClient.Delete(context.TODO(), vsc1)).To(Succeed()) + + claimCount-- + ensureClaimCount(apiReader, claimCount) + }) + }) + When("there are multiple SnapshotClass created with required labels", func() { + It("creates ClusterClaims", func() { + By("creating a SnapshotClass") + + vsc1 = baseVSC.DeepCopy() + vsc1.Name = "vsc1" + Expect(k8sClient.Create(context.TODO(), vsc1)).To(Succeed()) + + vsc2 = baseVSC.DeepCopy() + vsc2.Name = "vsc2" + Expect(k8sClient.Create(context.TODO(), vsc2)).To(Succeed()) + + claimCount += 2 + ensureClusterClaim(apiReader, "snapshot.class", "vsc1") + ensureClusterClaim(apiReader, "snapshot.class", "vsc2") + ensureClaimCount(apiReader, claimCount) + }) + }) + When("a SnapshotClass label is deleted", func() { + It("deletes the associated ClusterClaim", func() { + By("deleting a SnapshotClass label") + + vsc2.Labels = map[string]string{} + Expect(k8sClient.Update(context.TODO(), vsc2)).To(Succeed()) + + claimCount-- + ensureClaimCount(apiReader, claimCount) + ensureClusterClaim(apiReader, "snapshot.class", "vsc1") + }) + }) + When("there is a VolumeReplicationCLass created with required labels", func() { + It("creates a ClusterClaim", func() { + By("creating a VolumeReplicationClass") + + vrc1 = baseVRC.DeepCopy() + vrc1.Name = "vrc1" + Expect(k8sClient.Create(context.TODO(), vrc1)).To(Succeed()) + + claimCount++ + ensureClusterClaim(apiReader, "replication.class", "vrc1") + ensureClaimCount(apiReader, claimCount) + }) + }) + When("a VolumeReplicationClass with required labels is deleted", func() { + It("deletes the associated ClusterClaim", func() { + By("deleting a VolumeReplicationClass") + + Expect(k8sClient.Delete(context.TODO(), vrc1)).To(Succeed()) + + claimCount-- + ensureClaimCount(apiReader, claimCount) + }) + }) + When("there are multiple VolumeReplicationClass created with required labels", func() { + It("creates ClusterClaims", func() { + By("creating a VolumeReplicationClass") + + vrc1 = baseVRC.DeepCopy() + vrc1.Name = "vrc1" + Expect(k8sClient.Create(context.TODO(), vrc1)).To(Succeed()) + + vrc2 = baseVRC.DeepCopy() + vrc2.Name = "vrc2" + Expect(k8sClient.Create(context.TODO(), vrc2)).To(Succeed()) + + claimCount += 2 + ensureClusterClaim(apiReader, "replication.class", "vrc1") + ensureClusterClaim(apiReader, "replication.class", "vrc2") + ensureClaimCount(apiReader, claimCount) + }) + }) + When("a VolumeReplicationClass label is deleted", func() { + It("deletes the associated ClusterClaim", func() { + By("deleting a VolumeReplicationClass label") + + vrc2.Labels = map[string]string{} + Expect(k8sClient.Update(context.TODO(), vrc2)).To(Succeed()) + + claimCount-- + ensureClaimCount(apiReader, claimCount) + ensureClusterClaim(apiReader, "replication.class", "vrc1") + }) + }) }) })