Completion of a restore triggers secrets refresh

CHANGELOG/CHANGELOG-1.10.md

@@ -15,6 +15,8 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
 
 ## unreleased
 
+* [FEATURE] [#1080](https://github.com/k8ssandra/k8ssandra-operator/issues/1080) When a restore is completed, an annotation is added to the user secrets that will cause cass-operator to refresh them on the Cassandra side.
+
 ## v1.10.3 - 2023-11-15
 
 * [BUGFIX] [#1110](https://github.com/k8ssandra/k8ssandra-operator/issues/1110) Fix cluster name being set to "Test Cluster" when running Cassandra 4.1+

Makefile

@@ -167,8 +167,11 @@ endif
 
 ##@ Build
 
+DATE  ?= $(shell date)
+COMMIT ?= $(shell git rev-parse --short HEAD)
+
 build: generate fmt vet ## Build manager binary.
-	go build -o bin/manager main.go
+	go build --ldflags "-X \"main.version=${VERSION}\" -X \"main.date=${DATE}\" -X \"main.commit=${COMMIT}\"" -o bin/manager main.go
 
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go

apis/k8ssandra/v1alpha1/constants.go

@@ -59,6 +59,8 @@ const (
 	K8ssandraClusterNamespaceLabel = "k8ssandra.io/cluster-namespace"
 
 	DatacenterLabel = "k8ssandra.io/datacenter"
+	// Forces refresh of secrets which relate to roles and authn in Cassandra.
+	RefreshAnnotation = "k8ssandra.io/refresh"
 )
 
 var (

controllers/medusa/medusarestorejob_controller.go

@@ -176,7 +176,16 @@
 		return ctrl.Result{RequeueAfter: r.DefaultDelay}, err
 	}
 
-	request.Log.Info("The restore operation is complete")
+	request.Log.Info("The restore operation is complete for DC", "CassandraDatacenter", request.Datacenter.Name)
+
+	recRes := medusa.RefreshSecrets(request.Datacenter, ctx, r.Client, request.Log, r.DefaultDelay, request.RestoreJob.Status.StartTime)
+	switch {
+	case recRes.IsError():
+		return ctrl.Result{RequeueAfter: r.DefaultDelay}, recRes.GetError()
+	case recRes.IsRequeue():
+		return ctrl.Result{RequeueAfter: r.DefaultDelay}, nil
+	}
+
 	return ctrl.Result{}, nil
 }
 

controllers/medusa/medusarestorejob_controller_test.go

@@ -292,8 +292,9 @@ func testMedusaRestoreDatacenter(t *testing.T, ctx context.Context, f *framework
 		return !restore.Status.FinishTime.IsZero()
 	}, timeout, interval)
 
-	err = f.DeleteK8ssandraCluster(ctx, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}, timeout, interval)
+	err = f.DeleteK8ssandraCluster(ctx, client.ObjectKey{Namespace: dc.Namespace, Name: kc.Name}, timeout, interval)
 	require.NoError(err, "failed to delete K8ssandraCluster")
+
 }
 
 func testValidationErrorStopsRestore(t *testing.T, ctx context.Context, f *framework.Framework, namespace string) {

main.go

@@ -67,6 +67,15 @@ import (
 	// +kubebuilder:scaffold:imports
 )
 
+var (
+	version        = "dev"
+	commit         = "n/a"
+	date           = "n/a"
+	versionMessage = "#######################" +
+		fmt.Sprintf("#### version %s commit %s date %s ####", version, commit, date) +
+		"#######################"
+)
+
 var (
 	scheme   = runtime.NewScheme()
 	setupLog = ctrl.Log.WithName("setup")
@@ -114,6 +123,8 @@ func main() {
 		setupLog.Info("watch namespace configured", "namespace", watchNamespace)
 	}
 
+	setupLog.Info(versionMessage)
+
 	options := ctrl.Options{
 		Scheme:                 scheme,
 		MetricsBindAddress:     metricsAddr,

pkg/medusa/refresh_secrets.go

@@ -0,0 +1,63 @@
+package medusa
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/go-logr/logr"
+	cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1"
+	k8ssandraapi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1"
+	"github.com/k8ssandra/k8ssandra-operator/pkg/reconciliation"
+	"github.com/k8ssandra/k8ssandra-operator/pkg/result"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func RefreshSecrets(dc *cassdcapi.CassandraDatacenter, ctx context.Context, client client.Client, logger logr.Logger, requeueDelay time.Duration, restoreTimestamp metav1.Time) result.ReconcileResult {
+	logger.Info(fmt.Sprintf("Restore complete for DC %#v, Refreshing secrets", dc.ObjectMeta))
+	userSecrets := []string{}
+	for _, user := range dc.Spec.Users {
+		userSecrets = append(userSecrets, user.SecretName)
+	}
+	if dc.Spec.SuperuserSecretName == "" {
+		userSecrets = append(userSecrets, cassdcapi.CleanupForKubernetes(dc.Spec.ClusterName)+"-superuser") //default SU secret
+	} else {
+		userSecrets = append(userSecrets, dc.Spec.SuperuserSecretName)
+	}
+	logger.Info(fmt.Sprintf("refreshing user secrets for %v", userSecrets))
+	//  Both Reaper and medusa secrets go into the userSecrets, so they don't need special handling.
+	requeues := 0
+	for _, i := range userSecrets {
+		secret := &corev1.Secret{}
+		// We need to do our own retries here instead of delegating it back up to the reconciler, because of
+		// the nature (time based) of the annotation we're adding. Otherwise we never complete because the
+		// object on the server never matches the desired object with the new time.
+		err := client.Get(ctx, types.NamespacedName{Name: i, Namespace: dc.Namespace}, secret)
+
+		if err != nil {
+			logger.Error(err, fmt.Sprintf("Failed to get secret %s", i))
+			return result.Error(err)
+		}
+		if secret.ObjectMeta.Annotations == nil {
+			secret.ObjectMeta.Annotations = make(map[string]string)
+		}
+		secret.ObjectMeta.Annotations[k8ssandraapi.RefreshAnnotation] = restoreTimestamp.String()
+		recRes := reconciliation.ReconcileObject(ctx, client, requeueDelay, *secret)
+		switch {
+		case recRes.IsError():
+			return recRes
+		case recRes.IsRequeue():
+			requeues++
+			continue
+		case recRes.IsDone():
+			continue
+		}
+		if requeues > 0 {
+			return result.RequeueSoon(requeueDelay)
+		}
+	}
+	return result.Done()
+}

pkg/medusa/refresh_secrets_test.go

@@ -0,0 +1,72 @@
+package medusa
+
+import (
+	"context"
+	"testing"
+
+	"github.com/go-logr/logr"
+	cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1"
+	"github.com/k8ssandra/k8ssandra-operator/pkg/test"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func TestRefreshSecrets_defaultSUSecret(t *testing.T) {
+	fakeClient := test.NewFakeClientWRestMapper()
+	cassDC := test.NewCassandraDatacenter("dc1", "test")
+	cassDC.Spec.Users = []cassdcapi.CassandraUser{
+		{SecretName: "custom-user"},
+	}
+	assert.NoError(t, fakeClient.Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test"}}))
+	assert.NoError(t, fakeClient.Create(context.Background(), &cassDC))
+	secrets := []corev1.Secret{
+		{ObjectMeta: metav1.ObjectMeta{Name: "custom-user", Namespace: "test"}, Data: map[string][]byte{"username": []byte("test")}},
+		{ObjectMeta: metav1.ObjectMeta{Name: "test-cluster-superuser", Namespace: "test"}, Data: map[string][]byte{"username": []byte("test")}},
+	}
+	for _, i := range secrets {
+		assert.NoError(t, fakeClient.Create(context.Background(), &i))
+	}
+
+	recRes := RefreshSecrets(&cassDC, context.Background(), fakeClient, logr.Logger{}, 0, metav1.Now())
+	assert.True(t, recRes.IsDone())
+	suSecret := &corev1.Secret{}
+	assert.NoError(t, fakeClient.Get(context.Background(), types.NamespacedName{Name: "test-cluster-superuser", Namespace: "test"}, suSecret))
+	_, exists := suSecret.ObjectMeta.Annotations["k8ssandra.io/refresh"]
+	assert.True(t, exists)
+	userSecret := &corev1.Secret{}
+	assert.NoError(t, fakeClient.Get(context.Background(), types.NamespacedName{Name: "custom-user", Namespace: "test"}, userSecret))
+	_, exists = userSecret.ObjectMeta.Annotations["k8ssandra.io/refresh"]
+	assert.True(t, exists)
+}
+
+func TestRefreshSecrets_customSecrets(t *testing.T) {
+	fakeClient := test.NewFakeClientWRestMapper()
+	cassDC := test.NewCassandraDatacenter("dc1", "test")
+	cassDC.Spec.Users = []cassdcapi.CassandraUser{
+		{SecretName: "custom-user"},
+	}
+	cassDC.Spec.SuperuserSecretName = "cass-custom-superuser"
+	assert.NoError(t, fakeClient.Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test"}}))
+	assert.NoError(t, fakeClient.Create(context.Background(), &cassDC))
+	secrets := []corev1.Secret{
+		{ObjectMeta: metav1.ObjectMeta{Name: "custom-user", Namespace: "test"}},
+		{ObjectMeta: metav1.ObjectMeta{Name: "cass-custom-superuser", Namespace: "test"}},
+	}
+	for _, i := range secrets {
+		assert.NoError(t, fakeClient.Create(context.Background(), &i))
+	}
+
+	recRes := RefreshSecrets(&cassDC, context.Background(), fakeClient, logr.Logger{}, 0, metav1.Now())
+	assert.True(t, recRes.IsDone())
+	suSecret := &corev1.Secret{}
+	assert.NoError(t, fakeClient.Get(context.Background(), types.NamespacedName{Name: "cass-custom-superuser", Namespace: "test"}, suSecret))
+	_, exists := suSecret.ObjectMeta.Annotations["k8ssandra.io/refresh"]
+	assert.True(t, exists)
+	userSecret := &corev1.Secret{}
+	assert.NoError(t, fakeClient.Get(context.Background(), types.NamespacedName{Name: "custom-user", Namespace: "test"}, userSecret))
+	_, exists = userSecret.ObjectMeta.Annotations["k8ssandra.io/refresh"]
+	assert.True(t, exists)
+
+}

test/e2e/medusa_test.go

@@ -7,6 +7,7 @@ import (
 
 	cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1"
 	api "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1"
+	k8ssandraapi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1"
 	medusa "github.com/k8ssandra/k8ssandra-operator/apis/medusa/v1alpha1"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/cassandra"
 	medusapkg "github.com/k8ssandra/k8ssandra-operator/pkg/medusa"
@@ -211,6 +212,28 @@ func verifyRestoreJobFinished(t *testing.T, ctx context.Context, f *framework.E2
 
 		return !restore.Status.FinishTime.IsZero()
 	}, polling.medusaRestoreDone.timeout, polling.medusaRestoreDone.interval, "restore didn't finish within timeout")
+
+	require.Eventually(func() bool {
+		dc := &cassdcapi.CassandraDatacenter{}
+		err := f.Get(ctx, dcKey, dc)
+		if err != nil {
+			t.Log(err)
+			return false
+		}
+		superUserSecret := dc.Spec.SuperuserSecretName
+		if dc.Spec.SuperuserSecretName == "" {
+			superUserSecret = cassdcapi.CleanupForKubernetes(dcKey.Name) + "-superuser"
+		}
+		secret := &corev1.Secret{}
+		err = f.Get(ctx, framework.NewClusterKey(restoreClusterKey.K8sContext, restoreClusterKey.Namespace, superUserSecret), secret)
+		if err != nil {
+			t.Log(err)
+			return false
+		}
+		_, exists := secret.Annotations[k8ssandraapi.RefreshAnnotation]
+		return exists
+	}, polling.medusaRestoreDone.timeout, polling.medusaRestoreDone.interval, "superuser secret wasn't updated with refresh annotation")
+
 }
 
 func checkMedusaStandaloneDeploymentExists(t *testing.T, ctx context.Context, dcKey framework.ClusterKey, f *framework.E2eFramework, kc *api.K8ssandraCluster) {