From eb40485eb9261ec134dfa6e8d4ce709bed695304 Mon Sep 17 00:00:00 2001 From: Mario Ranftl Date: Tue, 7 Jan 2025 18:06:25 +0100 Subject: [PATCH] adds envx, switch to backup-ns.sh/weekly label to w04 only (without YYYY), snap labels, simulate test label generation --- Dockerfile | 6 +- README.md | 2 +- cmd/create.go | 2 +- cmd/delete.go | 2 +- go.mod | 2 +- internal/lib/init_test.go | 72 +++++++++ internal/lib/mysql_test.go | 3 +- internal/lib/postgres_test.go | 3 +- internal/lib/vs.go | 23 +-- internal/lib/vs_test.go | 144 +++++++++++++++++- internal/lib/vsc_test.go | 3 +- reference/lib/vs.sh | 4 +- .../TestGenerateVSLabelsRetainDaysJSON.golden | 8 + ...tGenerateVSLabelsRetainScheduleJSON.golden | 9 ++ 14 files changed, 255 insertions(+), 28 deletions(-) create mode 100644 test/testdata/snapshots/TestGenerateVSLabelsRetainDaysJSON.golden create mode 100644 test/testdata/snapshots/TestGenerateVSLabelsRetainScheduleJSON.golden diff --git a/Dockerfile b/Dockerfile index 025a8ec..d8ce5ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ -# which kubectl version to install (should be in sync with the kubernetes version used by GKE) # https://hub.docker.com/r/bitnami/kubectl/tags -FROM bitnami/kubectl:1.28 as kubectl +FROM bitnami/kubectl:1.29 as kubectl ### ----------------------- # --- Stage: development @@ -235,8 +234,9 @@ ENV PATH $PATH:$KREW_ROOT/bin # -> https://github.com/stern/stern # -> https://github.com/ahmetb/kubectx # -> https://github.com/patrickdappollonio/kubectl-slice +# -> https://github.com/majodev/kubectl-envx # hack: we chown to $USERNAME to avoid permission issues when running the container -RUN kubectl krew install stern ctx ns slice \ +RUN kubectl krew install stern ctx ns slice envx \ && chown -R $USERNAME $KREW_ROOT # typical k8s aliases/completions and other .bashrc modifications diff --git a/README.md b/README.md index 198c207..3d05b90 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ kubectl label vs/ "backup-ns.sh/monthly"- # Add a specific label daily/weekly/monthly kubectl label vs/ "backup-ns.sh/daily"="YYYY-MM-DD" -kubectl label vs/ "backup-ns.sh/weekly"="YYYY-w04" +kubectl label vs/ "backup-ns.sh/weekly"="w04" kubectl label vs/ "backup-ns.sh/monthly"="YYYY-MM" # Add a specific deleteAfter label (the pruner will delete the vs after the specified date)! diff --git a/cmd/create.go b/cmd/create.go index 57327a1..76a611c 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -78,7 +78,7 @@ func runCreate(_ *cobra.Command, _ []string) { runMySQLDump(config) } - vsLabels := lib.GenerateVSLabels(config.Namespace, config.PVCName, config.LabelVS) + vsLabels := lib.GenerateVSLabels(config.Namespace, config.PVCName, config.LabelVS, time.Now()) vsAnnotations := lib.GenerateVSAnnotations(lib.GetBAKEnvVars()) vsObject := lib.GenerateVSObject(config.Namespace, config.VSClassName, config.PVCName, vsName, vsLabels, vsAnnotations) diff --git a/cmd/delete.go b/cmd/delete.go index 8c17a36..0913266 100644 --- a/cmd/delete.go +++ b/cmd/delete.go @@ -40,7 +40,7 @@ func runDelete(_ *cobra.Command, args []string) { log.Printf("Using namespace '%s'.\n", namespace) - if err := lib.PruneVolumeSnapshot(namespace, volumeSnapshotName); err != nil { + if err := lib.PruneVolumeSnapshot(namespace, volumeSnapshotName, true); err != nil { log.Fatalf("Error deleting VolumeSnapshot: %v\n", err) } diff --git a/go.mod b/go.mod index ddc1dc0..51da7f8 100644 --- a/go.mod +++ b/go.mod @@ -7,12 +7,12 @@ require ( github.com/google/uuid v1.6.0 github.com/pmezard/go-difflib v1.0.0 github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 ) require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/internal/lib/init_test.go b/internal/lib/init_test.go index 9e888b1..67a3565 100644 --- a/internal/lib/init_test.go +++ b/internal/lib/init_test.go @@ -1,9 +1,13 @@ package lib_test import ( + "fmt" "log" "os/exec" "strings" + "sync" + + "github.com/allaboutapps/backup-ns/internal/lib" ) // Immediately error out if kubectl is currently not in the context of our kind test k8s cluster @@ -15,4 +19,72 @@ func init() { if context, _ := exec.Command("kubectl", "config", "current-context").Output(); !strings.Contains(string(context), "kind-backup-ns") { log.Fatalf("kubectl is not currently running within the context of the kind test k8s cluster named 'kind-backup-ns', exit now!") } + + cleanupTestVolumeSnapshots() +} + +func cleanupTestVolumeSnapshots() { + // Get all managed volume snapshots + vss, err := lib.GetManagedVolumeSnapshots() + if err != nil { + log.Fatalf("failed to get volume snapshots: %v", err) + } + + // Filter snapshots in test namespaces + testNamespaces := map[string]bool{ + "generic-test": true, + "postgres-test": true, + "mysql-test": true, + } + + var filteredVss []lib.NamespacedK8sObject + for _, vs := range vss { + if testNamespaces[vs.Namespace] { + filteredVss = append(filteredVss, vs) + } + } + + log.Printf("del %d test volume snapshots...\n", len(filteredVss)) + + const maxWorkers = 16 + var wg sync.WaitGroup + jobs := make(chan lib.NamespacedK8sObject, len(filteredVss)) + errors := make(chan error, len(filteredVss)) + + // Start workers + for w := 0; w < maxWorkers; w++ { + go func() { + for vs := range jobs { + if err := lib.PruneVolumeSnapshot(vs.Namespace, vs.Name, false); err != nil { + errors <- fmt.Errorf("failed to prune %s/%s: %w", vs.Namespace, vs.Name, err) + } + wg.Done() + } + }() + } + + // Queue jobs + for _, vs := range filteredVss { + wg.Add(1) + jobs <- vs + } + close(jobs) + + // Wait for completion and collect errors + go func() { + wg.Wait() + close(errors) + }() + + // Check for errors + var errs []error + for err := range errors { + errs = append(errs, err) + } + + if len(errs) > 0 { + log.Fatalf("errors during volume snapshot cleanup: %v", errs) + } + + log.Printf("deleted %d test volume snapshots\n", len(filteredVss)) } diff --git a/internal/lib/mysql_test.go b/internal/lib/mysql_test.go index e385123..76ea31b 100644 --- a/internal/lib/mysql_test.go +++ b/internal/lib/mysql_test.go @@ -5,6 +5,7 @@ import ( "os/exec" "path/filepath" "testing" + "time" "github.com/allaboutapps/backup-ns/internal/lib" ) @@ -53,7 +54,7 @@ func TestDumpAndRestoreMySQL(t *testing.T) { t.Fatal("backup MySQL failed: ", err) } - vsLabels := lib.GenerateVSLabels(namespace, "data", labelVSConfig) + vsLabels := lib.GenerateVSLabels(namespace, "data", labelVSConfig, time.Now()) vsAnnotations := lib.GenerateVSAnnotations(lib.GetBAKEnvVars()) vsObject := lib.GenerateVSObject(namespace, "csi-hostpath-snapclass", "data", vsName, vsLabels, vsAnnotations) diff --git a/internal/lib/postgres_test.go b/internal/lib/postgres_test.go index dfe3a36..72ad916 100644 --- a/internal/lib/postgres_test.go +++ b/internal/lib/postgres_test.go @@ -5,6 +5,7 @@ import ( "os/exec" "path/filepath" "testing" + "time" "github.com/allaboutapps/backup-ns/internal/lib" "github.com/allaboutapps/backup-ns/internal/test" @@ -53,7 +54,7 @@ func TestDumpAndRestorePostgres(t *testing.T) { t.Fatal("backup Postgres failed: ", err) } - vsLabels := lib.GenerateVSLabels(namespace, "data", labelVSConfig) + vsLabels := lib.GenerateVSLabels(namespace, "data", labelVSConfig, time.Now()) vsAnnotations := lib.GenerateVSAnnotations(map[string]string{ "BAK_NAMESPACE": namespace, "BAK_DB_POSTGRES": "true", diff --git a/internal/lib/vs.go b/internal/lib/vs.go index 71e4e2c..7a18b81 100644 --- a/internal/lib/vs.go +++ b/internal/lib/vs.go @@ -70,7 +70,7 @@ func GenerateVSName(vsNameTemplate string, pvcName string, vsRand string) (strin return buf.String(), nil } -func GenerateVSLabels(namespace, pvcName string, config LabelVSConfig) map[string]string { +func GenerateVSLabels(namespace, pvcName string, config LabelVSConfig, now time.Time) map[string]string { labels := map[string]string{ "backup-ns.sh/pvc": pvcName, "backup-ns.sh/type": config.Type, @@ -79,13 +79,13 @@ func GenerateVSLabels(namespace, pvcName string, config LabelVSConfig) map[strin labels["backup-ns.sh/pod"] = config.Pod } if config.Retain == "daily_weekly_monthly" { - now := time.Now() + now := now labels["backup-ns.sh/retain"] = "daily_weekly_monthly" dailyLabel := now.Format("2006-01-02") - _, week := time.Now().ISOWeek() - weeklyLabel := now.Format("2006-") + fmt.Sprintf("w%02d", week) + _, week := now.ISOWeek() + weeklyLabel := fmt.Sprintf("w%02d", week) monthlyLabel := now.Format("2006-01") if !volumeSnapshotWithLabelValueExists(namespace, "backup-ns.sh/daily", dailyLabel) { @@ -98,7 +98,7 @@ func GenerateVSLabels(namespace, pvcName string, config LabelVSConfig) map[strin labels["backup-ns.sh/monthly"] = monthlyLabel } } else if config.Retain == "days" { - deleteAfter := time.Now().AddDate(0, 0, config.RetainDays).Format("2006-01-02") + deleteAfter := now.AddDate(0, 0, config.RetainDays).Format("2006-01-02") labels["backup-ns.sh/retain"] = "days" labels["backup-ns.sh/retain-days"] = strconv.Itoa(config.RetainDays) labels["backup-ns.sh/delete-after"] = deleteAfter @@ -296,8 +296,13 @@ func CreateVolumeSnapshot(namespace string, dryRun bool, vsName string, vsObject return nil } -func deleteVolumeSnapshot(namespace, volumeSnapshotName string) error { - deleteCmd := exec.Command("kubectl", "delete", "volumesnapshot", volumeSnapshotName, "-n", namespace) +func deleteVolumeSnapshot(namespace, volumeSnapshotName string, wait bool) error { + args := []string{"delete", "volumesnapshot", volumeSnapshotName, "-n", namespace} + if !wait { + args = append(args, "--wait=false") + } + + deleteCmd := exec.Command("kubectl", args...) output, err := deleteCmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to delete VolumeSnapshot: %w, output: %s", err, output) @@ -309,7 +314,7 @@ func deleteVolumeSnapshot(namespace, volumeSnapshotName string) error { // Delete a VolumeSnapshot, its associated VolumeSnapshotContent and the underlying storage! // This is a destructive operation and should be used with caution! // This function will set the deletionPolicy of the VolumeSnapshotContent to "Delete" before deleting the VolumeSnapshot, thus ensuring the underlying storage is also deleted. -func PruneVolumeSnapshot(namespace, volumeSnapshotName string) error { +func PruneVolumeSnapshot(namespace, volumeSnapshotName string, wait bool) error { // Get the VolumeSnapshotContent name vscName, err := GetVolumeSnapshotContentName(namespace, volumeSnapshotName) if err != nil { @@ -322,7 +327,7 @@ func PruneVolumeSnapshot(namespace, volumeSnapshotName string) error { } // Delete the VolumeSnapshot - if err := deleteVolumeSnapshot(namespace, volumeSnapshotName); err != nil { + if err := deleteVolumeSnapshot(namespace, volumeSnapshotName, wait); err != nil { return err } diff --git a/internal/lib/vs_test.go b/internal/lib/vs_test.go index e0c921b..838d56d 100644 --- a/internal/lib/vs_test.go +++ b/internal/lib/vs_test.go @@ -1,11 +1,14 @@ package lib_test import ( + "encoding/json" "fmt" "os/exec" "testing" + "time" "github.com/allaboutapps/backup-ns/internal/lib" + "github.com/allaboutapps/backup-ns/internal/test" "github.com/stretchr/testify/require" ) @@ -14,12 +17,13 @@ func TestVolumeCreateAndDelete(t *testing.T) { namespace := "generic-test" labelVSConfig := lib.LabelVSConfig{ - Type: "adhoc", - Pod: "gotest", - Retain: "daily_weekly_monthly", + Type: "adhoc", + Pod: "gotest", + Retain: "days", + RetainDays: 30, } - vsLabels := lib.GenerateVSLabels(namespace, "data", labelVSConfig) + vsLabels := lib.GenerateVSLabels(namespace, "data", labelVSConfig, time.Now()) vsAnnotations := lib.GenerateVSAnnotations(lib.GetBAKEnvVars()) vsObject := lib.GenerateVSObject(namespace, "csi-hostpath-snapclass", "data", vsName, vsLabels, vsAnnotations) @@ -34,14 +38,41 @@ func TestVolumeCreateAndDelete(t *testing.T) { t.Fatal("get vs failed: ", err, string(output)) } - err = lib.PruneVolumeSnapshot(namespace, vsName) + err = lib.PruneVolumeSnapshot(namespace, vsName, false) if err != nil { t.Fatal("delete vs failed: ", err) } } -func TestVolumeCreateFailsNameSpace(t *testing.T) { +func TestGenerateVSLabelsRetainSchedule(t *testing.T) { + + labelVSConfig := lib.LabelVSConfig{ + Type: "cronjob", + Pod: "gotest", + Retain: "daily_weekly_monthly", + } + + vsLabels := lib.GenerateVSLabels("generic-test", "data", labelVSConfig, time.Date(2022, 5, 21, 0, 17, 0, 0, time.Local)) + + test.Snapshoter.SaveJSON(t, vsLabels) +} + +func TestGenerateVSLabelsRetainDays(t *testing.T) { + + labelVSConfig := lib.LabelVSConfig{ + Type: "cronjob", + Pod: "gotest", + Retain: "days", + RetainDays: 30, + } + + vsLabels := lib.GenerateVSLabels("generic-test", "data", labelVSConfig, time.Date(2022, 5, 21, 0, 17, 0, 0, time.Local)) + + test.Snapshoter.SaveJSON(t, vsLabels) +} + +func TestVolumeCreateFailsNamespace(t *testing.T) { vsName := fmt.Sprintf("test-backup-generic-%s", lib.GenerateRandomStringOrPanic(6)) namespace := "non-existant-namespace" // !!! @@ -51,10 +82,109 @@ func TestVolumeCreateFailsNameSpace(t *testing.T) { Retain: "daily_weekly_monthly", } - vsLabels := lib.GenerateVSLabels(namespace, "data", labelVSConfig) + vsLabels := lib.GenerateVSLabels(namespace, "data", labelVSConfig, time.Now()) vsAnnotations := lib.GenerateVSAnnotations(lib.GetBAKEnvVars()) vsObject := lib.GenerateVSObject(namespace, "csi-hostpath-snapclass", "data", vsName, vsLabels, vsAnnotations) require.Error(t, lib.CreateVolumeSnapshot(namespace, false, vsName, vsObject, true, "25s")) } + +func TestVolumeCreateSimulatedForWeek(t *testing.T) { + namespace := "generic-test" + pvcName := "data" + + testCases := []struct { + date time.Time + expectedLabels map[string]string + }{ + { + date: time.Date(2024, 12, 30, 10, 0, 0, 0, time.UTC), + expectedLabels: map[string]string{ + "backup-ns.sh/monthly": "2024-12", + "backup-ns.sh/daily": "2024-12-30", + "backup-ns.sh/weekly": "w01", + }, + }, + { + date: time.Date(2024, 12, 31, 10, 0, 0, 0, time.UTC), + expectedLabels: map[string]string{ + "backup-ns.sh/daily": "2024-12-31", + }, + }, + { + date: time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC), + expectedLabels: map[string]string{ + "backup-ns.sh/daily": "2025-01-01", + "backup-ns.sh/monthly": "2025-01", + }, + }, + { + date: time.Date(2025, 1, 2, 10, 0, 0, 0, time.UTC), + expectedLabels: map[string]string{ + "backup-ns.sh/daily": "2025-01-02", + }, + }, + { + date: time.Date(2025, 1, 5, 10, 0, 0, 0, time.UTC), + expectedLabels: map[string]string{ + "backup-ns.sh/daily": "2025-01-05", + }, + }, + { + date: time.Date(2025, 1, 6, 10, 0, 0, 0, time.UTC), + expectedLabels: map[string]string{ + "backup-ns.sh/daily": "2025-01-06", + "backup-ns.sh/weekly": "w02", + }, + }, + } + + labelVSConfig := lib.LabelVSConfig{ + Type: "cronjob", + Pod: "gotest", + Retain: "daily_weekly_monthly", + } + + for _, tc := range testCases { + t.Run(tc.date.Format("2006-01-02"), func(t *testing.T) { + // Generate unique snapshot name + vsName := fmt.Sprintf("test-backup-%s-%s", tc.date.Format("20060102"), lib.GenerateRandomStringOrPanic(6)) + + // Generate labels and create snapshot + labels := lib.GenerateVSLabels(namespace, pvcName, labelVSConfig, tc.date) + vsAnnotations := lib.GenerateVSAnnotations(lib.GetBAKEnvVars()) + vsObject := lib.GenerateVSObject(namespace, "csi-hostpath-snapclass", pvcName, vsName, labels, vsAnnotations) + + // Create the snapshot + err := lib.CreateVolumeSnapshot(namespace, false, vsName, vsObject, false, "0s") + require.NoError(t, err) + + // Get snapshot and verify labels + cmd := exec.Command("kubectl", "get", "volumesnapshot", vsName, "-n", namespace, "-o", "json") + output, err := cmd.CombinedOutput() + require.NoError(t, err) + + var vs map[string]interface{} + err = json.Unmarshal(output, &vs) + require.NoError(t, err) + + metadata := vs["metadata"].(map[string]interface{}) + actualLabels := metadata["labels"].(map[string]interface{}) + + // Verify expected labels exist + for key, expectedValue := range tc.expectedLabels { + require.Equal(t, expectedValue, actualLabels[key], "Label %s mismatch", key) + } + + // Verify other retention labels don't exist + allPossibleLabels := []string{"backup-ns.sh/daily", "backup-ns.sh/weekly", "backup-ns.sh/monthly"} + for _, label := range allPossibleLabels { + if _, expected := tc.expectedLabels[label]; !expected { + _, exists := actualLabels[label] + require.False(t, exists, "Label %s should not exist", label) + } + } + }) + } +} diff --git a/internal/lib/vsc_test.go b/internal/lib/vsc_test.go index bade210..a0830ce 100644 --- a/internal/lib/vsc_test.go +++ b/internal/lib/vsc_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os/exec" "testing" + "time" "github.com/allaboutapps/backup-ns/internal/lib" "github.com/allaboutapps/backup-ns/internal/test" @@ -21,7 +22,7 @@ func createTestVS(t *testing.T) (string /* namespace*/, string /* vsName */) { RetainDays: 0, } - vsLabels := lib.GenerateVSLabels(namespace, "data", labelVSConfig) + vsLabels := lib.GenerateVSLabels(namespace, "data", labelVSConfig, time.Now()) vsAnnotations := lib.GenerateVSAnnotations(lib.GetBAKEnvVars()) vsObject := lib.GenerateVSObject(namespace, "csi-hostpath-snapclass", "data", vsName, vsLabels, vsAnnotations) diff --git a/reference/lib/vs.sh b/reference/lib/vs.sh index 15ce450..c1ba594 100644 --- a/reference/lib/vs.sh +++ b/reference/lib/vs.sh @@ -35,7 +35,7 @@ backup-ns.sh/pod: \"${pod}\"" # The following labels are used: # backup-ns.sh/retain: "daily_weekly_monthly" # backup-ns.sh/monthly: e.g. "2024-04" -# backup-ns.sh/weekly: e.g. "2024-w15" # ISO: Monday as first day of week, see https://unix.stackexchange.com/questions/282609/how-to-use-the-date-command-to-display-week-number-of-the-year +# backup-ns.sh/weekly: e.g. "w15" # ISO: Monday as first day of week, see https://unix.stackexchange.com/questions/282609/how-to-use-the-date-command-to-display-week-number-of-the-year # backup-ns.sh/daily: e.g. "2024-04-11" #### # backup-ns.sh/hourly: e.g. "2024-04-0900" # disabled for now, as it is not really useful for most use-cases # @@ -53,7 +53,7 @@ vs_get_retain_labels_daily_weekly_monthly() { # hourly_label=$(date +"%Y-%m-%d-%H00") local daily_label; daily_label=$(date +"%Y-%m-%d") - local weekly_label; weekly_label=$(date +"%Y-w%0V") + local weekly_label; weekly_label=$(date +"w%0V") local monthly_label; monthly_label=$(date +"%Y-%m") local labels="" diff --git a/test/testdata/snapshots/TestGenerateVSLabelsRetainDaysJSON.golden b/test/testdata/snapshots/TestGenerateVSLabelsRetainDaysJSON.golden new file mode 100644 index 0000000..d0743e6 --- /dev/null +++ b/test/testdata/snapshots/TestGenerateVSLabelsRetainDaysJSON.golden @@ -0,0 +1,8 @@ +{ + "backup-ns.sh/delete-after": "2022-06-20", + "backup-ns.sh/pod": "gotest", + "backup-ns.sh/pvc": "data", + "backup-ns.sh/retain": "days", + "backup-ns.sh/retain-days": "30", + "backup-ns.sh/type": "cronjob" +} \ No newline at end of file diff --git a/test/testdata/snapshots/TestGenerateVSLabelsRetainScheduleJSON.golden b/test/testdata/snapshots/TestGenerateVSLabelsRetainScheduleJSON.golden new file mode 100644 index 0000000..e8278ba --- /dev/null +++ b/test/testdata/snapshots/TestGenerateVSLabelsRetainScheduleJSON.golden @@ -0,0 +1,9 @@ +{ + "backup-ns.sh/daily": "2022-05-21", + "backup-ns.sh/monthly": "2022-05", + "backup-ns.sh/pod": "gotest", + "backup-ns.sh/pvc": "data", + "backup-ns.sh/retain": "daily_weekly_monthly", + "backup-ns.sh/type": "cronjob", + "backup-ns.sh/weekly": "w20" +} \ No newline at end of file