dedicated mysql and postgres dump and restore commands

cmd/mysql_dump.go

@@ -23,6 +23,8 @@ var mysqlDumpCmd = &cobra.Command{
 		if !config.MySQL.Enabled {
 			log.Fatal("BAK_DB_MYSQL=true must be set.")
 		}
+
+		runMySQLDump(config)
 	},
 }
 

cmd/mysql_restore.go

@@ -0,0 +1,47 @@
+package cmd
+
+import (
+	"log"
+
+	"github.com/allaboutapps/backup-ns/internal/lib"
+	"github.com/spf13/cobra"
+)
+
+// mysqlRestoreCmd represents the restore command
+var mysqlRestoreCmd = &cobra.Command{
+	Use:   "restore",
+	Short: "Connects to the live mysql/mariadb container and creates a database restore",
+	// Long: `...`,
+	Run: func(_ *cobra.Command, _ []string) {
+		config := lib.LoadConfig()
+
+		if config.DryRun {
+			log.Println("Dry run mode is active, write operations are skipped!")
+		}
+
+		if !config.MySQL.Enabled {
+			log.Fatal("BAK_DB_MYSQL=true must be set.")
+		}
+
+		runMySQLRestore(config)
+	},
+}
+
+func init() {
+	mysqlCmd.AddCommand(mysqlRestoreCmd)
+}
+
+func runMySQLRestore(config lib.Config) {
+	if err := lib.EnsureResourceAvailable(config.Namespace, config.MySQL.ExecResource); err != nil {
+		log.Fatal(err)
+	}
+	if err := lib.EnsureMySQLAvailable(config.Namespace, config.MySQL); err != nil {
+		log.Fatal(err)
+	}
+
+	if err := lib.RestoreMySQL(config.Namespace, config.DryRun, config.MySQL); err != nil {
+		log.Fatal(err)
+	}
+
+	log.Printf("Finished mysql restore in namespace='%s'!", config.Namespace)
+}

cmd/postgres_restore.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"log"
+
+	"github.com/allaboutapps/backup-ns/internal/lib"
+	"github.com/spf13/cobra"
+)
+
+// postgresRestoreCmd represents the dump command
+var postgresRestoreCmd = &cobra.Command{
+	Use:   "restore",
+	Short: "Connects to the live postgres container and restores the database dump",
+	// Long: `...`,
+	Run: func(_ *cobra.Command, _ []string) {
+		config := lib.LoadConfig()
+
+		if config.DryRun {
+			log.Println("Dry run mode is active, write operations are skipped!")
+		}
+
+		if !config.Postgres.Enabled {
+			log.Fatal("BAK_DB_POSTGRES=true must be set.")
+		}
+
+		runPostgresRestore(config)
+	},
+}
+
+func init() {
+	postgresCmd.AddCommand(postgresRestoreCmd)
+}
+
+func runPostgresRestore(config lib.Config) {
+	if err := lib.EnsureResourceAvailable(config.Namespace, config.Postgres.ExecResource); err != nil {
+		log.Fatal(err)
+	}
+	if err := lib.EnsurePostgresAvailable(config.Namespace, config.Postgres); err != nil {
+		log.Fatal(err)
+	}
+
+	if err := lib.RestorePostgres(config.Namespace, config.DryRun, config.Postgres); err != nil {
+		log.Fatal(err)
+	}
+
+	log.Printf("Finished postgres restore in namespace='%s'!", config.Namespace)
+
+}

internal/lib/bak_env.go

@@ -41,20 +41,24 @@ type PostgresConfig struct {
 	ExecResource  string `json:"BAK_DB_POSTGRES_EXEC_RESOURCE"`
 	ExecContainer string `json:"BAK_DB_POSTGRES_EXEC_CONTAINER"`
 	DumpFile      string `json:"BAK_DB_POSTGRES_DUMP_FILE"`
+	Host          string `json:"BAK_DB_POSTGRES_HOST"`
+	Port          string `json:"BAK_DB_POSTGRES_PORT"`
 	User          string `json:"BAK_DB_POSTGRES_USER"`
 	Password      string `json:"-"` // sensitive
 	DB            string `json:"BAK_DB_POSTGRES_DB"`
 }
 
 type MySQLConfig struct {
-	Enabled       bool   `json:"BAK_DB_MYSQL"`
-	ExecResource  string `json:"BAK_DB_MYSQL_EXEC_RESOURCE"`
-	ExecContainer string `json:"BAK_DB_MYSQL_EXEC_CONTAINER"`
-	DumpFile      string `json:"BAK_DB_MYSQL_DUMP_FILE"`
-	Host          string `json:"BAK_DB_MYSQL_HOST"`
-	User          string `json:"BAK_DB_MYSQL_USER"`
-	Password      string `json:"-"` // sensitive
-	DB            string `json:"BAK_DB_MYSQL_DB"`
+	Enabled             bool   `json:"BAK_DB_MYSQL"`
+	ExecResource        string `json:"BAK_DB_MYSQL_EXEC_RESOURCE"`
+	ExecContainer       string `json:"BAK_DB_MYSQL_EXEC_CONTAINER"`
+	DumpFile            string `json:"BAK_DB_MYSQL_DUMP_FILE"`
+	Host                string `json:"BAK_DB_MYSQL_HOST"`
+	Port                string `json:"BAK_DB_MYSQL_PORT"`
+	User                string `json:"BAK_DB_MYSQL_USER"`
+	Password            string `json:"-"` // sensitive
+	DB                  string `json:"BAK_DB_MYSQL_DB"`
+	DefaultCharacterSet string `json:"BAK_DB_MYSQL_DEFAULT_CHARACTER_SET"`
 }
 
 type FlockConfig struct {
@@ -126,6 +130,12 @@ func LoadConfig() Config {
 			// The file inside the container to store the dump
 			DumpFile: util.GetEnv("BAK_DB_POSTGRES_DUMP_FILE", "/var/lib/postgresql/data/dump.sql.gz"),
 
+			// The postgresql host to use for connecting/creating/restoring the dump
+			Host: util.GetEnv("BAK_DB_POSTGRES_HOST", "127.0.0.1"),
+
+			// The postgresql host to use for connecting/creating/restoring the dump
+			Port: util.GetEnv("BAK_DB_POSTGRES_PORT", "5432"),
+
 			// The postgresql user to use for connecting/creating the dump (psql and pg_dump must be allowed)
 			// Read from inside the *container* by default (${POSTGRES_USER})
 			User: util.GetEnv("BAK_DB_POSTGRES_USER", "${POSTGRES_USER}"),
@@ -152,9 +162,12 @@ func LoadConfig() Config {
 			// The file inside the container to store the dump
 			DumpFile: util.GetEnv("BAK_DB_MYSQL_DUMP_FILE", "/var/lib/mysql/dump.sql.gz"),
 
-			// The mysql host to use for connecting/creating the dump
+			// The mysql host to use for connecting/creating/restoring the dump
 			Host: util.GetEnv("BAK_DB_MYSQL_HOST", "127.0.0.1"),
 
+			// The mysql host to use for connecting/creating/restoring the dump
+			Port: util.GetEnv("BAK_DB_MYSQL_PORT", "3306"),
+
 			// The mysql user to use for connecting/creating the dump
 			User: util.GetEnv("BAK_DB_MYSQL_USER", "root"),
 
@@ -165,6 +178,10 @@ func LoadConfig() Config {
 			// The mysql database to use for connecting/creating the dump
 			// Read from inside the *container* by default (${MYSQL_DATABASE})
 			DB: util.GetEnv("BAK_DB_MYSQL_DB", "${MYSQL_DATABASE}"),
+
+			// The mysql character set to use for connecting/creating the dump
+			// utf8 is by default active for backwards compatibility
+			DefaultCharacterSet: util.GetEnv("BAK_DB_MYSQL_DEFAULT_CHARACTER_SET", "utf8"),
 		},
 
 		Flock: FlockConfig{

internal/lib/mysql.go

@@ -14,6 +14,9 @@ var mysqlCheckScript string
 //go:embed templates/mysql_dump.sh.tmpl
 var mysqlDumpScript string
 
+//go:embed templates/mysql_restore.sh.tmpl
+var mysqlRestoreScript string
+
 func EnsureMySQLAvailable(namespace string, config MySQLConfig) error {
 	log.Printf("Checking if MySQL is available in namespace '%s'...", namespace)
 
@@ -49,3 +52,18 @@ func DumpMySQL(namespace string, dryRun bool, config MySQLConfig) error {
 
 	return KubectlExecTemplate(namespace, config.ExecResource, config.ExecContainer, tmpl, data)
 }
+
+func RestoreMySQL(namespace string, dryRun bool, config MySQLConfig) error {
+	if dryRun {
+		log.Println("Skipping MySQL restore - dry run mode is active")
+		return nil
+	}
+	log.Printf("Restoring MySQL database '%s' in namespace '%s'...", config.DB, namespace)
+
+	tmpl, err := template.New("mysql_backup").Parse(mysqlRestoreScript)
+	if err != nil {
+		return fmt.Errorf("failed to parse MySQL restore script template: %w", err)
+	}
+
+	return KubectlExecTemplate(namespace, config.ExecResource, config.ExecContainer, tmpl, config)
+}

internal/lib/mysql_test.go

@@ -9,19 +9,21 @@ import (
 	"github.com/allaboutapps/backup-ns/internal/lib"
 )
 
-func TestDumpMySQL(t *testing.T) {
+func TestDumpAndRestoreMySQL(t *testing.T) {
 	vsName := fmt.Sprintf("test-backup-mysql-%s", lib.GenerateRandomStringOrPanic(6))
 	namespace := "mysql-test"
 
 	mysqlConfig := lib.MySQLConfig{
-		Enabled:       true,
-		ExecResource:  "deployment/mysql",
-		ExecContainer: "mysql",
-		DumpFile:      "/var/lib/mysql/dump.sql.gz",
-		Host:          "127.0.0.1",
-		User:          "root",
-		Password:      "${MYSQL_ROOT_PASSWORD}",
-		DB:            "${MYSQL_DATABASE}",
+		Enabled:             true,
+		ExecResource:        "deployment/mysql",
+		ExecContainer:       "mysql",
+		DumpFile:            "/var/lib/mysql/dump.sql.gz",
+		Host:                "127.0.0.1",
+		Port:                "3306",
+		User:                "root",
+		Password:            "${MYSQL_ROOT_PASSWORD}",
+		DB:                  "${MYSQL_DATABASE}",
+		DefaultCharacterSet: "utf8",
 	}
 
 	labelVSConfig := lib.LabelVSConfig{
@@ -65,4 +67,8 @@ func TestDumpMySQL(t *testing.T) {
 	if err != nil {
 		t.Fatal("get vs failed: ", err, string(output))
 	}
+
+	if err := lib.RestoreMySQL(namespace, false, mysqlConfig); err != nil {
+		t.Fatal("restore Postgres failed: ", err)
+	}
 }

internal/lib/postgres.go

@@ -14,6 +14,9 @@ var postgresCheckScript string
 //go:embed templates/postgres_dump.sh.tmpl
 var postgresDumpScript string
 
+//go:embed templates/postgres_restore.sh.tmpl
+var postgresRestoreScript string
+
 func EnsurePostgresAvailable(namespace string, config PostgresConfig) error {
 	log.Printf("Checking if Postgres is available in namespace '%s'...", namespace)
 
@@ -49,3 +52,18 @@ func DumpPostgres(namespace string, dryRun bool, config PostgresConfig) error {
 
 	return KubectlExecTemplate(namespace, config.ExecResource, config.ExecContainer, tmpl, data)
 }
+
+func RestorePostgres(namespace string, dryRun bool, config PostgresConfig) error {
+	if dryRun {
+		log.Println("Skipping Postgres restore - dry run mode is active")
+		return nil
+	}
+	log.Printf("Restoring Postgres database '%s' in namespace '%s'...", config.DB, namespace)
+
+	tmpl, err := template.New("postgres_restore").Parse(postgresRestoreScript)
+	if err != nil {
+		return fmt.Errorf("failed to parse postgres restore script template: %w", err)
+	}
+
+	return KubectlExecTemplate(namespace, config.ExecResource, config.ExecContainer, tmpl, config)
+}

internal/lib/postgres_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/allaboutapps/backup-ns/internal/test"
 )
 
-func TestDumpPostgres(t *testing.T) {
+func TestDumpAndRestorePostgres(t *testing.T) {
 	vsName := fmt.Sprintf("test-backup-postgres-%s", lib.GenerateRandomStringOrPanic(6))
 	namespace := "postgres-test"
 
@@ -22,6 +22,8 @@ func TestDumpPostgres(t *testing.T) {
 		User:          "${POSTGRES_USER}",     // read inside container
 		Password:      "${POSTGRES_PASSWORD}", // read inside container
 		DB:            "${POSTGRES_DB}",       // read inside container
+		Host:          "127.0.0.1",
+		Port:          "5432",
 	}
 
 	labelVSConfig := lib.LabelVSConfig{
@@ -71,4 +73,9 @@ func TestDumpPostgres(t *testing.T) {
 	if err != nil {
 		t.Fatal("get vs failed: ", err, string(output))
 	}
+
+	if err := lib.RestorePostgres(namespace, false, postgresConfig); err != nil {
+		t.Fatal("restore Postgres failed: ", err)
+	}
+
 }

internal/lib/templates/mysql_check.sh.tmpl

@@ -13,7 +13,8 @@ mysqldump --version
 # check db is accessible (default password injected via above MYSQL_PWD)
 mysql \
     --host {{.Host}} \
+    --port {{.Port}} \
     --user {{.User}} \
-    --default-character-set=utf8 \
+    --default-character-set={{.DefaultCharacterSet}} \
     {{.DB}} \
     -e "SELECT 1;" >/dev/null

internal/lib/templates/mysql_dump.sh.tmpl

@@ -15,8 +15,9 @@ trap 'trap - SIGTERM && kill -- -$$' SIGTERM SIGPIPE
 # create dump and pipe to gzip archive (default password injected via above MYSQL_PWD)
 mysqldump \
     --host {{.Host}} \
+    --port {{.Port}} \
     --user {{.User}} \
-    --default-character-set=utf8 \
+    --default-character-set={{.DefaultCharacterSet}} \
     --add-locks \
     --set-charset \
     --compact \

internal/lib/templates/mysql_restore.sh.tmpl

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# inject default MYSQL_PWD into current env (before cmds are visible in logs)
+export MYSQL_PWD="{{.Password}}"
+
+set -Eeox pipefail
+
+# ensure the dump file exists...
+[ -s {{.DumpFile}} ] || exit 1
+
+# print dump file info
+ls -lha {{.DumpFile}}
+
+# restore from dump file
+gzip -dc {{.DumpFile}} | mysql \
+    --host={{.Host}} \
+    --port={{.Port}} \
+    --user={{.User}} \
+    --default-character-set={{.DefaultCharacterSet}} \
+    {{.DB}}

internal/lib/templates/postgres_check.sh.tmpl

@@ -11,4 +11,4 @@ psql --version
 pg_dump --version
 
 # check db is accessible
-psql --username={{.User}} {{.DB}} -c "SELECT 1;" >/dev/null
+psql --username={{.User}} {{.DB}} --host {{.Host}} --port {{.Port}} -c "SELECT 1;" >/dev/null

internal/lib/templates/postgres_dump.sh.tmpl

@@ -13,7 +13,7 @@ trap 'exit_code=$?; [ $exit_code -ne 0 ] && echo "TRAP!" && rm -f {{.DumpFile}}
 trap 'trap - SIGTERM && kill -- -$$' SIGTERM SIGPIPE
 
 # create dump and pipe to gzip archive
-pg_dump --username={{.User}} --format=p --clean --if-exists {{.DB}} | gzip -c > {{.DumpFile}}
+pg_dump --username={{.User}} --format=p --clean --if-exists {{.DB}} --host {{.Host}} --port {{.Port}} | gzip -c > {{.DumpFile}}
 
 # print dump file info
 ls -lha {{.DumpFile}}

internal/lib/templates/postgres_restore.sh.tmpl

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# inject default PGPASSWORD into current env (before cmds are visible in logs)
+export PGPASSWORD="{{.Password}}"
+
+set -Eeox pipefail
+
+# ensure the dump file exists...
+[ -s {{.DumpFile}} ] || exit 1
+
+# print dump file info
+ls -lha {{.DumpFile}}
+
+# restore from dump file
+gzip -dc {{.DumpFile}} | psql --host {{.Host}} --port {{.Port}} --username={{.User}} {{.DB}}

test/test_create.sh

@@ -12,4 +12,7 @@ BAK_VS_CLASS_NAME=csi-hostpath-snapclass BAK_DB_POSTGRES=true BAK_NAMESPACE=post
 BAK_VS_CLASS_NAME=csi-hostpath-snapclass BAK_DB_MYSQL=true BAK_NAMESPACE=mysql-test BAK_DB_MYSQL_EXEC_RESOURCE=deployment/mysql app create
 
 # BAK_DB_POSTGRES=true BAK_NAMESPACE=postgres-test BAK_DB_POSTGRES_EXEC_RESOURCE=deployment/postgres app postgres dump
-# BAK_DB_MYSQL=true BAK_NAMESPACE=mysql-test BAK_DB_MYSQL_EXEC_RESOURCE=deployment/mysql app mysql dump
+# BAK_DB_POSTGRES=true BAK_NAMESPACE=postgres-test BAK_DB_POSTGRES_EXEC_RESOURCE=deployment/postgres app postgres restore
+
+# BAK_DB_MYSQL=true BAK_NAMESPACE=mysql-test BAK_DB_MYSQL_EXEC_RESOURCE=deployment/mysql app mysql dump
+# BAK_DB_MYSQL=true BAK_NAMESPACE=mysql-test BAK_DB_MYSQL_EXEC_RESOURCE=deployment/mysql app mysql restore