feat: add --compress flag

Make rsync's `-z` flag configurable.

Also some CI improvements: dirtiness checks, auto-generation of USAGE.md.

Signed-off-by: Utku Ozdemir <[email protected]>

.github/workflows/build-rsync-image.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+        uses: actions/[email protected]
       - name: Set up QEMU
         uses: docker/[email protected]
       - name: Set up Docker Buildx

.github/workflows/build-sshd-image.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+        uses: actions/[email protected]
       - name: Set up QEMU
         uses: docker/[email protected]
       - name: Set up Docker Buildx

.github/workflows/build.yml

@@ -37,13 +37,18 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+        uses: actions/[email protected]
       - name: Install go
         uses: actions/[email protected]
         with:
           go-version-file: go.mod
-      - name: Ensure go.mod is already tidied
-        run: go mod tidy && git diff --no-patch --exit-code
+      - name: Install Task
+        uses: arduino/[email protected]
+        with:
+          # renovate: depName=go-task/task datasource=github-releases
+          version: 3.37.1
+      - name: Tidy go.mod
+        run: go mod tidy
       - name: Run linters
         uses: golangci/[email protected]
         with:
@@ -58,11 +63,15 @@ jobs:
           # renovate: depName=goreleaser/goreleaser datasource=github-releases
           version: v1.25.1
           args: build --snapshot --clean --single-target
+      - name: Update usage
+        run: task update-usage
+      - name: Check for dirty files
+        run: git diff --exit-code
   test:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+        uses: actions/[email protected]
       - name: Install go
         uses: actions/[email protected]
         with:

.github/workflows/release-rsync-image.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+        uses: actions/[email protected]
       - name: Set up QEMU
         uses: docker/[email protected]
       - name: Set up Docker Buildx

.github/workflows/release-sshd-image.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+        uses: actions/[email protected]
       - name: Set up QEMU
         uses: docker/[email protected]
       - name: Set up Docker Buildx

.github/workflows/release.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+        uses: actions/[email protected]
         with:
           # to be able to generate the full changelog:
           # https://github.com/goreleaser/goreleaser-action/issues/56#issuecomment-568718162

.goreleaser.yml

@@ -30,7 +30,6 @@ builds:
 
 archives:
   - id: pv-migrate-archive
-    rlcp: true
     # default name template except we use .Tag instead of .Version to keep the "v" prefix
     name_template: >-
       {{ .ProjectName }}_
@@ -107,14 +106,14 @@ docker_manifests:
 #  disable: true
 
 brews:
-  - tap:
+  - repository:
       owner: utkuozdemir
       name: homebrew-pv-migrate
       token: "{{ .Env.PRIVATE_ACCESS_TOKEN }}"
     commit_author:
       name: Utku Ozdemir
       email: [email protected]
-    folder: Formula
+    directory: Formula
     goarm: "7"
     homepage: https://github.com/utkuozdemir/pv-migrate
     description: Persistent volume migration plugin for Kubernetes
@@ -128,7 +127,7 @@ brews:
       fish_completion.install "completions/pv-migrate.fish"
 
 scoops:
-  - bucket:
+  - repository:
       owner: utkuozdemir
       name: scoop-pv-migrate
       token: "{{ .Env.PRIVATE_ACCESS_TOKEN }}"

README.md

@@ -71,7 +71,7 @@ the data from the old PVC to the new one with the desired StorageClass.
 
 ## Installation
 
-See [INSTALL.md](INSTALL.md) for various installation methods and shell completion configuration.
+See [INSTALL.md](INSTALL.md.gotmpl) for various installation methods and shell completion configuration.
 
 ## Usage
 

Taskfile.yml

@@ -12,6 +12,7 @@ tasks:
       - gofumpt -w -l .
       - golangci-lint run --fix ./...
       - shfmt -l -i 2 -ci -sr -w .
+
   lint:
     desc: lint code
     cmds:
@@ -24,6 +25,16 @@ tasks:
     cmds:
       - rm -rf {{.ROOT_DIR}}/dist/
 
+  update-usage:
+    desc: update usage
+    env:
+      USAGE:
+        sh: go run ./... --help
+    cmd: >-
+        docker run  -v {{.ROOT_DIR}}:/project -e USAGE
+        hairyhenderson/gomplate:stable
+        --file /project/USAGE.md.gotmpl --out /project/USAGE.md
+
   build:
     desc: build
     cmds:

USAGE.md

@@ -14,6 +14,7 @@ Available Commands:
   help        Help about any command
 
 Flags:
+      --compress                       compress data during migration ('-z' flag of rsync) (default true)
       --dest string                    destination PVC name
   -C, --dest-context string            context in the kubeconfig file of the destination PVC
   -d, --dest-delete-extraneous-files   delete extraneous files on the destination by using rsync's '--delete' flag

USAGE.md.gotmpl

@@ -0,0 +1,89 @@
+# Usage
+
+Root command:
+
+```
+{{ .Env.USAGE }}
+```
+
+The Kubernetes resources created by pv-migrate are sourced from a [Helm chart](helm/pv-migrate).
+
+You can pass raw values to the backing Helm chart
+using the `--helm-*` flags for further customization: container images,
+resources, serviceacccounts, additional annotations etc.
+
+## Strategies
+
+`pv-migrate` has multiple strategies implemented to carry out the migration operation. Those are the following:
+
+| Name    | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `mnt2`  | **Mount both** - Mounts both PVCs in a single pod and runs a regular rsync, without using SSH or the network. Only applicable if source and destination PVCs are in the same namespace and both can be mounted from a single pod.                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| `svc`   | **Service** - Runs rsync+ssh over a Kubernetes Service (`ClusterIP`). Only applicable when source and destination PVCs are in the same Kubernetes cluster.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| `lbsvc` | **Load Balancer Service** - Runs rsync+ssh over a Kubernetes Service of type `LoadBalancer`. Always applicable (will fail if `LoadBalancer` IP is not assigned for a long period).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| `local` | **Local Transfer** - Runs sshd on both source and destination, then uses a combination of `kubectl port-forward` logic and an SSH reverse proxy to tunnel all the traffic over the client device (the device which runs pv-migrate, e.g. your laptop). Requires `ssh` command to be available on the client device. <br/><br/>Note that this strategy is **experimental** (and not enabled by default), potentially can put heavy load on both apiservers and is not as resilient as others. It is recommended for small amounts of data and/or when the only access to both clusters seems to be through `kubectl` (e.g. for air-gapped clusters, on jump hosts etc.). |
+
+## Examples
+
+See the various examples below which copy the contents of the `old-pvc` into the `new-pvc`.
+
+### Example 1: In a single namespace (minimal example)
+
+```bash
+$ pv-migrate --source old-pvc --dest new-pvc
+```
+
+### Example 2: Between namespaces
+
+```bash
+$ pv-migrate \
+  --source-namespace source-ns --source old-pvc \
+  --dest-namespace dest-ns --dest new-pvc
+```
+
+### Example 3: Between different clusters
+
+```bash
+pv-migrate \
+  --source-kubeconfig /path/to/source/kubeconfig \
+  --source-context some-context \
+  --source-namespace source-ns \
+  --source old-pvc \
+  --dest-kubeconfig /path/to/dest/kubeconfig \
+  --dest-context some-other-context \
+  --dest-namespace dest-ns \
+  --dest-delete-extraneous-files \
+  --dest new-pvc
+```
+
+### Example 4: Using custom container images from custom repository
+
+```bash
+$ pv-migrate \
+  --helm-set rsync.image.repository=mycustomrepo/rsync \
+  --helm-set rsync.image.tag=v1.2.3 \
+  --helm-set sshd.image.repository=mycustomrepo/sshd \
+  --helm-set sshd.image.tag=v1.2.3 \
+  --source old-pvc \
+  --dest new-pvc
+```
+
+### Example 5: Enabling network policies (on clusters with deny-all traffic rules)
+
+```bash
+$ pv-migrate \
+  --helm-set sshd.networkPolicy.enabled=true \
+  --helm-set rsync.networkPolicy.enabled=true \
+  --source-namespace source-ns --source old-pvc \
+  --dest-namespace dest-ns --dest new-pvc
+```
+
+### Example 6: Passing additional rsync arguments
+
+```bash
+$ pv-migrate \
+  --helm-set rsync.extraArgs="--partial --inplace" \
+  --source old-pvc --dest new-pvc
+```
+
+**For further customization on the rendered manifests** (custom labels, annotations etc.), see the [Helm chart values](helm/pv-migrate).

app/migrate.go

@@ -48,6 +48,7 @@ const (
 	FlagSourceMountReadOnly       = "source-mount-read-only"
 	FlagStrategies                = "strategies"
 	FlagSSHKeyAlgorithm           = "ssh-key-algorithm"
+	FlagCompress                  = "compress"
 
 	FlagHelmTimeout   = "helm-timeout"
 	FlagHelmValues    = "helm-values"
@@ -207,6 +208,7 @@ func setMigrateCmdFlags(cmd *cobra.Command, logLevels, logFormats []string, lega
 			"Has no effect for mnt2 and local strategies")
 	flags.Duration(FlagLBSvcTimeout, lbSvcTimeoutDefault, fmt.Sprintf("timeout for the load balancer service to "+
 		"receive an external IP. Only used by the %s strategy", strategy.LbSvcStrategy))
+	flags.Bool(FlagCompress, true, "compress data during migration ('-z' flag of rsync)")
 
 	flags.DurationP(FlagHelmTimeout, "t", 1*time.Minute, "install/uninstall timeout for helm releases")
 	flags.StringSliceP(FlagHelmValues, "f", nil,
@@ -256,6 +258,7 @@ func runMigration(cmd *cobra.Command, args []string) error {
 	strs, _ := flags.GetStringSlice(FlagStrategies)
 	destHostOverride, _ := flags.GetString(FlagDestHostOverride)
 	lbSvcTimeout, _ := flags.GetDuration(FlagLBSvcTimeout)
+	compress, _ := flags.GetBool(FlagCompress)
 
 	deleteExtraneousFiles, _ := flags.GetBool(FlagDestDeleteExtraneousFiles)
 	request := migration.Request{
@@ -276,6 +279,7 @@ func runMigration(cmd *cobra.Command, args []string) error {
 		Strategies:            strs,
 		DestHostOverride:      destHostOverride,
 		LBSvcTimeout:          lbSvcTimeout,
+		Compress:              compress,
 	}
 
 	logger.Info("🚀 Starting migration")

integration/integration_test.go

@@ -243,7 +243,7 @@ func TestDeleteExtraneousFiles(t *testing.T) {
 	_, err := execInPod(ctx, mainClusterCli, ns1, "dest", generateExtraDataShellCommand)
 	require.NoError(t, err)
 
-	cmd := fmt.Sprintf("%s -d -i -n %s -N %s source dest", migrateLegacyCmdline, ns1, ns1)
+	cmd := fmt.Sprintf("%s --compress=false -d -i -n %s -N %s source dest", migrateLegacyCmdline, ns1, ns1)
 	require.NoError(t, runCliApp(ctx, cmd))
 
 	stdout, err := execInPod(ctx, mainClusterCli, ns1, "dest", printDataUIDGIDContentShellCommand)

migration/types.go

@@ -34,6 +34,7 @@ type Request struct {
 	Strategies            []string
 	DestHostOverride      string
 	LBSvcTimeout          time.Duration
+	Compress              bool
 }
 
 type Migration struct {

rsync/cmd.go

@@ -20,6 +20,7 @@ type Cmd struct {
 	DestSSHUser string
 	DestSSHHost string
 	DestPath    string
+	Compress    bool
 }
 
 func (c *Cmd) Build() (string, error) {
@@ -43,10 +44,14 @@ func (c *Cmd) Build() (string, error) {
 	sshArgsStr := fmt.Sprintf("\"%s\"", strings.Join(sshArgs, " "))
 
 	rsyncArgs := []string{
-		"-azv", "--info=progress2,misc0,flist0",
+		"-av", "--info=progress2,misc0,flist0",
 		"--no-inc-recursive", "-e", sshArgsStr,
 	}
 
+	if c.Compress {
+		rsyncArgs = append(rsyncArgs, "-z")
+	}
+
 	if c.NoChown {
 		rsyncArgs = append(rsyncArgs, "--no-o", "--no-g")
 	}

strategy/lbsvc.go

@@ -120,6 +120,7 @@ func installOnDest(attempt *migration.Attempt, releaseName, privateKey,
 		DestPath:   destPath,
 		SrcUseSSH:  true,
 		SrcSSHHost: sshHost,
+		Compress:   mig.Request.Compress,
 	}
 
 	rsyncCmdStr, err := rsyncCmd.Build()

strategy/local.go

@@ -155,6 +155,7 @@ func buildRsyncCmdLocal(mig *migration.Migration) (string, error) {
 		DestPath:    destPath,
 		DestUseSSH:  true,
 		DestSSHHost: "localhost",
+		Compress:    mig.Request.Compress,
 	}
 
 	cmd, err := rsyncCmd.Build()