From f3a1361dafc06fe7fc125dbe85112fa0fdf60513 Mon Sep 17 00:00:00 2001
From: shoemakn <shoebucket@gmail.com>
Date: Mon, 12 Feb 2024 12:20:34 -0500
Subject: [PATCH 01/15] feat(cloud-init): create clones

---
 Kubernetes/Cloud-Init/create-clones.sh | 148 +++++++++++++++++++++++++
 1 file changed, 148 insertions(+)
 create mode 100755 Kubernetes/Cloud-Init/create-clones.sh

diff --git a/Kubernetes/Cloud-Init/create-clones.sh b/Kubernetes/Cloud-Init/create-clones.sh
new file mode 100755
index 00000000..7286db0a
--- /dev/null
+++ b/Kubernetes/Cloud-Init/create-clones.sh
@@ -0,0 +1,148 @@
+#!/bin/bash
+
+###########################
+# DEFAULT VALUES          #
+###########################
+# Template VMID
+template_id=5000
+# Number of VMs to be created
+vm_number=3
+# The first VM id, smallest id is 100
+id=121
+# Name prefix of the first VM
+name=k3s
+
+drive_name=local-zfs
+disk_size=20G
+
+##################
+# Functions      #
+##################
+function run() {
+    get_user_variables
+    print_info # Prints information about what will be created based on defaults/user inputs
+    setup      # Do not worry it asks for confirmation before the setup/installation starts
+    start_vms  # You can choose to start all VMs if you want
+}
+
+function get_user_variables() {
+    echo -e "\e[36mWhich VMID template? \e[0m"
+    read -e -p "" -i $template_id template_id
+    echo -e "\e[36mHow many VM do you want to create? \e[0m"
+    read -e -p "" -i "$vm_number" vm_number
+    echo -e "\e[36mFirst VM ID? (minimum 100)\e[0m"
+    read -e -p "" -i $id id
+    echo -e "\e[36mVM name prefix? \e[0m"
+    read -e -p "" -i $name name
+    echo -e "\e[36mDisk Size? \e[0m"
+    read -e -p "" -i $disk_size disk_size
+}
+
+function print_info() {
+    echo -e "\e[36m\nThe following Virtual Machines will be created:\e[0m"
+    for ((i = 1; i <= $vm_number; i++)); do
+        if [[ $i -le 9 ]]; then
+            idx="0$i"
+        else
+            idx=$i
+        fi
+        echo -e "\e[32mVM ID: $(($id + $i - 1)), Name: $name-$idx\e[0m"
+    done
+    echo -e "\e[32mDisk size:\e[0m" "$disk_size""B"
+}
+
+function setup() {
+    yesno=n
+    echo -e "\e[36mDo you want to proceed with the setup? (y/n) \e[0m"
+    read -e -p "" -i $yesno yesno
+    case $yesno in
+    [Yy]*)
+        check_template
+        create_vms
+        ;;
+    [Nn]*)
+        echo -e "\e[31mInstallation aborted by user. No changes were made.\e[0m"
+        exit
+        ;;
+    *) ;;
+    esac
+}
+
+function check_template() {
+    # Make sure our template exists
+    if [[ $(qm list | grep $template_id | cut -c 1-10 | xargs) != $template_id ]]; then
+        echo -e "\e[31mVM Template $template_id not found. Script aborted. No changes were made.\e[0m"
+        exit 
+    fi
+}
+
+function create_vms() {
+    for ((i = 1; i <= $vm_number; i++)); do
+        # Create VM from the template
+        if [[ $i -le 9 ]]; then
+            idx="0$i"
+        else
+            idx=$i
+        fi
+        echo -e "\e[33mCreating Virtual Machine: $idx\e[0m"
+        echo "VM ID: $(($id + $i - 1)), Name: $name-$idx, Clone Of: $template_id"
+        qm clone $template_id $(($id + $i - 1)) \
+            --name $name-$idx \
+            --full
+        echo -e "\e[33mResizing disk to $disk_size...\e[0m"
+        qm disk resize $(($id + $i - 1)) scsi0 $disk_size
+    done
+}
+
+function start_vms() {
+    yesno=n
+    echo -e "\e[36mDo you want to start up the Virtual Machines now? (y/n) \e[0m"
+    read -e -p "" -i $yesno yesno
+    case $yesno in
+    [Yy]*)
+        # Start VMs
+        for ((i = 1; i <= $vm_number; i++)); do
+            if [[ $i -le 9 ]]; then
+                idx="0$i"
+            else
+                idx=$i
+            fi
+            echo -e "\e[33mStarting Virtual Machine $idx\e[0m"
+            qm start $(($id + $i - 1))
+        done
+
+        # Typically after the first cloud-init boot, there will be updates to apply
+        # Wait 60s and reboot
+        wait 60
+        for ((i = 1; i <= $vm_number; i++)); do
+            if [[ $i -le 9 ]]; then
+                idx="0$i"
+            else
+                idx=$i
+            fi
+            echo -e "\e[33mInitial reboot of Virtual Machine $idx to apply updates...\e[0m"
+            qm reboot $(($id + $i - 1))
+        done
+        
+        # Print VMs statuses
+        for ((i = 1; i <= $vm_number; i++)); do
+            if [[ $i -le 9 ]]; then
+                idx="0$i"
+            else
+                idx=$i
+            fi
+            echo -e "\e[33mVirtual Machine $idx status: \e[0m"
+            qm status $(($id + $i - 1))
+        done
+        ;;
+    [Nn]*)
+        exit
+        ;;
+    *) ;;
+    esac
+}
+
+#########################
+# Run the script        #
+#########################
+run

From 6f7048b2a90138d73085ccc9ac25a9a669e4fa6f Mon Sep 17 00:00:00 2001
From: shoemakn <shoebucket@gmail.com>
Date: Mon, 12 Feb 2024 21:40:52 -0500
Subject: [PATCH 02/15] feat(cloud-init): update clone creation

Allow for custom MAC addresses, check for existing template
---
 Kubernetes/Cloud-Init/create-clones.sh | 142 ++++++++++++++++++++-----
 1 file changed, 115 insertions(+), 27 deletions(-)

diff --git a/Kubernetes/Cloud-Init/create-clones.sh b/Kubernetes/Cloud-Init/create-clones.sh
index 7286db0a..42b575ed 100755
--- a/Kubernetes/Cloud-Init/create-clones.sh
+++ b/Kubernetes/Cloud-Init/create-clones.sh
@@ -6,14 +6,15 @@
 # Template VMID
 template_id=5000
 # Number of VMs to be created
-vm_number=3
+vm_number=5
 # The first VM id, smallest id is 100
-id=121
+id=1001
 # Name prefix of the first VM
 name=k3s
 
 drive_name=local-zfs
-disk_size=20G
+disk_size=10G
+custom_macs=()
 
 ##################
 # Functions      #
@@ -26,8 +27,19 @@ function run() {
 }
 
 function get_user_variables() {
-    echo -e "\e[36mWhich VMID template? \e[0m"
-    read -e -p "" -i $template_id template_id
+    cnt=1
+    while [ $cnt -eq 1 ] || [[ $(qm list | grep $template_id | cut -c 1-10 | xargs) != $template_id ]]; do
+        if [ $cnt -gt 1 ]; then
+            echo -e "\e[1;31mVM Template $template_id not found.\e[0m"
+        fi
+        echo -e "\e[36mWhich VMID template? (q to quit)\e[0m"
+        read -e -p "" -i $template_id template_id
+        if [ $template_id == "q" ]; then
+            echo -e "\e[31mInstallation aborted by user. No changes were made.\e[0m"
+            exit
+        fi
+        cnt+=1
+    done
     echo -e "\e[36mHow many VM do you want to create? \e[0m"
     read -e -p "" -i "$vm_number" vm_number
     echo -e "\e[36mFirst VM ID? (minimum 100)\e[0m"
@@ -36,6 +48,35 @@ function get_user_variables() {
     read -e -p "" -i $name name
     echo -e "\e[36mDisk Size? \e[0m"
     read -e -p "" -i $disk_size disk_size
+    customize=n
+    echo -e "\e[36mCustomize MAC addresses? \e[0m"
+    read -e -p "" -i $customize customize
+    case $customize in
+    [Yy]*)
+        for ((i = 1; i <= $vm_number; i++)); do
+            if [[ $i -le 9 ]]; then
+                idx="0$i"
+            else
+                idx=$i
+            fi
+            # Get a random valid MAC address for PVE as a suggestion
+            # tweaked from https://serverfault.com/questions/40712/what-range-of-mac-addresses-can-i-safely-use-for-my-virtual-machines#comment1485911_40720
+            # loop until the proper length mac address is formed since /dev/urandom is a stream
+            mac_start=$(cat /etc/pve/qemu-server/$template_id.conf | grep virtio | grep net | cut -c 14-22)
+            mac=""
+            until [[ ${#mac} == 17 ]]
+            do
+                mac="$mac_start$(od -vAn -N6 -tu1 < /dev/urandom | tr -d -c '[:digit:]A-F' | fold -w 12 | sed -E -n -e '/^.[26AE]/s/(..)/\1:/gp' | sed -e 's/:$//g')"
+            done
+
+            echo -n -e "  \e[46m$name-$idx\e[0m\e[36m MAC address:\e[0m "
+            read -e -p "" -i $mac mac
+            custom_macs+=($mac)
+        done
+        ;;
+    *) ;;
+    esac
+    
 }
 
 function print_info() {
@@ -46,7 +87,15 @@ function print_info() {
         else
             idx=$i
         fi
-        echo -e "\e[32mVM ID: $(($id + $i - 1)), Name: $name-$idx\e[0m"
+        echo -n -e "\e[32mVM ID: $(($id + $i - 1)), Name: $name-$idx"
+        case $customize in
+        [Yy]*)
+            echo -e ", MAC: ${custom_macs[$i-1]}\e[0m"
+            ;;
+        *)
+            echo "\e[0m"
+            ;;
+        esac
     done
     echo -e "\e[32mDisk size:\e[0m" "$disk_size""B"
 }
@@ -57,7 +106,6 @@ function setup() {
     read -e -p "" -i $yesno yesno
     case $yesno in
     [Yy]*)
-        check_template
         create_vms
         ;;
     [Nn]*)
@@ -68,14 +116,6 @@ function setup() {
     esac
 }
 
-function check_template() {
-    # Make sure our template exists
-    if [[ $(qm list | grep $template_id | cut -c 1-10 | xargs) != $template_id ]]; then
-        echo -e "\e[31mVM Template $template_id not found. Script aborted. No changes were made.\e[0m"
-        exit 
-    fi
-}
-
 function create_vms() {
     for ((i = 1; i <= $vm_number; i++)); do
         # Create VM from the template
@@ -85,12 +125,22 @@ function create_vms() {
             idx=$i
         fi
         echo -e "\e[33mCreating Virtual Machine: $idx\e[0m"
-        echo "VM ID: $(($id + $i - 1)), Name: $name-$idx, Clone Of: $template_id"
+        echo "VM ID: $(($id + $i - 1)), Name: $name-$idx, Cloning ID: $template_id"
         qm clone $template_id $(($id + $i - 1)) \
             --name $name-$idx \
             --full
         echo -e "\e[33mResizing disk to $disk_size...\e[0m"
         qm disk resize $(($id + $i - 1)) scsi0 $disk_size
+        case $customize in
+        [Yy]*)
+            echo -e "\e[33mSetting MAC address to ${custom_macs[$i-1]}...\e[0m"
+            oldMac=$(cat /etc/pve/qemu-server/$(($id + $i - 1)).conf | grep virtio | grep net | cut -c 14-30)
+            sed -i "s/$oldMac/${custom_macs[$i-1]}/g" /etc/pve/qemu-server/$(($id + $i - 1)).conf
+            ;;
+        *) ;;
+        esac
+
+        print_vm_status
     done
 }
 
@@ -112,8 +162,32 @@ function start_vms() {
         done
 
         # Typically after the first cloud-init boot, there will be updates to apply
-        # Wait 60s and reboot
-        wait 60
+        # Wait 60s before offering
+        echo -e "\e[33mWaiting for VMs to start...(60s)\e[0m"
+        for ((i=60; i >- 0; i--)); do
+            echo -e "\e[1A\e[K\e[33mWaiting for VMs to start...($i s)\e[0m"
+            sleep 1
+        done
+
+        print_vm_status
+
+        echo -e "\n\e[94;43;5mMake any needed updates to DHCP reservations now...\e[0m\n"
+
+        restart_vms
+        ;;
+    [Nn]*)
+        exit
+        ;;
+    *) ;;
+    esac
+}
+
+function restart_vms() {
+    yesno=n
+    echo -e "\e[36mDo you want to restart the Virtual Machines now to apply updates? (y/n) \e[0m"
+    read -e -p "" -i $yesno yesno
+    case $yesno in
+    [Yy]*)
         for ((i = 1; i <= $vm_number; i++)); do
             if [[ $i -le 9 ]]; then
                 idx="0$i"
@@ -124,16 +198,13 @@ function start_vms() {
             qm reboot $(($id + $i - 1))
         done
         
-        # Print VMs statuses
-        for ((i = 1; i <= $vm_number; i++)); do
-            if [[ $i -le 9 ]]; then
-                idx="0$i"
-            else
-                idx=$i
-            fi
-            echo -e "\e[33mVirtual Machine $idx status: \e[0m"
-            qm status $(($id + $i - 1))
+        echo -e "\e[33mWaiting for VMs to restart...(30s)\e[0m"
+        for ((i=30; i >- 0; i--)); do
+            echo -e "\e[1A\e[K\e[33mWaiting for VMs to start...($i s)\e[0m"
+            sleep 1
         done
+
+        print_vm_status
         ;;
     [Nn]*)
         exit
@@ -142,6 +213,23 @@ function start_vms() {
     esac
 }
 
+function print_vm_status() {
+        # Print VMs info
+        echo -e "\n\n\e[95m   Virtual Machine  \e[0m|\e[35m    MAC Address    \e[0m|\e[35m  Status\e[0m"
+        echo -e "-----------------------------------------------------"
+        for ((i = 1; i <= $vm_number; i++)); do
+            if [[ $i -le 9 ]]; then
+                idx="0$i"
+            else
+                idx=$i
+            fi
+            printf "%18.18s" "$name-$idx"
+            echo -n "  | $(cat /etc/pve/qemu-server/$(($id + $i - 1)).conf | grep virtio | grep net | cut -c 14-30) |  "
+            qm status $(($id + $i - 1)) | sed 's/status: //'
+            echo -e "\n\n"
+        done
+}
+
 #########################
 # Run the script        #
 #########################

From fdd6eba671be6541bc174ffeb4354111a77960c5 Mon Sep 17 00:00:00 2001
From: shoemakn <shoebucket@gmail.com>
Date: Tue, 13 Feb 2024 00:50:56 -0500
Subject: [PATCH 03/15] feat(cloud-init): create-template added

---
 Kubernetes/Cloud-Init/create-template.sh | 282 +++++++++++++++++++++++
 1 file changed, 282 insertions(+)
 create mode 100755 Kubernetes/Cloud-Init/create-template.sh

diff --git a/Kubernetes/Cloud-Init/create-template.sh b/Kubernetes/Cloud-Init/create-template.sh
new file mode 100755
index 00000000..64b4116e
--- /dev/null
+++ b/Kubernetes/Cloud-Init/create-template.sh
@@ -0,0 +1,282 @@
+#!/bin/bash
+
+# for Debian this must be installed for Longhorn to work
+# sudo apt-get install -y open-iscsi
+
+###########################
+# DEFAULT VALUES          #
+###########################
+os_options=("Debian" "Ubuntu")
+os="Debian"
+# Proxmox path to the template folder
+template_path="/var/lib/vz/template"
+# Proxmox certificate path
+cert_path="/root/.ssh"
+# The first VM id, smallest id is 100
+id=5000
+# Name prefix of the first VM
+name=cloud-template
+
+drive_name=local-zfs
+agent=1
+disk_size=5G
+memory=2048
+socket=1
+core=2
+
+# IP for the first VM
+ip=192.168.0.21
+gateway=192.168.0.1
+
+# ssh certificate name variable
+cert_name=id_rsa
+
+# User settings
+user=$USER
+password=password
+
+ubuntu_url=http://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64-disk-kvm.img
+ubuntu_filename=jammy-server-cloudimg-amd64-disk-kvm.img
+
+debian_url=https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2
+debian_filename=debian-12-genericcloud-amd64.qcow2
+
+os_url=https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2
+os_filename=debian-12-genericcloud-amd64.qcow2
+
+##################
+# Functions      #
+##################
+function run() {
+    get_user_variables
+    print_info # Prints information about what will be created based on defaults/user inputs
+    setup      # Do not worry it asks for confirmation before the setup/installation starts
+}
+
+function get_user_variables() {
+    echo -e -n "\e[36mWhich OS cloud image would you like to use?\n\e[0m"
+    PS3=""
+    select option in "${os_options[@]}"; do
+        # Check if the user selected an option
+        if [[ -n "$option" ]]; then
+            # Do something with the selected option
+            case $option in
+            "Debian") ;;
+            "Ubuntu") ;;
+            *)
+                echo -e "\e[31mInvalid option selected. Exiting...\e[0m"
+                exit
+                ;;
+            esac
+        else
+            # No option was selected
+            echo -e "\e[31mNo option was selected. Exiting...\e[0m"
+            exit
+        fi
+        # Set the selected Operating system
+        os=$option
+        # Exit the select loop
+        break
+    done
+    yesno=y
+    echo -e "\e[36mDo you want to add qemu agent to the VM images? (y/n) \e[0m"
+    read -e -p "" -i $yesno yesno
+    case $yesno in
+    [Yy]*)
+        echo "Agent will be installed."
+        agent=1
+        ;;
+    *) 
+        echo "Agent will not be installed."
+        agent=0
+        ;;
+    esac
+    echo -e "\e[36mVM Template ID? (minimum 100)\e[0m"
+    read -e -p "" -i $id id
+    echo -e "\e[36mVM name? \e[0m"
+    read -e -p "" -i $name name
+    echo -e "\e[36mDisk Size? \e[0m"
+    read -e -p "" -i $disk_size disk_size
+    echo -e "\e[36mMemory Size? \e[0m"
+    read -e -p "" -i $memory memory
+    echo -e "\e[36mNumber of cpu sockets? \e[0m"
+    read -e -p "" -i $socket socket
+    echo -e "\e[36mNumber of processor cores per socket? \e[0m"
+    read -e -p "" -i $core core
+    echo -e "\e[36mUser name? \e[0m"
+    read -e -p "" -i $user user
+    echo -e "\e[36mUser password? \e[0m"
+    read -e -p "" -i $password password
+    echo -e "\e[36mCertification name? \e[0m"
+    read -e -p "" -i $cert_name cert_name
+    echo -e "\e[36mDrive name to store images? \e[0m"
+    read -e -p "" -i $drive_name drive_name
+}
+
+function print_info() {
+    echo -e "\e[36m\nThe following Virtual Machine Template will be created:\e[0m"
+    echo -e "\e[32mVM ID: $id, Name: $name\e[0m"
+    echo -e "\e[36m\nCommon VM parameters:\e[0m"
+    echo -e "\e[32mOS cloud image:\e[0m" "$os"
+    echo -e "\e[32mQEMU Agent:\e[0m" $yesno
+    echo -e "\e[32mPublic Proxmox Certificate:\e[0m" "$cert_path/$cert_name.pub\n"
+    echo -e "\e[32mDisk size:\e[0m" "$disk_size""B"
+    echo -e "\e[32mMemory size:\e[0m" "$memory""GB"
+    echo -e "\e[32mCPU sockets:\e[0m" "$socket"
+    echo -e "\e[32mCPU cores per socket:\e[0m" "$core"
+    echo -e "\e[32mDrive name:\e[0m" "$drive_name"
+}
+
+function setup() {
+    yesno=n
+    echo -e "\e[36mDo you want to proceed with the setup? (y/n) \e[0m"
+    read -e -p "" -i $yesno yesno
+    case $yesno in
+    [Yy]*)
+        get_os_image
+        if [[ $agent == 1 ]]; then
+            add_qemu_agent
+        fi
+        create_vms
+        ;;
+    [Nn]*)
+        echo -e "\e[31mInstallation aborted by user. No changes were made.\e[0m"
+        exit
+        ;;
+    *) ;;
+    esac
+}
+
+function get_os_image() {
+    case $os in
+    "Debian")
+        os_url=$debian_url
+        os_filename=$debian_filename
+        # Check if the directory exists.
+        if [ ! -d "$template_path/qcow" ]; then
+            mkdir $template_path/qcow
+        fi
+        cd $template_path/qcow
+        ;;
+    "Ubuntu")
+        os_url=$ubuntu_url
+        os_filename=$ubuntu_filename
+        # Check if the directory exists.
+        if [ ! -d "$template_path/iso" ]; then
+            mkdir $template_path/iso
+        fi
+        cd $template_path/iso
+        ;;
+    *)
+        echo -e "\e[31Invalid option.\e[0m"
+        ;;
+    esac
+
+    # Check if the os image file already exists.
+    # If not then download it.
+    if [ ! -f "$os_filename" ]; then
+        # Download the selected os cloud image
+        echo -e "\e[33mDownloading $os cloud image ...\e[0m"
+        wget $os_url
+    fi
+
+}
+
+function check_guestfs_tools() {
+    REQUIRED_PKG="libguestfs-tools"
+    PKG_OK=$(dpkg-query -W --showformat='${Status}\n' $REQUIRED_PKG|grep "install ok installed")
+    echo -e "\e[33mChecking for $REQUIRED_PKG:\e[0m $PKG_OK"
+    if [ "" = "$PKG_OK" ]; then
+        echo "$REQUIRED_PKG not found. Setting up $REQUIRED_PKG"
+        apt install $REQUIRED_PKG -y
+    fi
+}
+
+function add_qemu_agent() {
+    case $os in
+    "Debian")
+        os_url=$debian_url
+        os_filename=$debian_filename
+        cd $template_path/qcow
+        os_newfilename=$(echo $os_filename | sed -E -e "s/\.qcow2/\-qemu\.qcow2/")
+        ;;
+    "Ubuntu")
+        os_url=$ubuntu_url
+        os_filename=$ubuntu_filename
+        cd $template_path/iso
+        os_newfilename=$(echo $os_filename | sed -E -e "s/\.img/\-qemu\.img/")
+        ;;
+    *)
+        echo -e "\e[31Invalid option.\e[0m"
+        ;;
+    esac
+
+    # Check if the os image file already exists.
+    # If not then set it up
+    if [ ! -f "$os_newfilename" ]; then
+        echo -e "\e[33mAdding QEMU Guest Agent.\e[0m"
+        check_guestfs_tools
+        cp $os_filename $os_newfilename -f
+        virt-customize -a $os_newfilename --install qemu-guest-agent
+        $os_filename = $os_newfilename
+    fi
+
+}
+
+# Only runs if you uncomment the function in `create_vms`. Please be careful
+function destroy_existing_vms() {
+    # Stop and destroy Virtual Machine if it already exists
+    # TODO: Put confirmation before doing anything
+    qm stop $id
+    qm destroy $id --destroy-unreferenced-disks --purge
+}
+
+function create_vms() {
+    # Stop and destroy Virtual Machine if it already exists.
+    # Be really careful with this only uncomment if you know what are you doing. !!!
+    #
+    #  destroy_existing_vms
+    #
+    # #############################
+    # Create VM from the cloud image
+    echo -e "\e[33mCreating Virtual Machine:\e[0m"
+    echo "VM ID: $id, Name: $name"
+    qm create $id \
+        --name $name \
+        --cpu cputype=host \
+        --socket $socket \
+        --core $core \
+        --numa 1 \
+        --memory $memory \
+        --balloon 0 \
+        --net0 virtio,bridge=vmbr0 \
+        --ipconfig0 ip=dhcp,ip6=dhcp \
+        --cipassword $password \
+        --ciuser $user \
+        --ciupgrade 1 \
+        --sshkeys $cert_path/$cert_name.pub \
+        --agent=$agent
+
+
+    echo -e "\e[33mImporting disk image.\e[0m"
+    qm importdisk $id $os_filename $drive_name
+    qm set $id --scsihw virtio-scsi-pci --scsi0 $drive_name:vm-$id-disk-0
+
+    echo -e "\e[33mResizing disk\e[0m"
+    qm disk resize $id scsi0 $disk_size
+
+    echo -e "\e[33mImporting cloud image and setting boot parameters.\e[0m"
+    qm set $id --ide2 $drive_name:cloudinit
+    qm set $id --boot c --bootdisk scsi0
+
+    echo -e "\e[33mConnecting serial VGA socket.\e[0m"
+    qm set $id --serial0 socket --vga serial0
+
+    echo -e "\e[33mConverting to template.\e[0m"
+    qm template $id
+}
+
+#########################
+# Run the script        #
+#########################
+run

From 8daaabd934ff3fe3eab4677b97b8a0e28b047fca Mon Sep 17 00:00:00 2001
From: shoemakn <shoebucket@gmail.com>
Date: Tue, 13 Feb 2024 00:56:09 -0500
Subject: [PATCH 04/15] feat(cloud-init): update create-clones

Simplify print_vm_status calls
---
 Kubernetes/Cloud-Init/create-clones.sh | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/Kubernetes/Cloud-Init/create-clones.sh b/Kubernetes/Cloud-Init/create-clones.sh
index 42b575ed..1c8af9f3 100755
--- a/Kubernetes/Cloud-Init/create-clones.sh
+++ b/Kubernetes/Cloud-Init/create-clones.sh
@@ -24,6 +24,7 @@ function run() {
     print_info # Prints information about what will be created based on defaults/user inputs
     setup      # Do not worry it asks for confirmation before the setup/installation starts
     start_vms  # You can choose to start all VMs if you want
+    print_vm_status
 }
 
 function get_user_variables() {
@@ -107,6 +108,9 @@ function setup() {
     case $yesno in
     [Yy]*)
         create_vms
+        print_vm_status
+
+        echo -e "\n\e[94;43;5mMake any needed updates to DHCP reservations now...\e[0m\n"
         ;;
     [Nn]*)
         echo -e "\e[31mInstallation aborted by user. No changes were made.\e[0m"
@@ -139,8 +143,6 @@ function create_vms() {
             ;;
         *) ;;
         esac
-
-        print_vm_status
     done
 }
 
@@ -169,10 +171,6 @@ function start_vms() {
             sleep 1
         done
 
-        print_vm_status
-
-        echo -e "\n\e[94;43;5mMake any needed updates to DHCP reservations now...\e[0m\n"
-
         restart_vms
         ;;
     [Nn]*)
@@ -203,8 +201,6 @@ function restart_vms() {
             echo -e "\e[1A\e[K\e[33mWaiting for VMs to start...($i s)\e[0m"
             sleep 1
         done
-
-        print_vm_status
         ;;
     [Nn]*)
         exit
@@ -226,8 +222,8 @@ function print_vm_status() {
             printf "%18.18s" "$name-$idx"
             echo -n "  | $(cat /etc/pve/qemu-server/$(($id + $i - 1)).conf | grep virtio | grep net | cut -c 14-30) |  "
             qm status $(($id + $i - 1)) | sed 's/status: //'
-            echo -e "\n\n"
         done
+        echo -e "\n\n"
 }
 
 #########################

From ec9ae8fe847e452489486281fbea531a1c33e817 Mon Sep 17 00:00:00 2001
From: shoemakn <shoebucket@gmail.com>
Date: Tue, 13 Feb 2024 00:56:20 -0500
Subject: [PATCH 05/15] update readme

---
 Kubernetes/Cloud-Init/readme.md | 45 +++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 14 deletions(-)

diff --git a/Kubernetes/Cloud-Init/readme.md b/Kubernetes/Cloud-Init/readme.md
index dc2f845b..5a5cca0e 100644
--- a/Kubernetes/Cloud-Init/readme.md
+++ b/Kubernetes/Cloud-Init/readme.md
@@ -1,14 +1,31 @@
-1. Download the ISO using the GUI (tested on https://cloud-images.ubuntu.com/lunar/current/lunar-server-cloudimg-amd64-disk-kvm.img)
-1. Create the VM via CLI
-```
-qm create 5000 --memory 2048 --core 2 --name ubuntu-cloud --net0 virtio,bridge=vmbr0
-cd /var/lib/vz/template/iso/
-qm importdisk 5000 lunar-server-cloudimg-amd64-disk-kvm.img <YOUR STORAGE HERE>
-qm set 5000 --scsihw virtio-scsi-pci --scsi0 <YOUR STORAGE HERE>:vm-5000-disk-0
-qm set 5000 --ide2 <YOUR STORAGE HERE>:cloudinit
-qm set 5000 --boot c --bootdisk scsi0
-qm set 5000 --serial0 socket --vga serial0
-```
-3. Expand the VM disk size to a suitable size (suggested 10 GB)
-4. Create the Cloud-Init template 
-5. Deploy new VMs by cloning the template (full clone)
+# Now Automated via Scripts
+### 1. Create the Template with [create-template.sh](create-template.sh)
+#### Parameters:
+* Debian vs Ubuntu
+* Automatically add the qemu-guest agent
+* VM ID
+* VM Name
+* Final Disk size
+* Memory size
+* CPU sockets & cores/socket
+* cloud image username/password
+* cloud image public cert name
+* storage location
+
+Check script for setting paths to downloadable cloud images; currently set for _bookworm_ for Debian and _jammy_ for Ubuntu.
+____
+### 2. Clone the Template with [create-clones.sh](create-clones.sh)
+#### Parameters:
+* Template VM ID (script checks for validity)
+* Number of clones to create
+* First VM ID (uses sequential IDs)
+* Final disk size
+* Customize MAC Addresses for DHCP reservation
+
+#### Features/Notes:
+* If customizing MAC addresses, script will suggest same prefix as the template for random valid new addresses
+* If starting up all VMs
+    * Script will wait 60s to allow for full boot time & auto-updates
+        * __Note:__ if your internet connection is slower, you may want to increase this wait time (lines 168-169) to allow for downloads
+    * After wait time, script will suggest reboots on all VMs for updates made
+* Script prints all VM clone info and pauses between boots to allow for you to apply DHCP reservations before next boot

From 5264aa4779d68090203eed6dbc06c10ead9ebc6e Mon Sep 17 00:00:00 2001
From: shoemakn <shoebucket@gmail.com>
Date: Tue, 13 Feb 2024 11:17:13 -0500
Subject: [PATCH 06/15] feat(k3s): update params

---
 Kubernetes/K3S-Deploy/k3s.sh | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/Kubernetes/K3S-Deploy/k3s.sh b/Kubernetes/K3S-Deploy/k3s.sh
index 2319e470..18025b96 100644
--- a/Kubernetes/K3S-Deploy/k3s.sh
+++ b/Kubernetes/K3S-Deploy/k3s.sh
@@ -21,17 +21,17 @@ echo -e " \033[32;5m                                                           \
 #############################################
 
 # Version of Kube-VIP to deploy
-KVVERSION="v0.6.3"
+KVVERSION="v0.7.0"
 
 # K3S Version
-k3sVersion="v1.26.10+k3s2"
+k3sVersion="v1.27.10+k3s2"
 
 # Set the IP addresses of the master and work nodes
-master1=192.168.3.21
-master2=192.168.3.22
-master3=192.168.3.23
-worker1=192.168.3.24
-worker2=192.168.3.25
+master1=192.168.11.81
+master2=192.168.11.82
+master3=192.168.11.83
+worker1=192.168.11.84
+worker2=192.168.11.85
 
 # User of remote machines
 user=ubuntu
@@ -40,7 +40,7 @@ user=ubuntu
 interface=eth0
 
 # Set the virtual IP address (VIP)
-vip=192.168.3.50
+vip=192.168.11.90
 
 # Array of master nodes
 masters=($master2 $master3)
@@ -55,7 +55,7 @@ all=($master1 $master2 $master3 $worker1 $worker2)
 allnomaster1=($master2 $master3 $worker1 $worker2)
 
 #Loadbalancer IP range
-lbrange=192.168.3.60-192.168.3.80
+lbrange=192.168.11.91-192.168.11.100
 
 #ssh certificate name variable
 certName=id_rsa
@@ -174,8 +174,7 @@ done
 kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
 
 # Step 8: Install Metallb
-kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml
-kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml
+kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.14.3/config/manifests/metallb-native.yaml
 # Download ipAddressPool and configure using lbrange above
 curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/ipAddressPool
 cat ipAddressPool | sed 's/$lbrange/'$lbrange'/g' > $HOME/ipAddressPool.yaml

From b7c92186d48ddc4521b2991cdb75036096ae3c80 Mon Sep 17 00:00:00 2001
From: shoemakn <shoebucket@gmail.com>
Date: Sat, 16 Mar 2024 21:09:05 -0400
Subject: [PATCH 07/15] feat(k3s): certs already copied

---
 Kubernetes/K3S-Deploy/k3s.sh | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/Kubernetes/K3S-Deploy/k3s.sh b/Kubernetes/K3S-Deploy/k3s.sh
index 18025b96..b570a860 100644
--- a/Kubernetes/K3S-Deploy/k3s.sh
+++ b/Kubernetes/K3S-Deploy/k3s.sh
@@ -68,9 +68,10 @@ sudo timedatectl set-ntp off
 sudo timedatectl set-ntp on
 
 # Move SSH certs to ~/.ssh and change permissions
-cp /home/$user/{$certName,$certName.pub} /home/$user/.ssh
-chmod 600 /home/$user/.ssh/$certName 
-chmod 644 /home/$user/.ssh/$certName.pub
+# NAS - these were already copied over during install
+# cp /home/$user/{$certName,$certName.pub} /home/$user/.ssh
+# chmod 600 /home/$user/.ssh/$certName 
+# chmod 644 /home/$user/.ssh/$certName.pub
 
 # Install k3sup to local machine if not already present
 if ! command -v k3sup version &> /dev/null
@@ -96,9 +97,10 @@ fi
 echo "StrictHostKeyChecking no" > ~/.ssh/config
 
 #add ssh keys for all nodes
-for node in "${all[@]}"; do
-  ssh-copy-id $user@$node
-done
+# NAS - again, done already
+# for node in "${all[@]}"; do
+#   ssh-copy-id $user@$node
+# done
 
 # Install policycoreutils for each node
 for newnode in "${all[@]}"; do

From c9889badd5a9c8f38973eda2281a5d518bf9486c Mon Sep 17 00:00:00 2001
From: Nick S <shoebucket@gmail.com>
Date: Tue, 19 Mar 2024 01:14:49 +0000
Subject: [PATCH 08/15] new .gitignore

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 .gitignore

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..4d36f198
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+Kubernetes/K3S-Deploy/kubectl
+Kubernetes/K3S-Deploy/kube-vip.yaml
+Kubernetes/K3S-Deploy/ipAddressPool.yaml
+Kubernetes/K3S-Deploy/k3sup

From f459eed7113e4ae01f9af91e62687e436db09ac6 Mon Sep 17 00:00:00 2001
From: Nick S <shoebucket@gmail.com>
Date: Tue, 19 Mar 2024 01:16:00 +0000
Subject: [PATCH 09/15] feat(k3s-deploy): fixes & updates

- use KVVERSION for kube-vip
- update for my environment
- update for running via cloned repository
---
 Kubernetes/K3S-Deploy/k3s.sh   | 27 +++++++++++++++------------
 Kubernetes/K3S-Deploy/kube-vip |  6 +++---
 2 files changed, 18 insertions(+), 15 deletions(-)
 mode change 100644 => 100755 Kubernetes/K3S-Deploy/k3s.sh

diff --git a/Kubernetes/K3S-Deploy/k3s.sh b/Kubernetes/K3S-Deploy/k3s.sh
old mode 100644
new mode 100755
index b570a860..e206f5af
--- a/Kubernetes/K3S-Deploy/k3s.sh
+++ b/Kubernetes/K3S-Deploy/k3s.sh
@@ -21,10 +21,10 @@ echo -e " \033[32;5m                                                           \
 #############################################
 
 # Version of Kube-VIP to deploy
-KVVERSION="v0.7.0"
+KVVERSION="v0.7.2"
 
 # K3S Version
-k3sVersion="v1.27.10+k3s2"
+k3sVersion="v1.27.11+k3s1"
 
 # Set the IP addresses of the master and work nodes
 master1=192.168.11.81
@@ -97,10 +97,9 @@ fi
 echo "StrictHostKeyChecking no" > ~/.ssh/config
 
 #add ssh keys for all nodes
-# NAS - again, done already
-# for node in "${all[@]}"; do
-#   ssh-copy-id $user@$node
-# done
+for node in "${all[@]}"; do
+  ssh-copy-id $user@$node
+done
 
 # Install policycoreutils for each node
 for newnode in "${all[@]}"; do
@@ -131,11 +130,12 @@ echo -e " \033[32;5mFirst Node bootstrapped successfully!\033[0m"
 kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
 
 # Step 3: Download kube-vip
-curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/kube-vip
-cat kube-vip | sed 's/$interface/'$interface'/g; s/$vip/'$vip'/g' > $HOME/kube-vip.yaml
+# NAS - running this from our cloned repository; no need to download
+# curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/kube-vip
+cat kube-vip | sed 's/$interface/'$interface'/g; s/$vip/'$vip'/g; s/$KVVERSION'$KVVERSION'/g' > ./kube-vip.yaml
 
 # Step 4: Copy kube-vip.yaml to master1
-scp -i ~/.ssh/$certName $HOME/kube-vip.yaml $user@$master1:~/kube-vip.yaml
+scp -i ~/.ssh/$certName ./kube-vip.yaml $user@$master1:~/kube-vip.yaml
 
 
 # Step 5: Connect to Master1 and move kube-vip.yaml
@@ -178,8 +178,9 @@ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provi
 # Step 8: Install Metallb
 kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.14.3/config/manifests/metallb-native.yaml
 # Download ipAddressPool and configure using lbrange above
-curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/ipAddressPool
-cat ipAddressPool | sed 's/$lbrange/'$lbrange'/g' > $HOME/ipAddressPool.yaml
+# NAS - running this from our cloned repository; no need to download
+# curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/ipAddressPool
+cat ipAddressPool | sed 's/$lbrange/'$lbrange'/g' > ./ipAddressPool.yaml
 
 # Step 9: Test with Nginx
 kubectl apply -f https://raw.githubusercontent.com/inlets/inlets-operator/master/contrib/nginx-sample-deployment.yaml -n default
@@ -197,7 +198,9 @@ kubectl wait --namespace metallb-system \
                 --selector=component=controller \
                 --timeout=120s
 kubectl apply -f ipAddressPool.yaml
-kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/l2Advertisement.yaml
+# NAS - running this from our cloned repository; no need to download
+# kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/l2Advertisement.yaml
+kubectl apply -f l2Advertisement.yaml
 
 kubectl get nodes
 kubectl get svc
diff --git a/Kubernetes/K3S-Deploy/kube-vip b/Kubernetes/K3S-Deploy/kube-vip
index 83dcbc57..76cd0b4a 100644
--- a/Kubernetes/K3S-Deploy/kube-vip
+++ b/Kubernetes/K3S-Deploy/kube-vip
@@ -4,7 +4,7 @@ metadata:
   creationTimestamp: null
   labels:
     app.kubernetes.io/name: kube-vip-ds
-    app.kubernetes.io/version: v0.6.3
+    app.kubernetes.io/version: $KVVERSION
   name: kube-vip-ds
   namespace: kube-system
 spec:
@@ -16,7 +16,7 @@ spec:
       creationTimestamp: null
       labels:
         app.kubernetes.io/name: kube-vip-ds
-        app.kubernetes.io/version: v0.6.3
+        app.kubernetes.io/version: $KVVERSION
     spec:
       affinity:
         nodeAffinity:
@@ -64,7 +64,7 @@ spec:
           value: $vip
         - name: prometheus_server
           value: :2112
-        image: ghcr.io/kube-vip/kube-vip:v0.6.3
+        image: ghcr.io/kube-vip/kube-vip:$KVVERSION
         imagePullPolicy: Always
         name: kube-vip
         resources: {}

From c3ca60581ed1b20d3b8682f5182624d53a5a0d51 Mon Sep 17 00:00:00 2001
From: Nick S <shoebucket@gmail.com>
Date: Tue, 19 Mar 2024 17:47:51 +0000
Subject: [PATCH 10/15] feat(longhorn): update for NAS

- use k3s version instead of channel
- check longhorn version
---
 Kubernetes/Longhorn/longhorn-K3S.sh           |   22 +-
 .../{longhorn.yaml => longhorn-v1.5.3.yaml}   |    0
 Kubernetes/Longhorn/longhorn-v1.6.0.yaml      | 4571 +++++++++++++++++
 3 files changed, 4586 insertions(+), 7 deletions(-)
 mode change 100644 => 100755 Kubernetes/Longhorn/longhorn-K3S.sh
 rename Kubernetes/Longhorn/{longhorn.yaml => longhorn-v1.5.3.yaml} (100%)
 create mode 100644 Kubernetes/Longhorn/longhorn-v1.6.0.yaml

diff --git a/Kubernetes/Longhorn/longhorn-K3S.sh b/Kubernetes/Longhorn/longhorn-K3S.sh
old mode 100644
new mode 100755
index c3a867dd..22b44780
--- a/Kubernetes/Longhorn/longhorn-K3S.sh
+++ b/Kubernetes/Longhorn/longhorn-K3S.sh
@@ -19,13 +19,19 @@ echo -e " \033[32;2m                                                           \
 #############################################
 # YOU SHOULD ONLY NEED TO EDIT THIS SECTION #
 #############################################
+
+# K3S Version
+k3sVersion="v1.27.11+k3s1"
+
+longhornVersion="v1.5.3"
+
 # Set the IP addresses of master1
-master1=192.168.3.21
+master1=192.168.11.81
 
 # Set the IP addresses of your Longhorn nodes
-longhorn1=192.168.3.26
-longhorn2=192.168.3.27
-longhorn3=192.168.3.28
+longhorn1=192.168.11.86
+longhorn2=192.168.11.87
+longhorn3=192.168.11.88
 
 # User of remote machines
 user=ubuntu
@@ -34,7 +40,7 @@ user=ubuntu
 interface=eth0
 
 # Set the virtual IP address (VIP)
-vip=192.168.3.50
+vip=192.168.11.90
 
 # Array of longhorn nodes
 storage=($longhorn1 $longhorn2 $longhorn3)
@@ -69,7 +75,7 @@ for newnode in "${storage[@]}"; do
     --ip $newnode \
     --user $user \
     --sudo \
-    --k3s-channel stable \
+    --k3s-version $k3sVersion \
     --server-ip $master1 \
     --k3s-extra-args "--node-label \"longhorn=true\"" \
     --ssh-key $HOME/.ssh/$certName
@@ -77,7 +83,9 @@ for newnode in "${storage[@]}"; do
 done
 
 # Step 2: Install Longhorn (using modified Official to pin to Longhorn Nodes)
-kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/Longhorn/longhorn.yaml
+# NAS - running this from our cloned repository; no need to download
+#kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/Longhorn/longhorn.yaml
+kubectl apply -f longhorn-$longhornVersion.yaml
 kubectl get pods \
 --namespace longhorn-system \
 --watch
diff --git a/Kubernetes/Longhorn/longhorn.yaml b/Kubernetes/Longhorn/longhorn-v1.5.3.yaml
similarity index 100%
rename from Kubernetes/Longhorn/longhorn.yaml
rename to Kubernetes/Longhorn/longhorn-v1.5.3.yaml
diff --git a/Kubernetes/Longhorn/longhorn-v1.6.0.yaml b/Kubernetes/Longhorn/longhorn-v1.6.0.yaml
new file mode 100644
index 00000000..c3c2e015
--- /dev/null
+++ b/Kubernetes/Longhorn/longhorn-v1.6.0.yaml
@@ -0,0 +1,4571 @@
+---
+# Builtin: "helm template" does not respect --create-namespace
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-system
+---
+# Source: longhorn/templates/priorityclass.yaml
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: "longhorn-critical"
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+description: "Ensure Longhorn pods have the highest priority to prevent any unexpected eviction by the Kubernetes scheduler under node pressure"
+globalDefault: false
+preemptionPolicy: PreemptLowerPriority
+value: 1000000000
+---
+# Source: longhorn/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-service-account
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+---
+# Source: longhorn/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-ui-service-account
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+---
+# Source: longhorn/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-support-bundle
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+---
+# Source: longhorn/templates/default-setting.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: longhorn-default-setting
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+data:
+  default-setting.yaml: |-
+    priority-class: longhorn-critical
+---
+# Source: longhorn/templates/storageclass.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: longhorn-storageclass
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+data:
+  storageclass.yaml: |
+    kind: StorageClass
+    apiVersion: storage.k8s.io/v1
+    metadata:
+      name: longhorn
+      annotations:
+        storageclass.kubernetes.io/is-default-class: "true"
+    provisioner: driver.longhorn.io
+    allowVolumeExpansion: true
+    reclaimPolicy: "Delete"
+    volumeBindingMode: Immediate
+    parameters:
+      numberOfReplicas: "3"
+      staleReplicaTimeout: "30"
+      fromBackup: ""
+      fsType: "ext4"
+      dataLocality: "disabled"
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: backingimagedatasources.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: BackingImageDataSource
+    listKind: BackingImageDataSourceList
+    plural: backingimagedatasources
+    shortNames:
+    - lhbids
+    singular: backingimagedatasource
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The current state of the pod used to provision the backing image file from source
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The data source type
+      jsonPath: .spec.sourceType
+      name: SourceType
+      type: string
+    - description: The node the backing image file will be prepared on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The disk the backing image file will be prepared on
+      jsonPath: .spec.diskUUID
+      name: DiskUUID
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: BackingImageDataSource is where Longhorn stores backing image data source object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The system generated UUID of the provisioned backing image file
+      jsonPath: .spec.uuid
+      name: UUID
+      type: string
+    - description: The current state of the pod used to provision the backing image file from source
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The data source type
+      jsonPath: .spec.sourceType
+      name: SourceType
+      type: string
+    - description: The backing image file size
+      jsonPath: .status.size
+      name: Size
+      type: string
+    - description: The node the backing image file will be prepared on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The disk the backing image file will be prepared on
+      jsonPath: .spec.diskUUID
+      name: DiskUUID
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: BackingImageDataSource is where Longhorn stores backing image data source object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackingImageDataSourceSpec defines the desired state of the Longhorn backing image data source
+            properties:
+              checksum:
+                type: string
+              diskPath:
+                type: string
+              diskUUID:
+                type: string
+              fileTransferred:
+                type: boolean
+              nodeID:
+                type: string
+              parameters:
+                additionalProperties:
+                  type: string
+                type: object
+              sourceType:
+                enum:
+                - download
+                - upload
+                - export-from-volume
+                - restore
+                type: string
+              uuid:
+                type: string
+            type: object
+          status:
+            description: BackingImageDataSourceStatus defines the observed state of the Longhorn backing image data source
+            properties:
+              checksum:
+                type: string
+              currentState:
+                type: string
+              ip:
+                type: string
+              message:
+                type: string
+              ownerID:
+                type: string
+              progress:
+                type: integer
+              runningParameters:
+                additionalProperties:
+                  type: string
+                nullable: true
+                type: object
+              size:
+                format: int64
+                type: integer
+              storageIP:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: backingimagemanagers.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: BackingImageManager
+    listKind: BackingImageManagerList
+    plural: backingimagemanagers
+    shortNames:
+    - lhbim
+    singular: backingimagemanager
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The current state of the manager
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The image the manager pod will use
+      jsonPath: .spec.image
+      name: Image
+      type: string
+    - description: The node the manager is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The disk the manager is responsible for
+      jsonPath: .spec.diskUUID
+      name: DiskUUID
+      type: string
+    - description: The disk path the manager is using
+      jsonPath: .spec.diskPath
+      name: DiskPath
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: BackingImageManager is where Longhorn stores backing image manager object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The current state of the manager
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The image the manager pod will use
+      jsonPath: .spec.image
+      name: Image
+      type: string
+    - description: The node the manager is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The disk the manager is responsible for
+      jsonPath: .spec.diskUUID
+      name: DiskUUID
+      type: string
+    - description: The disk path the manager is using
+      jsonPath: .spec.diskPath
+      name: DiskPath
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: BackingImageManager is where Longhorn stores backing image manager object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackingImageManagerSpec defines the desired state of the Longhorn backing image manager
+            properties:
+              backingImages:
+                additionalProperties:
+                  type: string
+                type: object
+              diskPath:
+                type: string
+              diskUUID:
+                type: string
+              image:
+                type: string
+              nodeID:
+                type: string
+            type: object
+          status:
+            description: BackingImageManagerStatus defines the observed state of the Longhorn backing image manager
+            properties:
+              apiMinVersion:
+                type: integer
+              apiVersion:
+                type: integer
+              backingImageFileMap:
+                additionalProperties:
+                  properties:
+                    currentChecksum:
+                      type: string
+                    message:
+                      type: string
+                    name:
+                      type: string
+                    progress:
+                      type: integer
+                    senderManagerAddress:
+                      type: string
+                    sendingReference:
+                      type: integer
+                    size:
+                      format: int64
+                      type: integer
+                    state:
+                      type: string
+                    uuid:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              currentState:
+                type: string
+              ip:
+                type: string
+              ownerID:
+                type: string
+              storageIP:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: backingimages.longhorn.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: longhorn-conversion-webhook
+          namespace: longhorn-system
+          path: /v1/webhook/conversion
+          port: 9501
+      conversionReviewVersions:
+      - v1beta2
+      - v1beta1
+  group: longhorn.io
+  names:
+    kind: BackingImage
+    listKind: BackingImageList
+    plural: backingimages
+    shortNames:
+    - lhbi
+    singular: backingimage
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The backing image name
+      jsonPath: .spec.image
+      name: Image
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: BackingImage is where Longhorn stores backing image object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The system generated UUID
+      jsonPath: .status.uuid
+      name: UUID
+      type: string
+    - description: The source of the backing image file data
+      jsonPath: .spec.sourceType
+      name: SourceType
+      type: string
+    - description: The backing image file size in each disk
+      jsonPath: .status.size
+      name: Size
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: BackingImage is where Longhorn stores backing image object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackingImageSpec defines the desired state of the Longhorn backing image
+            properties:
+              checksum:
+                type: string
+              disks:
+                additionalProperties:
+                  type: string
+                type: object
+              sourceParameters:
+                additionalProperties:
+                  type: string
+                type: object
+              sourceType:
+                enum:
+                - download
+                - upload
+                - export-from-volume
+                - restore
+                type: string
+            type: object
+          status:
+            description: BackingImageStatus defines the observed state of the Longhorn backing image status
+            properties:
+              checksum:
+                type: string
+              diskFileStatusMap:
+                additionalProperties:
+                  properties:
+                    lastStateTransitionTime:
+                      type: string
+                    message:
+                      type: string
+                    progress:
+                      type: integer
+                    state:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              diskLastRefAtMap:
+                additionalProperties:
+                  type: string
+                nullable: true
+                type: object
+              ownerID:
+                type: string
+              size:
+                format: int64
+                type: integer
+              uuid:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    longhorn-manager: ""
+  name: backupbackingimages.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: BackupBackingImage
+    listKind: BackupBackingImageList
+    plural: backupbackingimages
+    shortNames:
+    - lhbbi
+    singular: backupbackingimage
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The backing image name
+      jsonPath: .status.backingImage
+      name: BackingImage
+      type: string
+    - description: The backing image size
+      jsonPath: .status.size
+      name: Size
+      type: string
+    - description: The backing image backup upload finished time
+      jsonPath: .status.backupCreatedAt
+      name: BackupCreatedAt
+      type: string
+    - description: The backing image backup state
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: BackupBackingImage is where Longhorn stores backing image backup object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackupBackingImageSpec defines the desired state of the Longhorn backing image backup
+            properties:
+              labels:
+                additionalProperties:
+                  type: string
+                description: The labels of backing image backup.
+                type: object
+              syncRequestedAt:
+                description: The time to request run sync the remote backing image backup.
+                format: date-time
+                nullable: true
+                type: string
+              userCreated:
+                description: Is this CR created by user through API or UI. Required
+                type: boolean
+            required:
+            - userCreated
+            type: object
+          status:
+            description: BackupBackingImageStatus defines the observed state of the Longhorn backing image backup
+            properties:
+              backingImage:
+                description: The backing image name.
+                type: string
+              backupCreatedAt:
+                description: The backing image backup upload finished time.
+                type: string
+              checksum:
+                description: The checksum of the backing image.
+                type: string
+              compressionMethod:
+                description: Compression method
+                type: string
+              error:
+                description: The error message when taking the backing image backup.
+                type: string
+              labels:
+                additionalProperties:
+                  type: string
+                description: The labels of backing image backup.
+                nullable: true
+                type: object
+              lastSyncedAt:
+                description: The last time that the backing image backup was synced with the remote backup target.
+                format: date-time
+                nullable: true
+                type: string
+              managerAddress:
+                description: The address of the backing image manager that runs backing image backup.
+                type: string
+              messages:
+                additionalProperties:
+                  type: string
+                description: The error messages when listing or inspecting backing image backup.
+                nullable: true
+                type: object
+              ownerID:
+                description: The node ID on which the controller is responsible to reconcile this CR.
+                type: string
+              progress:
+                description: The backing image backup progress.
+                type: integer
+              size:
+                description: The backing image size.
+                format: int64
+                type: integer
+              state:
+                description: The backing image backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown".
+                type: string
+              url:
+                description: The backing image backup URL.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: backups.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: Backup
+    listKind: BackupList
+    plural: backups
+    shortNames:
+    - lhb
+    singular: backup
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The snapshot name
+      jsonPath: .status.snapshotName
+      name: SnapshotName
+      type: string
+    - description: The snapshot size
+      jsonPath: .status.size
+      name: SnapshotSize
+      type: string
+    - description: The snapshot creation time
+      jsonPath: .status.snapshotCreatedAt
+      name: SnapshotCreatedAt
+      type: string
+    - description: The backup state
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The backup last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: Backup is where Longhorn stores backup object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The snapshot name
+      jsonPath: .status.snapshotName
+      name: SnapshotName
+      type: string
+    - description: The snapshot size
+      jsonPath: .status.size
+      name: SnapshotSize
+      type: string
+    - description: The snapshot creation time
+      jsonPath: .status.snapshotCreatedAt
+      name: SnapshotCreatedAt
+      type: string
+    - description: The backup state
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The backup last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Backup is where Longhorn stores backup object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackupSpec defines the desired state of the Longhorn backup
+            properties:
+              labels:
+                additionalProperties:
+                  type: string
+                description: The labels of snapshot backup.
+                type: object
+              snapshotName:
+                description: The snapshot name.
+                type: string
+              syncRequestedAt:
+                description: The time to request run sync the remote backup.
+                format: date-time
+                nullable: true
+                type: string
+            type: object
+          status:
+            description: BackupStatus defines the observed state of the Longhorn backup
+            properties:
+              backupCreatedAt:
+                description: The snapshot backup upload finished time.
+                type: string
+              compressionMethod:
+                description: Compression method
+                type: string
+              error:
+                description: The error message when taking the snapshot backup.
+                type: string
+              labels:
+                additionalProperties:
+                  type: string
+                description: The labels of snapshot backup.
+                nullable: true
+                type: object
+              lastSyncedAt:
+                description: The last time that the backup was synced with the remote backup target.
+                format: date-time
+                nullable: true
+                type: string
+              messages:
+                additionalProperties:
+                  type: string
+                description: The error messages when calling longhorn engine on listing or inspecting backups.
+                nullable: true
+                type: object
+              ownerID:
+                description: The node ID on which the controller is responsible to reconcile this backup CR.
+                type: string
+              progress:
+                description: The snapshot backup progress.
+                type: integer
+              replicaAddress:
+                description: The address of the replica that runs snapshot backup.
+                type: string
+              size:
+                description: The snapshot size.
+                type: string
+              snapshotCreatedAt:
+                description: The snapshot creation time.
+                type: string
+              snapshotName:
+                description: The snapshot name.
+                type: string
+              state:
+                description: The backup creation state. Can be "", "InProgress", "Completed", "Error", "Unknown".
+                type: string
+              url:
+                description: The snapshot backup URL.
+                type: string
+              volumeBackingImageName:
+                description: The volume's backing image name.
+                type: string
+              volumeCreated:
+                description: The volume creation time.
+                type: string
+              volumeName:
+                description: The volume name.
+                type: string
+              volumeSize:
+                description: The volume size.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: backuptargets.longhorn.io
+spec:
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: longhorn-conversion-webhook
+          namespace: longhorn-system
+          path: /v1/webhook/conversion
+          port: 9501
+      conversionReviewVersions:
+      - v1beta2
+      - v1beta1
+  group: longhorn.io
+  names:
+    kind: BackupTarget
+    listKind: BackupTargetList
+    plural: backuptargets
+    shortNames:
+    - lhbt
+    singular: backuptarget
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The backup target URL
+      jsonPath: .spec.backupTargetURL
+      name: URL
+      type: string
+    - description: The backup target credential secret
+      jsonPath: .spec.credentialSecret
+      name: Credential
+      type: string
+    - description: The backup target poll interval
+      jsonPath: .spec.pollInterval
+      name: LastBackupAt
+      type: string
+    - description: Indicate whether the backup target is available or not
+      jsonPath: .status.available
+      name: Available
+      type: boolean
+    - description: The backup target last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: BackupTarget is where Longhorn stores backup target object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The backup target URL
+      jsonPath: .spec.backupTargetURL
+      name: URL
+      type: string
+    - description: The backup target credential secret
+      jsonPath: .spec.credentialSecret
+      name: Credential
+      type: string
+    - description: The backup target poll interval
+      jsonPath: .spec.pollInterval
+      name: LastBackupAt
+      type: string
+    - description: Indicate whether the backup target is available or not
+      jsonPath: .status.available
+      name: Available
+      type: boolean
+    - description: The backup target last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: BackupTarget is where Longhorn stores backup target object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackupTargetSpec defines the desired state of the Longhorn backup target
+            properties:
+              backupTargetURL:
+                description: The backup target URL.
+                type: string
+              credentialSecret:
+                description: The backup target credential secret.
+                type: string
+              pollInterval:
+                description: The interval that the cluster needs to run sync with the backup target.
+                type: string
+              syncRequestedAt:
+                description: The time to request run sync the remote backup target.
+                format: date-time
+                nullable: true
+                type: string
+            type: object
+          status:
+            description: BackupTargetStatus defines the observed state of the Longhorn backup target
+            properties:
+              available:
+                description: Available indicates if the remote backup target is available or not.
+                type: boolean
+              conditions:
+                description: Records the reason on why the backup target is unavailable.
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status is the status of the condition. Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              lastSyncedAt:
+                description: The last time that the controller synced with the remote backup target.
+                format: date-time
+                nullable: true
+                type: string
+              ownerID:
+                description: The node ID on which the controller is responsible to reconcile this backup target CR.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: backupvolumes.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: BackupVolume
+    listKind: BackupVolumeList
+    plural: backupvolumes
+    shortNames:
+    - lhbv
+    singular: backupvolume
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The backup volume creation time
+      jsonPath: .status.createdAt
+      name: CreatedAt
+      type: string
+    - description: The backup volume last backup name
+      jsonPath: .status.lastBackupName
+      name: LastBackupName
+      type: string
+    - description: The backup volume last backup time
+      jsonPath: .status.lastBackupAt
+      name: LastBackupAt
+      type: string
+    - description: The backup volume last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: BackupVolume is where Longhorn stores backup volume object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The backup volume creation time
+      jsonPath: .status.createdAt
+      name: CreatedAt
+      type: string
+    - description: The backup volume last backup name
+      jsonPath: .status.lastBackupName
+      name: LastBackupName
+      type: string
+    - description: The backup volume last backup time
+      jsonPath: .status.lastBackupAt
+      name: LastBackupAt
+      type: string
+    - description: The backup volume last synced time
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: BackupVolume is where Longhorn stores backup volume object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: BackupVolumeSpec defines the desired state of the Longhorn backup volume
+            properties:
+              syncRequestedAt:
+                description: The time to request run sync the remote backup volume.
+                format: date-time
+                nullable: true
+                type: string
+            type: object
+          status:
+            description: BackupVolumeStatus defines the observed state of the Longhorn backup volume
+            properties:
+              backingImageChecksum:
+                description: the backing image checksum.
+                type: string
+              backingImageName:
+                description: The backing image name.
+                type: string
+              createdAt:
+                description: The backup volume creation time.
+                type: string
+              dataStored:
+                description: The backup volume block count.
+                type: string
+              labels:
+                additionalProperties:
+                  type: string
+                description: The backup volume labels.
+                nullable: true
+                type: object
+              lastBackupAt:
+                description: The latest volume backup time.
+                type: string
+              lastBackupName:
+                description: The latest volume backup name.
+                type: string
+              lastModificationTime:
+                description: The backup volume config last modification time.
+                format: date-time
+                nullable: true
+                type: string
+              lastSyncedAt:
+                description: The last time that the backup volume was synced into the cluster.
+                format: date-time
+                nullable: true
+                type: string
+              messages:
+                additionalProperties:
+                  type: string
+                description: The error messages when call longhorn engine on list or inspect backup volumes.
+                nullable: true
+                type: object
+              ownerID:
+                description: The node ID on which the controller is responsible to reconcile this backup volume CR.
+                type: string
+              size:
+                description: The backup volume size.
+                type: string
+              storageClassName:
+                description: the storage class name of pv/pvc binding with the volume.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: engineimages.longhorn.io
+spec:
+  preserveUnknownFields: false
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: longhorn-conversion-webhook
+          namespace: longhorn-system
+          path: /v1/webhook/conversion
+          port: 9501
+      conversionReviewVersions:
+      - v1beta2
+      - v1beta1
+  group: longhorn.io
+  names:
+    kind: EngineImage
+    listKind: EngineImageList
+    plural: engineimages
+    shortNames:
+    - lhei
+    singular: engineimage
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: State of the engine image
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The Longhorn engine image
+      jsonPath: .spec.image
+      name: Image
+      type: string
+    - description: Number of resources using the engine image
+      jsonPath: .status.refCount
+      name: RefCount
+      type: integer
+    - description: The build date of the engine image
+      jsonPath: .status.buildDate
+      name: BuildDate
+      type: date
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: EngineImage is where Longhorn stores engine image object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: Compatibility of the engine image
+      jsonPath: .status.incompatible
+      name: Incompatible
+      type: boolean
+    - description: State of the engine image
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The Longhorn engine image
+      jsonPath: .spec.image
+      name: Image
+      type: string
+    - description: Number of resources using the engine image
+      jsonPath: .status.refCount
+      name: RefCount
+      type: integer
+    - description: The build date of the engine image
+      jsonPath: .status.buildDate
+      name: BuildDate
+      type: date
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: EngineImage is where Longhorn stores engine image object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: EngineImageSpec defines the desired state of the Longhorn engine image
+            properties:
+              image:
+                minLength: 1
+                type: string
+            required:
+            - image
+            type: object
+          status:
+            description: EngineImageStatus defines the observed state of the Longhorn engine image
+            properties:
+              buildDate:
+                type: string
+              cliAPIMinVersion:
+                type: integer
+              cliAPIVersion:
+                type: integer
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status is the status of the condition. Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              controllerAPIMinVersion:
+                type: integer
+              controllerAPIVersion:
+                type: integer
+              dataFormatMinVersion:
+                type: integer
+              dataFormatVersion:
+                type: integer
+              gitCommit:
+                type: string
+              incompatible:
+                type: boolean
+              noRefSince:
+                type: string
+              nodeDeploymentMap:
+                additionalProperties:
+                  type: boolean
+                nullable: true
+                type: object
+              ownerID:
+                type: string
+              refCount:
+                type: integer
+              state:
+                type: string
+              version:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: engines.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: Engine
+    listKind: EngineList
+    plural: engines
+    shortNames:
+    - lhe
+    singular: engine
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The current state of the engine
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The node that the engine is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The instance manager of the engine
+      jsonPath: .status.instanceManagerName
+      name: InstanceManager
+      type: string
+    - description: The current image of the engine
+      jsonPath: .status.currentImage
+      name: Image
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: Engine is where Longhorn stores engine object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The data engine of the engine
+      jsonPath: .spec.dataEngine
+      name: Data Engine
+      type: string
+    - description: The current state of the engine
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The node that the engine is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The instance manager of the engine
+      jsonPath: .status.instanceManagerName
+      name: InstanceManager
+      type: string
+    - description: The current image of the engine
+      jsonPath: .status.currentImage
+      name: Image
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Engine is where Longhorn stores engine object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: EngineSpec defines the desired state of the Longhorn engine
+            properties:
+              active:
+                type: boolean
+              backendStoreDriver:
+                description: 'Deprecated: Replaced by field `dataEngine`.'
+                type: string
+              backupVolume:
+                type: string
+              dataEngine:
+                enum:
+                - v1
+                - v2
+                type: string
+              desireState:
+                type: string
+              disableFrontend:
+                type: boolean
+              engineImage:
+                description: 'Deprecated: Replaced by field `image`.'
+                type: string
+              frontend:
+                enum:
+                - blockdev
+                - iscsi
+                - nvmf
+                - ""
+                type: string
+              image:
+                type: string
+              logRequested:
+                type: boolean
+              nodeID:
+                type: string
+              replicaAddressMap:
+                additionalProperties:
+                  type: string
+                type: object
+              requestedBackupRestore:
+                type: string
+              requestedDataSource:
+                type: string
+              revisionCounterDisabled:
+                type: boolean
+              salvageRequested:
+                type: boolean
+              snapshotMaxCount:
+                type: integer
+              snapshotMaxSize:
+                format: int64
+                type: string
+              unmapMarkSnapChainRemovedEnabled:
+                type: boolean
+              upgradedReplicaAddressMap:
+                additionalProperties:
+                  type: string
+                type: object
+              volumeName:
+                type: string
+              volumeSize:
+                format: int64
+                type: string
+            type: object
+          status:
+            description: EngineStatus defines the observed state of the Longhorn engine
+            properties:
+              backupStatus:
+                additionalProperties:
+                  properties:
+                    backupURL:
+                      type: string
+                    error:
+                      type: string
+                    progress:
+                      type: integer
+                    replicaAddress:
+                      type: string
+                    snapshotName:
+                      type: string
+                    state:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              cloneStatus:
+                additionalProperties:
+                  properties:
+                    error:
+                      type: string
+                    fromReplicaAddress:
+                      type: string
+                    isCloning:
+                      type: boolean
+                    progress:
+                      type: integer
+                    snapshotName:
+                      type: string
+                    state:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status is the status of the condition. Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              currentImage:
+                type: string
+              currentReplicaAddressMap:
+                additionalProperties:
+                  type: string
+                nullable: true
+                type: object
+              currentSize:
+                format: int64
+                type: string
+              currentState:
+                type: string
+              endpoint:
+                type: string
+              instanceManagerName:
+                type: string
+              ip:
+                type: string
+              isExpanding:
+                type: boolean
+              lastExpansionError:
+                type: string
+              lastExpansionFailedAt:
+                type: string
+              lastRestoredBackup:
+                type: string
+              logFetched:
+                type: boolean
+              ownerID:
+                type: string
+              port:
+                type: integer
+              purgeStatus:
+                additionalProperties:
+                  properties:
+                    error:
+                      type: string
+                    isPurging:
+                      type: boolean
+                    progress:
+                      type: integer
+                    state:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              rebuildStatus:
+                additionalProperties:
+                  properties:
+                    error:
+                      type: string
+                    fromReplicaAddress:
+                      type: string
+                    isRebuilding:
+                      type: boolean
+                    progress:
+                      type: integer
+                    state:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              replicaModeMap:
+                additionalProperties:
+                  type: string
+                nullable: true
+                type: object
+              restoreStatus:
+                additionalProperties:
+                  properties:
+                    backupURL:
+                      type: string
+                    currentRestoringBackup:
+                      type: string
+                    error:
+                      type: string
+                    filename:
+                      type: string
+                    isRestoring:
+                      type: boolean
+                    lastRestored:
+                      type: string
+                    progress:
+                      type: integer
+                    state:
+                      type: string
+                  type: object
+                nullable: true
+                type: object
+              salvageExecuted:
+                type: boolean
+              snapshotMaxCount:
+                type: integer
+              snapshotMaxSize:
+                format: int64
+                type: string
+              snapshots:
+                additionalProperties:
+                  properties:
+                    children:
+                      additionalProperties:
+                        type: boolean
+                      nullable: true
+                      type: object
+                    created:
+                      type: string
+                    labels:
+                      additionalProperties:
+                        type: string
+                      nullable: true
+                      type: object
+                    name:
+                      type: string
+                    parent:
+                      type: string
+                    removed:
+                      type: boolean
+                    size:
+                      type: string
+                    usercreated:
+                      type: boolean
+                  type: object
+                nullable: true
+                type: object
+              snapshotsError:
+                type: string
+              started:
+                type: boolean
+              storageIP:
+                type: string
+              unmapMarkSnapChainRemovedEnabled:
+                type: boolean
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: instancemanagers.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: InstanceManager
+    listKind: InstanceManagerList
+    plural: instancemanagers
+    shortNames:
+    - lhim
+    singular: instancemanager
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The state of the instance manager
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The type of the instance manager (engine or replica)
+      jsonPath: .spec.type
+      name: Type
+      type: string
+    - description: The node that the instance manager is running on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: InstanceManager is where Longhorn stores instance manager object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The data engine of the instance manager
+      jsonPath: .spec.dataEngine
+      name: Data Engine
+      type: string
+    - description: The state of the instance manager
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The type of the instance manager (engine or replica)
+      jsonPath: .spec.type
+      name: Type
+      type: string
+    - description: The node that the instance manager is running on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: InstanceManager is where Longhorn stores instance manager object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: InstanceManagerSpec defines the desired state of the Longhorn instance manager
+            properties:
+              dataEngine:
+                type: string
+              image:
+                type: string
+              nodeID:
+                type: string
+              type:
+                enum:
+                - aio
+                - engine
+                - replica
+                type: string
+            type: object
+          status:
+            description: InstanceManagerStatus defines the observed state of the Longhorn instance manager
+            properties:
+              apiMinVersion:
+                type: integer
+              apiVersion:
+                type: integer
+              proxyApiMinVersion:
+                type: integer
+              proxyApiVersion:
+                type: integer
+              currentState:
+                type: string
+              instanceEngines:
+                additionalProperties:
+                  properties:
+                    spec:
+                      properties:
+                        backendStoreDriver:
+                          description: 'Deprecated: Replaced by field `dataEngine`.'
+                          type: string
+                        dataEngine:
+                          type: string
+                        name:
+                          type: string
+                      type: object
+                    status:
+                      properties:
+                        conditions:
+                          additionalProperties:
+                            type: boolean
+                          type: object
+                        endpoint:
+                          type: string
+                        errorMsg:
+                          type: string
+                        listen:
+                          type: string
+                        portEnd:
+                          format: int32
+                          type: integer
+                        portStart:
+                          format: int32
+                          type: integer
+                        resourceVersion:
+                          format: int64
+                          type: integer
+                        state:
+                          type: string
+                        type:
+                          type: string
+                      type: object
+                  type: object
+                nullable: true
+                type: object
+              instanceReplicas:
+                additionalProperties:
+                  properties:
+                    spec:
+                      properties:
+                        backendStoreDriver:
+                          description: 'Deprecated: Replaced by field `dataEngine`.'
+                          type: string
+                        dataEngine:
+                          type: string
+                        name:
+                          type: string
+                      type: object
+                    status:
+                      properties:
+                        conditions:
+                          additionalProperties:
+                            type: boolean
+                          type: object
+                        endpoint:
+                          type: string
+                        errorMsg:
+                          type: string
+                        listen:
+                          type: string
+                        portEnd:
+                          format: int32
+                          type: integer
+                        portStart:
+                          format: int32
+                          type: integer
+                        resourceVersion:
+                          format: int64
+                          type: integer
+                        state:
+                          type: string
+                        type:
+                          type: string
+                      type: object
+                  type: object
+                nullable: true
+                type: object
+              instances:
+                additionalProperties:
+                  properties:
+                    spec:
+                      properties:
+                        backendStoreDriver:
+                          description: 'Deprecated: Replaced by field `dataEngine`.'
+                          type: string
+                        dataEngine:
+                          type: string
+                        name:
+                          type: string
+                      type: object
+                    status:
+                      properties:
+                        conditions:
+                          additionalProperties:
+                            type: boolean
+                          type: object
+                        endpoint:
+                          type: string
+                        errorMsg:
+                          type: string
+                        listen:
+                          type: string
+                        portEnd:
+                          format: int32
+                          type: integer
+                        portStart:
+                          format: int32
+                          type: integer
+                        resourceVersion:
+                          format: int64
+                          type: integer
+                        state:
+                          type: string
+                        type:
+                          type: string
+                      type: object
+                  type: object
+                nullable: true
+                description: 'Deprecated: Replaced by InstanceEngines and InstanceReplicas'
+                type: object
+              ip:
+                type: string
+              ownerID:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: nodes.longhorn.io
+spec:
+  preserveUnknownFields: false
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: longhorn-conversion-webhook
+          namespace: longhorn-system
+          path: /v1/webhook/conversion
+          port: 9501
+      conversionReviewVersions:
+      - v1beta2
+      - v1beta1
+  group: longhorn.io
+  names:
+    kind: Node
+    listKind: NodeList
+    plural: nodes
+    shortNames:
+    - lhn
+    singular: node
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: Indicate whether the node is ready
+      jsonPath: .status.conditions['Ready']['status']
+      name: Ready
+      type: string
+    - description: Indicate whether the user disabled/enabled replica scheduling for the node
+      jsonPath: .spec.allowScheduling
+      name: AllowScheduling
+      type: boolean
+    - description: Indicate whether Longhorn can schedule replicas on the node
+      jsonPath: .status.conditions['Schedulable']['status']
+      name: Schedulable
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: Node is where Longhorn stores Longhorn node object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: Indicate whether the node is ready
+      jsonPath: .status.conditions[?(@.type=='Ready')].status
+      name: Ready
+      type: string
+    - description: Indicate whether the user disabled/enabled replica scheduling for the node
+      jsonPath: .spec.allowScheduling
+      name: AllowScheduling
+      type: boolean
+    - description: Indicate whether Longhorn can schedule replicas on the node
+      jsonPath: .status.conditions[?(@.type=='Schedulable')].status
+      name: Schedulable
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Node is where Longhorn stores Longhorn node object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: NodeSpec defines the desired state of the Longhorn node
+            properties:
+              allowScheduling:
+                description: Allow scheduling replicas on the node.
+                type: boolean
+              disks:
+                additionalProperties:
+                  properties:
+                    allowScheduling:
+                      type: boolean
+                    diskType:
+                      enum:
+                      - filesystem
+                      - block
+                      type: string
+                    evictionRequested:
+                      type: boolean
+                    path:
+                      type: string
+                    storageReserved:
+                      format: int64
+                      type: integer
+                    tags:
+                      items:
+                        type: string
+                      type: array
+                  type: object
+                type: object
+              evictionRequested:
+                type: boolean
+              instanceManagerCPURequest:
+                type: integer
+              name:
+                type: string
+              tags:
+                items:
+                  type: string
+                type: array
+            type: object
+          status:
+            description: NodeStatus defines the observed state of the Longhorn node
+            properties:
+              autoEvicting:
+                type: boolean
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status is the status of the condition. Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              diskStatus:
+                additionalProperties:
+                  properties:
+                    conditions:
+                      items:
+                        properties:
+                          lastProbeTime:
+                            description: Last time we probed the condition.
+                            type: string
+                          lastTransitionTime:
+                            description: Last time the condition transitioned from one status to another.
+                            type: string
+                          message:
+                            description: Human-readable message indicating details about last transition.
+                            type: string
+                          reason:
+                            description: Unique, one-word, CamelCase reason for the condition's last transition.
+                            type: string
+                          status:
+                            description: Status is the status of the condition. Can be True, False, Unknown.
+                            type: string
+                          type:
+                            description: Type is the type of the condition.
+                            type: string
+                        type: object
+                      nullable: true
+                      type: array
+                    diskType:
+                      type: string
+                    diskUUID:
+                      type: string
+                    filesystemType:
+                      type: string
+                    scheduledReplica:
+                      additionalProperties:
+                        format: int64
+                        type: integer
+                      nullable: true
+                      type: object
+                    storageAvailable:
+                      format: int64
+                      type: integer
+                    storageMaximum:
+                      format: int64
+                      type: integer
+                    storageScheduled:
+                      format: int64
+                      type: integer
+                  type: object
+                description: The status of the disks on the node.
+                nullable: true
+                type: object
+              region:
+                description: The Region of the node.
+                type: string
+              snapshotCheckStatus:
+                description: The status of the snapshot integrity check.
+                properties:
+                  lastPeriodicCheckedAt:
+                    format: date-time
+                    type: string
+                type: object
+              zone:
+                description: The Zone of the node.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: orphans.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: Orphan
+    listKind: OrphanList
+    plural: orphans
+    shortNames:
+    - lho
+    singular: orphan
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The type of the orphan
+      jsonPath: .spec.orphanType
+      name: Type
+      type: string
+    - description: The node that the orphan is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Orphan is where Longhorn stores orphan object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: OrphanSpec defines the desired state of the Longhorn orphaned data
+            properties:
+              nodeID:
+                description: The node ID on which the controller is responsible to reconcile this orphan CR.
+                type: string
+              orphanType:
+                description: The type of the orphaned data. Can be "replica".
+                type: string
+              parameters:
+                additionalProperties:
+                  type: string
+                description: The parameters of the orphaned data
+                type: object
+            type: object
+          status:
+            description: OrphanStatus defines the observed state of the Longhorn orphaned data
+            properties:
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status is the status of the condition. Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              ownerID:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    longhorn-manager: ""
+  name: recurringjobs.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: RecurringJob
+    listKind: RecurringJobList
+    plural: recurringjobs
+    shortNames:
+    - lhrj
+    singular: recurringjob
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume
+      jsonPath: .spec.groups
+      name: Groups
+      type: string
+    - description: Should be one of "backup" or "snapshot"
+      jsonPath: .spec.task
+      name: Task
+      type: string
+    - description: The cron expression represents recurring job scheduling
+      jsonPath: .spec.cron
+      name: Cron
+      type: string
+    - description: The number of snapshots/backups to keep for the volume
+      jsonPath: .spec.retain
+      name: Retain
+      type: integer
+    - description: The concurrent job to run by each cron job
+      jsonPath: .spec.concurrency
+      name: Concurrency
+      type: integer
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    - description: Specify the labels
+      jsonPath: .spec.labels
+      name: Labels
+      type: string
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: RecurringJob is where Longhorn stores recurring job object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: Sets groupings to the jobs. When set to "default" group will be added to the volume label when no other job label exist in volume
+      jsonPath: .spec.groups
+      name: Groups
+      type: string
+    - description: Should be one of "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim"
+      jsonPath: .spec.task
+      name: Task
+      type: string
+    - description: The cron expression represents recurring job scheduling
+      jsonPath: .spec.cron
+      name: Cron
+      type: string
+    - description: The number of snapshots/backups to keep for the volume
+      jsonPath: .spec.retain
+      name: Retain
+      type: integer
+    - description: The concurrent job to run by each cron job
+      jsonPath: .spec.concurrency
+      name: Concurrency
+      type: integer
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    - description: Specify the labels
+      jsonPath: .spec.labels
+      name: Labels
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: RecurringJob is where Longhorn stores recurring job object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: RecurringJobSpec defines the desired state of the Longhorn recurring job
+            properties:
+              concurrency:
+                description: The concurrency of taking the snapshot/backup.
+                type: integer
+              cron:
+                description: The cron setting.
+                type: string
+              groups:
+                description: The recurring job group.
+                items:
+                  type: string
+                type: array
+              labels:
+                additionalProperties:
+                  type: string
+                description: The label of the snapshot/backup.
+                type: object
+              name:
+                description: The recurring job name.
+                type: string
+              retain:
+                description: The retain count of the snapshot/backup.
+                type: integer
+              task:
+                description: The recurring job task. Can be "snapshot", "snapshot-force-create", "snapshot-cleanup", "snapshot-delete", "backup", "backup-force-create" or "filesystem-trim"
+                enum:
+                - snapshot
+                - snapshot-force-create
+                - snapshot-cleanup
+                - snapshot-delete
+                - backup
+                - backup-force-create
+                - filesystem-trim
+                type: string
+            type: object
+          status:
+            description: RecurringJobStatus defines the observed state of the Longhorn recurring job
+            properties:
+              ownerID:
+                description: The owner ID which is responsible to reconcile this recurring job CR.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: replicas.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: Replica
+    listKind: ReplicaList
+    plural: replicas
+    shortNames:
+    - lhr
+    singular: replica
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The current state of the replica
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The node that the replica is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The disk that the replica is on
+      jsonPath: .spec.diskID
+      name: Disk
+      type: string
+    - description: The instance manager of the replica
+      jsonPath: .status.instanceManagerName
+      name: InstanceManager
+      type: string
+    - description: The current image of the replica
+      jsonPath: .status.currentImage
+      name: Image
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: Replica is where Longhorn stores replica object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The data engine of the replica
+      jsonPath: .spec.dataEngine
+      name: Data Engine
+      type: string
+    - description: The current state of the replica
+      jsonPath: .status.currentState
+      name: State
+      type: string
+    - description: The node that the replica is on
+      jsonPath: .spec.nodeID
+      name: Node
+      type: string
+    - description: The disk that the replica is on
+      jsonPath: .spec.diskID
+      name: Disk
+      type: string
+    - description: The instance manager of the replica
+      jsonPath: .status.instanceManagerName
+      name: InstanceManager
+      type: string
+    - description: The current image of the replica
+      jsonPath: .status.currentImage
+      name: Image
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Replica is where Longhorn stores replica object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: ReplicaSpec defines the desired state of the Longhorn replica
+            properties:
+              active:
+                type: boolean
+              backendStoreDriver:
+                description: 'Deprecated: Replaced by field `dataEngine`.'
+                type: string
+              backingImage:
+                type: string
+              dataDirectoryName:
+                type: string
+              dataEngine:
+                enum:
+                - v1
+                - v2
+                type: string
+              desireState:
+                type: string
+              diskID:
+                type: string
+              diskPath:
+                type: string
+              engineImage:
+                description: 'Deprecated: Replaced by field `image`.'
+                type: string
+              engineName:
+                type: string
+              evictionRequested:
+                type: boolean
+              failedAt:
+                type: string
+              hardNodeAffinity:
+                type: string
+              healthyAt:
+                type: string
+              image:
+                type: string
+              logRequested:
+                type: boolean
+              nodeID:
+                type: string
+              rebuildRetryCount:
+                type: integer
+              revisionCounterDisabled:
+                type: boolean
+              salvageRequested:
+                type: boolean
+              snapshotMaxCount:
+                type: integer
+              snapshotMaxSize:
+                format: int64
+                type: string
+              unmapMarkDiskChainRemovedEnabled:
+                type: boolean
+              volumeName:
+                type: string
+              volumeSize:
+                format: int64
+                type: string
+            type: object
+          status:
+            description: ReplicaStatus defines the observed state of the Longhorn replica
+            properties:
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status is the status of the condition. Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              currentImage:
+                type: string
+              currentState:
+                type: string
+              evictionRequested:
+                description: 'Deprecated: Replaced by field `spec.evictionRequested`.'
+                type: boolean
+              instanceManagerName:
+                type: string
+              ip:
+                type: string
+              logFetched:
+                type: boolean
+              ownerID:
+                type: string
+              port:
+                type: integer
+              salvageExecuted:
+                type: boolean
+              started:
+                type: boolean
+              storageIP:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: settings.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: Setting
+    listKind: SettingList
+    plural: settings
+    shortNames:
+    - lhs
+    singular: setting
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The value of the setting
+      jsonPath: .value
+      name: Value
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: Setting is where Longhorn stores setting object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          value:
+            type: string
+        required:
+        - value
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The value of the setting
+      jsonPath: .value
+      name: Value
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Setting is where Longhorn stores setting object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          value:
+            description: The value of the setting.
+            type: string
+        required:
+        - value
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: sharemanagers.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: ShareManager
+    listKind: ShareManagerList
+    plural: sharemanagers
+    shortNames:
+    - lhsm
+    singular: sharemanager
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The state of the share manager
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The node that the share manager is owned by
+      jsonPath: .status.ownerID
+      name: Node
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: ShareManager is where Longhorn stores share manager object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The state of the share manager
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The node that the share manager is owned by
+      jsonPath: .status.ownerID
+      name: Node
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: ShareManager is where Longhorn stores share manager object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: ShareManagerSpec defines the desired state of the Longhorn share manager
+            properties:
+              image:
+                description: Share manager image used for creating a share manager pod
+                type: string
+            type: object
+          status:
+            description: ShareManagerStatus defines the observed state of the Longhorn share manager
+            properties:
+              endpoint:
+                description: NFS endpoint that can access the mounted filesystem of the volume
+                type: string
+              ownerID:
+                description: The node ID on which the controller is responsible to reconcile this share manager resource
+                type: string
+              state:
+                description: The state of the share manager resource
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: snapshots.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: Snapshot
+    listKind: SnapshotList
+    plural: snapshots
+    shortNames:
+    - lhsnap
+    singular: snapshot
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The volume that this snapshot belongs to
+      jsonPath: .spec.volume
+      name: Volume
+      type: string
+    - description: Timestamp when the point-in-time snapshot was taken
+      jsonPath: .status.creationTime
+      name: CreationTime
+      type: string
+    - description: Indicates if the snapshot is ready to be used to restore/backup a volume
+      jsonPath: .status.readyToUse
+      name: ReadyToUse
+      type: boolean
+    - description: Represents the minimum size of volume required to rehydrate from this snapshot
+      jsonPath: .status.restoreSize
+      name: RestoreSize
+      type: string
+    - description: The actual size of the snapshot
+      jsonPath: .status.size
+      name: Size
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Snapshot is the Schema for the snapshots API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: SnapshotSpec defines the desired state of Longhorn Snapshot
+            properties:
+              createSnapshot:
+                description: require creating a new snapshot
+                type: boolean
+              labels:
+                additionalProperties:
+                  type: string
+                description: The labels of snapshot
+                nullable: true
+                type: object
+              volume:
+                description: the volume that this snapshot belongs to. This field is immutable after creation. Required
+                type: string
+            required:
+            - volume
+            type: object
+          status:
+            description: SnapshotStatus defines the observed state of Longhorn Snapshot
+            properties:
+              checksum:
+                type: string
+              children:
+                additionalProperties:
+                  type: boolean
+                nullable: true
+                type: object
+              creationTime:
+                type: string
+              error:
+                type: string
+              labels:
+                additionalProperties:
+                  type: string
+                nullable: true
+                type: object
+              markRemoved:
+                type: boolean
+              ownerID:
+                type: string
+              parent:
+                type: string
+              readyToUse:
+                type: boolean
+              restoreSize:
+                format: int64
+                type: integer
+              size:
+                format: int64
+                type: integer
+              userCreated:
+                type: boolean
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: supportbundles.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: SupportBundle
+    listKind: SupportBundleList
+    plural: supportbundles
+    shortNames:
+    - lhbundle
+    singular: supportbundle
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The state of the support bundle
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The issue URL
+      jsonPath: .spec.issueURL
+      name: Issue
+      type: string
+    - description: A brief description of the issue
+      jsonPath: .spec.description
+      name: Description
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: SupportBundle is where Longhorn stores support bundle object
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: SupportBundleSpec defines the desired state of the Longhorn SupportBundle
+            properties:
+              description:
+                description: A brief description of the issue
+                type: string
+              issueURL:
+                description: The issue URL
+                nullable: true
+                type: string
+              nodeID:
+                description: The preferred responsible controller node ID.
+                type: string
+            required:
+            - description
+            type: object
+          status:
+            description: SupportBundleStatus defines the observed state of the Longhorn SupportBundle
+            properties:
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status is the status of the condition. Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                type: array
+              filename:
+                type: string
+              filesize:
+                format: int64
+                type: integer
+              image:
+                description: The support bundle manager image
+                type: string
+              managerIP:
+                description: The support bundle manager IP
+                type: string
+              ownerID:
+                description: The current responsible controller node ID
+                type: string
+              progress:
+                type: integer
+              state:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: systembackups.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: SystemBackup
+    listKind: SystemBackupList
+    plural: systembackups
+    shortNames:
+    - lhsb
+    singular: systembackup
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The system backup Longhorn version
+      jsonPath: .status.version
+      name: Version
+      type: string
+    - description: The system backup state
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The system backup creation time
+      jsonPath: .status.createdAt
+      name: Created
+      type: string
+    - description: The last time that the system backup was synced into the cluster
+      jsonPath: .status.lastSyncedAt
+      name: LastSyncedAt
+      type: string
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: SystemBackup is where Longhorn stores system backup object
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: SystemBackupSpec defines the desired state of the Longhorn SystemBackup
+            properties:
+              volumeBackupPolicy:
+                description: The create volume backup policy Can be "if-not-present", "always" or "disabled"
+                nullable: true
+                type: string
+            type: object
+          status:
+            description: SystemBackupStatus defines the observed state of the Longhorn SystemBackup
+            properties:
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status is the status of the condition. Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              createdAt:
+                description: The system backup creation time.
+                format: date-time
+                type: string
+              gitCommit:
+                description: The saved Longhorn manager git commit.
+                nullable: true
+                type: string
+              lastSyncedAt:
+                description: The last time that the system backup was synced into the cluster.
+                format: date-time
+                nullable: true
+                type: string
+              managerImage:
+                description: The saved manager image.
+                type: string
+              ownerID:
+                description: The node ID of the responsible controller to reconcile this SystemBackup.
+                type: string
+              state:
+                description: The system backup state.
+                type: string
+              version:
+                description: The saved Longhorn version.
+                nullable: true
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: systemrestores.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: SystemRestore
+    listKind: SystemRestoreList
+    plural: systemrestores
+    shortNames:
+    - lhsr
+    singular: systemrestore
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The system restore state
+      jsonPath: .status.state
+      name: State
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: SystemRestore is where Longhorn stores system restore object
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: SystemRestoreSpec defines the desired state of the Longhorn SystemRestore
+            properties:
+              systemBackup:
+                description: The system backup name in the object store.
+                type: string
+            required:
+            - systemBackup
+            type: object
+          status:
+            description: SystemRestoreStatus defines the observed state of the Longhorn SystemRestore
+            properties:
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status is the status of the condition. Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              ownerID:
+                description: The node ID of the responsible controller to reconcile this SystemRestore.
+                type: string
+              sourceURL:
+                description: The source system backup URL.
+                type: string
+              state:
+                description: The system restore state.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: volumes.longhorn.io
+spec:
+  preserveUnknownFields: false
+  conversion:
+    strategy: Webhook
+    webhook:
+      clientConfig:
+        service:
+          name: longhorn-conversion-webhook
+          namespace: longhorn-system
+          path: /v1/webhook/conversion
+          port: 9501
+      conversionReviewVersions:
+      - v1beta2
+      - v1beta1
+  group: longhorn.io
+  names:
+    kind: Volume
+    listKind: VolumeList
+    plural: volumes
+    shortNames:
+    - lhv
+    singular: volume
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: The state of the volume
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The robustness of the volume
+      jsonPath: .status.robustness
+      name: Robustness
+      type: string
+    - description: The scheduled condition of the volume
+      jsonPath: .status.conditions['scheduled']['status']
+      name: Scheduled
+      type: string
+    - description: The size of the volume
+      jsonPath: .spec.size
+      name: Size
+      type: string
+    - description: The node that the volume is currently attaching to
+      jsonPath: .status.currentNodeID
+      name: Node
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: Volume is where Longhorn stores volume object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            x-kubernetes-preserve-unknown-fields: true
+          status:
+            x-kubernetes-preserve-unknown-fields: true
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: The data engine of the volume
+      jsonPath: .spec.dataEngine
+      name: Data Engine
+      type: string
+    - description: The state of the volume
+      jsonPath: .status.state
+      name: State
+      type: string
+    - description: The robustness of the volume
+      jsonPath: .status.robustness
+      name: Robustness
+      type: string
+    - description: The scheduled condition of the volume
+      jsonPath: .status.conditions[?(@.type=='Schedulable')].status
+      name: Scheduled
+      type: string
+    - description: The size of the volume
+      jsonPath: .spec.size
+      name: Size
+      type: string
+    - description: The node that the volume is currently attaching to
+      jsonPath: .status.currentNodeID
+      name: Node
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta2
+    schema:
+      openAPIV3Schema:
+        description: Volume is where Longhorn stores volume object.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: VolumeSpec defines the desired state of the Longhorn volume
+            properties:
+              Standby:
+                type: boolean
+              accessMode:
+                enum:
+                - rwo
+                - rwx
+                type: string
+              backendStoreDriver:
+                description: 'Deprecated: Replaced by field `dataEngine`.'
+                type: string
+              backingImage:
+                type: string
+              backupCompressionMethod:
+                enum:
+                - none
+                - lz4
+                - gzip
+                type: string
+              dataEngine:
+                enum:
+                - v1
+                - v2
+                type: string
+              dataLocality:
+                enum:
+                - disabled
+                - best-effort
+                - strict-local
+                type: string
+              dataSource:
+                type: string
+              disableFrontend:
+                type: boolean
+              diskSelector:
+                items:
+                  type: string
+                type: array
+              encrypted:
+                type: boolean
+              engineImage:
+                description: 'Deprecated: Replaced by field `image`.'
+                type: string
+              fromBackup:
+                type: string
+              frontend:
+                enum:
+                - blockdev
+                - iscsi
+                - nvmf
+                - ""
+                type: string
+              image:
+                type: string
+              lastAttachedBy:
+                type: string
+              migratable:
+                type: boolean
+              migrationNodeID:
+                type: string
+              nodeID:
+                type: string
+              nodeSelector:
+                items:
+                  type: string
+                type: array
+              numberOfReplicas:
+                type: integer
+              offlineReplicaRebuilding:
+                description: OfflineReplicaRebuilding is used to determine if the offline replica rebuilding feature is enabled or not
+                enum:
+                - ignored
+                - disabled
+                - enabled
+                type: string
+              replicaAutoBalance:
+                enum:
+                - ignored
+                - disabled
+                - least-effort
+                - best-effort
+                type: string
+              replicaDiskSoftAntiAffinity:
+                description: Replica disk soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same disk.
+                enum:
+                - ignored
+                - enabled
+                - disabled
+                type: string
+              replicaSoftAntiAffinity:
+                description: Replica soft anti affinity of the volume. Set enabled to allow replicas to be scheduled on the same node.
+                enum:
+                - ignored
+                - enabled
+                - disabled
+                type: string
+              replicaZoneSoftAntiAffinity:
+                description: Replica zone soft anti affinity of the volume. Set enabled to allow replicas to be scheduled in the same zone.
+                enum:
+                - ignored
+                - enabled
+                - disabled
+                type: string
+              restoreVolumeRecurringJob:
+                enum:
+                - ignored
+                - enabled
+                - disabled
+                type: string
+              revisionCounterDisabled:
+                type: boolean
+              size:
+                format: int64
+                type: string
+              snapshotDataIntegrity:
+                enum:
+                - ignored
+                - disabled
+                - enabled
+                - fast-check
+                type: string
+              snapshotMaxCount:
+                type: integer
+              snapshotMaxSize:
+                format: int64
+                type: string
+              staleReplicaTimeout:
+                type: integer
+              unmapMarkSnapChainRemoved:
+                enum:
+                - ignored
+                - disabled
+                - enabled
+                type: string
+            type: object
+          status:
+            description: VolumeStatus defines the observed state of the Longhorn volume
+            properties:
+              actualSize:
+                format: int64
+                type: integer
+              cloneStatus:
+                properties:
+                  snapshot:
+                    type: string
+                  sourceVolume:
+                    type: string
+                  state:
+                    type: string
+                type: object
+              conditions:
+                items:
+                  properties:
+                    lastProbeTime:
+                      description: Last time we probed the condition.
+                      type: string
+                    lastTransitionTime:
+                      description: Last time the condition transitioned from one status to another.
+                      type: string
+                    message:
+                      description: Human-readable message indicating details about last transition.
+                      type: string
+                    reason:
+                      description: Unique, one-word, CamelCase reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status is the status of the condition. Can be True, False, Unknown.
+                      type: string
+                    type:
+                      description: Type is the type of the condition.
+                      type: string
+                  type: object
+                nullable: true
+                type: array
+              currentImage:
+                type: string
+              currentMigrationNodeID:
+                description: the node that this volume is currently migrating to
+                type: string
+              currentNodeID:
+                type: string
+              expansionRequired:
+                type: boolean
+              frontendDisabled:
+                type: boolean
+              isStandby:
+                type: boolean
+              kubernetesStatus:
+                properties:
+                  lastPVCRefAt:
+                    type: string
+                  lastPodRefAt:
+                    type: string
+                  namespace:
+                    description: determine if PVC/Namespace is history or not
+                    type: string
+                  pvName:
+                    type: string
+                  pvStatus:
+                    type: string
+                  pvcName:
+                    type: string
+                  workloadsStatus:
+                    description: determine if Pod/Workload is history or not
+                    items:
+                      properties:
+                        podName:
+                          type: string
+                        podStatus:
+                          type: string
+                        workloadName:
+                          type: string
+                        workloadType:
+                          type: string
+                      type: object
+                    nullable: true
+                    type: array
+                type: object
+              lastBackup:
+                type: string
+              lastBackupAt:
+                type: string
+              lastDegradedAt:
+                type: string
+              offlineReplicaRebuildingRequired:
+                type: boolean
+              ownerID:
+                type: string
+              pendingNodeID:
+                description: Deprecated.
+                type: string
+              remountRequestedAt:
+                type: string
+              restoreInitiated:
+                type: boolean
+              restoreRequired:
+                type: boolean
+              robustness:
+                type: string
+              shareEndpoint:
+                type: string
+              shareState:
+                type: string
+              state:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/crds.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.7.0
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    longhorn-manager: ""
+  name: volumeattachments.longhorn.io
+spec:
+  group: longhorn.io
+  names:
+    kind: VolumeAttachment
+    listKind: VolumeAttachmentList
+    plural: volumeattachments
+    shortNames:
+      - lhva
+    singular: volumeattachment
+  scope: Namespaced
+  versions:
+    - additionalPrinterColumns:
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1beta2
+      schema:
+        openAPIV3Schema:
+          description: VolumeAttachment stores attachment information of a Longhorn volume
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: VolumeAttachmentSpec defines the desired state of Longhorn VolumeAttachment
+              properties:
+                attachmentTickets:
+                  additionalProperties:
+                    properties:
+                      generation:
+                        description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.
+                        format: int64
+                        type: integer
+                      id:
+                        description: The unique ID of this attachment. Used to differentiate different attachments of the same volume.
+                        type: string
+                      nodeID:
+                        description: The node that this attachment is requesting
+                        type: string
+                      parameters:
+                        additionalProperties:
+                          type: string
+                        description: Optional additional parameter for this attachment
+                        type: object
+                      type:
+                        type: string
+                    type: object
+                  type: object
+                volume:
+                  description: The name of Longhorn volume of this VolumeAttachment
+                  type: string
+              required:
+                - volume
+              type: object
+            status:
+              description: VolumeAttachmentStatus defines the observed state of Longhorn VolumeAttachment
+              properties:
+                attachmentTicketStatuses:
+                  additionalProperties:
+                    properties:
+                      conditions:
+                        description: Record any error when trying to fulfill this attachment
+                        items:
+                          properties:
+                            lastProbeTime:
+                              description: Last time we probed the condition.
+                              type: string
+                            lastTransitionTime:
+                              description: Last time the condition transitioned from one status to another.
+                              type: string
+                            message:
+                              description: Human-readable message indicating details about last transition.
+                              type: string
+                            reason:
+                              description: Unique, one-word, CamelCase reason for the condition's last transition.
+                              type: string
+                            status:
+                              description: Status is the status of the condition. Can be True, False, Unknown.
+                              type: string
+                            type:
+                              description: Type is the type of the condition.
+                              type: string
+                          type: object
+                        nullable: true
+                        type: array
+                      generation:
+                        description: A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.
+                        format: int64
+                        type: integer
+                      id:
+                        description: The unique ID of this attachment. Used to differentiate different attachments of the same volume.
+                        type: string
+                      satisfied:
+                        description: Indicate whether this attachment ticket has been satisfied
+                        type: boolean
+                    required:
+                      - conditions
+                      - satisfied
+                    type: object
+                  type: object
+              type: object
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# Source: longhorn/templates/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: longhorn-role
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+rules:
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - "*"
+- apiGroups: [""]
+  resources: ["pods", "events", "persistentvolumes", "persistentvolumeclaims","persistentvolumeclaims/status", "nodes", "proxy/nodes", "pods/log", "secrets", "services", "endpoints", "configmaps", "serviceaccounts"]
+  verbs: ["*"]
+- apiGroups: [""]
+  resources: ["namespaces"]
+  verbs: ["get", "list"]
+- apiGroups: ["apps"]
+  resources: ["daemonsets", "statefulsets", "deployments"]
+  verbs: ["*"]
+- apiGroups: ["batch"]
+  resources: ["jobs", "cronjobs"]
+  verbs: ["*"]
+- apiGroups: ["policy"]
+  resources: ["poddisruptionbudgets", "podsecuritypolicies"]
+  verbs: ["*"]
+- apiGroups: ["scheduling.k8s.io"]
+  resources: ["priorityclasses"]
+  verbs: ["watch", "list"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["storageclasses", "volumeattachments", "volumeattachments/status", "csinodes", "csidrivers"]
+  verbs: ["*"]
+- apiGroups: ["snapshot.storage.k8s.io"]
+  resources: ["volumesnapshotclasses", "volumesnapshots", "volumesnapshotcontents", "volumesnapshotcontents/status"]
+  verbs: ["*"]
+- apiGroups: ["longhorn.io"]
+  resources: ["volumes", "volumes/status", "engines", "engines/status", "replicas", "replicas/status", "settings",
+              "engineimages", "engineimages/status", "nodes", "nodes/status", "instancemanagers", "instancemanagers/status",
+              "sharemanagers", "sharemanagers/status", "backingimages", "backingimages/status",
+              "backingimagemanagers", "backingimagemanagers/status", "backingimagedatasources", "backingimagedatasources/status",
+              "backuptargets", "backuptargets/status", "backupvolumes", "backupvolumes/status", "backups", "backups/status",
+              "recurringjobs", "recurringjobs/status", "orphans", "orphans/status", "snapshots", "snapshots/status",
+              "supportbundles", "supportbundles/status", "systembackups", "systembackups/status", "systemrestores", "systemrestores/status",
+              "volumeattachments", "volumeattachments/status", "backupbackingimages", "backupbackingimages/status"]
+  verbs: ["*"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["*"]
+- apiGroups: ["metrics.k8s.io"]
+  resources: ["pods", "nodes"]
+  verbs: ["get", "list"]
+- apiGroups: ["apiregistration.k8s.io"]
+  resources: ["apiservices"]
+  verbs: ["list", "watch"]
+- apiGroups: ["admissionregistration.k8s.io"]
+  resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"]
+  verbs: ["get", "list", "create", "patch", "delete"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["roles", "rolebindings", "clusterrolebindings", "clusterroles"]
+  verbs: ["*"]
+---
+# Source: longhorn/templates/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: longhorn-bind
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: longhorn-role
+subjects:
+- kind: ServiceAccount
+  name: longhorn-service-account
+  namespace: longhorn-system
+---
+# Source: longhorn/templates/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: longhorn-support-bundle
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: longhorn-support-bundle
+  namespace: longhorn-system
+---
+# Source: longhorn/templates/daemonset-sa.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    app: longhorn-manager
+  name: longhorn-backend
+  namespace: longhorn-system
+spec:
+  type: ClusterIP
+  selector:
+    app: longhorn-manager
+  ports:
+  - name: manager
+    port: 9500
+    targetPort: manager
+---
+# Source: longhorn/templates/deployment-ui.yaml
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    app: longhorn-ui
+  name: longhorn-frontend
+  namespace: longhorn-system
+spec:
+  type: ClusterIP
+  selector:
+    app: longhorn-ui
+  ports:
+  - name: http
+    port: 80
+    targetPort: http
+    nodePort: null
+---
+# Source: longhorn/templates/services.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    app: longhorn-conversion-webhook
+  name: longhorn-conversion-webhook
+  namespace: longhorn-system
+spec:
+  type: ClusterIP
+  selector:
+    app: longhorn-manager
+  ports:
+  - name: conversion-webhook
+    port: 9501
+    targetPort: conversion-wh
+---
+# Source: longhorn/templates/services.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    app: longhorn-admission-webhook
+  name: longhorn-admission-webhook
+  namespace: longhorn-system
+spec:
+  type: ClusterIP
+  selector:
+    app: longhorn-manager
+  ports:
+  - name: admission-webhook
+    port: 9502
+    targetPort: admission-wh
+---
+# Source: longhorn/templates/services.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    app: longhorn-recovery-backend
+  name: longhorn-recovery-backend
+  namespace: longhorn-system
+spec:
+  type: ClusterIP
+  selector:
+    app: longhorn-manager
+  ports:
+  - name: recovery-backend
+    port: 9503
+    targetPort: recov-backend
+---
+# Source: longhorn/templates/services.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+  name: longhorn-engine-manager
+  namespace: longhorn-system
+spec:
+  clusterIP: None
+  selector:
+    longhorn.io/component: instance-manager
+    longhorn.io/instance-manager-type: engine
+---
+# Source: longhorn/templates/services.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+  name: longhorn-replica-manager
+  namespace: longhorn-system
+spec:
+  clusterIP: None
+  selector:
+    longhorn.io/component: instance-manager
+    longhorn.io/instance-manager-type: replica
+---
+# Source: longhorn/templates/daemonset-sa.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    app: longhorn-manager
+  name: longhorn-manager
+  namespace: longhorn-system
+spec:
+  selector:
+    matchLabels:
+      app: longhorn-manager
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: longhorn
+        app.kubernetes.io/instance: longhorn
+        app.kubernetes.io/version: v1.6.0
+        app: longhorn-manager
+    spec:
+      containers:
+      - name: longhorn-manager
+        image: longhornio/longhorn-manager:v1.6.0
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          privileged: true
+        command:
+        - longhorn-manager
+        - -d
+        - daemon
+        - --engine-image
+        - "longhornio/longhorn-engine:v1.6.0"
+        - --instance-manager-image
+        - "longhornio/longhorn-instance-manager:v1.6.0"
+        - --share-manager-image
+        - "longhornio/longhorn-share-manager:v1.6.0"
+        - --backing-image-manager-image
+        - "longhornio/backing-image-manager:v1.6.0"
+        - --support-bundle-manager-image
+        - "longhornio/support-bundle-kit:v0.0.33"
+        - --manager-image
+        - "longhornio/longhorn-manager:v1.6.0"
+        - --service-account
+        - longhorn-service-account
+        - --upgrade-version-check
+        ports:
+        - containerPort: 9500
+          name: manager
+        - containerPort: 9501
+          name: conversion-wh
+        - containerPort: 9502
+          name: admission-wh
+        - containerPort: 9503
+          name: recov-backend
+        readinessProbe:
+          httpGet:
+            path: /v1/healthz
+            port: 9501
+            scheme: HTTPS
+        volumeMounts:
+        - name: dev
+          mountPath: /host/dev/
+        - name: proc
+          mountPath: /host/proc/
+        - name: longhorn
+          mountPath: /var/lib/longhorn/
+          mountPropagation: Bidirectional
+        - name: longhorn-grpc-tls
+          mountPath: /tls-files/
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+      volumes:
+      - name: dev
+        hostPath:
+          path: /dev/
+      - name: proc
+        hostPath:
+          path: /proc/
+      - name: longhorn
+        hostPath:
+          path: /var/lib/longhorn/
+      - name: longhorn-grpc-tls
+        secret:
+          secretName: longhorn-grpc-tls
+          optional: true
+      priorityClassName: "longhorn-critical"
+      serviceAccountName: longhorn-service-account
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: "100%"
+---
+# Source: longhorn/templates/deployment-driver.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: longhorn-driver-deployer
+  namespace: longhorn-system
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: longhorn-driver-deployer
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: longhorn
+        app.kubernetes.io/instance: longhorn
+        app.kubernetes.io/version: v1.6.0
+        app: longhorn-driver-deployer
+    spec:
+      initContainers:
+        - name: wait-longhorn-manager
+          image: longhornio/longhorn-manager:v1.6.0
+          command: ['sh', '-c', 'while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done']
+      containers:
+        - name: longhorn-driver-deployer
+          image: longhornio/longhorn-manager:v1.6.0
+          imagePullPolicy: IfNotPresent
+          command:
+          - longhorn-manager
+          - -d
+          - deploy-driver
+          - --manager-image
+          - "longhornio/longhorn-manager:v1.6.0"
+          - --manager-url
+          - http://longhorn-backend:9500/v1
+          env:
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: SERVICE_ACCOUNT
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.serviceAccountName
+          - name: CSI_ATTACHER_IMAGE
+            value: "longhornio/csi-attacher:v4.4.2"
+          - name: CSI_PROVISIONER_IMAGE
+            value: "longhornio/csi-provisioner:v3.6.2"
+          - name: CSI_NODE_DRIVER_REGISTRAR_IMAGE
+            value: "longhornio/csi-node-driver-registrar:v2.9.2"
+          - name: CSI_RESIZER_IMAGE
+            value: "longhornio/csi-resizer:v1.9.2"
+          - name: CSI_SNAPSHOTTER_IMAGE
+            value: "longhornio/csi-snapshotter:v6.3.2"
+          - name: CSI_LIVENESS_PROBE_IMAGE
+            value: "longhornio/livenessprobe:v2.11.0"
+      priorityClassName: "longhorn-critical"
+      serviceAccountName: longhorn-service-account
+      securityContext:
+        runAsUser: 0
+---
+# Source: longhorn/templates/deployment-ui.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: longhorn
+    app.kubernetes.io/instance: longhorn
+    app.kubernetes.io/version: v1.6.0
+    app: longhorn-ui
+  name: longhorn-ui
+  namespace: longhorn-system
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: longhorn-ui
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: longhorn
+        app.kubernetes.io/instance: longhorn
+        app.kubernetes.io/version: v1.6.0
+        app: longhorn-ui
+    spec:
+      serviceAccountName: longhorn-ui-service-account
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 1
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                  - longhorn-ui
+              topologyKey: kubernetes.io/hostname
+      containers:
+      - name: longhorn-ui
+        image: longhornio/longhorn-ui:v1.6.0
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+        - name : nginx-cache
+          mountPath: /var/cache/nginx/
+        - name : nginx-config
+          mountPath: /var/config/nginx/
+        - name: var-run
+          mountPath: /var/run/
+        ports:
+        - containerPort: 8000
+          name: http
+        env:
+          - name: LONGHORN_MANAGER_IP
+            value: "http://longhorn-backend:9500"
+          - name: LONGHORN_UI_PORT
+            value: "8000"
+      volumes:
+      - emptyDir: {}
+        name: nginx-cache
+      - emptyDir: {}
+        name: nginx-config
+      - emptyDir: {}
+        name: var-run
+      priorityClassName: "longhorn-critical"
+---
+# Source: longhorn/templates/validate-psp-install.yaml
+#

From 30611262d77c983c73da0b258245925dad7ff97f Mon Sep 17 00:00:00 2001
From: Nick S <shoebucket@gmail.com>
Date: Sat, 23 Mar 2024 03:49:04 +0000
Subject: [PATCH 11/15] update .gitignore for secret values

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index 4d36f198..60fe6a3e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,6 @@ Kubernetes/K3S-Deploy/kubectl
 Kubernetes/K3S-Deploy/kube-vip.yaml
 Kubernetes/K3S-Deploy/ipAddressPool.yaml
 Kubernetes/K3S-Deploy/k3sup
+Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Issuers/letsencrypt-production.yaml
+Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Issuers/secret-cf-token.yaml
+Kubernetes/Traefik-PiHole/Helm/Traefik/Dashboard/secret-dashboard.yaml

From 1124f0855aff640de8ea17b7ee45c3337810b113 Mon Sep 17 00:00:00 2001
From: Nick S <shoebucket@gmail.com>
Date: Sat, 23 Mar 2024 03:50:48 +0000
Subject: [PATCH 12/15] feat(traefik): update for my environment

---
 ...tion.yaml => shoemakernet-production.yaml} | 10 +--
 .../Helm/Traefik/Dashboard/ingress.yaml       |  4 +-
 .../Traefik-PiHole/Helm/Traefik/values.yaml   |  2 +-
 Kubernetes/Traefik-PiHole/deploy.sh           | 68 +++++++++++--------
 4 files changed, 48 insertions(+), 36 deletions(-)
 rename Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/{jimsgarage-production.yaml => shoemakernet-production.yaml} (50%)
 mode change 100644 => 100755 Kubernetes/Traefik-PiHole/deploy.sh

diff --git a/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/jimsgarage-production.yaml b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/shoemakernet-production.yaml
similarity index 50%
rename from Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/jimsgarage-production.yaml
rename to Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/shoemakernet-production.yaml
index f7c0d7fc..a98ae532 100644
--- a/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/jimsgarage-production.yaml
+++ b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/shoemakernet-production.yaml
@@ -2,14 +2,14 @@
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
-  name: jimsgarage # change to your domain
+  name: shoemakernet # change to your domain
   namespace: traefik # add to traefik namespace so it can use it (you DO NOT need it in each app namespace!!!)
 spec:
-  secretName: jimsgarage-tls # change to your secretname
+  secretName: shoemakernet-tls # change to your secretname
   issuerRef:
     name: letsencrypt-production
     kind: ClusterIssuer
-  commonName: "*.jimsgarage.co.uk" # change to your domain
+  commonName: "*.shoe-maker.net" # change to your domain
   dnsNames:
-  - "*.jimsgarage.co.uk" # change to your domain
-  - jimsgarage.co.uk # change to your domain
+  - "*.shoe-maker.net" # change to your domain
+  - shoe-maker.net # change to your domain
diff --git a/Kubernetes/Traefik-PiHole/Helm/Traefik/Dashboard/ingress.yaml b/Kubernetes/Traefik-PiHole/Helm/Traefik/Dashboard/ingress.yaml
index ae808923..ef124c59 100644
--- a/Kubernetes/Traefik-PiHole/Helm/Traefik/Dashboard/ingress.yaml
+++ b/Kubernetes/Traefik-PiHole/Helm/Traefik/Dashboard/ingress.yaml
@@ -9,7 +9,7 @@ spec:
   entryPoints:
     - websecure
   routes:
-    - match: Host(`traefik2.jimsgarage.co.uk`) # change this to your domain
+    - match: Host(`traefik.shoe-maker.net`) # change this to your domain
       kind: Rule
       middlewares:
         - name: traefik-dashboard-basicauth
@@ -18,4 +18,4 @@ spec:
         - name: api@internal
           kind: TraefikService
   tls:
-    secretName: jimsgarage-tls # change this to your secret!!!
+    secretName: shoemakernet-tls # change this to your secret!!!
diff --git a/Kubernetes/Traefik-PiHole/Helm/Traefik/values.yaml b/Kubernetes/Traefik-PiHole/Helm/Traefik/values.yaml
index cb6b1f33..c23e7461 100644
--- a/Kubernetes/Traefik-PiHole/Helm/Traefik/values.yaml
+++ b/Kubernetes/Traefik-PiHole/Helm/Traefik/values.yaml
@@ -50,6 +50,6 @@ service:
   annotations: {}
   labels: {}
   spec:
-    loadBalancerIP: 192.168.3.65 # this should be an IP in the Kube-VIP range
+    loadBalancerIP: 192.168.11.100 # this should be an IP in the Kube-VIP range
   loadBalancerSourceRanges: []
   externalIPs: []
diff --git a/Kubernetes/Traefik-PiHole/deploy.sh b/Kubernetes/Traefik-PiHole/deploy.sh
old mode 100644
new mode 100755
index 66f83a77..f1e66987
--- a/Kubernetes/Traefik-PiHole/deploy.sh
+++ b/Kubernetes/Traefik-PiHole/deploy.sh
@@ -19,22 +19,22 @@ echo -e " \033[32;2m                                                           \
 # and https://github.com/traefik/traefik-helm-chart
 
 # Step 0: Clone repository
-DESTINATION=~/Helm/Traefik
-if [ ! -d "`eval echo ${DESTINATION//>}`" ]; then
-    sudo apt install unzip -y
-    mkdir jimsgarage
-    mkdir Helm
-    mkdir Manifest
-    curl -L -o master.zip https://github.com/JamesTurland/JimsGarage/archive/refs/heads/main.zip
-    unzip master.zip -d ~/jimsgarage
-    cp -r ~/jimsgarage/JimsGarage-main/Kubernetes/Traefik-PiHole/* ~/
-    rm master.zip
-    rm -r ~/jimsgarage
-    echo -e " \033[32;5mRepo cloned - EDIT FILES!!!\033[0m"
-    exit
-else
-    echo -e " \033[32;5mRepo already exists, continuing...\033[0m"
-fi
+# DESTINATION=~/Helm/Traefik
+# if [ ! -d "`eval echo ${DESTINATION//>}`" ]; then
+#     sudo apt install unzip -y
+#     mkdir jimsgarage
+#     mkdir Helm
+#     mkdir Manifest
+#     curl -L -o master.zip https://github.com/JamesTurland/JimsGarage/archive/refs/heads/main.zip
+#     unzip master.zip -d ~/jimsgarage
+#     cp -r ~/jimsgarage/JimsGarage-main/Kubernetes/Traefik-PiHole/* ~/
+#     rm master.zip
+#     rm -r ~/jimsgarage
+#     echo -e " \033[32;5mRepo cloned - EDIT FILES!!!\033[0m"
+#     exit
+# else
+#     echo -e " \033[32;5mRepo already exists, continuing...\033[0m"
+# fi
 
 # Step 1: Check dependencies
 # Helm
@@ -56,6 +56,14 @@ then
 else
     echo -e " \033[32;5mKubectl already installed\033[0m"
 fi
+# jq
+if ! command -v jq --version &> /dev/null
+then
+    echo -e " \033[31;5mJQ not found, installing\033[0m"
+    sudo apt install jq -y
+else
+    echo -e " \033[32;5mJQ already installed\033[0m"
+fi
 
 # Step 2: Add Helm Repos
 helm repo add traefik https://helm.traefik.io/traefik
@@ -64,26 +72,30 @@ helm repo add crowdsec https://crowdsecurity.github.io/helm-charts
 helm repo update
 
 # Step 3: Create Traefik namespace
-kubectl create namespace traefik
+namespaceStatus=$(kubectl get ns traefik -o json | jq .status.phase -r)
+if [ $namespaceStatus != "Active" ]
+then
+    kubectl create namespace traefik
+fi
 
 # Step 4: Install Traefik
-helm install --namespace=traefik traefik traefik/traefik -f ~/Helm/Traefik/values.yaml
+helm install --namespace=traefik traefik traefik/traefik -f ./Helm/Traefik/values.yaml
 
 # Step 5: Check Traefik deployment
 kubectl get svc -n traefik
 kubectl get pods -n traefik
 
 # Step 6: Apply Middleware
-kubectl apply -f ~/Helm/Traefik/default-headers.yaml
+kubectl apply -f ./Helm/Traefik/default-headers.yaml
 
 # Step 7: Create Secret for Traefik Dashboard
-kubectl apply -f ~/Helm/Traefik/Dashboard/secret-dashboard.yaml
+kubectl apply -f ./Helm/Traefik/Dashboard/secret-dashboard.yaml
 
 # Step 8: Apply Middleware
-kubectl apply -f ~/Helm/Traefik/Dashboard/middleware.yaml
+kubectl apply -f ./Helm/Traefik/Dashboard/middleware.yaml
 
 # Step 9: Apply Ingress to Access Service
-kubectl apply -f ~/Helm/Traefik/Dashboard/ingress.yaml
+kubectl apply -f ./Helm/Traefik/Dashboard/ingress.yaml
 
 # Step 10: Install Cert-Manager (should already have this with Rancher deployment)
 # Check if we already have it by querying namespace
@@ -96,7 +108,7 @@ then
     cert-manager \
     jetstack/cert-manager \
     --namespace cert-manager \
-    --values ~/Helm/Traefik/Cert-Manager/values.yaml
+    --values ./Helm/Traefik/Cert-Manager/values.yaml
 else
     echo "Cert-Manager is not present, installing..."
     kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.2/cert-manager.crds.yaml
@@ -109,18 +121,18 @@ else
 fi
 
 # Step 11: Apply secret for certificate (Cloudflare)
-kubectl apply -f ~/Helm/Traefik/Cert-Manager/Issuers/secret-cf-token.yaml
+kubectl apply -f ./Helm/Traefik/Cert-Manager/Issuers/secret-cf-token.yaml
 
 # Step 12: Apply production certificate issuer (technically you should use the staging to test as per documentation)
-kubectl apply -f ~/Helm/Traefik/Cert-Manager/Issuers/letsencrypt-production.yaml
+kubectl apply -f ./Helm/Traefik/Cert-Manager/Issuers/letsencrypt-production.yaml
 
 # Step 13: Apply production certificate
-kubectl apply -f ~/Helm/Traefik/Cert-Manager/Certificates/Production/jimsgarage-production.yaml
+kubectl apply -f ./Helm/Traefik/Cert-Manager/Certificates/Production/shoemakernet-production.yaml
 
 # Step 14: Create PiHole namespace
-kubectl create namespace pihole
+#kubectl create namespace pihole
 
 # Step 15: Deploy PiHole
-kubectl apply -f ~/Manifest/PiHole
+#kubectl apply -f ./Manifest/PiHole
 
 echo -e " \033[32;5mScript finished. Be sure to create PVC for PiHole in Longhorn UI\033[0m"

From dc9fc9ba9020c5d1fd246173c9de890b016c0634 Mon Sep 17 00:00:00 2001
From: Nick S <shoebucket@gmail.com>
Date: Wed, 27 Mar 2024 03:14:56 +0000
Subject: [PATCH 13/15] feat(traefik): automatic certs and ingress

---
 .../local-shoemakernet-production.yaml        | 15 +++++++++++++
 .../traefik-local-shoemaker-net.yaml          | 14 +++++++++++++
 .../Staging/shoemakernet-staging.yaml         | 15 +++++++++++++
 .../Issuers/letsencrypt-production.yaml       |  6 +++---
 .../Issuers/letsencrypt-staging.yaml          | 21 +++++++++++++++++++
 .../Helm/Traefik/Cert-Manager/values.yaml     |  5 +++++
 .../Helm/Traefik/Dashboard/ingress.yaml       |  6 ++++--
 7 files changed, 77 insertions(+), 5 deletions(-)
 create mode 100644 Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/local-shoemakernet-production.yaml
 create mode 100644 Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/traefik-local-shoemaker-net.yaml
 create mode 100644 Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Staging/shoemakernet-staging.yaml
 create mode 100644 Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Issuers/letsencrypt-staging.yaml

diff --git a/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/local-shoemakernet-production.yaml b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/local-shoemakernet-production.yaml
new file mode 100644
index 00000000..b9919410
--- /dev/null
+++ b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/local-shoemakernet-production.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: local-shoemaker-net # change to your domain
+  namespace: traefik # add to traefik namespace so it can use it (you DO NOT need it in each app namespace!!!)
+spec:
+  secretName: local-shoemaker-net-tls # change to your secretname
+  issuerRef:
+    name: letsencrypt-production
+    kind: ClusterIssuer
+  commonName: "*.local.shoe-maker.net" # change to your domain
+  dnsNames:
+  - "local.shoe-maker.net" # change to your domain
+  - "*.local.shoe-maker.net" # change to your domain
diff --git a/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/traefik-local-shoemaker-net.yaml b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/traefik-local-shoemaker-net.yaml
new file mode 100644
index 00000000..e4dc765d
--- /dev/null
+++ b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Production/traefik-local-shoemaker-net.yaml
@@ -0,0 +1,14 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: traefik-local-shoemaker-net # change to your domain
+  namespace: traefik # add to traefik namespace so it can use it (you DO NOT need it in each app namespace!!!)
+spec:
+  secretName: traefik-local-shoemaker-net-tls # change to your secretname
+  issuerRef:
+    name: letsencrypt-production
+    kind: ClusterIssuer
+  commonName: "traefik.local.shoe-maker.net" # change to your domain
+  dnsNames:
+  - "traefik.local.shoe-maker.net" # change to your domain
diff --git a/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Staging/shoemakernet-staging.yaml b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Staging/shoemakernet-staging.yaml
new file mode 100644
index 00000000..cce7677c
--- /dev/null
+++ b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Certificates/Staging/shoemakernet-staging.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: local-shoemaker-net-staging # change to your domain
+  namespace: traefik # add to traefik namespace so it can use it (you DO NOT need it in each app namespace!!!)
+spec:
+  secretName: local-shoemaker-net-staging-tls # change to your secretname
+  issuerRef:
+    name: letsencrypt-staging
+    kind: ClusterIssuer
+  commonName: "*.local.shoe-maker.net" # change to your domain
+  dnsNames:
+  - "local.shoe-maker.net" # change to your domain
+  - "*.local.shoe-maker.net" # change to your domain
diff --git a/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Issuers/letsencrypt-production.yaml b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Issuers/letsencrypt-production.yaml
index 223c5009..cf51ce77 100644
--- a/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Issuers/letsencrypt-production.yaml
+++ b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Issuers/letsencrypt-production.yaml
@@ -6,16 +6,16 @@ metadata:
 spec:
   acme:
     server: https://acme-v02.api.letsencrypt.org/directory
-    email: your@email.com # add your email 
+    email: shoebucket@gmail.com # add your email 
     privateKeySecretRef:
       name: letsencrypt-production
     solvers:
       - dns01:
           cloudflare:
-            email: your@email.com # add your email to your cloudflare account
+            email: shoebucket@gmail.com # add your email to your cloudflare account
             apiTokenSecretRef:
               name: cloudflare-token-secret
               key: cloudflare-token
         selector:
           dnsZones:
-            - "your-domain.com" # change to your zone on CloudFlare
+            - "shoe-maker.net" # change to your zone on CloudFlare
diff --git a/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Issuers/letsencrypt-staging.yaml b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Issuers/letsencrypt-staging.yaml
new file mode 100644
index 00000000..57ad0a80
--- /dev/null
+++ b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Issuers/letsencrypt-staging.yaml
@@ -0,0 +1,21 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-staging
+spec:
+  acme:
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    email: shoebucket@gmail.com # add your email 
+    privateKeySecretRef:
+      name: letsencrypt-staging
+    solvers:
+      - dns01:
+          cloudflare:
+            email: shoebucket@gmail.com # add your email to your cloudflare account
+            apiTokenSecretRef:
+              name: cloudflare-token-secret
+              key: cloudflare-token
+        selector:
+          dnsZones:
+            - "shoe-maker.net" # change to your zone on CloudFlare
diff --git a/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/values.yaml b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/values.yaml
index a9abcebc..a7c8f4e2 100644
--- a/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/values.yaml
+++ b/Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/values.yaml
@@ -8,3 +8,8 @@ podDnsConfig:
   nameservers:
     - 1.1.1.1
     - 9.9.9.9
+
+tolerations:
+  - key: "node-role.kubernetes.io/master"
+    operator: "Exists"
+    effect: "NoSchedule"
diff --git a/Kubernetes/Traefik-PiHole/Helm/Traefik/Dashboard/ingress.yaml b/Kubernetes/Traefik-PiHole/Helm/Traefik/Dashboard/ingress.yaml
index ef124c59..46f5d83e 100644
--- a/Kubernetes/Traefik-PiHole/Helm/Traefik/Dashboard/ingress.yaml
+++ b/Kubernetes/Traefik-PiHole/Helm/Traefik/Dashboard/ingress.yaml
@@ -9,13 +9,15 @@ spec:
   entryPoints:
     - websecure
   routes:
-    - match: Host(`traefik.shoe-maker.net`) # change this to your domain
+    - match: Host(`traefik.local.shoe-maker.net`) # change this to your domain
       kind: Rule
       middlewares:
         - name: traefik-dashboard-basicauth
           namespace: traefik
+        - name: default-headers
+          namespace: traefik
       services:
         - name: api@internal
           kind: TraefikService
   tls:
-    secretName: shoemakernet-tls # change this to your secret!!!
+    secretName: traefik-local-shoemaker-net-tls # change this to your secret!!!

From a8634e410f38e9b7d75a8953f043aec69e7743d8 Mon Sep 17 00:00:00 2001
From: Nick S <shoebucket@gmail.com>
Date: Wed, 27 Mar 2024 03:17:01 +0000
Subject: [PATCH 14/15] update .gitignore

---
 .gitignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 60fe6a3e..e674ca14 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,5 @@ Kubernetes/K3S-Deploy/kubectl
 Kubernetes/K3S-Deploy/kube-vip.yaml
 Kubernetes/K3S-Deploy/ipAddressPool.yaml
 Kubernetes/K3S-Deploy/k3sup
-Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Issuers/letsencrypt-production.yaml
 Kubernetes/Traefik-PiHole/Helm/Traefik/Cert-Manager/Issuers/secret-cf-token.yaml
 Kubernetes/Traefik-PiHole/Helm/Traefik/Dashboard/secret-dashboard.yaml

From 731a037cc08a307e8a380ce60f23f2e7785f065f Mon Sep 17 00:00:00 2001
From: shoemakn <shoebucket@gmail.com>
Date: Thu, 28 Mar 2024 00:11:44 -0400
Subject: [PATCH 15/15] fix(cloud-init): coloration on message

---
 Kubernetes/Cloud-Init/create-clones.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Kubernetes/Cloud-Init/create-clones.sh b/Kubernetes/Cloud-Init/create-clones.sh
index 1c8af9f3..e06ea789 100755
--- a/Kubernetes/Cloud-Init/create-clones.sh
+++ b/Kubernetes/Cloud-Init/create-clones.sh
@@ -94,7 +94,7 @@ function print_info() {
             echo -e ", MAC: ${custom_macs[$i-1]}\e[0m"
             ;;
         *)
-            echo "\e[0m"
+            echo "\e[32m\e[0m"
             ;;
         esac
     done