feat: COSI controller Addon (#1008)

**What problem does this PR solve?**:
This PR adds the COSI controller Addon, without any providers.
The chart comes from
https://github.com/mesosphere/charts/tree/master/stable/cosi

The addon does not take in any configuration so the API to enable it is
just:
```yaml
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
  name: <NAME>
spec:
  topology:
    variables:
      - name: clusterConfig
        value:
          addons:
            cosi: {}
```
Once we add providers, the API can be similar to what it is with CSI
with a `providers:` field.

**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->
New unit tests.
Also deployed it on a Nutanix Cluster and the Nutanix COSI driver from
nutanix-cloud-native/cosi-driver-nutanix#375

Create the credentials Secret
```
apiVersion: v1
kind: Secret
metadata:
  annotations:
  labels:
  name: objectstorage-provisioner
  namespace: cosi-driver-nutanix
stringData:
  ACCESS_KEY: "<>"
  ACCOUNT_NAME: "<>"
  ENDPOINT: "http://10.1.1.1:80"
  PC_SECRET: "<>:9440:<>:<>"
  SECRET_KEY: "<>"
type: Opaque
```

```
NUTANIX_COSI_DIR=<>
helm install cosi-driver -n cosi-driver-nutanix --create-namespace --set=secret.enabled=false --set=cosiController.enabled=false --skip-crds$NUTANIX_COSI_DIR/charts/

kubectl apply -f $NUTANIX_COSI_DIR/project/examples/bucketclass.yaml
kubectl apply -f $NUTANIX_COSI_DIR/project/examples/bucketclaim.yaml
kubectl apply -f $NUTANIX_COSI_DIR/project/examples/bucketaccessclass.yaml
kubectl apply -f $NUTANIX_COSI_DIR/project/examples/bucketaccess.yaml

# From nutanix-cloud-native/cosi-driver-nutanix#374
kubectl apply -f https://raw.githubusercontent.com/nutanix-cloud-native/cosi-driver-nutanix/944ce9ba66bc34e59bd94519a88da8bf9fae3d0c/project/examples/awscliapppod.yaml
```
The Secret was correctly generated and the test Pod wrote and read to a
new bucket.
```
$ kubectl logs awscli
Defaulted container "awscli" out of: awscli, write-aws-credentials (init), write-test-file (init)
+ aws s3 ls
2025-01-07 20:43:31 sample-bucketclass8ce19295-f704-45bb-a533-15591c55a747
++ cat /tmp/test-directory/file.txt
+ readonly BUCKET_NAME=sample-bucketclass8ce19295-f704-45bb-a533-15591c55a747
+ BUCKET_NAME=sample-bucketclass8ce19295-f704-45bb-a533-15591c55a747
++ date +%Y%m%d_%H%M%S
+ readonly FILE_NAME=20250107_205233.txt
+ FILE_NAME=20250107_205233.txt
+ aws s3 cp /tmp/test-directory/file.txt s3://sample-bucketclass8ce19295-f704-45bb-a533-15591c55a747/20250107_205233.txt
upload: ../tmp/test-directory/file.txt to s3://sample-bucketclass8ce19295-f704-45bb-a533-15591c55a747/20250107_205233.txt
+ aws s3 cp s3://sample-bucketclass8ce19295-f704-45bb-a533-15591c55a747/20250107_205233.txt -
sample-bucketclass8ce19295-f704-45bb-a533-15591c55a747
```

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

api/v1alpha1/addon_types.go

@@ -70,13 +70,19 @@ type DockerAddons struct {
 
 	// +kubebuilder:validation:Optional
 	CSI *DockerCSI `json:"csi,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	COSI *DockerCOSI `json:"cosi,omitempty"`
 }
 
 type NutanixAddons struct {
 	GenericAddons `json:",inline"`
 
 	// +kubebuilder:validation:Optional
 	CSI *NutanixCSI `json:"csi,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	COSI *NutanixCOSI `json:"cosi,omitempty"`
 }
 
 type GenericAddons struct {
@@ -96,8 +102,6 @@ type GenericAddons struct {
 	ServiceLoadBalancer *ServiceLoadBalancer `json:"serviceLoadBalancer,omitempty"`
 }
 
-// +kubebuilder:validation:Optional
-// +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon
 type AddonStrategy string
 
 // CNI required for providing CNI configuration.
@@ -109,13 +113,15 @@ type CNI struct {
 
 	// Addon strategy used to deploy the CNI provider to the workload cluster.
 	// +kubebuilder:default=HelmAddon
+	// +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon
 	Strategy *AddonStrategy `json:"strategy,omitempty"`
 }
 
 // NFD tells us to enable or disable the node feature discovery addon.
 type NFD struct {
 	// Addon strategy used to deploy Node Feature Discovery (NFD) to the workload cluster.
 	// +kubebuilder:default=HelmAddon
+	// +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon
 	Strategy *AddonStrategy `json:"strategy,omitempty"`
 }
 
@@ -124,6 +130,7 @@ type ClusterAutoscaler struct {
 	// Addon strategy used to deploy cluster-autoscaler to the management cluster
 	// targeting the workload cluster.
 	// +kubebuilder:default=HelmAddon
+	// +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon
 	Strategy *AddonStrategy `json:"strategy,omitempty"`
 }
 
@@ -136,9 +143,17 @@ type GenericCSI struct {
 	SnapshotController *SnapshotController `json:"snapshotController,omitempty"`
 }
 
+type GenericCOSI struct {
+	// Addon strategy used to deploy the COSI controller to the workload cluster.
+	// +kubebuilder:default=HelmAddon
+	// +kubebuilder:validation:Enum=HelmAddon
+	Strategy *AddonStrategy `json:"strategy,omitempty"`
+}
+
 type SnapshotController struct {
 	// Addon strategy used to deploy the snapshot controller to the workload cluster.
 	// +kubebuilder:default=HelmAddon
+	// +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon
 	Strategy *AddonStrategy `json:"strategy,omitempty"`
 }
 
@@ -197,6 +212,7 @@ type CSIProvider struct {
 
 	// Addon strategy used to deploy the CSI provider to the workload cluster.
 	// +kubebuilder:default=HelmAddon
+	// +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon
 	Strategy *AddonStrategy `json:"strategy,omitempty"`
 
 	// The reference to any secret used by the CSI Provider.
@@ -231,6 +247,14 @@ type CSICredentials struct {
 	SecretRef LocalObjectReference `json:"secretRef"`
 }
 
+type DockerCOSI struct {
+	GenericCOSI `json:",inline"`
+}
+
+type NutanixCOSI struct {
+	GenericCOSI `json:",inline"`
+}
+
 // CCM tells us to enable or disable the cloud provider interface.
 type CCM struct {
 	// A reference to the Secret for credential information for the target Prism Central instance
@@ -239,6 +263,7 @@ type CCM struct {
 
 	// Addon strategy used to deploy the CCM to the workload cluster.
 	// +kubebuilder:default=HelmAddon
+	// +kubebuilder:validation:Enum=ClusterResourceSet;HelmAddon
 	Strategy *AddonStrategy `json:"strategy,omitempty"`
 }
 

api/v1alpha1/constants.go

@@ -22,7 +22,8 @@ const (
 	CNIVariableName = "cni"
 	// NFDVariableName is the NFD external patch variable name.
 	NFDVariableName = "nfd"
-
+	// COSIVariableName is the COSI external patch variable name.
+	COSIVariableName = "cosi"
 	// ClusterAutoscalerVariableName is the cluster-autoscaler external patch variable name.
 	ClusterAutoscalerVariableName = "clusterAutoscaler"
 	// ServiceLoadBalancerVariableName is the Service LoadBalancer config patch variable name.

api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml

@@ -104,6 +104,15 @@ spec:
                       required:
                         - provider
                       type: object
+                    cosi:
+                      properties:
+                        strategy:
+                          default: HelmAddon
+                          description: Addon strategy used to deploy the COSI controller to the workload cluster.
+                          enum:
+                            - HelmAddon
+                          type: string
+                      type: object
                     csi:
                       properties:
                         defaultStorage:

api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml

@@ -104,6 +104,15 @@ spec:
                       required:
                         - provider
                       type: object
+                    cosi:
+                      properties:
+                        strategy:
+                          default: HelmAddon
+                          description: Addon strategy used to deploy the COSI controller to the workload cluster.
+                          enum:
+                            - HelmAddon
+                          type: string
+                      type: object
                     csi:
                       properties:
                         defaultStorage:

api/variables/aggregate_types.go

@@ -53,10 +53,16 @@ type Addons struct {
 	carenv1.GenericAddons `json:",inline"`
 
 	CSI *CSI `json:"csi,omitempty"`
+
+	COSI *COSI `json:"cosi,omitempty"`
 }
 
 type CSI struct {
 	carenv1.GenericCSI `json:",inline"`
 
 	Providers map[string]carenv1.CSIProvider `json:"providers"`
 }
+
+type COSI struct {
+	carenv1.GenericCOSI `json:",inline"`
+}

charts/cluster-api-runtime-extensions-nutanix/README.md

@@ -74,6 +74,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
 | hooks.cni.cilium.crsStrategy.defaultCiliumConfigMap.name | string | `"cilium"` |  |
 | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |
 | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cilium-cni-helm-values-template"` |  |
+| hooks.cosi.controller.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |
+| hooks.cosi.controller.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cosi-controller-helm-values-template"` |  |
 | hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |
 | hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-aws-ebs-csi-helm-values-template"` |  |
 | hooks.csi.local-path.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |

charts/cluster-api-runtime-extensions-nutanix/templates/cosi/controller/manifests/helm-addon-installation.yaml

@@ -0,0 +1,12 @@
+# Copyright 2025 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+{{- if (index .Values.hooks.cosi "controller").helmAddonStrategy.defaultValueTemplateConfigMap.create }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: '{{ (index .Values.hooks.cosi "controller").helmAddonStrategy.defaultValueTemplateConfigMap.name }}'
+data:
+  values.yaml: |-
+    {{- .Files.Get "addons/cosi/controller/values-template.yaml" | nindent 4 }}
+{{- end -}}

charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml

@@ -41,6 +41,7 @@ spec:
         - --csi.local-path.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "local-path").helmAddonStrategy.defaultValueTemplateConfigMap.name }}
         - --csi.snapshot-controller.helm-addon.default-values-template-configmap-name={{ (index .Values.hooks.csi "snapshot-controller").helmAddonStrategy.defaultValueTemplateConfigMap.name }}
         - --ccm.aws.helm-addon.default-values-template-configmap-name={{ .Values.hooks.ccm.aws.helmAddonStrategy.defaultValueTemplateConfigMap.name }}
+        - --cosi.controller.helm-addon.default-values-template-configmap-name={{ .Values.hooks.cosi.controller.helmAddonStrategy.defaultValueTemplateConfigMap.name }}
         {{- range $k, $v := .Values.hooks.ccm.aws.k8sMinorVersionToCCMVersion }}
         - --ccm.aws.aws-ccm-versions={{ $k }}={{ $v }}
         {{- end }}

charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml

@@ -23,6 +23,10 @@ data:
     ChartName: cluster-autoscaler
     ChartVersion: 9.43.2
     RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/autoscaler{{ end }}'
+  cosi-controller: |
+    ChartName: cosi
+    ChartVersion: 0.0.1-alpha.1
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://mesosphere.github.io/charts/stable/{{ end }}'
   local-path-provisioner-csi: |
     ChartName: local-path-provisioner
     ChartVersion: 0.0.30

charts/cluster-api-runtime-extensions-nutanix/values.schema.json

@@ -361,6 +361,32 @@
                     },
                     "type": "object"
                 },
+                "cosi": {
+                    "properties": {
+                        "controller": {
+                            "properties": {
+                                "helmAddonStrategy": {
+                                    "properties": {
+                                        "defaultValueTemplateConfigMap": {
+                                            "properties": {
+                                                "create": {
+                                                    "type": "boolean"
+                                                },
+                                                "name": {
+                                                    "type": "string"
+                                                }
+                                            },
+                                            "type": "object"
+                                        }
+                                    },
+                                    "type": "object"
+                                }
+                            },
+                            "type": "object"
+                        }
+                    },
+                    "type": "object"
+                },
                 "csi": {
                     "properties": {
                         "aws-ebs": {

charts/cluster-api-runtime-extensions-nutanix/values.yaml

@@ -107,6 +107,12 @@ hooks:
       defaultTemplateConfigMap:
         create: true
         name: default-kube-vip-template
+  cosi:
+    controller:
+      helmAddonStrategy:
+        defaultValueTemplateConfigMap:
+          create: true
+          name: default-cosi-controller-helm-values-template
 
 helmAddonsConfigMap: default-helm-addons-config
 

docs/content/addons/cosi.md

@@ -0,0 +1,32 @@
++++
+title = " Container Object Storage Interface (COSI)"
+icon = "fa-solid fa-eye"
++++
+
+By leveraging CAPI cluster lifecycle hooks, this handler deploys [Container Object Storage Interface] (COSI)
+on the new cluster at the `AfterControlPlaneInitialized` phase.
+
+Deployment of COSI is opt-in via the [provider-specific cluster configuration]({{< ref ".." >}}).
+
+The hook uses the [Cluster API Add-on Provider for Helm] to deploy the COSI resources.
+
+## Example
+
+To enable deployment of COSI on a cluster, specify the following values:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          addons:
+            cosi: {}
+```
+
+[Container Object Storage Interface]: https://kubernetes.io/blog/2022/09/02/cosi-kubernetes-object-storage-management/
+[Cluster API Add-on Provider for Helm]: https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm

examples/capi-quick-start/docker-cluster-calico-crs.yaml

@@ -27,6 +27,7 @@ spec:
           cni:
             provider: Calico
             strategy: ClusterResourceSet
+          cosi: {}
           csi:
             defaultStorage:
               provider: local-path

examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml

@@ -25,6 +25,7 @@ spec:
           clusterAutoscaler: {}
           cni:
             provider: Calico
+          cosi: {}
           csi:
             defaultStorage:
               provider: local-path