From 7662213d84cee3de428b20d2cc307263c7753f08 Mon Sep 17 00:00:00 2001 From: faiq Date: Mon, 18 Mar 2024 16:36:01 -0600 Subject: [PATCH 01/31] feat: adds nutanix csi to api --- api/v1alpha1/addon_types.go | 39 ++++++++++++++++++++++----- api/v1alpha1/clusterconfig_types.go | 3 ++- api/v1alpha1/zz_generated.deepcopy.go | 31 ++++++++++++++++++++- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index c88edf1e7..ba5bc61af 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -137,16 +137,45 @@ func (ClusterAutoscaler) VariableSchema() clusterv1.VariableSchema { type CSIProviders struct { // +optional Providers []CSIProvider `json:"providers,omitempty"` - // +optional - DefaultClassName string `json:"defaultClassName,omitempty"` } type CSIProvider struct { + // +optional Name string `json:"name,omitempty"` + + // +optional + StorageClassConfig *StorageClassConfig `json:"storageClassConfig,omitempty"` +} + +type StorageClassConfig struct { + // +optional + DefaultClassName string `json:"defaultClassName,omitempty"` + + // +optional + Parameters map[string]string `json:"parameters,omitempty"` +} + +func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { + supportedCSIProviders := []string{CSIProviderAWSEBS, CSIProviderNutanix} + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "defaultClassName": { + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON(supportedCSIProviders...), + }, + "parameters": { + Type: "object", + XPreserveUnknownFields: true, + }, + }, + }, + } } func (CSIProviders) VariableSchema() clusterv1.VariableSchema { - supportedCSIProviders := []string{CSIProviderAWSEBS} + supportedCSIProviders := []string{CSIProviderAWSEBS, CSIProviderNutanix} return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Type: "object", @@ -164,10 +193,6 @@ func (CSIProviders) VariableSchema() clusterv1.VariableSchema { }, }, }, - "defaultClassName": { - Type: "string", - Enum: variables.MustMarshalValuesToEnumJSON(supportedCSIProviders...), - }, }, }, } diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index 7bbbef0d8..75e02b61c 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -18,7 +18,8 @@ const ( CNIProviderCalico = "Calico" CNIProviderCilium = "Cilium" - CSIProviderAWSEBS = "aws-ebs" + CSIProviderAWSEBS = "aws-ebs" + CSIProviderNutanix = "nutanix" CCMProviderAWS = "aws" ) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 01692d804..ea0b63e27 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -254,6 +254,11 @@ func (in *CNI) DeepCopy() *CNI { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CSIProvider) DeepCopyInto(out *CSIProvider) { *out = *in + if in.StorageClassConfig != nil { + in, out := &in.StorageClassConfig, &out.StorageClassConfig + *out = new(StorageClassConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIProvider. @@ -272,7 +277,9 @@ func (in *CSIProviders) DeepCopyInto(out *CSIProviders) { if in.Providers != nil { in, out := &in.Providers, &out.Providers *out = make([]CSIProvider, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } } @@ -730,6 +737,28 @@ func (in *SecurityGroup) DeepCopy() *SecurityGroup { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageClassConfig) DeepCopyInto(out *StorageClassConfig) { + *out = *in + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageClassConfig. +func (in *StorageClassConfig) DeepCopy() *StorageClassConfig { + if in == nil { + return nil + } + out := new(StorageClassConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SubnetSpec) DeepCopyInto(out *SubnetSpec) { *out = *in From e32da135251827f2bdd5dd343f04ff5db7829dbe Mon Sep 17 00:00:00 2001 From: faiq Date: Tue, 19 Mar 2024 12:56:56 -0600 Subject: [PATCH 02/31] feat: minor API tweaks and adds tooling for chart --- api/v1alpha1/addon_types.go | 70 ++- api/v1alpha1/zz_generated.deepcopy.go | 11 +- .../manifests/helm-addon-installation.yaml | 9 + .../manifests/nutanix-csi-configmap.yaml | 543 ++++++++++++++++++ .../values.yaml | 6 + .../kustomize/nutanix-csi/helm-values.yaml | 1 + .../nutanix-csi/kustomization.yaml.tmpl | 20 + hack/addons/update-nutanix-csi.sh | 42 ++ make/addons.mk | 17 + 9 files changed, 706 insertions(+), 13 deletions(-) create mode 100644 charts/capi-runtime-extensions/templates/csi/nutanix/manifests/helm-addon-installation.yaml create mode 100644 charts/capi-runtime-extensions/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml create mode 100644 hack/addons/kustomize/nutanix-csi/helm-values.yaml create mode 100644 hack/addons/kustomize/nutanix-csi/kustomization.yaml.tmpl create mode 100755 hack/addons/update-nutanix-csi.sh diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index ba5bc61af..025570b9c 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -4,6 +4,7 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/variables" @@ -137,6 +138,9 @@ func (ClusterAutoscaler) VariableSchema() clusterv1.VariableSchema { type CSIProviders struct { // +optional Providers []CSIProvider `json:"providers,omitempty"` + + // +optional + DefaultStorageClassName string `json:"defaultStorageClassName,omitempty"` } type CSIProvider struct { @@ -144,12 +148,18 @@ type CSIProvider struct { Name string `json:"name,omitempty"` // +optional - StorageClassConfig *StorageClassConfig `json:"storageClassConfig,omitempty"` + StorageClassConfig []StorageClassConfig `json:"storageClassConfig,omitempty"` + + // +optional + Strategy AddonStrategy `json:"strategy,omitempty"` + + // +optional + Credentials *corev1.SecretReference `json:"credentials,omitempty"` } type StorageClassConfig struct { // +optional - DefaultClassName string `json:"defaultClassName,omitempty"` + Name string `json:"name,omitempty"` // +optional Parameters map[string]string `json:"parameters,omitempty"` @@ -161,7 +171,7 @@ func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ - "defaultClassName": { + "name": { Type: "string", Enum: variables.MustMarshalValuesToEnumJSON(supportedCSIProviders...), }, @@ -174,6 +184,44 @@ func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { } } +func (CSIProvider) VariableSchema() clusterv1.VariableSchema { + supportedCSIProviders := []string{CSIProviderAWSEBS, CSIProviderNutanix} + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "name": { + Description: "Name of the CSI Provider", + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON( + supportedCSIProviders...), + }, + "strategy": { + Description: "Addon strategy used to deploy the CSI provider to the workload cluster", + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON( + AddonStrategyClusterResourceSet, + AddonStrategyHelmAddon, + ), + }, + "credentials": { + Type: "object", + Description: "The reference to any secret used by the CSI Provider.", + Properties: map[string]clusterv1.JSONSchemaProps{ + "name": { + Type: "string", + }, + "namespace": { + Type: "string", + }, + }, + }, + "storageClassConfig": StorageClassConfig{}.VariableSchema().OpenAPIV3Schema, + }, + }, + } +} + func (CSIProviders) VariableSchema() clusterv1.VariableSchema { supportedCSIProviders := []string{CSIProviderAWSEBS, CSIProviderNutanix} return clusterv1.VariableSchema{ @@ -183,16 +231,16 @@ func (CSIProviders) VariableSchema() clusterv1.VariableSchema { "providers": { Type: "array", Items: &clusterv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "name": { - Type: "string", - Enum: variables.MustMarshalValuesToEnumJSON( - supportedCSIProviders...), - }, - }, + Type: "object", + Properties: CSIProvider{}.VariableSchema().OpenAPIV3Schema.Properties, }, }, + "defaultStorageClassName": { + Type: "string", + Description: "Storage Class that will be made default for the cluster.", + Enum: variables.MustMarshalValuesToEnumJSON( + supportedCSIProviders...), + }, }, }, } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ea0b63e27..29687e997 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -256,8 +256,15 @@ func (in *CSIProvider) DeepCopyInto(out *CSIProvider) { *out = *in if in.StorageClassConfig != nil { in, out := &in.StorageClassConfig, &out.StorageClassConfig - *out = new(StorageClassConfig) - (*in).DeepCopyInto(*out) + *out = make([]StorageClassConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(v1.SecretReference) + **out = **in } } diff --git a/charts/capi-runtime-extensions/templates/csi/nutanix/manifests/helm-addon-installation.yaml b/charts/capi-runtime-extensions/templates/csi/nutanix/manifests/helm-addon-installation.yaml new file mode 100644 index 000000000..a92e3ff95 --- /dev/null +++ b/charts/capi-runtime-extensions/templates/csi/nutanix/manifests/helm-addon-installation.yaml @@ -0,0 +1,9 @@ +{{- if .Values.hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: '{{ .Values.hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.name }}' +data: + values.yaml: |- + createSecret: false +{{- end -}} diff --git a/charts/capi-runtime-extensions/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml b/charts/capi-runtime-extensions/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml new file mode 100644 index 000000000..03c2539b2 --- /dev/null +++ b/charts/capi-runtime-extensions/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml @@ -0,0 +1,543 @@ +# Copyright 2023 D2iQ, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +#================================================================= +# DO NOT EDIT THIS FILE +# IT HAS BEEN GENERATED BY /hack/addons/update-nutanix-csi.sh +#================================================================= +apiVersion: v1 +data: + nutanix-csi.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: nutanix-csi-controller + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: nutanix-csi-node + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: nutanix-csi-controller-role + namespace: nutanix-system + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - update + - patch + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - watch + - update + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots/status + verbs: + - update + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - delete + - update + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: nutanix-csi-node-role + namespace: nutanix-system + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - update + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: nutanix-csi-controller-binding + namespace: nutanix-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nutanix-csi-controller-role + subjects: + - kind: ServiceAccount + name: nutanix-csi-controller + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: nutanix-csi-node-binding + namespace: nutanix-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nutanix-csi-node-role + subjects: + - kind: ServiceAccount + name: nutanix-csi-node + namespace: kube-system + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app: nutanix-csi-metrics + name: nutanix-csi-metrics + namespace: kube-system + spec: + ports: + - name: provisioner + port: 9809 + protocol: TCP + targetPort: 9809 + - name: resizer + port: 9810 + protocol: TCP + targetPort: 9810 + selector: + app: nutanix-csi-controller + type: ClusterIP + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nutanix-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: nutanix-csi-controller + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: nutanix-csi-controller + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app: nutanix-csi-controller + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - --csi-address=$(ADDRESS) + - --timeout=60s + - --worker-threads=16 + - --extra-create-metadata=true + - --default-fstype=ext4 + - --http-endpoint=:9809 + - --v=2 + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.2 + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --v=2 + - --csi-address=$(ADDRESS) + - --timeout=60s + - --leader-election=true + - --handle-volume-inuse-error=false + - --http-endpoint=:9810 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.9.2 + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + requests: + cpu: 5m + memory: 30Mi + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --leader-election=true + - --logtostderr=true + - --timeout=300s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v3.0.3 + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + requests: + cpu: 5m + memory: 30Mi + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --endpoint=$(CSI_ENDPOINT) + - --nodeid=$(NODE_ID) + - --drivername=csi.nutanix.com + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: quay.io/karbon/ntnx-csi:v2.6.6 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: http-endpoint + initialDelaySeconds: 10 + periodSeconds: 2 + timeoutSeconds: 3 + name: nutanix-csi-plugin + ports: + - containerPort: 9807 + name: http-endpoint + protocol: TCP + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: true + privileged: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - mountPath: /host + name: root-dir + - args: + - --csi-address=/csi/csi.sock + - --http-endpoint=:9807 + image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + requests: + cpu: 5m + memory: 20Mi + volumeMounts: + - mountPath: /csi + name: socket-dir + hostNetwork: true + priorityClassName: system-cluster-critical + serviceAccount: nutanix-csi-controller + volumes: + - emptyDir: {} + name: socket-dir + - hostPath: + path: / + type: Directory + name: root-dir + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: nutanix-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: nutanix-csi-node + template: + metadata: + labels: + app: nutanix-csi-node + spec: + containers: + - args: + - --v=2 + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.nutanix.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.1 + imagePullPolicy: IfNotPresent + name: driver-registrar + resources: + requests: + cpu: 100m + memory: 20Mi + volumeMounts: + - mountPath: /csi/ + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --endpoint=$(CSI_ENDPOINT) + - --nodeid=$(NODE_ID) + - --drivername=csi.nutanix.com + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + image: quay.io/karbon/ntnx-csi:v2.6.6 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: http-endpoint + initialDelaySeconds: 10 + periodSeconds: 2 + timeoutSeconds: 3 + name: nutanix-csi-node + ports: + - containerPort: 9808 + name: http-endpoint + protocol: TCP + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: true + privileged: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /etc/iscsi + name: iscsi-dir + - mountPath: /host + mountPropagation: Bidirectional + name: root-dir + - args: + - --csi-address=/csi/csi.sock + - --http-endpoint=:9808 + image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + requests: + cpu: 5m + memory: 20Mi + volumeMounts: + - mountPath: /csi + name: plugin-dir + hostNetwork: true + priorityClassName: system-cluster-critical + serviceAccount: nutanix-csi-node + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/csi.nutanix.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: /dev + name: device-dir + - hostPath: + path: /etc/iscsi + type: Directory + name: iscsi-dir + - hostPath: + path: / + type: Directory + name: root-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + name: csi.nutanix.com + spec: + attachRequired: false + podInfoOnMount: true +kind: ConfigMap +metadata: + creationTimestamp: null + name: nutanix-csi diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.yaml b/charts/cluster-api-runtime-extensions-nutanix/values.yaml index 81576e8ba..9a2a4d7ba 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/values.yaml @@ -35,6 +35,12 @@ hooks: defaultValueTemplateConfigMap: create: true name: default-cilium-cni-helm-values-template + csi: + nutanix: + helmAddonStrategy: + defaultValueTemplateConfigMap: + create: true + name: default-nutanix-csi-helm-values-template nfd: crsStrategy: defaultInstallationConfigMap: diff --git a/hack/addons/kustomize/nutanix-csi/helm-values.yaml b/hack/addons/kustomize/nutanix-csi/helm-values.yaml new file mode 100644 index 000000000..a16eb51a5 --- /dev/null +++ b/hack/addons/kustomize/nutanix-csi/helm-values.yaml @@ -0,0 +1 @@ +createSecret: false diff --git a/hack/addons/kustomize/nutanix-csi/kustomization.yaml.tmpl b/hack/addons/kustomize/nutanix-csi/kustomization.yaml.tmpl new file mode 100644 index 000000000..bc838e438 --- /dev/null +++ b/hack/addons/kustomize/nutanix-csi/kustomization.yaml.tmpl @@ -0,0 +1,20 @@ +# Copyright 2023 D2iQ, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +metadata: + name: nutanix-csi-kustomize + +namespace: kube-system + +helmCharts: +- name: nutanix-csi-storage + repo: https://nutanix.github.io/helm/ + releaseName: nutanix-csi-storage + version: ${NUTANIX_CSI_CHART_VERSION} + valuesFile: helm-values.yaml + includeCRDs: true + skipTests: true + namespace: nutanix-system diff --git a/hack/addons/update-nutanix-csi.sh b/hack/addons/update-nutanix-csi.sh new file mode 100755 index 000000000..5ce837e21 --- /dev/null +++ b/hack/addons/update-nutanix-csi.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR + +# shellcheck source=hack/common.sh +source "${SCRIPT_DIR}/../common.sh" + +if [ -z "${NUTANIX_CSI_CHART_VERSION:-}" ]; then + echo "Missing environment variable: NUTANIX_CSI_CHART_VERSION" + exit 1 +fi + +ASSETS_DIR="$(mktemp -d -p "${TMPDIR:-/tmp}")" +readonly ASSETS_DIR +trap_add "rm -rf ${ASSETS_DIR}" EXIT + +readonly FILE_NAME="nutanix-csi.yaml" + +readonly KUSTOMIZE_BASE_DIR="${SCRIPT_DIR}/kustomize/nutanix-csi" +mkdir -p "${ASSETS_DIR}/nutanix-csi" +envsubst -no-unset <"${KUSTOMIZE_BASE_DIR}/kustomization.yaml.tmpl" >"${ASSETS_DIR}/nutanix-csi/kustomization.yaml" +cp -r "${KUSTOMIZE_BASE_DIR}"/*.yaml "${ASSETS_DIR}/nutanix-csi/" + +kustomize build --enable-helm "${ASSETS_DIR}/nutanix-csi/" >"${ASSETS_DIR}/${FILE_NAME}" + +kubectl create configmap nutanix-csi --dry-run=client --output yaml \ + --from-file "${ASSETS_DIR}/${FILE_NAME}" \ + >"${ASSETS_DIR}/nutanix-csi-configmap.yaml" + +# add warning not to edit file directly +cat <"${GIT_REPO_ROOT}/charts/capi-runtime-extensions/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml" +$(cat "${GIT_REPO_ROOT}/hack/license-header.yaml.txt") + +#================================================================= +# DO NOT EDIT THIS FILE +# IT HAS BEEN GENERATED BY /hack/addons/update-nutanix-csi.sh +#================================================================= +$(cat "${ASSETS_DIR}/nutanix-csi-configmap.yaml") +EOF diff --git a/make/addons.mk b/make/addons.mk index 85ad54575..4cc4d8f42 100644 --- a/make/addons.mk +++ b/make/addons.mk @@ -7,6 +7,7 @@ export NODE_FEATURE_DISCOVERY_VERSION := $(shell goprintconst -file pkg/handlers export CLUSTER_AUTOSCALER_VERSION := 9.35.0 export AWS_CSI_SNAPSHOT_CONTROLLER_VERSION := v6.3.3 export AWS_EBS_CSI_CHART_VERSION := v2.28.1 +<<<<<<< HEAD # a map of AWS CCM versions export AWS_CCM_VERSION_127 := v1.27.1 export AWS_CCM_CHART_VERSION_127 := 0.0.8 @@ -15,6 +16,17 @@ export AWS_CCM_CHART_VERSION_128 := 0.0.8 .PHONY: addons.sync addons.sync: $(addprefix update-addon.,calico cilium nfd cluster-autoscaler aws-ebs-csi aws-ccm.127 aws-ccm.128) +======= +export NUTANIX_CSI_CHART_VERSION := v2.6.6 +# a map of AWS CPI versions +export AWS_CPI_VERSION_127 := v1.27.1 +export AWS_CPI_CHART_VERSION_127 := 0.0.8 +export AWS_CPI_VERSION_128 := v1.28.1 +export AWS_CPI_CHART_VERSION_128 := 0.0.8 + +.PHONY: addons.sync +addons.sync: $(addprefix update-addon.,calico cilium nfd cluster-autoscaler aws-ebs-csi aws-cpi.127 aws-cpi.128 nutanix-csi) +>>>>>>> 625d067 (feat: minor API tweaks and adds tooling for chart) .PHONY: update-addon.calico update-addon.calico: ; $(info $(M) updating calico manifests) @@ -39,3 +51,8 @@ update-addon.aws-ebs-csi: ; $(info $(M) updating aws ebs csi manifests) .PHONY: update-addon.aws-ccm.% update-addon.aws-ccm.%: ; $(info $(M) updating aws ccm $* manifests) ./hack/addons/update-aws-ccm.sh $(AWS_CCM_VERSION_$*) $(AWS_CCM_CHART_VERSION_$*) + +.PHONY: update-addon.nutanix-csi +update-addon.nutanix-csi: ; $(info $(M) updating nutanix csi manifests) + ./hack/addons/update-nutanix-csi.sh + From d9f9e3ed0ee275aab0ab02ea0f31e9bf9b75d34b Mon Sep 17 00:00:00 2001 From: faiq Date: Wed, 20 Mar 2024 17:43:54 -0600 Subject: [PATCH 03/31] feat: more api changes --- api/v1alpha1/addon_types.go | 101 ++++++++++---- api/v1alpha1/zz_generated.deepcopy.go | 20 +++ .../manifests/aws-ebs-csi-configmap.yaml | 10 -- .../kustomize/aws-ebs-csi/helm-values.yaml | 7 - .../generic/lifecycle/csi/aws-ebs/handler.go | 124 ++++++++++++++++-- pkg/handlers/generic/lifecycle/csi/handler.go | 77 +---------- 6 files changed, 216 insertions(+), 123 deletions(-) diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index 025570b9c..101ef8d27 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -10,6 +10,18 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/variables" ) +const ( + AddonStrategyClusterResourceSet AddonStrategy = "ClusterResourceSet" + AddonStrategyHelmAddon AddonStrategy = "HelmAddon" + + VolumeBindingImmediate string = "Immmediate" + VolumeBindingWaitForFirstConsumer string = "WaitForFirstConsumer" + + VolumeReclaimRecycle string = "Recycle" + VolumeReclaimDelete string = "Delete" + VolumeReclaimRetain string = "Retain" +) + type Addons struct { // +optional CNI *CNI `json:"cni,omitempty"` @@ -45,11 +57,6 @@ func (Addons) VariableSchema() clusterv1.VariableSchema { type AddonStrategy string -const ( - AddonStrategyClusterResourceSet AddonStrategy = "ClusterResourceSet" - AddonStrategyHelmAddon AddonStrategy = "HelmAddon" -) - // CNI required for providing CNI configuration. type CNI struct { // +optional @@ -135,50 +142,76 @@ func (ClusterAutoscaler) VariableSchema() clusterv1.VariableSchema { } } +type DefaultStorage struct { + ProviderName string `json:"providerName"` + StorageClassConfigName string `json:"storageClassConfigName"` +} + type CSIProviders struct { // +optional Providers []CSIProvider `json:"providers,omitempty"` - // +optional - DefaultStorageClassName string `json:"defaultStorageClassName,omitempty"` + DefaultStorage *DefaultStorage `json:"defaultStorage,omitempty"` } type CSIProvider struct { - // +optional - Name string `json:"name,omitempty"` + Name string `json:"name"` // +optional StorageClassConfig []StorageClassConfig `json:"storageClassConfig,omitempty"` - // +optional - Strategy AddonStrategy `json:"strategy,omitempty"` + Strategy AddonStrategy `json:"strategy"` // +optional Credentials *corev1.SecretReference `json:"credentials,omitempty"` } type StorageClassConfig struct { - // +optional - Name string `json:"name,omitempty"` + Name string `json:"name"` // +optional Parameters map[string]string `json:"parameters,omitempty"` + + // +optional + ReclaimPolicy string `json:"reclaimPolicy,omitempty"` + + // +optional + VolumeBindingMode string `json:"volumeBindingMode,omitempty"` } func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { supportedCSIProviders := []string{CSIProviderAWSEBS, CSIProviderNutanix} + supportedReclaimPolicies := []string{ + VolumeReclaimRecycle, + VolumeReclaimDelete, + VolumeReclaimRetain, + } + supportedBindingModes := []string{ + VolumeBindingImmediate, + VolumeBindingWaitForFirstConsumer, + } return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ "name": { - Type: "string", - Enum: variables.MustMarshalValuesToEnumJSON(supportedCSIProviders...), + Type: "string", + Description: "Name of storage class config.", + Enum: variables.MustMarshalValuesToEnumJSON(supportedCSIProviders...), }, "parameters": { Type: "object", + Description: "Parameters passed into the storage class object.", XPreserveUnknownFields: true, }, + "reclaimPolicy": { + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON(supportedReclaimPolicies...), + }, + "volumeBindingMode": { + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON(supportedBindingModes...), + }, }, }, } @@ -216,14 +249,41 @@ func (CSIProvider) VariableSchema() clusterv1.VariableSchema { }, }, }, - "storageClassConfig": StorageClassConfig{}.VariableSchema().OpenAPIV3Schema, + "storageClassConfig": { + Type: "array", + Items: &clusterv1.JSONSchemaProps{ + Type: "object", + Properties: StorageClassConfig{}.VariableSchema().OpenAPIV3Schema.Properties, + }, + }, }, }, } } -func (CSIProviders) VariableSchema() clusterv1.VariableSchema { +func (DefaultStorage) VariableSchema() clusterv1.VariableSchema { supportedCSIProviders := []string{CSIProviderAWSEBS, CSIProviderNutanix} + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "object", + Description: "A tuple of provider name and storage class ", + Properties: map[string]clusterv1.JSONSchemaProps{ + "providerName": { + Type: "string", + Description: "Name of the CSI Provider for the default storage class", + Enum: variables.MustMarshalValuesToEnumJSON( + supportedCSIProviders...), + }, + "storageClassConfigName": { + Type: "string", + Description: "Name of storage class config in any of the provider objects", + }, + }, + }, + } +} + +func (CSIProviders) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Type: "object", @@ -235,12 +295,7 @@ func (CSIProviders) VariableSchema() clusterv1.VariableSchema { Properties: CSIProvider{}.VariableSchema().OpenAPIV3Schema.Properties, }, }, - "defaultStorageClassName": { - Type: "string", - Description: "Storage Class that will be made default for the cluster.", - Enum: variables.MustMarshalValuesToEnumJSON( - supportedCSIProviders...), - }, + "defaultStorage": DefaultStorage{}.VariableSchema().OpenAPIV3Schema, }, }, } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 29687e997..d55db46f9 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -288,6 +288,11 @@ func (in *CSIProviders) DeepCopyInto(out *CSIProviders) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.DefaultStorage != nil { + in, out := &in.DefaultStorage, &out.DefaultStorage + *out = new(DefaultStorage) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIProviders. @@ -372,6 +377,21 @@ func (in *ClusterConfigSpec) DeepCopy() *ClusterConfigSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DefaultStorage) DeepCopyInto(out *DefaultStorage) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DefaultStorage. +func (in *DefaultStorage) DeepCopy() *DefaultStorage { + if in == nil { + return nil + } + out := new(DefaultStorage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DockerNodeSpec) DeepCopyInto(out *DockerNodeSpec) { *out = *in diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml index 09e7ce18e..5529a361f 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml @@ -8,16 +8,6 @@ apiVersion: v1 data: aws-ebs-csi.yaml: | - apiVersion: storage.k8s.io/v1 - kind: StorageClass - metadata: - name: ebs-sc - parameters: - csi.storage.k8s.io/fstype: ext4 - type: gp3 - provisioner: ebs.csi.aws.com - volumeBindingMode: WaitForFirstConsumer - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: diff --git a/hack/addons/kustomize/aws-ebs-csi/helm-values.yaml b/hack/addons/kustomize/aws-ebs-csi/helm-values.yaml index d1cf5632d..33bb95823 100644 --- a/hack/addons/kustomize/aws-ebs-csi/helm-values.yaml +++ b/hack/addons/kustomize/aws-ebs-csi/helm-values.yaml @@ -27,10 +27,3 @@ node: sidecars: snapshotter: forceEnable: true -storageClasses: -- metadata: - name: ebs-sc - volumeBindingMode: WaitForFirstConsumer - parameters: - csi.storage.k8s.io/fstype: ext4 - type: gp3 diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index 0d75a2212..25ea8f42e 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -7,16 +7,38 @@ import ( "context" "fmt" + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + lifecycleutils "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/lifecycle/utils" "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) +const ( + variableRootName = "csi" + kindStorageClass = "StorageClass" + awsEBSProvisionerName = "ebs.csi.aws.com" +) + +var ( + defualtStorageClassKey = "storageclass.kubernetes.io/is-default-class" + defaultStorageClassMap = map[string]string{ + defualtStorageClassKey: "true", + } + defaultParams = map[string]string{ + "csi.storage.k8s.io/fstype": "ext4", + "type": "gp3", + } +) + type AWSEBSConfig struct { *options.GlobalOptions defaultAWSEBSConfigMapName string @@ -46,10 +68,83 @@ func New( } } -func (a *AWSEBS) EnsureCSIConfigMapForCluster( +func (a *AWSEBS) Apply( ctx context.Context, - cluster *clusterv1.Cluster, -) (*corev1.ConfigMap, error) { + provider v1alpha1.CSIProvider, + defaultStorageConfig *v1alpha1.DefaultStorage, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, +) error { + strategy := provider.Strategy + switch strategy { + case v1alpha1.AddonStrategyClusterResourceSet: + err := a.handleCRSApply(ctx, defaultStorageConfig, req) + if err != nil { + return err + } + case v1alpha1.AddonStrategyHelmAddon: + fallthrough + default: + return fmt.Errorf("Stategy %s not implemented", strategy) + } + return a.createStorageClasses(ctx, provider.StorageClassConfig, defaultStorageConfig) +} + +func (a *AWSEBS) createStorageClasses(ctx context.Context, + configs []v1alpha1.StorageClassConfig, + defaultStorageConfig *v1alpha1.DefaultStorage, +) error { + for _, c := range configs { + var volumeBindingMode *storagev1.VolumeBindingMode + switch c.VolumeBindingMode { + case v1alpha1.VolumeBindingImmediate: + volumeBindingMode = ptr.To(storagev1.VolumeBindingImmediate) + case v1alpha1.VolumeBindingWaitForFirstConsumer: + fallthrough + default: + volumeBindingMode = ptr.To(storagev1.VolumeBindingWaitForFirstConsumer) + } + var reclaimPolicy *corev1.PersistentVolumeReclaimPolicy + switch c.ReclaimPolicy { + case v1alpha1.VolumeReclaimRecycle: + reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimRecycle) + case v1alpha1.VolumeReclaimDelete: + reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimDelete) + case v1alpha1.VolumeReclaimRetain: + reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimRetain) + } + params := defaultParams + if c.Parameters != nil { + params = c.DeepCopy().Parameters + } + setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && + v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName + sc := storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.Name, + Namespace: a.config.DefaultsNamespace(), + }, + Provisioner: awsEBSProvisionerName, + Parameters: params, + VolumeBindingMode: volumeBindingMode, + ReclaimPolicy: reclaimPolicy, + } + if setAsDefault { + sc.ObjectMeta.Annotations = defaultStorageClassMap + } + if err := client.ServerSideApply(ctx, a.client, &sc); err != nil { + return fmt.Errorf( + "failed to create storage class %w", + err, + ) + } + } + return nil +} + +func (a *AWSEBS) handleCRSApply(ctx context.Context, + defaultStorageConfig *v1alpha1.DefaultStorage, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, +) error { awsEBSCSIConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: a.config.DefaultsNamespace(), @@ -61,22 +156,31 @@ func (a *AWSEBS) EnsureCSIConfigMapForCluster( ) err := a.client.Get(ctx, defaultAWSEBSCSIConfigMapObjName, awsEBSCSIConfigMap) if err != nil { - return nil, fmt.Errorf( + return fmt.Errorf( "failed to retrieve default AWS EBS CSI manifests ConfigMap %q: %w", defaultAWSEBSCSIConfigMapObjName, err, ) } - - awsEBSConfigMap := generateAWSEBSCSIConfigMap(awsEBSCSIConfigMap, cluster) - if err := client.ServerSideApply(ctx, a.client, awsEBSConfigMap); err != nil { - return nil, fmt.Errorf( + cluster := req.Cluster + cm := generateAWSEBSCSIConfigMap(awsEBSCSIConfigMap, &cluster) + if err := client.ServerSideApply(ctx, a.client, cm); err != nil { + return fmt.Errorf( "failed to apply AWS EBS CSI manifests ConfigMap: %w", err, ) } - - return awsEBSConfigMap, nil + err = lifecycleutils.EnsureCRSForClusterFromConfigMaps( + ctx, + cm.Name, + a.client, + &req.Cluster, + cm, + ) + if err != nil { + return err + } + return nil } func generateAWSEBSCSIConfigMap( diff --git a/pkg/handlers/generic/lifecycle/csi/handler.go b/pkg/handlers/generic/lifecycle/csi/handler.go index c084c05f1..80f1c6372 100644 --- a/pkg/handlers/generic/lifecycle/csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/handler.go @@ -7,11 +7,7 @@ import ( "context" "fmt" - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" - utilyaml "sigs.k8s.io/cluster-api/util/yaml" ctrl "sigs.k8s.io/controller-runtime" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -20,23 +16,14 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/lifecycle" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" - lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" ) const ( variableRootName = "csi" - kindStorageClass = "StorageClass" -) - -var ( - defualtStorageClassKey = "storageclass.kubernetes.io/is-default-class" - defaultStorageClassMap = map[string]string{ - defualtStorageClassKey: "true", - } ) type CSIProvider interface { - EnsureCSIConfigMapForCluster(context.Context, *clusterv1.Cluster) (*corev1.ConfigMap, error) + Apply(context.Context, v1alpha1.CSIProvider, *v1alpha1.DefaultStorage, *runtimehooksv1.AfterControlPlaneInitializedRequest) error } type CSIHandler struct { @@ -117,73 +104,17 @@ func (c *CSIHandler) AfterControlPlaneInitialized( ) continue } - log.Info(fmt.Sprintf("Creating config map for csi provider %s", provider)) - cm, err := handler.EnsureCSIConfigMapForCluster(ctx, &req.Cluster) + log.Info(fmt.Sprintf("Creating csi provider %s", provider)) + err = handler.Apply(ctx, provider, csiProviders.DefaultStorage, req) if err != nil { log.Error( err, fmt.Sprintf( - "failed to ensure %s csi driver installation manifests ConfigMap", + "failed to create %s csi driver object.", provider.Name, ), ) resp.SetStatus(runtimehooksv1.ResponseStatusFailure) } - if cm != nil { - if provider.Name == csiProviders.DefaultClassName { - log.Info("Setting default storage class ", provider, csiProviders.DefaultClassName) - err = setDefaultStorageClass(log, cm) - if err != nil { - log.Error(err, "failed to set default storage class") - resp.SetStatus(runtimehooksv1.ResponseStatusFailure) - } - if err := c.client.Update(ctx, cm); err != nil { - log.Error(err, "failed to apply default storage class annotation to configmap") - resp.SetStatus(runtimehooksv1.ResponseStatusFailure) - } - } - err = lifecycleutils.EnsureCRSForClusterFromConfigMaps( - ctx, - cm.Name, - c.client, - &req.Cluster, - cm, - ) - if err != nil { - log.Error( - err, - fmt.Sprintf( - "failed to ensure %s csi driver installation manifests ConfigMap", - provider.Name, - ), - ) - resp.SetStatus(runtimehooksv1.ResponseStatusFailure) - } - } - } -} - -func setDefaultStorageClass( - log logr.Logger, - cm *corev1.ConfigMap, -) error { - for k, contents := range cm.Data { - objs, err := utilyaml.ToUnstructured([]byte(contents)) - if err != nil { - log.Error(err, "failed to parse yaml") - continue - } - for i := range objs { - obj := objs[i] - if obj.GetKind() == kindStorageClass { - obj.SetAnnotations(defaultStorageClassMap) - } - } - rawObjs, err := utilyaml.FromUnstructured(objs) - if err != nil { - return fmt.Errorf("failed to convert unstructured objects back to string %w", err) - } - cm.Data[k] = string(rawObjs) } - return nil } From 39cf44f06077970c8ece4c51aee0e20eb49b8a67 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 21 Mar 2024 10:46:29 -0600 Subject: [PATCH 04/31] fix: adds proper permissions and api fixes --- api/v1alpha1/addon_types.go | 2 - .../templates/role.yaml | 10 ++ .../clusterautoscaler/strategy_helmaddon.go | 32 +---- .../generic/lifecycle/csi/aws-ebs/handler.go | 7 +- pkg/handlers/generic/lifecycle/csi/doc.go | 1 + .../lifecycle/csi/nutanix-csi/handler.go | 135 ++++++++++++++++++ pkg/handlers/generic/lifecycle/handlers.go | 13 +- pkg/handlers/generic/lifecycle/utils/utils.go | 26 ++++ 8 files changed, 193 insertions(+), 33 deletions(-) create mode 100644 pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index 101ef8d27..df4ff06bb 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -180,7 +180,6 @@ type StorageClassConfig struct { } func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { - supportedCSIProviders := []string{CSIProviderAWSEBS, CSIProviderNutanix} supportedReclaimPolicies := []string{ VolumeReclaimRecycle, VolumeReclaimDelete, @@ -197,7 +196,6 @@ func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { "name": { Type: "string", Description: "Name of storage class config.", - Enum: variables.MustMarshalValuesToEnumJSON(supportedCSIProviders...), }, "parameters": { Type: "object", diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml index 9f909c5e3..9e47fb55a 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml @@ -69,3 +69,13 @@ rules: - get - list - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclass + verbs: + - create + - get + - list + - patch + - update diff --git a/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go b/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go index 007d19ab4..7ee2ed2a3 100644 --- a/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go +++ b/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go @@ -9,7 +9,6 @@ import ( "github.com/go-logr/logr" "github.com/spf13/pflag" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" @@ -18,6 +17,7 @@ import ( caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" ) const ( @@ -53,7 +53,10 @@ func (s helmAddonStrategy) apply( log logr.Logger, ) error { log.Info("Retrieving cluster-autoscaler installation values template for cluster") - valuesTemplateConfigMap, err := s.retrieveValuesTemplateConfigMap(ctx, defaultsNamespace) + valuesTemplateConfigMap, err := utils.RetrieveValuesTemplateConfigMap(ctx, + s.client, + s.config.defaultValuesTemplateConfigMapName, + defaultsNamespace) if err != nil { return fmt.Errorf( "failed to retrieve cluster-autoscaler installation values template ConfigMap for cluster: %w", @@ -108,28 +111,3 @@ func (s helmAddonStrategy) apply( return nil } - -func (s helmAddonStrategy) retrieveValuesTemplateConfigMap( - ctx context.Context, - defaultsNamespace string, -) (*corev1.ConfigMap, error) { - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: defaultsNamespace, - Name: s.config.defaultValuesTemplateConfigMapName, - }, - } - configMapObjName := ctrlclient.ObjectKeyFromObject( - configMap, - ) - err := s.client.Get(ctx, configMapObjName, configMap) - if err != nil { - return nil, fmt.Errorf( - "failed to retrieve installation values template ConfigMap %q: %w", - configMapObjName, - err, - ) - } - - return configMap, nil -} diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index 25ea8f42e..e37ee1e7b 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -77,7 +77,7 @@ func (a *AWSEBS) Apply( strategy := provider.Strategy switch strategy { case v1alpha1.AddonStrategyClusterResourceSet: - err := a.handleCRSApply(ctx, defaultStorageConfig, req) + err := a.handleCRSApply(ctx, req) if err != nil { return err } @@ -119,6 +119,10 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName sc := storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: "storageclass", + APIVersion: storagev1.SchemeGroupVersion.String(), + }, ObjectMeta: metav1.ObjectMeta{ Name: c.Name, Namespace: a.config.DefaultsNamespace(), @@ -142,7 +146,6 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, } func (a *AWSEBS) handleCRSApply(ctx context.Context, - defaultStorageConfig *v1alpha1.DefaultStorage, req *runtimehooksv1.AfterControlPlaneInitializedRequest, ) error { awsEBSCSIConfigMap := &corev1.ConfigMap{ diff --git a/pkg/handlers/generic/lifecycle/csi/doc.go b/pkg/handlers/generic/lifecycle/csi/doc.go index 9a2cc0ff4..238c82185 100644 --- a/pkg/handlers/generic/lifecycle/csi/doc.go +++ b/pkg/handlers/generic/lifecycle/csi/doc.go @@ -9,4 +9,5 @@ // // +kubebuilder:rbac:groups=addons.cluster.x-k8s.io,resources=clusterresourcesets,verbs=watch;list;get;create;patch;update;delete // +kubebuilder:rbac:groups="",resources=configmaps,verbs=watch;list;get;create;patch;update;delete +// +kubebuilder:rbac:groups="storage.k8s.io",resources=storageclass,verbs=list;get;create;patch;update package csi diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go new file mode 100644 index 000000000..ed5ed1cff --- /dev/null +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -0,0 +1,135 @@ +package nutanix + +import ( + "context" + "fmt" + + "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" + "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/k8s/client" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/lifecycle/utils" + "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/options" + "github.com/spf13/pflag" + + caaphv1 "github.com/d2iq-labs/capi-runtime-extensions/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const ( + defaultHelmRepositoryURL = "https://nutanix.github.io/helm/" + defaultHelmChartVersion = "v2.6.6" + defaultHelmChartName = "nutanix-csi-storage" + defaultHelmReleaseNameTemplate = "nutanix-csi-storage-%s" +) + +type NutanixCSIConfig struct { + *options.GlobalOptions + defaultValuesTemplateConfigMapName string +} + +func (n *NutanixCSIConfig) AddFlags(prefix string, flags *pflag.FlagSet) { + flags.StringVar( + &n.defaultValuesTemplateConfigMapName, + prefix+".default-values-template-configmap-name", + "default-nutanix-csi-helm-values-template", + "default values ConfigMap name", + ) +} + +type NutanixCSI struct { + client ctrlclient.Client + config *NutanixCSIConfig +} + +func New( + c ctrlclient.Client, + cfg *NutanixCSIConfig, +) *NutanixCSI { + return &NutanixCSI{ + client: c, + config: cfg, + } +} + +func (n *NutanixCSI) Apply( + ctx context.Context, + provider v1alpha1.CSIProvider, + defaultStorageConfig *v1alpha1.DefaultStorage, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, +) error { + strategy := provider.Strategy + switch strategy { + case v1alpha1.AddonStrategyHelmAddon: + err := n.handleHelmAddonApply(ctx, req) + if err != nil { + return err + } + case v1alpha1.AddonStrategyClusterResourceSet: + fallthrough + default: + return fmt.Errorf("Stategy %s not implemented", strategy) + } + return n.createStorageClasses(ctx, provider.StorageClassConfig, defaultStorageConfig) +} + +func (n *NutanixCSI) handleHelmAddonApply( + ctx context.Context, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, +) error { + valuesTemplateConfigMap, err := utils.RetrieveValuesTemplateConfigMap(ctx, + n.client, + n.config.defaultValuesTemplateConfigMapName, + n.config.DefaultsNamespace()) + if err != nil { + return fmt.Errorf( + "failed to retrieve nutanix csi installation values template ConfigMap for cluster: %w", + err, + ) + } + values := valuesTemplateConfigMap.Data["values.yaml"] + + hcp := &caaphv1.HelmChartProxy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: caaphv1.GroupVersion.String(), + Kind: "HelmChartProxy", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.Cluster.Namespace, + Name: "nutanix-csi-" + req.Cluster.Name, + }, + Spec: caaphv1.HelmChartProxySpec{ + RepoURL: defaultHelmRepositoryURL, + ChartName: defaultHelmChartName, + ClusterSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{capiv1.ClusterNameLabel: req.Cluster.Name}, + }, + ReleaseNamespace: req.Cluster.Namespace, + ReleaseName: fmt.Sprintf(defaultHelmReleaseNameTemplate, req.Cluster.Name), + Version: defaultHelmChartVersion, + ValuesTemplate: values, + }, + } + + if err = controllerutil.SetOwnerReference(&req.Cluster, hcp, n.client.Scheme()); err != nil { + return fmt.Errorf( + "failed to set owner reference on nutanix-csi installation HelmChartProxy: %w", + err, + ) + } + + if err = client.ServerSideApply(ctx, n.client, hcp); err != nil { + return fmt.Errorf("failed to apply nutanix-csi installation HelmChartProxy: %w", err) + } + + return nil +} + +func (n *NutanixCSI) createStorageClasses(ctx context.Context, + configs []v1alpha1.StorageClassConfig, + defaultStorageConfig *v1alpha1.DefaultStorage, +) error { + return nil +} diff --git a/pkg/handlers/generic/lifecycle/handlers.go b/pkg/handlers/generic/lifecycle/handlers.go index 366e78196..33d08e130 100644 --- a/pkg/handlers/generic/lifecycle/handlers.go +++ b/pkg/handlers/generic/lifecycle/handlers.go @@ -16,6 +16,7 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/cni/cilium" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi" awsebs "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/aws-ebs" + nutanixcsi "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/nutanix-csi" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/nfd" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/servicelbgc" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" @@ -27,7 +28,8 @@ type Handlers struct { nfdConfig *nfd.Config clusterAutoscalerConfig *clusterautoscaler.Config ebsConfig *awsebs.AWSEBSConfig - awsccmConfig *awsccm.AWSCCMConfig + nutnaixCSIConfig *nutanixcsi.NutanixCSIConfig + awsCPIConfig *awscpi.AWSCPIConfig } func New(globalOptions *options.GlobalOptions) *Handlers { @@ -38,12 +40,14 @@ func New(globalOptions *options.GlobalOptions) *Handlers { clusterAutoscalerConfig: &clusterautoscaler.Config{GlobalOptions: globalOptions}, ebsConfig: &awsebs.AWSEBSConfig{GlobalOptions: globalOptions}, awsccmConfig: &awsccm.AWSCCMConfig{GlobalOptions: globalOptions}, + nutnaixCSIConfig: &nutanixcsi.NutanixCSIConfig{GlobalOptions: globalOptions}, } } func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named { csiHandlers := map[string]csi.CSIProvider{ - v1alpha1.CSIProviderAWSEBS: awsebs.New(mgr.GetClient(), h.ebsConfig), + v1alpha1.CSIProviderAWSEBS: awsebs.New(mgr.GetClient(), h.ebsConfig), + v1alpha1.CSIProviderNutanix: nutanixcsi.New(mgr.GetClient(), h.nutnaixCSIConfig), } ccmHandlers := map[string]ccm.CCMProvider{ v1alpha1.CCMProviderAWS: awsccm.New(mgr.GetClient(), h.awsccmConfig), @@ -66,5 +70,10 @@ func (h *Handlers) AddFlags(flagSet *pflag.FlagSet) { h.calicoCNIConfig.AddFlags("cni.calico", flagSet) h.ciliumCNIConfig.AddFlags("cni.cilium", flagSet) h.ebsConfig.AddFlags("awsebs", pflag.CommandLine) +<<<<<<< HEAD h.awsccmConfig.AddFlags("awsccm", pflag.CommandLine) +======= + h.nutnaixCSIConfig.AddFlags("nutanixcsi", flagSet) + h.awsCPIConfig.AddFlags("awscpi", pflag.CommandLine) +>>>>>>> f7ff2e9 (fix: adds proper permissions and api fixes) } diff --git a/pkg/handlers/generic/lifecycle/utils/utils.go b/pkg/handlers/generic/lifecycle/utils/utils.go index 00915b7ab..33553499b 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils.go +++ b/pkg/handlers/generic/lifecycle/utils/utils.go @@ -86,3 +86,29 @@ func EnsureNamespace(ctx context.Context, c ctrlclient.Client, name string) erro return nil } + +func RetrieveValuesTemplateConfigMap( + ctx context.Context, + c ctrlclient.Client, + configMapName, + defaultsNamespace string, +) (*corev1.ConfigMap, error) { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultsNamespace, + Name: configMapName, + }, + } + configMapObjName := ctrlclient.ObjectKeyFromObject( + configMap, + ) + err := c.Get(ctx, configMapObjName, configMap) + if err != nil { + return nil, fmt.Errorf( + "failed to retrieve installation values template ConfigMap %q: %w", + configMapObjName, + err, + ) + } + return configMap, nil +} From c6733ba3d0a38ccdffa7c2d870731c3d9f6dc0b8 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 21 Mar 2024 11:26:57 -0600 Subject: [PATCH 05/31] fix: adds configuration for new storage to examples/ --- examples/capi-quick-start/aws-cluster-calico-crs.yaml | 6 ++++++ .../capi-quick-start/aws-cluster-calico-helm-addon.yaml | 6 ++++++ examples/capi-quick-start/aws-cluster-cilium-crs.yaml | 6 ++++++ .../capi-quick-start/aws-cluster-cilium-helm-addon.yaml | 6 ++++++ hack/examples/patches/aws/csi.yaml | 6 ++++++ 5 files changed, 30 insertions(+) diff --git a/examples/capi-quick-start/aws-cluster-calico-crs.yaml b/examples/capi-quick-start/aws-cluster-calico-crs.yaml index 847222c10..09c3a5a39 100644 --- a/examples/capi-quick-start/aws-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-crs.yaml @@ -28,8 +28,14 @@ spec: provider: Calico strategy: ClusterResourceSet csi: + defaultStorageConfig: + providerName: aws-ebs + storageClassConfigName: aws-ebs-sc providers: - name: aws-ebs + storageClassConfig: + - name: aws-ebs-sc + strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet aws: diff --git a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml index ac26e8051..15c7dfd4a 100644 --- a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml @@ -28,8 +28,14 @@ spec: provider: Calico strategy: HelmAddon csi: + defaultStorageConfig: + providerName: aws-ebs + storageClassConfigName: aws-ebs-sc providers: - name: aws-ebs + storageClassConfig: + - name: aws-ebs-sc + strategy: ClusterResourceSet nfd: strategy: HelmAddon aws: diff --git a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml index 2fd77b867..bc832e0b5 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml @@ -28,8 +28,14 @@ spec: provider: Cilium strategy: ClusterResourceSet csi: + defaultStorageConfig: + providerName: aws-ebs + storageClassConfigName: aws-ebs-sc providers: - name: aws-ebs + storageClassConfig: + - name: aws-ebs-sc + strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet aws: diff --git a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml index 4bd929d2f..563dfa1d6 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml @@ -28,8 +28,14 @@ spec: provider: Cilium strategy: HelmAddon csi: + defaultStorageConfig: + providerName: aws-ebs + storageClassConfigName: aws-ebs-sc providers: - name: aws-ebs + storageClassConfig: + - name: aws-ebs-sc + strategy: ClusterResourceSet nfd: strategy: HelmAddon aws: diff --git a/hack/examples/patches/aws/csi.yaml b/hack/examples/patches/aws/csi.yaml index 1d395d192..f92b2f939 100644 --- a/hack/examples/patches/aws/csi.yaml +++ b/hack/examples/patches/aws/csi.yaml @@ -4,5 +4,11 @@ - op: "add" path: "/spec/topology/variables/0/value/addons/csi" value: + defaultStorageConfig: + providerName: aws-ebs + storageClassConfigName: aws-ebs-sc providers: - name: aws-ebs + storageClassConfig: + - name: aws-ebs-sc + strategy: ClusterResourceSet From d5c5ec2868c881a91e400c4fe0839785b2efbd91 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 21 Mar 2024 11:32:55 -0600 Subject: [PATCH 06/31] fix: don't use deepcopy --- .../templates/role.yaml | 2 +- .../aws-cluster-calico-crs.yaml | 2 +- .../aws-cluster-calico-helm-addon.yaml | 2 +- .../aws-cluster-cilium-crs.yaml | 2 +- .../aws-cluster-cilium-helm-addon.yaml | 2 +- hack/examples/patches/aws/csi.yaml | 2 +- .../generic/lifecycle/csi/aws-ebs/handler.go | 24 ++++++++++++++----- pkg/handlers/generic/lifecycle/csi/doc.go | 2 +- .../lifecycle/csi/nutanix-csi/handler.go | 10 ++++---- 9 files changed, 30 insertions(+), 18 deletions(-) diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml index 9e47fb55a..e3cd174f7 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml @@ -72,7 +72,7 @@ rules: - apiGroups: - storage.k8s.io resources: - - storageclass + - storageclasses verbs: - create - get diff --git a/examples/capi-quick-start/aws-cluster-calico-crs.yaml b/examples/capi-quick-start/aws-cluster-calico-crs.yaml index 09c3a5a39..68221eaff 100644 --- a/examples/capi-quick-start/aws-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-crs.yaml @@ -28,7 +28,7 @@ spec: provider: Calico strategy: ClusterResourceSet csi: - defaultStorageConfig: + defaultStorage: providerName: aws-ebs storageClassConfigName: aws-ebs-sc providers: diff --git a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml index 15c7dfd4a..c7dde4e5f 100644 --- a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml @@ -28,7 +28,7 @@ spec: provider: Calico strategy: HelmAddon csi: - defaultStorageConfig: + defaultStorage: providerName: aws-ebs storageClassConfigName: aws-ebs-sc providers: diff --git a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml index bc832e0b5..88ee71752 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml @@ -28,7 +28,7 @@ spec: provider: Cilium strategy: ClusterResourceSet csi: - defaultStorageConfig: + defaultStorage: providerName: aws-ebs storageClassConfigName: aws-ebs-sc providers: diff --git a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml index 563dfa1d6..91fd7923b 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml @@ -28,7 +28,7 @@ spec: provider: Cilium strategy: HelmAddon csi: - defaultStorageConfig: + defaultStorage: providerName: aws-ebs storageClassConfigName: aws-ebs-sc providers: diff --git a/hack/examples/patches/aws/csi.yaml b/hack/examples/patches/aws/csi.yaml index f92b2f939..fccddd2ad 100644 --- a/hack/examples/patches/aws/csi.yaml +++ b/hack/examples/patches/aws/csi.yaml @@ -4,7 +4,7 @@ - op: "add" path: "/spec/topology/variables/0/value/addons/csi" value: - defaultStorageConfig: + defaultStorage: providerName: aws-ebs storageClassConfigName: aws-ebs-sc providers: diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index e37ee1e7b..fd8010b6d 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -7,15 +7,17 @@ import ( "context" "fmt" - "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" - lifecycleutils "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/lifecycle/utils" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/remote" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + capiutil "sigs.k8s.io/cluster-api/util" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" @@ -86,11 +88,12 @@ func (a *AWSEBS) Apply( default: return fmt.Errorf("Stategy %s not implemented", strategy) } - return a.createStorageClasses(ctx, provider.StorageClassConfig, defaultStorageConfig) + return a.createStorageClasses(ctx, provider.StorageClassConfig, req.Cluster, defaultStorageConfig) } func (a *AWSEBS) createStorageClasses(ctx context.Context, configs []v1alpha1.StorageClassConfig, + cluster clusterv1.Cluster, defaultStorageConfig *v1alpha1.DefaultStorage, ) error { for _, c := range configs { @@ -114,13 +117,17 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, } params := defaultParams if c.Parameters != nil { - params = c.DeepCopy().Parameters + m := make(map[string]string) + for k, v := range c.Parameters { + m[k] = v + } + params = m } setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName sc := storagev1.StorageClass{ TypeMeta: metav1.TypeMeta{ - Kind: "storageclass", + Kind: "StorageClass", APIVersion: storagev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ @@ -135,7 +142,12 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, if setAsDefault { sc.ObjectMeta.Annotations = defaultStorageClassMap } - if err := client.ServerSideApply(ctx, a.client, &sc); err != nil { + workloadClient, err := remote.NewClusterClient(ctx, "", a.client, capiutil.ObjectKey(&cluster)) + if err != nil { + return err + } + + if err := client.ServerSideApply(ctx, workloadClient, &sc); err != nil { return fmt.Errorf( "failed to create storage class %w", err, diff --git a/pkg/handlers/generic/lifecycle/csi/doc.go b/pkg/handlers/generic/lifecycle/csi/doc.go index 238c82185..41a048f24 100644 --- a/pkg/handlers/generic/lifecycle/csi/doc.go +++ b/pkg/handlers/generic/lifecycle/csi/doc.go @@ -9,5 +9,5 @@ // // +kubebuilder:rbac:groups=addons.cluster.x-k8s.io,resources=clusterresourcesets,verbs=watch;list;get;create;patch;update;delete // +kubebuilder:rbac:groups="",resources=configmaps,verbs=watch;list;get;create;patch;update;delete -// +kubebuilder:rbac:groups="storage.k8s.io",resources=storageclass,verbs=list;get;create;patch;update +// +kubebuilder:rbac:groups="storage.k8s.io",resources=storageclasses,verbs=list;get;create;patch;update package csi diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index ed5ed1cff..f5584ac8b 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -4,13 +4,13 @@ import ( "context" "fmt" - "github.com/d2iq-labs/capi-runtime-extensions/api/v1alpha1" - "github.com/d2iq-labs/capi-runtime-extensions/common/pkg/k8s/client" - "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/generic/lifecycle/utils" - "github.com/d2iq-labs/capi-runtime-extensions/pkg/handlers/options" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" "github.com/spf13/pflag" - caaphv1 "github.com/d2iq-labs/capi-runtime-extensions/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" + caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" From 80a80dbe0976fb83d39cc90300133040e79d8817 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 21 Mar 2024 17:29:49 -0600 Subject: [PATCH 07/31] fix: rename files to put the configmaps in the right place --- .../templates/csi/nutanix/manifests/helm-addon-installation.yaml | 0 .../templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename charts/{capi-runtime-extensions => cluster-api-runtime-extensions-nutanix}/templates/csi/nutanix/manifests/helm-addon-installation.yaml (100%) rename charts/{capi-runtime-extensions => cluster-api-runtime-extensions-nutanix}/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml (100%) diff --git a/charts/capi-runtime-extensions/templates/csi/nutanix/manifests/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/helm-addon-installation.yaml similarity index 100% rename from charts/capi-runtime-extensions/templates/csi/nutanix/manifests/helm-addon-installation.yaml rename to charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/helm-addon-installation.yaml diff --git a/charts/capi-runtime-extensions/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml similarity index 100% rename from charts/capi-runtime-extensions/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml rename to charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml From 2280da18829b979f471131fc94431927cb5bf4d1 Mon Sep 17 00:00:00 2001 From: faiq Date: Tue, 26 Mar 2024 12:55:28 -0600 Subject: [PATCH 08/31] refactor: apply suggestion from review --- pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index fd8010b6d..12ea241b3 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -6,6 +6,7 @@ package aws import ( "context" "fmt" + "maps" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" @@ -117,11 +118,7 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, } params := defaultParams if c.Parameters != nil { - m := make(map[string]string) - for k, v := range c.Parameters { - m[k] = v - } - params = m + params = maps.Clone(c.Parameters) } setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName From efdeab15fd12cb05283317d276ff92e4733b75e2 Mon Sep 17 00:00:00 2001 From: faiq Date: Tue, 26 Mar 2024 13:34:49 -0600 Subject: [PATCH 09/31] fix: linting errors --- api/v1alpha1/zz_generated.deepcopy.go | 2 +- .../generic/lifecycle/csi/aws-ebs/handler.go | 24 +++++++++----- pkg/handlers/generic/lifecycle/csi/handler.go | 7 +++- .../generic/lifecycle/csi/handler_test.go | 33 +------------------ .../lifecycle/csi/nutanix-csi/handler.go | 15 ++++----- 5 files changed, 31 insertions(+), 50 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d55db46f9..467ea9763 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -9,7 +9,7 @@ package v1alpha1 import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index 12ea241b3..7546a2164 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -8,8 +8,6 @@ import ( "fmt" "maps" - "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" - lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -21,7 +19,9 @@ import ( capiutil "sigs.k8s.io/cluster-api/util" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) @@ -85,16 +85,20 @@ func (a *AWSEBS) Apply( return err } case v1alpha1.AddonStrategyHelmAddon: - fallthrough default: - return fmt.Errorf("Stategy %s not implemented", strategy) + return fmt.Errorf("stategy %s not implemented", strategy) } - return a.createStorageClasses(ctx, provider.StorageClassConfig, req.Cluster, defaultStorageConfig) + return a.createStorageClasses( + ctx, + provider.StorageClassConfig, + &req.Cluster, + defaultStorageConfig, + ) } func (a *AWSEBS) createStorageClasses(ctx context.Context, configs []v1alpha1.StorageClassConfig, - cluster clusterv1.Cluster, + cluster *clusterv1.Cluster, defaultStorageConfig *v1alpha1.DefaultStorage, ) error { for _, c := range configs { @@ -103,7 +107,6 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, case v1alpha1.VolumeBindingImmediate: volumeBindingMode = ptr.To(storagev1.VolumeBindingImmediate) case v1alpha1.VolumeBindingWaitForFirstConsumer: - fallthrough default: volumeBindingMode = ptr.To(storagev1.VolumeBindingWaitForFirstConsumer) } @@ -139,7 +142,12 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, if setAsDefault { sc.ObjectMeta.Annotations = defaultStorageClassMap } - workloadClient, err := remote.NewClusterClient(ctx, "", a.client, capiutil.ObjectKey(&cluster)) + workloadClient, err := remote.NewClusterClient( + ctx, + "", + a.client, + capiutil.ObjectKey(cluster), + ) if err != nil { return err } diff --git a/pkg/handlers/generic/lifecycle/csi/handler.go b/pkg/handlers/generic/lifecycle/csi/handler.go index 80f1c6372..14209bec3 100644 --- a/pkg/handlers/generic/lifecycle/csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/handler.go @@ -23,7 +23,12 @@ const ( ) type CSIProvider interface { - Apply(context.Context, v1alpha1.CSIProvider, *v1alpha1.DefaultStorage, *runtimehooksv1.AfterControlPlaneInitializedRequest) error + Apply( + context.Context, + v1alpha1.CSIProvider, + *v1alpha1.DefaultStorage, + *runtimehooksv1.AfterControlPlaneInitializedRequest, + ) error } type CSIHandler struct { diff --git a/pkg/handlers/generic/lifecycle/csi/handler_test.go b/pkg/handlers/generic/lifecycle/csi/handler_test.go index de5015895..b6a14183f 100644 --- a/pkg/handlers/generic/lifecycle/csi/handler_test.go +++ b/pkg/handlers/generic/lifecycle/csi/handler_test.go @@ -4,26 +4,12 @@ package csi import ( - "strings" "testing" - "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var startAWSConfigMap = ` -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ebs-sc -parameters: - csi.storage.k8s.io/fstype: ext4 - type: gp3 -provisioner: ebs.csi.aws.com -volumeBindingMode: WaitForFirstConsumer -` - func Test_setDefaultStorageClass(t *testing.T) { tests := []struct { name string @@ -37,30 +23,13 @@ func Test_setDefaultStorageClass(t *testing.T) { Name: "test1", Namespace: "default", }, - Data: map[string]string{ - "aws-ebs-csi.yaml": startAWSConfigMap, - }, + Data: map[string]string{}, }, key: "aws-ebs-csi.yaml", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := setDefaultStorageClass( - logr.Discard(), - tt.startConfigMap, - ) - if err != nil { - t.Fatal("failed to set default storage class", err) - } - if !strings.Contains(tt.startConfigMap.Data[tt.key], defualtStorageClassKey) { - t.Logf( - "expected %s to containe %s", - tt.startConfigMap.Data[tt.key], - defualtStorageClassKey, - ) - t.Fail() - } }) } } diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index f5584ac8b..96a2e4c79 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -4,18 +4,18 @@ import ( "context" "fmt" - "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" - "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" - "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" - "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" "github.com/spf13/pflag" - - caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) const ( @@ -68,9 +68,8 @@ func (n *NutanixCSI) Apply( return err } case v1alpha1.AddonStrategyClusterResourceSet: - fallthrough default: - return fmt.Errorf("Stategy %s not implemented", strategy) + return fmt.Errorf("stategy %s not implemented", strategy) } return n.createStorageClasses(ctx, provider.StorageClassConfig, defaultStorageConfig) } From e4c3aa56c2a68dfb6001ec1ff6f6ac2daa80d65e Mon Sep 17 00:00:00 2001 From: faiq Date: Tue, 26 Mar 2024 14:45:22 -0600 Subject: [PATCH 10/31] feat: adds nutanix driver create storage class --- .../generic/lifecycle/csi/aws-ebs/handler.go | 77 ++-------------- .../lifecycle/csi/nutanix-csi/handler.go | 27 +++++- pkg/handlers/generic/lifecycle/utils/utils.go | 87 +++++++++++++++++++ 3 files changed, 117 insertions(+), 74 deletions(-) diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index 7546a2164..70153104e 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -6,17 +6,12 @@ package aws import ( "context" "fmt" - "maps" "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/remote" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" - capiutil "sigs.k8s.io/cluster-api/util" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" @@ -25,23 +20,6 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) -const ( - variableRootName = "csi" - kindStorageClass = "StorageClass" - awsEBSProvisionerName = "ebs.csi.aws.com" -) - -var ( - defualtStorageClassKey = "storageclass.kubernetes.io/is-default-class" - defaultStorageClassMap = map[string]string{ - defualtStorageClassKey: "true", - } - defaultParams = map[string]string{ - "csi.storage.k8s.io/fstype": "ext4", - "type": "gp3", - } -) - type AWSEBSConfig struct { *options.GlobalOptions defaultAWSEBSConfigMapName string @@ -102,61 +80,18 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, defaultStorageConfig *v1alpha1.DefaultStorage, ) error { for _, c := range configs { - var volumeBindingMode *storagev1.VolumeBindingMode - switch c.VolumeBindingMode { - case v1alpha1.VolumeBindingImmediate: - volumeBindingMode = ptr.To(storagev1.VolumeBindingImmediate) - case v1alpha1.VolumeBindingWaitForFirstConsumer: - default: - volumeBindingMode = ptr.To(storagev1.VolumeBindingWaitForFirstConsumer) - } - var reclaimPolicy *corev1.PersistentVolumeReclaimPolicy - switch c.ReclaimPolicy { - case v1alpha1.VolumeReclaimRecycle: - reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimRecycle) - case v1alpha1.VolumeReclaimDelete: - reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimDelete) - case v1alpha1.VolumeReclaimRetain: - reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimRetain) - } - params := defaultParams - if c.Parameters != nil { - params = maps.Clone(c.Parameters) - } setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName - sc := storagev1.StorageClass{ - TypeMeta: metav1.TypeMeta{ - Kind: "StorageClass", - APIVersion: storagev1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: c.Name, - Namespace: a.config.DefaultsNamespace(), - }, - Provisioner: awsEBSProvisionerName, - Parameters: params, - VolumeBindingMode: volumeBindingMode, - ReclaimPolicy: reclaimPolicy, - } - if setAsDefault { - sc.ObjectMeta.Annotations = defaultStorageClassMap - } - workloadClient, err := remote.NewClusterClient( + err := lifecycleutils.CreateStorageClass( ctx, - "", a.client, - capiutil.ObjectKey(cluster), + c, + cluster, + a.config.GlobalOptions.DefaultsNamespace(), + setAsDefault, ) if err != nil { - return err - } - - if err := client.ServerSideApply(ctx, workloadClient, &sc); err != nil { - return fmt.Errorf( - "failed to create storage class %w", - err, - ) + return fmt.Errorf("failed to create storageclass %w", err) } } return nil diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index 96a2e4c79..49f2fafbd 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -6,7 +6,7 @@ import ( "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -71,7 +71,12 @@ func (n *NutanixCSI) Apply( default: return fmt.Errorf("stategy %s not implemented", strategy) } - return n.createStorageClasses(ctx, provider.StorageClassConfig, defaultStorageConfig) + return n.createStorageClasses( + ctx, + provider.StorageClassConfig, + &req.Cluster, + defaultStorageConfig, + ) } func (n *NutanixCSI) handleHelmAddonApply( @@ -103,7 +108,7 @@ func (n *NutanixCSI) handleHelmAddonApply( RepoURL: defaultHelmRepositoryURL, ChartName: defaultHelmChartName, ClusterSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{capiv1.ClusterNameLabel: req.Cluster.Name}, + MatchLabels: map[string]string{clusterv1.ClusterNameLabel: req.Cluster.Name}, }, ReleaseNamespace: req.Cluster.Namespace, ReleaseName: fmt.Sprintf(defaultHelmReleaseNameTemplate, req.Cluster.Name), @@ -128,7 +133,23 @@ func (n *NutanixCSI) handleHelmAddonApply( func (n *NutanixCSI) createStorageClasses(ctx context.Context, configs []v1alpha1.StorageClassConfig, + cluster *clusterv1.Cluster, defaultStorageConfig *v1alpha1.DefaultStorage, ) error { + for _, c := range configs { + setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && + v1alpha1.CSIProviderNutanix == defaultStorageConfig.ProviderName + err := utils.CreateStorageClass( + ctx, + n.client, + c, + cluster, + n.config.GlobalOptions.DefaultsNamespace(), + setAsDefault, + ) + if err != nil { + return fmt.Errorf("failed to create storageclass %w", err) + } + } return nil } diff --git a/pkg/handlers/generic/lifecycle/utils/utils.go b/pkg/handlers/generic/lifecycle/utils/utils.go index 33553499b..06887648c 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils.go +++ b/pkg/handlers/generic/lifecycle/utils/utils.go @@ -6,17 +6,39 @@ package utils import ( "context" "fmt" + "maps" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/remote" crsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" + capiutil "sigs.k8s.io/cluster-api/util" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" ) +const ( + kindStorageClass = "StorageClass" + awsEBSProvisionerName = "ebs.csi.aws.com" +) + +var ( + defualtStorageClassKey = "storageclass.kubernetes.io/is-default-class" + defaultStorageClassMap = map[string]string{ + defualtStorageClassKey: "true", + } + defaultParams = map[string]string{ + "csi.storage.k8s.io/fstype": "ext4", + "type": "gp3", + } +) + func EnsureCRSForClusterFromConfigMaps( ctx context.Context, crsName string, @@ -112,3 +134,68 @@ func RetrieveValuesTemplateConfigMap( } return configMap, nil } + +func CreateStorageClass( + ctx context.Context, + cl ctrlclient.Client, + storageConfig v1alpha1.StorageClassConfig, + cluster *clusterv1.Cluster, + defaultsNamespace string, + isDefault bool, +) error { + var volumeBindingMode *storagev1.VolumeBindingMode + switch storageConfig.VolumeBindingMode { + case v1alpha1.VolumeBindingImmediate: + volumeBindingMode = ptr.To(storagev1.VolumeBindingImmediate) + case v1alpha1.VolumeBindingWaitForFirstConsumer: + default: + volumeBindingMode = ptr.To(storagev1.VolumeBindingWaitForFirstConsumer) + } + var reclaimPolicy *corev1.PersistentVolumeReclaimPolicy + switch storageConfig.ReclaimPolicy { + case v1alpha1.VolumeReclaimRecycle: + reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimRecycle) + case v1alpha1.VolumeReclaimDelete: + reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimDelete) + case v1alpha1.VolumeReclaimRetain: + reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimRetain) + } + params := defaultParams + if storageConfig.Parameters != nil { + params = maps.Clone(storageConfig.Parameters) + } + sc := storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: "StorageClass", + APIVersion: storagev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: storageConfig.Name, + Namespace: defaultsNamespace, + }, + Provisioner: awsEBSProvisionerName, + Parameters: params, + VolumeBindingMode: volumeBindingMode, + ReclaimPolicy: reclaimPolicy, + } + if isDefault { + sc.ObjectMeta.Annotations = defaultStorageClassMap + } + workloadClient, err := remote.NewClusterClient( + ctx, + "", + cl, + capiutil.ObjectKey(cluster), + ) + if err != nil { + return err + } + + if err := client.ServerSideApply(ctx, workloadClient, &sc); err != nil { + return fmt.Errorf( + "failed to create storage class %w", + err, + ) + } + return nil +} From e58e2f85d1e2dc1c335f450ab55e7b588158e1cf Mon Sep 17 00:00:00 2001 From: faiq Date: Tue, 26 Mar 2024 14:53:54 -0600 Subject: [PATCH 11/31] fix: pre-commit errors --- charts/cluster-api-runtime-extensions-nutanix/README.md | 2 ++ .../csi/nutanix/manifests/helm-addon-installation.yaml | 3 +++ hack/addons/kustomize/nutanix-csi/helm-values.yaml | 3 +++ hack/addons/update-nutanix-csi.sh | 2 +- pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go | 3 +++ 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/charts/cluster-api-runtime-extensions-nutanix/README.md b/charts/cluster-api-runtime-extensions-nutanix/README.md index a90733b90..3049ea2a1 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/README.md +++ b/charts/cluster-api-runtime-extensions-nutanix/README.md @@ -49,6 +49,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix | hooks.cni.cilium.crsStrategy.defaultCiliumConfigMap.name | string | `"cilium"` | | | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cilium-cni-helm-values-template"` | | +| hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | +| hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-nutanix-csi-helm-values-template"` | | | hooks.nfd.crsStrategy.defaultInstallationConfigMap.name | string | `"node-feature-discovery"` | | | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-nfd-helm-values-template"` | | diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/helm-addon-installation.yaml index a92e3ff95..b62635034 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/helm-addon-installation.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/helm-addon-installation.yaml @@ -1,3 +1,6 @@ +# Copyright 2023 D2iQ, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + {{- if .Values.hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.create }} apiVersion: v1 kind: ConfigMap diff --git a/hack/addons/kustomize/nutanix-csi/helm-values.yaml b/hack/addons/kustomize/nutanix-csi/helm-values.yaml index a16eb51a5..eebd55d74 100644 --- a/hack/addons/kustomize/nutanix-csi/helm-values.yaml +++ b/hack/addons/kustomize/nutanix-csi/helm-values.yaml @@ -1 +1,4 @@ +# Copyright 2023 D2iQ, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + createSecret: false diff --git a/hack/addons/update-nutanix-csi.sh b/hack/addons/update-nutanix-csi.sh index 5ce837e21..e18414a86 100755 --- a/hack/addons/update-nutanix-csi.sh +++ b/hack/addons/update-nutanix-csi.sh @@ -31,7 +31,7 @@ kubectl create configmap nutanix-csi --dry-run=client --output yaml \ >"${ASSETS_DIR}/nutanix-csi-configmap.yaml" # add warning not to edit file directly -cat <"${GIT_REPO_ROOT}/charts/capi-runtime-extensions/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml" +cat <"${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml" $(cat "${GIT_REPO_ROOT}/hack/license-header.yaml.txt") #================================================================= diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index 49f2fafbd..5ea0085e8 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -1,3 +1,6 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + package nutanix import ( From 79defdec7e5557acba3fccef7e2a7880fb8cbecf Mon Sep 17 00:00:00 2001 From: faiq Date: Wed, 27 Mar 2024 10:35:38 -0600 Subject: [PATCH 12/31] fix: pass in provisioner name --- .../generic/lifecycle/csi/aws-ebs/handler.go | 5 +++ .../generic/lifecycle/csi/handler_test.go | 35 ------------------- .../lifecycle/csi/nutanix-csi/handler.go | 2 ++ pkg/handlers/generic/lifecycle/utils/utils.go | 10 +++--- 4 files changed, 12 insertions(+), 40 deletions(-) delete mode 100644 pkg/handlers/generic/lifecycle/csi/handler_test.go diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index 70153104e..7c669d773 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -20,6 +20,10 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) +const ( + awsEBSProvisionerName = "ebs.csi.aws.com" +) + type AWSEBSConfig struct { *options.GlobalOptions defaultAWSEBSConfigMapName string @@ -88,6 +92,7 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, c, cluster, a.config.GlobalOptions.DefaultsNamespace(), + awsEBSProvisionerName, setAsDefault, ) if err != nil { diff --git a/pkg/handlers/generic/lifecycle/csi/handler_test.go b/pkg/handlers/generic/lifecycle/csi/handler_test.go deleted file mode 100644 index b6a14183f..000000000 --- a/pkg/handlers/generic/lifecycle/csi/handler_test.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2023 D2iQ, Inc. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package csi - -import ( - "testing" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func Test_setDefaultStorageClass(t *testing.T) { - tests := []struct { - name string - startConfigMap *corev1.ConfigMap - key string - }{ - { - name: "aws config map", - startConfigMap: &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test1", - Namespace: "default", - }, - Data: map[string]string{}, - }, - key: "aws-ebs-csi.yaml", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - }) - } -} diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index 5ea0085e8..4a1331ba3 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -26,6 +26,7 @@ const ( defaultHelmChartVersion = "v2.6.6" defaultHelmChartName = "nutanix-csi-storage" defaultHelmReleaseNameTemplate = "nutanix-csi-storage-%s" + nutanixCSIProvisionerName = "csi.nutanix.com" ) type NutanixCSIConfig struct { @@ -147,6 +148,7 @@ func (n *NutanixCSI) createStorageClasses(ctx context.Context, n.client, c, cluster, + nutanixCSIProvisionerName, n.config.GlobalOptions.DefaultsNamespace(), setAsDefault, ) diff --git a/pkg/handlers/generic/lifecycle/utils/utils.go b/pkg/handlers/generic/lifecycle/utils/utils.go index 06887648c..72dbf2f5a 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils.go +++ b/pkg/handlers/generic/lifecycle/utils/utils.go @@ -24,8 +24,7 @@ import ( ) const ( - kindStorageClass = "StorageClass" - awsEBSProvisionerName = "ebs.csi.aws.com" + kindStorageClass = "StorageClass" ) var ( @@ -140,7 +139,8 @@ func CreateStorageClass( cl ctrlclient.Client, storageConfig v1alpha1.StorageClassConfig, cluster *clusterv1.Cluster, - defaultsNamespace string, + defaultsNamespace, + provisionerName string, isDefault bool, ) error { var volumeBindingMode *storagev1.VolumeBindingMode @@ -166,14 +166,14 @@ func CreateStorageClass( } sc := storagev1.StorageClass{ TypeMeta: metav1.TypeMeta{ - Kind: "StorageClass", + Kind: kindStorageClass, APIVersion: storagev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ Name: storageConfig.Name, Namespace: defaultsNamespace, }, - Provisioner: awsEBSProvisionerName, + Provisioner: provisionerName, Parameters: params, VolumeBindingMode: volumeBindingMode, ReclaimPolicy: reclaimPolicy, From 7d0c094e138d2f23dd54eca92f03b7dbd782847a Mon Sep 17 00:00:00 2001 From: faiq Date: Wed, 27 Mar 2024 17:28:53 -0600 Subject: [PATCH 13/31] refactor: deploy storage class as a cluster resource set --- api/v1alpha1/clusterconfig_types.go | 8 +- .../generic/lifecycle/csi/aws-ebs/handler.go | 33 ++++--- .../lifecycle/csi/nutanix-csi/handler.go | 34 ++++--- pkg/handlers/generic/lifecycle/utils/utils.go | 67 ++++++++----- .../generic/lifecycle/utils/utils_test.go | 95 +++++++++++++++++++ 5 files changed, 185 insertions(+), 52 deletions(-) create mode 100644 pkg/handlers/generic/lifecycle/utils/utils_test.go diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index 75e02b61c..c664fc665 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -14,9 +14,13 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/openapi/patterns" ) +type StorageDriver string + const ( - CNIProviderCalico = "Calico" - CNIProviderCilium = "Cilium" + CNIProviderCalico = "Calico" + CNIProviderCilium = "Cilium" + AWSEBSDriver StorageDriver = "ebs.csi.aws.com" + NutanixDriver StorageDriver = "csi.nutanix.com" CSIProviderAWSEBS = "aws-ebs" CSIProviderNutanix = "nutanix" diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index 7c669d773..15305348c 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -20,10 +21,6 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) -const ( - awsEBSProvisionerName = "ebs.csi.aws.com" -) - type AWSEBSConfig struct { *options.GlobalOptions defaultAWSEBSConfigMapName string @@ -83,23 +80,33 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, cluster *clusterv1.Cluster, defaultStorageConfig *v1alpha1.DefaultStorage, ) error { + allStorageClasses := make([]runtime.Object, 0, len(configs)) for _, c := range configs { setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName - err := lifecycleutils.CreateStorageClass( - ctx, - a.client, + allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass( c, cluster, a.config.GlobalOptions.DefaultsNamespace(), - awsEBSProvisionerName, + v1alpha1.CSIProviderAWSEBS, setAsDefault, - ) - if err != nil { - return fmt.Errorf("failed to create storageclass %w", err) - } + )) } - return nil + cm, err := lifecycleutils.CreateConfigMapForCRS( + "aws-storageclass-cm", + a.config.DefaultsNamespace(), + allStorageClasses..., + ) + if err != nil { + return err + } + return lifecycleutils.EnsureCRSForClusterFromConfigMaps( + ctx, + "aws-storageclass-crs", + a.client, + cluster, + cm, + ) } func (a *AWSEBS) handleCRSApply(ctx context.Context, diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index 4a1331ba3..4f2d67a8b 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -17,7 +18,7 @@ import ( caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" - "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" + lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) @@ -26,7 +27,6 @@ const ( defaultHelmChartVersion = "v2.6.6" defaultHelmChartName = "nutanix-csi-storage" defaultHelmReleaseNameTemplate = "nutanix-csi-storage-%s" - nutanixCSIProvisionerName = "csi.nutanix.com" ) type NutanixCSIConfig struct { @@ -87,7 +87,7 @@ func (n *NutanixCSI) handleHelmAddonApply( ctx context.Context, req *runtimehooksv1.AfterControlPlaneInitializedRequest, ) error { - valuesTemplateConfigMap, err := utils.RetrieveValuesTemplateConfigMap(ctx, + valuesTemplateConfigMap, err := lifecycleutils.RetrieveValuesTemplateConfigMap(ctx, n.client, n.config.defaultValuesTemplateConfigMapName, n.config.DefaultsNamespace()) @@ -140,21 +140,31 @@ func (n *NutanixCSI) createStorageClasses(ctx context.Context, cluster *clusterv1.Cluster, defaultStorageConfig *v1alpha1.DefaultStorage, ) error { + allStorageClasses := make([]runtime.Object, 0, len(configs)) for _, c := range configs { setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && v1alpha1.CSIProviderNutanix == defaultStorageConfig.ProviderName - err := utils.CreateStorageClass( - ctx, - n.client, + allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass( c, cluster, - nutanixCSIProvisionerName, n.config.GlobalOptions.DefaultsNamespace(), + v1alpha1.NutanixDriver, setAsDefault, - ) - if err != nil { - return fmt.Errorf("failed to create storageclass %w", err) - } + )) } - return nil + cm, err := lifecycleutils.CreateConfigMapForCRS( + "nutanix-storageclass-cm", + n.config.DefaultsNamespace(), + allStorageClasses..., + ) + if err != nil { + return err + } + return lifecycleutils.EnsureCRSForClusterFromConfigMaps( + ctx, + "nutanix-storageclass-crs", + n.client, + cluster, + cm, + ) } diff --git a/pkg/handlers/generic/lifecycle/utils/utils.go b/pkg/handlers/generic/lifecycle/utils/utils.go index 72dbf2f5a..7c58c8017 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils.go +++ b/pkg/handlers/generic/lifecycle/utils/utils.go @@ -11,11 +11,12 @@ import ( corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/remote" crsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" - capiutil "sigs.k8s.io/cluster-api/util" + utilyaml "sigs.k8s.io/cluster-api/util/yaml" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -24,7 +25,8 @@ import ( ) const ( - kindStorageClass = "StorageClass" + kindStorageClass = "StorageClass" + defaultCRSConfigMapKey = "custom-resources.yaml" ) var ( @@ -135,20 +137,17 @@ func RetrieveValuesTemplateConfigMap( } func CreateStorageClass( - ctx context.Context, - cl ctrlclient.Client, storageConfig v1alpha1.StorageClassConfig, cluster *clusterv1.Cluster, - defaultsNamespace, - provisionerName string, + defaultsNamespace string, + provisionerName v1alpha1.StorageDriver, isDefault bool, -) error { +) *storagev1.StorageClass { var volumeBindingMode *storagev1.VolumeBindingMode switch storageConfig.VolumeBindingMode { case v1alpha1.VolumeBindingImmediate: volumeBindingMode = ptr.To(storagev1.VolumeBindingImmediate) case v1alpha1.VolumeBindingWaitForFirstConsumer: - default: volumeBindingMode = ptr.To(storagev1.VolumeBindingWaitForFirstConsumer) } var reclaimPolicy *corev1.PersistentVolumeReclaimPolicy @@ -173,7 +172,7 @@ func CreateStorageClass( Name: storageConfig.Name, Namespace: defaultsNamespace, }, - Provisioner: provisionerName, + Provisioner: string(provisionerName), Parameters: params, VolumeBindingMode: volumeBindingMode, ReclaimPolicy: reclaimPolicy, @@ -181,21 +180,39 @@ func CreateStorageClass( if isDefault { sc.ObjectMeta.Annotations = defaultStorageClassMap } - workloadClient, err := remote.NewClusterClient( - ctx, - "", - cl, - capiutil.ObjectKey(cluster), - ) - if err != nil { - return err - } + return &sc +} - if err := client.ServerSideApply(ctx, workloadClient, &sc); err != nil { - return fmt.Errorf( - "failed to create storage class %w", - err, - ) +func CreateConfigMapForCRS(configMapName, configMapNamespace string, + objs ...runtime.Object, +) (*corev1.ConfigMap, error) { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: configMapNamespace, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + Data: make(map[string]string), + } + l := make([][]byte, 0, len(objs)) + for _, v := range objs { + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(v) + if err != nil { + return nil, err + } + objYaml, err := utilyaml.FromUnstructured([]unstructured.Unstructured{ + { + Object: obj, + }, + }) + if err != nil { + return nil, err + } + l = append(l, objYaml) } - return nil + cm.Data[defaultCRSConfigMapKey] = fmt.Sprintf("|\n%s", string(utilyaml.JoinYaml(l...))) + return cm, nil } diff --git a/pkg/handlers/generic/lifecycle/utils/utils_test.go b/pkg/handlers/generic/lifecycle/utils/utils_test.go new file mode 100644 index 000000000..fccea15b8 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/utils/utils_test.go @@ -0,0 +1,95 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func TestCreateConfigMapForCRS(t *testing.T) { + tests := []struct { + name string + testCMName string + testNamespace string + objs []runtime.Object + expectedCM corev1.ConfigMap + }{ + { + name: "multiple storage class objects", + testCMName: "test", + testNamespace: "default", + objs: []runtime.Object{ + &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: kindStorageClass, + APIVersion: storagev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + }, + &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: kindStorageClass, + APIVersion: storagev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-2", + Namespace: "default", + }, + }, + }, + expectedCM: corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + Data: map[string]string{ + defaultCRSConfigMapKey: `| +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + creationTimestamp: null + name: test + namespace: default +provisioner: "" +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + creationTimestamp: null + name: test-2 + namespace: default +provisioner: ""`, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cm, err := CreateConfigMapForCRS(tt.testCMName, tt.testNamespace, tt.objs...) + if err != nil { + t.Errorf("failed to create cm with error %v", err) + } + data, ok := cm.Data[defaultCRSConfigMapKey] + if !ok { + t.Errorf("expected %s to exist in cm.Data. got %v", defaultCRSConfigMapKey, cm.Data) + } + expected := tt.expectedCM.Data[defaultCRSConfigMapKey] + if data != expected { + t.Errorf("expected %s \n got %s", expected, data) + } + }) + } +} From a54e97c5476e2ae99dddf65f0a6b667e6e804603 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 28 Mar 2024 09:11:10 -0600 Subject: [PATCH 14/31] refactor: change name of example sc --- examples/capi-quick-start/aws-cluster-calico-crs.yaml | 2 +- examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml | 2 +- examples/capi-quick-start/aws-cluster-cilium-crs.yaml | 2 +- examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml | 2 +- hack/examples/patches/aws/csi.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/capi-quick-start/aws-cluster-calico-crs.yaml b/examples/capi-quick-start/aws-cluster-calico-crs.yaml index 68221eaff..fe1ffdef1 100644 --- a/examples/capi-quick-start/aws-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-crs.yaml @@ -34,7 +34,7 @@ spec: providers: - name: aws-ebs storageClassConfig: - - name: aws-ebs-sc + - name: aws-ebs strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet diff --git a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml index c7dde4e5f..8041814b5 100644 --- a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml @@ -34,7 +34,7 @@ spec: providers: - name: aws-ebs storageClassConfig: - - name: aws-ebs-sc + - name: aws-ebs strategy: ClusterResourceSet nfd: strategy: HelmAddon diff --git a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml index 88ee71752..cf11f7edd 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml @@ -34,7 +34,7 @@ spec: providers: - name: aws-ebs storageClassConfig: - - name: aws-ebs-sc + - name: aws-ebs strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet diff --git a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml index 91fd7923b..bc7efdc8a 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml @@ -34,7 +34,7 @@ spec: providers: - name: aws-ebs storageClassConfig: - - name: aws-ebs-sc + - name: aws-ebs strategy: ClusterResourceSet nfd: strategy: HelmAddon diff --git a/hack/examples/patches/aws/csi.yaml b/hack/examples/patches/aws/csi.yaml index fccddd2ad..f76374834 100644 --- a/hack/examples/patches/aws/csi.yaml +++ b/hack/examples/patches/aws/csi.yaml @@ -10,5 +10,5 @@ providers: - name: aws-ebs storageClassConfig: - - name: aws-ebs-sc + - name: aws-ebs strategy: ClusterResourceSet From dda31d8284dec3dc777b541cf0a882615a75e837 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 28 Mar 2024 09:16:16 -0600 Subject: [PATCH 15/31] fix: typo for default storage class key --- .../generic/lifecycle/clusterautoscaler/strategy_helmaddon.go | 3 ++- pkg/handlers/generic/lifecycle/utils/utils.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go b/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go index 7ee2ed2a3..a22e69d2f 100644 --- a/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go +++ b/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go @@ -53,7 +53,8 @@ func (s helmAddonStrategy) apply( log logr.Logger, ) error { log.Info("Retrieving cluster-autoscaler installation values template for cluster") - valuesTemplateConfigMap, err := utils.RetrieveValuesTemplateConfigMap(ctx, + valuesTemplateConfigMap, err := utils.RetrieveValuesTemplateConfigMap( + ctx, s.client, s.config.defaultValuesTemplateConfigMapName, defaultsNamespace) diff --git a/pkg/handlers/generic/lifecycle/utils/utils.go b/pkg/handlers/generic/lifecycle/utils/utils.go index 7c58c8017..10c38bee6 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils.go +++ b/pkg/handlers/generic/lifecycle/utils/utils.go @@ -30,9 +30,9 @@ const ( ) var ( - defualtStorageClassKey = "storageclass.kubernetes.io/is-default-class" + defaultStorageClassKey = "storageclass.kubernetes.io/is-default-class" defaultStorageClassMap = map[string]string{ - defualtStorageClassKey: "true", + defaultStorageClassKey: "true", } defaultParams = map[string]string{ "csi.storage.k8s.io/fstype": "ext4", From 832cc64fc9c077cb7016e437f760c197477312a7 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 28 Mar 2024 11:03:33 -0600 Subject: [PATCH 16/31] fix: addon defaults for csi --- api/v1alpha1/addon_types.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index df4ff06bb..270ca65dc 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -5,6 +5,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/variables" @@ -203,12 +204,14 @@ func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { XPreserveUnknownFields: true, }, "reclaimPolicy": { - Type: "string", - Enum: variables.MustMarshalValuesToEnumJSON(supportedReclaimPolicies...), + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON(supportedReclaimPolicies...), + Default: variables.MustMarshal(VolumeReclaimDelete), }, "volumeBindingMode": { - Type: "string", - Enum: variables.MustMarshalValuesToEnumJSON(supportedBindingModes...), + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON(supportedBindingModes...), + Default: variables.MustMarshal(VolumeBindingWaitForFirstConsumer), }, }, }, @@ -250,8 +253,8 @@ func (CSIProvider) VariableSchema() clusterv1.VariableSchema { "storageClassConfig": { Type: "array", Items: &clusterv1.JSONSchemaProps{ - Type: "object", - Properties: StorageClassConfig{}.VariableSchema().OpenAPIV3Schema.Properties, + Type: "object", + Items: ptr.To(StorageClassConfig{}.VariableSchema().OpenAPIV3Schema), }, }, }, @@ -289,8 +292,8 @@ func (CSIProviders) VariableSchema() clusterv1.VariableSchema { "providers": { Type: "array", Items: &clusterv1.JSONSchemaProps{ - Type: "object", - Properties: CSIProvider{}.VariableSchema().OpenAPIV3Schema.Properties, + Type: "object", + Items: ptr.To(CSIProvider{}.VariableSchema().OpenAPIV3Schema), }, }, "defaultStorage": DefaultStorage{}.VariableSchema().OpenAPIV3Schema, From 868022f7c6f24d8ba4622a03beaf78dc1f00c70b Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 28 Mar 2024 12:29:54 -0600 Subject: [PATCH 17/31] fix: sets the Items correctly --- api/v1alpha1/addon_types.go | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index 270ca65dc..19bd443d6 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -15,12 +15,12 @@ const ( AddonStrategyClusterResourceSet AddonStrategy = "ClusterResourceSet" AddonStrategyHelmAddon AddonStrategy = "HelmAddon" - VolumeBindingImmediate string = "Immmediate" - VolumeBindingWaitForFirstConsumer string = "WaitForFirstConsumer" + VolumeBindingImmediate = "Immmediate" + VolumeBindingWaitForFirstConsumer = "WaitForFirstConsumer" - VolumeReclaimRecycle string = "Recycle" - VolumeReclaimDelete string = "Delete" - VolumeReclaimRetain string = "Retain" + VolumeReclaimRecycle = "Recycle" + VolumeReclaimDelete = "Delete" + VolumeReclaimRetain = "Retain" ) type Addons struct { @@ -199,9 +199,11 @@ func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { Description: "Name of storage class config.", }, "parameters": { - Type: "object", - Description: "Parameters passed into the storage class object.", - XPreserveUnknownFields: true, + Type: "object", + Description: "Parameters passed into the storage class object.", + AdditionalProperties: &clusterv1.JSONSchemaProps{ + Type: "string", + }, }, "reclaimPolicy": { Type: "string", @@ -251,11 +253,8 @@ func (CSIProvider) VariableSchema() clusterv1.VariableSchema { }, }, "storageClassConfig": { - Type: "array", - Items: &clusterv1.JSONSchemaProps{ - Type: "object", - Items: ptr.To(StorageClassConfig{}.VariableSchema().OpenAPIV3Schema), - }, + Type: "array", + Items: ptr.To(StorageClassConfig{}.VariableSchema().OpenAPIV3Schema), }, }, }, @@ -290,11 +289,8 @@ func (CSIProviders) VariableSchema() clusterv1.VariableSchema { Type: "object", Properties: map[string]clusterv1.JSONSchemaProps{ "providers": { - Type: "array", - Items: &clusterv1.JSONSchemaProps{ - Type: "object", - Items: ptr.To(CSIProvider{}.VariableSchema().OpenAPIV3Schema), - }, + Type: "array", + Items: ptr.To(CSIProvider{}.VariableSchema().OpenAPIV3Schema), }, "defaultStorage": DefaultStorage{}.VariableSchema().OpenAPIV3Schema, }, From 25b05fca89e890429bde8754faac48ca66828570 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 28 Mar 2024 12:37:47 -0600 Subject: [PATCH 18/31] fix: examples --- examples/capi-quick-start/aws-cluster-calico-crs.yaml | 2 +- examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml | 2 +- examples/capi-quick-start/aws-cluster-cilium-crs.yaml | 2 +- examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml | 2 +- hack/examples/patches/aws/csi.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/capi-quick-start/aws-cluster-calico-crs.yaml b/examples/capi-quick-start/aws-cluster-calico-crs.yaml index fe1ffdef1..790b4a640 100644 --- a/examples/capi-quick-start/aws-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-crs.yaml @@ -30,7 +30,7 @@ spec: csi: defaultStorage: providerName: aws-ebs - storageClassConfigName: aws-ebs-sc + storageClassConfigName: aws-ebs providers: - name: aws-ebs storageClassConfig: diff --git a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml index 8041814b5..52bf9648a 100644 --- a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml @@ -30,7 +30,7 @@ spec: csi: defaultStorage: providerName: aws-ebs - storageClassConfigName: aws-ebs-sc + storageClassConfigName: aws-ebs providers: - name: aws-ebs storageClassConfig: diff --git a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml index cf11f7edd..e9541e789 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml @@ -30,7 +30,7 @@ spec: csi: defaultStorage: providerName: aws-ebs - storageClassConfigName: aws-ebs-sc + storageClassConfigName: aws-ebs providers: - name: aws-ebs storageClassConfig: diff --git a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml index bc7efdc8a..e803994e8 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml @@ -30,7 +30,7 @@ spec: csi: defaultStorage: providerName: aws-ebs - storageClassConfigName: aws-ebs-sc + storageClassConfigName: aws-ebs providers: - name: aws-ebs storageClassConfig: diff --git a/hack/examples/patches/aws/csi.yaml b/hack/examples/patches/aws/csi.yaml index f76374834..0bbe4c5bd 100644 --- a/hack/examples/patches/aws/csi.yaml +++ b/hack/examples/patches/aws/csi.yaml @@ -6,7 +6,7 @@ value: defaultStorage: providerName: aws-ebs - storageClassConfigName: aws-ebs-sc + storageClassConfigName: aws-ebs providers: - name: aws-ebs storageClassConfig: From 59bf818e983fbd945db78c35bde5bcc6680e33e9 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 28 Mar 2024 13:22:28 -0600 Subject: [PATCH 19/31] fix: actually apply cm --- pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go | 4 ++++ pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index 15305348c..f3ab9c2ee 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -100,6 +100,10 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, if err != nil { return err } + err = client.ServerSideApply(ctx, a.client, cm) + if err != nil { + return err + } return lifecycleutils.EnsureCRSForClusterFromConfigMaps( ctx, "aws-storageclass-crs", diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index 4f2d67a8b..0e6b1abf9 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -160,6 +160,10 @@ func (n *NutanixCSI) createStorageClasses(ctx context.Context, if err != nil { return err } + err = client.ServerSideApply(ctx, n.client, cm) + if err != nil { + return err + } return lifecycleutils.EnsureCRSForClusterFromConfigMaps( ctx, "nutanix-storageclass-crs", From 3eee6ce2b224aae64006b41117980e1c4a834828 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 28 Mar 2024 13:35:03 -0600 Subject: [PATCH 20/31] fix: storage class crs --- pkg/handlers/generic/lifecycle/utils/utils.go | 2 +- pkg/handlers/generic/lifecycle/utils/utils_test.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/handlers/generic/lifecycle/utils/utils.go b/pkg/handlers/generic/lifecycle/utils/utils.go index 10c38bee6..d87e225c4 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils.go +++ b/pkg/handlers/generic/lifecycle/utils/utils.go @@ -213,6 +213,6 @@ func CreateConfigMapForCRS(configMapName, configMapNamespace string, } l = append(l, objYaml) } - cm.Data[defaultCRSConfigMapKey] = fmt.Sprintf("|\n%s", string(utilyaml.JoinYaml(l...))) + cm.Data[defaultCRSConfigMapKey] = string(utilyaml.JoinYaml(l...)) return cm, nil } diff --git a/pkg/handlers/generic/lifecycle/utils/utils_test.go b/pkg/handlers/generic/lifecycle/utils/utils_test.go index fccea15b8..26ec622e6 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils_test.go +++ b/pkg/handlers/generic/lifecycle/utils/utils_test.go @@ -56,8 +56,7 @@ func TestCreateConfigMapForCRS(t *testing.T) { Kind: "ConfigMap", }, Data: map[string]string{ - defaultCRSConfigMapKey: `| -apiVersion: storage.k8s.io/v1 + defaultCRSConfigMapKey: `apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: creationTimestamp: null From f0397e5e2fed98e5e1334a9f83bf644dd64c9e58 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 28 Mar 2024 13:58:23 -0600 Subject: [PATCH 21/31] fix: use a unique storage class cm per cluster --- pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go | 2 +- pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index f3ab9c2ee..8aa171907 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -93,7 +93,7 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, )) } cm, err := lifecycleutils.CreateConfigMapForCRS( - "aws-storageclass-cm", + fmt.Sprintf("aws-storageclass-cm-%s", cluster.Name), a.config.DefaultsNamespace(), allStorageClasses..., ) diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index 0e6b1abf9..8869fac3e 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -153,7 +153,7 @@ func (n *NutanixCSI) createStorageClasses(ctx context.Context, )) } cm, err := lifecycleutils.CreateConfigMapForCRS( - "nutanix-storageclass-cm", + fmt.Sprintf("nutanix-storageclass-cm-%s", cluster.Name), n.config.DefaultsNamespace(), allStorageClasses..., ) From c57e70924c41d76407b4f10f656d85fa973c3254 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 28 Mar 2024 14:07:51 -0600 Subject: [PATCH 22/31] refactor: rename CSIProviders to CSI --- api/v1alpha1/addon_types.go | 8 +-- api/v1alpha1/zz_generated.deepcopy.go | 60 ++++++++++++------- .../templates/role.yaml | 2 +- make/addons.mk | 13 +--- pkg/handlers/generic/lifecycle/csi/handler.go | 2 +- 5 files changed, 46 insertions(+), 39 deletions(-) diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index 19bd443d6..b550445a4 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -37,7 +37,7 @@ type Addons struct { CCM *CCM `json:"ccm,omitempty"` // +optional - CSIProviders *CSIProviders `json:"csi,omitempty"` + CSIProviders *CSI `json:"csi,omitempty"` } func (Addons) VariableSchema() clusterv1.VariableSchema { @@ -49,7 +49,7 @@ func (Addons) VariableSchema() clusterv1.VariableSchema { "cni": CNI{}.VariableSchema().OpenAPIV3Schema, "nfd": NFD{}.VariableSchema().OpenAPIV3Schema, "clusterAutoscaler": ClusterAutoscaler{}.VariableSchema().OpenAPIV3Schema, - "csi": CSIProviders{}.VariableSchema().OpenAPIV3Schema, + "csi": CSI{}.VariableSchema().OpenAPIV3Schema, "ccm": CCM{}.VariableSchema().OpenAPIV3Schema, }, }, @@ -148,7 +148,7 @@ type DefaultStorage struct { StorageClassConfigName string `json:"storageClassConfigName"` } -type CSIProviders struct { +type CSI struct { // +optional Providers []CSIProvider `json:"providers,omitempty"` // +optional @@ -283,7 +283,7 @@ func (DefaultStorage) VariableSchema() clusterv1.VariableSchema { } } -func (CSIProviders) VariableSchema() clusterv1.VariableSchema { +func (CSI) VariableSchema() clusterv1.VariableSchema { return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Type: "object", diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 467ea9763..ec26a8beb 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -206,7 +206,7 @@ func (in *Addons) DeepCopyInto(out *Addons) { } if in.CSIProviders != nil { in, out := &in.CSIProviders, &out.CSIProviders - *out = new(CSIProviders) + *out = new(CSI) (*in).DeepCopyInto(*out) } } @@ -252,34 +252,24 @@ func (in *CNI) DeepCopy() *CNI { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CSIProvider) DeepCopyInto(out *CSIProvider) { +<<<<<<< HEAD +======= +func (in *CPI) DeepCopyInto(out *CPI) { *out = *in - if in.StorageClassConfig != nil { - in, out := &in.StorageClassConfig, &out.StorageClassConfig - *out = make([]StorageClassConfig, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Credentials != nil { - in, out := &in.Credentials, &out.Credentials - *out = new(v1.SecretReference) - **out = **in - } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIProvider. -func (in *CSIProvider) DeepCopy() *CSIProvider { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CPI. +func (in *CPI) DeepCopy() *CPI { if in == nil { return nil } - out := new(CSIProvider) + out := new(CPI) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CSIProviders) DeepCopyInto(out *CSIProviders) { +func (in *CSI) DeepCopyInto(out *CSI) { *out = *in if in.Providers != nil { in, out := &in.Providers, &out.Providers @@ -295,12 +285,40 @@ func (in *CSIProviders) DeepCopyInto(out *CSIProviders) { } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIProviders. -func (in *CSIProviders) DeepCopy() *CSIProviders { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSI. +func (in *CSI) DeepCopy() *CSI { if in == nil { return nil } - out := new(CSIProviders) + out := new(CSI) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +>>>>>>> 1020f52 (refactor: rename CSIProviders to CSI) +func (in *CSIProvider) DeepCopyInto(out *CSIProvider) { + *out = *in + if in.StorageClassConfig != nil { + in, out := &in.StorageClassConfig, &out.StorageClassConfig + *out = make([]StorageClassConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(v1.SecretReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIProvider. +func (in *CSIProvider) DeepCopy() *CSIProvider { + if in == nil { + return nil + } + out := new(CSIProvider) in.DeepCopyInto(out) return out } diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml index e3cd174f7..84702cdfb 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml @@ -4,7 +4,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "chart.name" . }}-manager-role + name: cluster-api-runtime-extensions-nutanix-manager-role rules: - apiGroups: - "" diff --git a/make/addons.mk b/make/addons.mk index 4cc4d8f42..64c1b42db 100644 --- a/make/addons.mk +++ b/make/addons.mk @@ -7,7 +7,7 @@ export NODE_FEATURE_DISCOVERY_VERSION := $(shell goprintconst -file pkg/handlers export CLUSTER_AUTOSCALER_VERSION := 9.35.0 export AWS_CSI_SNAPSHOT_CONTROLLER_VERSION := v6.3.3 export AWS_EBS_CSI_CHART_VERSION := v2.28.1 -<<<<<<< HEAD +export NUTANIX_CSI_CHART_VERSION := v2.6.6 # a map of AWS CCM versions export AWS_CCM_VERSION_127 := v1.27.1 export AWS_CCM_CHART_VERSION_127 := 0.0.8 @@ -16,17 +16,6 @@ export AWS_CCM_CHART_VERSION_128 := 0.0.8 .PHONY: addons.sync addons.sync: $(addprefix update-addon.,calico cilium nfd cluster-autoscaler aws-ebs-csi aws-ccm.127 aws-ccm.128) -======= -export NUTANIX_CSI_CHART_VERSION := v2.6.6 -# a map of AWS CPI versions -export AWS_CPI_VERSION_127 := v1.27.1 -export AWS_CPI_CHART_VERSION_127 := 0.0.8 -export AWS_CPI_VERSION_128 := v1.28.1 -export AWS_CPI_CHART_VERSION_128 := 0.0.8 - -.PHONY: addons.sync -addons.sync: $(addprefix update-addon.,calico cilium nfd cluster-autoscaler aws-ebs-csi aws-cpi.127 aws-cpi.128 nutanix-csi) ->>>>>>> 625d067 (feat: minor API tweaks and adds tooling for chart) .PHONY: update-addon.calico update-addon.calico: ; $(info $(M) updating calico manifests) diff --git a/pkg/handlers/generic/lifecycle/csi/handler.go b/pkg/handlers/generic/lifecycle/csi/handler.go index 14209bec3..e7bccfb3b 100644 --- a/pkg/handlers/generic/lifecycle/csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/handler.go @@ -72,7 +72,7 @@ func (c *CSIHandler) AfterControlPlaneInitialized( ) varMap := variables.ClusterVariablesToVariablesMap(req.Cluster.Spec.Topology.Variables) resp.SetStatus(runtimehooksv1.ResponseStatusSuccess) - csiProviders, found, err := variables.Get[v1alpha1.CSIProviders]( + csiProviders, found, err := variables.Get[v1alpha1.CSI]( varMap, c.variableName, c.variablePath...) From 8ceebeff2ebcb61c80a3c4ca2afadcd341fa42c7 Mon Sep 17 00:00:00 2001 From: faiq Date: Thu, 28 Mar 2024 16:02:34 -0600 Subject: [PATCH 23/31] fix: more unit tests --- api/v1alpha1/clusterconfig_types.go | 10 +- .../generic/lifecycle/csi/aws-ebs/handler.go | 3 +- .../lifecycle/csi/nutanix-csi/handler.go | 3 +- pkg/handlers/generic/lifecycle/utils/utils.go | 10 +- .../generic/lifecycle/utils/utils_test.go | 95 +++++++++++++++++++ 5 files changed, 108 insertions(+), 13 deletions(-) diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index c664fc665..83242d1b9 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -14,13 +14,13 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/openapi/patterns" ) -type StorageDriver string +type StorageProvisioner string const ( - CNIProviderCalico = "Calico" - CNIProviderCilium = "Cilium" - AWSEBSDriver StorageDriver = "ebs.csi.aws.com" - NutanixDriver StorageDriver = "csi.nutanix.com" + CNIProviderCalico = "Calico" + CNIProviderCilium = "Cilium" + AWSEBSProvisioner StorageProvisioner = "ebs.csi.aws.com" + NutanixProvisioner StorageProvisioner = "csi.nutanix.com" CSIProviderAWSEBS = "aws-ebs" CSIProviderNutanix = "nutanix" diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index 8aa171907..528d50cef 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -86,9 +86,8 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass( c, - cluster, a.config.GlobalOptions.DefaultsNamespace(), - v1alpha1.CSIProviderAWSEBS, + v1alpha1.AWSEBSProvisioner, setAsDefault, )) } diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index 8869fac3e..bb721f366 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -146,9 +146,8 @@ func (n *NutanixCSI) createStorageClasses(ctx context.Context, v1alpha1.CSIProviderNutanix == defaultStorageConfig.ProviderName allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass( c, - cluster, n.config.GlobalOptions.DefaultsNamespace(), - v1alpha1.NutanixDriver, + v1alpha1.NutanixProvisioner, setAsDefault, )) } diff --git a/pkg/handlers/generic/lifecycle/utils/utils.go b/pkg/handlers/generic/lifecycle/utils/utils.go index d87e225c4..6906be3b3 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils.go +++ b/pkg/handlers/generic/lifecycle/utils/utils.go @@ -34,7 +34,7 @@ var ( defaultStorageClassMap = map[string]string{ defaultStorageClassKey: "true", } - defaultParams = map[string]string{ + defaultAWSStorageClassParams = map[string]string{ "csi.storage.k8s.io/fstype": "ext4", "type": "gp3", } @@ -138,9 +138,8 @@ func RetrieveValuesTemplateConfigMap( func CreateStorageClass( storageConfig v1alpha1.StorageClassConfig, - cluster *clusterv1.Cluster, defaultsNamespace string, - provisionerName v1alpha1.StorageDriver, + provisionerName v1alpha1.StorageProvisioner, isDefault bool, ) *storagev1.StorageClass { var volumeBindingMode *storagev1.VolumeBindingMode @@ -159,7 +158,10 @@ func CreateStorageClass( case v1alpha1.VolumeReclaimRetain: reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimRetain) } - params := defaultParams + var params map[string]string + if provisionerName == v1alpha1.AWSEBSProvisioner { + params = defaultAWSStorageClassParams + } if storageConfig.Parameters != nil { params = maps.Clone(storageConfig.Parameters) } diff --git a/pkg/handlers/generic/lifecycle/utils/utils_test.go b/pkg/handlers/generic/lifecycle/utils/utils_test.go index 26ec622e6..feae34c02 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils_test.go +++ b/pkg/handlers/generic/lifecycle/utils/utils_test.go @@ -6,12 +6,107 @@ package utils import ( "testing" + "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" ) +func TestCreateStorageClass(t *testing.T) { + tests := []struct { + name string + defaultsNamespace string + storageConfig v1alpha1.StorageClassConfig + expectedStorageClass *storagev1.StorageClass + provisioner v1alpha1.StorageProvisioner + isDefault bool + }{ + { + name: "defaulting with AWS", + storageConfig: v1alpha1.StorageClassConfig{ + Name: "aws-ebs", + ReclaimPolicy: v1alpha1.VolumeReclaimDelete, + VolumeBindingMode: v1alpha1.VolumeBindingWaitForFirstConsumer, + Parameters: nil, + }, + expectedStorageClass: &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: kindStorageClass, + APIVersion: storagev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "aws-ebs", + Namespace: "default", + }, + Parameters: defaultAWSStorageClassParams, + ReclaimPolicy: ptr.To(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: ptr.To(storagev1.VolumeBindingWaitForFirstConsumer), + Provisioner: string(v1alpha1.AWSEBSProvisioner), + }, + provisioner: v1alpha1.AWSEBSProvisioner, + defaultsNamespace: "default", + }, + { + name: "nutanix for nutanix files", + storageConfig: v1alpha1.StorageClassConfig{ + Name: "nutanix-volumes", + ReclaimPolicy: v1alpha1.VolumeReclaimDelete, + VolumeBindingMode: v1alpha1.VolumeBindingWaitForFirstConsumer, + Parameters: map[string]string{ + "csi.storage.k8s.io/fstype": "ext4", + "flashMode": "ENABLED", + "storageContainer": "storage-container-name", + "chapAuth": "ENABLED", + "storageType": "NutanixVolumes", + "whitelistIPMode": "ENABLED", + "whitelistIPAddr": "1.1.1.1", + }, + }, + expectedStorageClass: &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: kindStorageClass, + APIVersion: storagev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "nutanix-volumes", + Namespace: "default", + }, + Parameters: map[string]string{ + "csi.storage.k8s.io/fstype": "ext4", + "flashMode": "ENABLED", + "storageContainer": "storage-container-name", + "chapAuth": "ENABLED", + "storageType": "NutanixVolumes", + "whitelistIPMode": "ENABLED", + "whitelistIPAddr": "1.1.1.1", + }, + ReclaimPolicy: ptr.To(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: ptr.To(storagev1.VolumeBindingWaitForFirstConsumer), + Provisioner: string(v1alpha1.NutanixProvisioner), + }, + provisioner: v1alpha1.NutanixProvisioner, + defaultsNamespace: "default", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sc := CreateStorageClass( + tt.storageConfig, + tt.defaultsNamespace, + tt.provisioner, + false, + ) + if diff := cmp.Diff(sc, tt.expectedStorageClass); diff != "" { + t.Errorf("CreateStorageClass() mismatch (-want +got):\n%s", diff) + } + }) + } +} + func TestCreateConfigMapForCRS(t *testing.T) { tests := []struct { name string From 3269c5b05db6b44109b91c32ee801bd530111a6e Mon Sep 17 00:00:00 2001 From: faiq Date: Fri, 29 Mar 2024 08:50:15 -0600 Subject: [PATCH 24/31] fix: gomod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c1df13beb..c55396282 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api v0.0.0-00010101000000-000000000000 github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common v0.0.0-00010101000000-000000000000 github.com/go-logr/logr v1.4.1 + github.com/google/go-cmp v0.6.0 github.com/onsi/ginkgo/v2 v2.16.0 github.com/onsi/gomega v1.31.1 github.com/spf13/pflag v1.0.5 @@ -71,7 +72,6 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/cel-go v0.17.7 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-github/v53 v53.2.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect From f963e9790f513122c970a5e61d86bcc0159cfa09 Mon Sep 17 00:00:00 2001 From: faiq Date: Fri, 29 Mar 2024 09:29:25 -0600 Subject: [PATCH 25/31] test: adds a unit test for no params --- .../generic/lifecycle/utils/utils_test.go | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pkg/handlers/generic/lifecycle/utils/utils_test.go b/pkg/handlers/generic/lifecycle/utils/utils_test.go index feae34c02..af23ce985 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils_test.go +++ b/pkg/handlers/generic/lifecycle/utils/utils_test.go @@ -91,6 +91,29 @@ func TestCreateStorageClass(t *testing.T) { provisioner: v1alpha1.NutanixProvisioner, defaultsNamespace: "default", }, + { + name: "nutanix defaults", + storageConfig: v1alpha1.StorageClassConfig{ + Name: "nutanix-volumes", + ReclaimPolicy: v1alpha1.VolumeReclaimDelete, + VolumeBindingMode: v1alpha1.VolumeBindingWaitForFirstConsumer, + }, + expectedStorageClass: &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: kindStorageClass, + APIVersion: storagev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "nutanix-volumes", + Namespace: "default", + }, + ReclaimPolicy: ptr.To(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: ptr.To(storagev1.VolumeBindingWaitForFirstConsumer), + Provisioner: string(v1alpha1.NutanixProvisioner), + }, + provisioner: v1alpha1.NutanixProvisioner, + defaultsNamespace: "default", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 9ec05d13ee12c20c339d989d62189fb8ee05daa3 Mon Sep 17 00:00:00 2001 From: faiq Date: Fri, 29 Mar 2024 12:05:44 -0600 Subject: [PATCH 26/31] feat: create crs from provider credentials --- pkg/handlers/generic/lifecycle/ccm/handler.go | 2 +- .../clusterautoscaler/strategy_crs.go | 2 +- .../lifecycle/cni/calico/strategy_crs.go | 2 +- .../lifecycle/cni/cilium/strategy_crs.go | 2 +- .../generic/lifecycle/csi/aws-ebs/handler.go | 4 +-- .../lifecycle/csi/nutanix-csi/handler.go | 33 ++++++++++++++++++- .../generic/lifecycle/nfd/strategy_crs.go | 2 +- pkg/handlers/generic/lifecycle/utils/utils.go | 29 ++++++++++++---- 8 files changed, 62 insertions(+), 14 deletions(-) diff --git a/pkg/handlers/generic/lifecycle/ccm/handler.go b/pkg/handlers/generic/lifecycle/ccm/handler.go index 43360aa1b..04dd32f85 100644 --- a/pkg/handlers/generic/lifecycle/ccm/handler.go +++ b/pkg/handlers/generic/lifecycle/ccm/handler.go @@ -114,7 +114,7 @@ func (c *CCMHandler) AfterControlPlaneInitialized( ) return } - err = lifecycleutils.EnsureCRSForClusterFromConfigMaps(ctx, cm.Name, c.client, &req.Cluster, cm) + err = lifecycleutils.EnsureCRSForClusterFromObjects(ctx, cm.Name, c.client, &req.Cluster, cm) if err != nil { log.Error( err, diff --git a/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_crs.go b/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_crs.go index 981d4ec01..432efba62 100644 --- a/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_crs.go +++ b/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_crs.go @@ -121,7 +121,7 @@ func (s crsStrategy) apply( ) } - if err = utils.EnsureCRSForClusterFromConfigMaps(ctx, cm.Name, s.client, targetCluster, cm); err != nil { + if err = utils.EnsureCRSForClusterFromObjects(ctx, cm.Name, s.client, targetCluster, cm); err != nil { return fmt.Errorf( "failed to apply cluster-autoscaler installation ClusterResourceSet: %w", err, diff --git a/pkg/handlers/generic/lifecycle/cni/calico/strategy_crs.go b/pkg/handlers/generic/lifecycle/cni/calico/strategy_crs.go index a08a686e2..27a366137 100644 --- a/pkg/handlers/generic/lifecycle/cni/calico/strategy_crs.go +++ b/pkg/handlers/generic/lifecycle/cni/calico/strategy_crs.go @@ -150,7 +150,7 @@ func (s crsStrategy) ensureCNICRSForCluster( ) } - if err := utils.EnsureCRSForClusterFromConfigMaps(ctx, cm.Name, s.client, cluster, tigeraConfigMap, cm); err != nil { + if err := utils.EnsureCRSForClusterFromObjects(ctx, cm.Name, s.client, cluster, tigeraConfigMap, cm); err != nil { return fmt.Errorf( "failed to apply Calico CNI installation ClusterResourceSet: %w", err, diff --git a/pkg/handlers/generic/lifecycle/cni/cilium/strategy_crs.go b/pkg/handlers/generic/lifecycle/cni/cilium/strategy_crs.go index 28f07a77c..6a9b805ac 100644 --- a/pkg/handlers/generic/lifecycle/cni/cilium/strategy_crs.go +++ b/pkg/handlers/generic/lifecycle/cni/cilium/strategy_crs.go @@ -87,7 +87,7 @@ func (s crsStrategy) apply( ) } - if err := utils.EnsureCRSForClusterFromConfigMaps(ctx, cm.Name, s.client, cluster, cm); err != nil { + if err := utils.EnsureCRSForClusterFromObjects(ctx, cm.Name, s.client, cluster, cm); err != nil { return fmt.Errorf( "failed to apply Cilium CNI installation ClusterResourceSet: %w", err, diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index 528d50cef..90072b3a2 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -103,7 +103,7 @@ func (a *AWSEBS) createStorageClasses(ctx context.Context, if err != nil { return err } - return lifecycleutils.EnsureCRSForClusterFromConfigMaps( + return lifecycleutils.EnsureCRSForClusterFromObjects( ctx, "aws-storageclass-crs", a.client, @@ -140,7 +140,7 @@ func (a *AWSEBS) handleCRSApply(ctx context.Context, err, ) } - err = lifecycleutils.EnsureCRSForClusterFromConfigMaps( + err = lifecycleutils.EnsureCRSForClusterFromObjects( ctx, cm.Name, a.client, diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index bb721f366..090a9e06d 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -75,6 +76,36 @@ func (n *NutanixCSI) Apply( default: return fmt.Errorf("stategy %s not implemented", strategy) } + if provider.Credentials != nil { + sec := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: provider.Credentials.Name, + Name: provider.Credentials.Namespace, + }, + } + err := n.client.Get( + ctx, + ctrlclient.ObjectKeyFromObject(sec), + sec, + ) + if err != nil { + return err + } + err = lifecycleutils.EnsureCRSForClusterFromObjects( + ctx, + fmt.Sprintf("nutanix-csi-credentials-crs-%s", req.Cluster.Name), + n.client, + &req.Cluster, + sec, + ) + if err != nil { + return err + } + } return n.createStorageClasses( ctx, provider.StorageClassConfig, @@ -163,7 +194,7 @@ func (n *NutanixCSI) createStorageClasses(ctx context.Context, if err != nil { return err } - return lifecycleutils.EnsureCRSForClusterFromConfigMaps( + return lifecycleutils.EnsureCRSForClusterFromObjects( ctx, "nutanix-storageclass-crs", n.client, diff --git a/pkg/handlers/generic/lifecycle/nfd/strategy_crs.go b/pkg/handlers/generic/lifecycle/nfd/strategy_crs.go index 461db5318..26498d482 100644 --- a/pkg/handlers/generic/lifecycle/nfd/strategy_crs.go +++ b/pkg/handlers/generic/lifecycle/nfd/strategy_crs.go @@ -87,7 +87,7 @@ func (s crsStrategy) apply( ) } - if err := utils.EnsureCRSForClusterFromConfigMaps(ctx, cm.Name, s.client, cluster, cm); err != nil { + if err := utils.EnsureCRSForClusterFromObjects(ctx, cm.Name, s.client, cluster, cm); err != nil { return fmt.Errorf( "failed to apply NFD installation ClusterResourceSet: %w", err, diff --git a/pkg/handlers/generic/lifecycle/utils/utils.go b/pkg/handlers/generic/lifecycle/utils/utils.go index 6906be3b3..885d45c65 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils.go +++ b/pkg/handlers/generic/lifecycle/utils/utils.go @@ -40,18 +40,35 @@ var ( } ) -func EnsureCRSForClusterFromConfigMaps( +func EnsureCRSForClusterFromObjects( ctx context.Context, crsName string, c ctrlclient.Client, cluster *clusterv1.Cluster, - configMaps ...*corev1.ConfigMap, + objects ...runtime.Object, ) error { - resources := make([]crsv1.ResourceRef, 0, len(configMaps)) - for _, cm := range configMaps { + resources := make([]crsv1.ResourceRef, 0, len(objects)) + for _, obj := range objects { + var name string + var kind crsv1.ClusterResourceSetResourceKind + cm, ok := obj.(*corev1.ConfigMap) + if !ok { + sec, secOk := obj.(*corev1.Secret) + if !secOk { + return fmt.Errorf( + "cannot create ClusterResourceSet with obj %v only secrets and configmaps are supported", + obj, + ) + } + name = sec.Name + kind = crsv1.SecretClusterResourceSetResourceKind + } else { + name = cm.Name + kind = crsv1.ConfigMapClusterResourceSetResourceKind + } resources = append(resources, crsv1.ResourceRef{ - Kind: string(crsv1.ConfigMapClusterResourceSetResourceKind), - Name: cm.Name, + Name: name, + Kind: string(kind), }) } From 13aa430c05968f9108d194277c1d454b71ab1b8c Mon Sep 17 00:00:00 2001 From: faiq Date: Fri, 29 Mar 2024 12:34:08 -0600 Subject: [PATCH 27/31] feat: set a default storage class if 1 provider and 1 storage class config --- .../generic/lifecycle/csi/aws-ebs/handler.go | 5 ++++- pkg/handlers/generic/lifecycle/csi/handler.go | 13 ++++++++++++- .../generic/lifecycle/csi/nutanix-csi/handler.go | 6 +++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index 90072b3a2..188e051b5 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -54,6 +54,7 @@ func (a *AWSEBS) Apply( ctx context.Context, provider v1alpha1.CSIProvider, defaultStorageConfig *v1alpha1.DefaultStorage, + hasOneProviderAndOneStorageClass bool, req *runtimehooksv1.AfterControlPlaneInitializedRequest, ) error { strategy := provider.Strategy @@ -71,6 +72,7 @@ func (a *AWSEBS) Apply( ctx, provider.StorageClassConfig, &req.Cluster, + hasOneProviderAndOneStorageClass, defaultStorageConfig, ) } @@ -78,12 +80,13 @@ func (a *AWSEBS) Apply( func (a *AWSEBS) createStorageClasses(ctx context.Context, configs []v1alpha1.StorageClassConfig, cluster *clusterv1.Cluster, + hasOneProviderAndOneStorageClass bool, defaultStorageConfig *v1alpha1.DefaultStorage, ) error { allStorageClasses := make([]runtime.Object, 0, len(configs)) for _, c := range configs { setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && - v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName + v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName || hasOneProviderAndOneStorageClass allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass( c, a.config.GlobalOptions.DefaultsNamespace(), diff --git a/pkg/handlers/generic/lifecycle/csi/handler.go b/pkg/handlers/generic/lifecycle/csi/handler.go index e7bccfb3b..59774aa56 100644 --- a/pkg/handlers/generic/lifecycle/csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/handler.go @@ -27,6 +27,7 @@ type CSIProvider interface { context.Context, v1alpha1.CSIProvider, *v1alpha1.DefaultStorage, + bool, // this is a bool which keeps track of the case where a user has only one provider and one storageclass *runtimehooksv1.AfterControlPlaneInitializedRequest, ) error } @@ -98,6 +99,10 @@ func (c *CSIHandler) AfterControlPlaneInitialized( ) return } + hasOneProviderAndOneStorageClass := len(csiProviders.Providers) == 1 && + len(csiProviders.Providers[0].StorageClassConfig) == 1 && + csiProviders.DefaultStorage == nil + for _, provider := range csiProviders.Providers { handler, ok := c.ProviderHandler[provider.Name] if !ok { @@ -110,7 +115,13 @@ func (c *CSIHandler) AfterControlPlaneInitialized( continue } log.Info(fmt.Sprintf("Creating csi provider %s", provider)) - err = handler.Apply(ctx, provider, csiProviders.DefaultStorage, req) + err = handler.Apply( + ctx, + provider, + csiProviders.DefaultStorage, + hasOneProviderAndOneStorageClass, + req, + ) if err != nil { log.Error( err, diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index 090a9e06d..756dabd45 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -63,6 +63,7 @@ func (n *NutanixCSI) Apply( ctx context.Context, provider v1alpha1.CSIProvider, defaultStorageConfig *v1alpha1.DefaultStorage, + hasOneProviderAndOneStorageClass bool, req *runtimehooksv1.AfterControlPlaneInitializedRequest, ) error { strategy := provider.Strategy @@ -110,6 +111,7 @@ func (n *NutanixCSI) Apply( ctx, provider.StorageClassConfig, &req.Cluster, + hasOneProviderAndOneStorageClass, defaultStorageConfig, ) } @@ -169,12 +171,14 @@ func (n *NutanixCSI) handleHelmAddonApply( func (n *NutanixCSI) createStorageClasses(ctx context.Context, configs []v1alpha1.StorageClassConfig, cluster *clusterv1.Cluster, + hasOneProviderAndOneStorageClass bool, defaultStorageConfig *v1alpha1.DefaultStorage, ) error { allStorageClasses := make([]runtime.Object, 0, len(configs)) for _, c := range configs { setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && - v1alpha1.CSIProviderNutanix == defaultStorageConfig.ProviderName + v1alpha1.CSIProviderNutanix == defaultStorageConfig.ProviderName || + hasOneProviderAndOneStorageClass allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass( c, n.config.GlobalOptions.DefaultsNamespace(), From 6845f969f13271c77cc3e911dc6d27d93bd44d7b Mon Sep 17 00:00:00 2001 From: faiq Date: Fri, 29 Mar 2024 12:57:03 -0600 Subject: [PATCH 28/31] refactor: use default storage config obj --- .../generic/lifecycle/csi/aws-ebs/handler.go | 5 +---- pkg/handlers/generic/lifecycle/csi/handler.go | 12 ++++++++---- .../generic/lifecycle/csi/nutanix-csi/handler.go | 6 +----- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index 188e051b5..90072b3a2 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -54,7 +54,6 @@ func (a *AWSEBS) Apply( ctx context.Context, provider v1alpha1.CSIProvider, defaultStorageConfig *v1alpha1.DefaultStorage, - hasOneProviderAndOneStorageClass bool, req *runtimehooksv1.AfterControlPlaneInitializedRequest, ) error { strategy := provider.Strategy @@ -72,7 +71,6 @@ func (a *AWSEBS) Apply( ctx, provider.StorageClassConfig, &req.Cluster, - hasOneProviderAndOneStorageClass, defaultStorageConfig, ) } @@ -80,13 +78,12 @@ func (a *AWSEBS) Apply( func (a *AWSEBS) createStorageClasses(ctx context.Context, configs []v1alpha1.StorageClassConfig, cluster *clusterv1.Cluster, - hasOneProviderAndOneStorageClass bool, defaultStorageConfig *v1alpha1.DefaultStorage, ) error { allStorageClasses := make([]runtime.Object, 0, len(configs)) for _, c := range configs { setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && - v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName || hasOneProviderAndOneStorageClass + v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass( c, a.config.GlobalOptions.DefaultsNamespace(), diff --git a/pkg/handlers/generic/lifecycle/csi/handler.go b/pkg/handlers/generic/lifecycle/csi/handler.go index 59774aa56..01f3e2b66 100644 --- a/pkg/handlers/generic/lifecycle/csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/handler.go @@ -27,7 +27,6 @@ type CSIProvider interface { context.Context, v1alpha1.CSIProvider, *v1alpha1.DefaultStorage, - bool, // this is a bool which keeps track of the case where a user has only one provider and one storageclass *runtimehooksv1.AfterControlPlaneInitializedRequest, ) error } @@ -99,9 +98,15 @@ func (c *CSIHandler) AfterControlPlaneInitialized( ) return } - hasOneProviderAndOneStorageClass := len(csiProviders.Providers) == 1 && + if len(csiProviders.Providers) == 1 && + csiProviders.Providers[0].StorageClassConfig != nil && len(csiProviders.Providers[0].StorageClassConfig) == 1 && - csiProviders.DefaultStorage == nil + csiProviders.DefaultStorage == nil { + csiProviders.DefaultStorage = &v1alpha1.DefaultStorage{ + ProviderName: csiProviders.Providers[0].Name, + StorageClassConfigName: csiProviders.Providers[0].StorageClassConfig[0].Name, + } + } for _, provider := range csiProviders.Providers { handler, ok := c.ProviderHandler[provider.Name] @@ -119,7 +124,6 @@ func (c *CSIHandler) AfterControlPlaneInitialized( ctx, provider, csiProviders.DefaultStorage, - hasOneProviderAndOneStorageClass, req, ) if err != nil { diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index 756dabd45..090a9e06d 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -63,7 +63,6 @@ func (n *NutanixCSI) Apply( ctx context.Context, provider v1alpha1.CSIProvider, defaultStorageConfig *v1alpha1.DefaultStorage, - hasOneProviderAndOneStorageClass bool, req *runtimehooksv1.AfterControlPlaneInitializedRequest, ) error { strategy := provider.Strategy @@ -111,7 +110,6 @@ func (n *NutanixCSI) Apply( ctx, provider.StorageClassConfig, &req.Cluster, - hasOneProviderAndOneStorageClass, defaultStorageConfig, ) } @@ -171,14 +169,12 @@ func (n *NutanixCSI) handleHelmAddonApply( func (n *NutanixCSI) createStorageClasses(ctx context.Context, configs []v1alpha1.StorageClassConfig, cluster *clusterv1.Cluster, - hasOneProviderAndOneStorageClass bool, defaultStorageConfig *v1alpha1.DefaultStorage, ) error { allStorageClasses := make([]runtime.Object, 0, len(configs)) for _, c := range configs { setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && - v1alpha1.CSIProviderNutanix == defaultStorageConfig.ProviderName || - hasOneProviderAndOneStorageClass + v1alpha1.CSIProviderNutanix == defaultStorageConfig.ProviderName allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass( c, n.config.GlobalOptions.DefaultsNamespace(), From 5ab09f8ce4de67e4f94413c3277f19e94b0d0c3c Mon Sep 17 00:00:00 2001 From: faiq Date: Fri, 29 Mar 2024 13:39:43 -0600 Subject: [PATCH 29/31] feat: allow volume expansion --- api/v1alpha1/addon_types.go | 18 +++++++++++--- pkg/handlers/generic/lifecycle/utils/utils.go | 9 +++---- .../generic/lifecycle/utils/utils_test.go | 24 +++++++++++-------- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index b550445a4..a21f1353c 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -178,6 +178,9 @@ type StorageClassConfig struct { // +optional VolumeBindingMode string `json:"volumeBindingMode,omitempty"` + + // +optional + AllowExpansion bool `json:"allowExpansion,omitempty"` } func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { @@ -192,7 +195,8 @@ func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { } return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Type: "object", + Type: "object", + Required: []string{"name"}, Properties: map[string]clusterv1.JSONSchemaProps{ "name": { Type: "string", @@ -215,6 +219,11 @@ func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { Enum: variables.MustMarshalValuesToEnumJSON(supportedBindingModes...), Default: variables.MustMarshal(VolumeBindingWaitForFirstConsumer), }, + "allowExpansion": { + Type: "boolean", + Default: variables.MustMarshal(false), + Description: "If the storage class should allow volume expanding", + }, }, }, } @@ -224,7 +233,8 @@ func (CSIProvider) VariableSchema() clusterv1.VariableSchema { supportedCSIProviders := []string{CSIProviderAWSEBS, CSIProviderNutanix} return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Type: "object", + Type: "object", + Required: []string{"name", "strategy"}, Properties: map[string]clusterv1.JSONSchemaProps{ "name": { Description: "Name of the CSI Provider", @@ -267,12 +277,14 @@ func (DefaultStorage) VariableSchema() clusterv1.VariableSchema { OpenAPIV3Schema: clusterv1.JSONSchemaProps{ Type: "object", Description: "A tuple of provider name and storage class ", + Required: []string{"providerName", "storageClassConfigName"}, Properties: map[string]clusterv1.JSONSchemaProps{ "providerName": { Type: "string", Description: "Name of the CSI Provider for the default storage class", Enum: variables.MustMarshalValuesToEnumJSON( - supportedCSIProviders...), + supportedCSIProviders..., + ), }, "storageClassConfigName": { Type: "string", diff --git a/pkg/handlers/generic/lifecycle/utils/utils.go b/pkg/handlers/generic/lifecycle/utils/utils.go index 885d45c65..7da8bc5b2 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils.go +++ b/pkg/handlers/generic/lifecycle/utils/utils.go @@ -191,10 +191,11 @@ func CreateStorageClass( Name: storageConfig.Name, Namespace: defaultsNamespace, }, - Provisioner: string(provisionerName), - Parameters: params, - VolumeBindingMode: volumeBindingMode, - ReclaimPolicy: reclaimPolicy, + Provisioner: string(provisionerName), + Parameters: params, + VolumeBindingMode: volumeBindingMode, + ReclaimPolicy: reclaimPolicy, + AllowVolumeExpansion: ptr.To(storageConfig.AllowExpansion), } if isDefault { sc.ObjectMeta.Annotations = defaultStorageClassMap diff --git a/pkg/handlers/generic/lifecycle/utils/utils_test.go b/pkg/handlers/generic/lifecycle/utils/utils_test.go index af23ce985..420ebe562 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils_test.go +++ b/pkg/handlers/generic/lifecycle/utils/utils_test.go @@ -32,6 +32,7 @@ func TestCreateStorageClass(t *testing.T) { ReclaimPolicy: v1alpha1.VolumeReclaimDelete, VolumeBindingMode: v1alpha1.VolumeBindingWaitForFirstConsumer, Parameters: nil, + AllowExpansion: true, }, expectedStorageClass: &storagev1.StorageClass{ TypeMeta: metav1.TypeMeta{ @@ -42,10 +43,11 @@ func TestCreateStorageClass(t *testing.T) { Name: "aws-ebs", Namespace: "default", }, - Parameters: defaultAWSStorageClassParams, - ReclaimPolicy: ptr.To(corev1.PersistentVolumeReclaimDelete), - VolumeBindingMode: ptr.To(storagev1.VolumeBindingWaitForFirstConsumer), - Provisioner: string(v1alpha1.AWSEBSProvisioner), + Parameters: defaultAWSStorageClassParams, + ReclaimPolicy: ptr.To(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: ptr.To(storagev1.VolumeBindingWaitForFirstConsumer), + Provisioner: string(v1alpha1.AWSEBSProvisioner), + AllowVolumeExpansion: ptr.To(true), }, provisioner: v1alpha1.AWSEBSProvisioner, defaultsNamespace: "default", @@ -84,9 +86,10 @@ func TestCreateStorageClass(t *testing.T) { "whitelistIPMode": "ENABLED", "whitelistIPAddr": "1.1.1.1", }, - ReclaimPolicy: ptr.To(corev1.PersistentVolumeReclaimDelete), - VolumeBindingMode: ptr.To(storagev1.VolumeBindingWaitForFirstConsumer), - Provisioner: string(v1alpha1.NutanixProvisioner), + ReclaimPolicy: ptr.To(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: ptr.To(storagev1.VolumeBindingWaitForFirstConsumer), + Provisioner: string(v1alpha1.NutanixProvisioner), + AllowVolumeExpansion: ptr.To(false), }, provisioner: v1alpha1.NutanixProvisioner, defaultsNamespace: "default", @@ -107,9 +110,10 @@ func TestCreateStorageClass(t *testing.T) { Name: "nutanix-volumes", Namespace: "default", }, - ReclaimPolicy: ptr.To(corev1.PersistentVolumeReclaimDelete), - VolumeBindingMode: ptr.To(storagev1.VolumeBindingWaitForFirstConsumer), - Provisioner: string(v1alpha1.NutanixProvisioner), + ReclaimPolicy: ptr.To(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: ptr.To(storagev1.VolumeBindingWaitForFirstConsumer), + Provisioner: string(v1alpha1.NutanixProvisioner), + AllowVolumeExpansion: ptr.To(false), }, provisioner: v1alpha1.NutanixProvisioner, defaultsNamespace: "default", From 05faebb781346ffab925e4d0db72dbdf7021741a Mon Sep 17 00:00:00 2001 From: faiq Date: Fri, 29 Mar 2024 14:01:20 -0600 Subject: [PATCH 30/31] feat: deploy snapshot chart with nutanix --- api/v1alpha1/zz_generated.deepcopy.go | 18 ------- .../templates/role.yaml | 2 +- make/addons.mk | 1 - pkg/handlers/generic/lifecycle/csi/handler.go | 7 ++- .../lifecycle/csi/nutanix-csi/handler.go | 53 ++++++++++++++++--- pkg/handlers/generic/lifecycle/handlers.go | 6 +-- 6 files changed, 51 insertions(+), 36 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ec26a8beb..4f33138e6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -251,23 +251,6 @@ func (in *CNI) DeepCopy() *CNI { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -<<<<<<< HEAD -======= -func (in *CPI) DeepCopyInto(out *CPI) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CPI. -func (in *CPI) DeepCopy() *CPI { - if in == nil { - return nil - } - out := new(CPI) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CSI) DeepCopyInto(out *CSI) { *out = *in @@ -296,7 +279,6 @@ func (in *CSI) DeepCopy() *CSI { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. ->>>>>>> 1020f52 (refactor: rename CSIProviders to CSI) func (in *CSIProvider) DeepCopyInto(out *CSIProvider) { *out = *in if in.StorageClassConfig != nil { diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml index 84702cdfb..e3cd174f7 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml @@ -4,7 +4,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: cluster-api-runtime-extensions-nutanix-manager-role + name: {{ include "chart.name" . }}-manager-role rules: - apiGroups: - "" diff --git a/make/addons.mk b/make/addons.mk index 64c1b42db..c09890922 100644 --- a/make/addons.mk +++ b/make/addons.mk @@ -44,4 +44,3 @@ update-addon.aws-ccm.%: ; $(info $(M) updating aws ccm $* manifests) .PHONY: update-addon.nutanix-csi update-addon.nutanix-csi: ; $(info $(M) updating nutanix csi manifests) ./hack/addons/update-nutanix-csi.sh - diff --git a/pkg/handlers/generic/lifecycle/csi/handler.go b/pkg/handlers/generic/lifecycle/csi/handler.go index 01f3e2b66..fd07767c6 100644 --- a/pkg/handlers/generic/lifecycle/csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/handler.go @@ -99,7 +99,6 @@ func (c *CSIHandler) AfterControlPlaneInitialized( return } if len(csiProviders.Providers) == 1 && - csiProviders.Providers[0].StorageClassConfig != nil && len(csiProviders.Providers[0].StorageClassConfig) == 1 && csiProviders.DefaultStorage == nil { csiProviders.DefaultStorage = &v1alpha1.DefaultStorage{ @@ -113,13 +112,13 @@ func (c *CSIHandler) AfterControlPlaneInitialized( if !ok { log.V(4).Info( fmt.Sprintf( - "Skipping CSI handler, for provider given in %q. Provider handler not given ", - provider, + "Skipping CSI handler, for provider given in %s. Provider handler not given.", + provider.Name, ), ) continue } - log.Info(fmt.Sprintf("Creating csi provider %s", provider)) + log.Info(fmt.Sprintf("Creating csi provider %s", provider.Name)) err = handler.Apply( ctx, provider, diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index 090a9e06d..90190bef1 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -24,10 +24,14 @@ import ( ) const ( - defaultHelmRepositoryURL = "https://nutanix.github.io/helm/" - defaultHelmChartVersion = "v2.6.6" - defaultHelmChartName = "nutanix-csi-storage" - defaultHelmReleaseNameTemplate = "nutanix-csi-storage-%s" + defaultHelmRepositoryURL = "https://nutanix.github.io/helm/" + defaultStorageHelmChartVersion = "v2.6.6" + defaultStorageHelmChartName = "nutanix-csi-storage" + defaultStorageHelmReleaseNameTemplate = "nutanix-csi-storage-%s" + + defaultSnapshotHelmChartVersion = "v6.3.2" + defaultSnapshotHelmChartName = "nutanix-csi-snapshot" + defaultSnapshotHelmReleaseNameTemplate = "nutanix-csi-snapshot-%s" ) type NutanixCSIConfig struct { @@ -141,13 +145,13 @@ func (n *NutanixCSI) handleHelmAddonApply( }, Spec: caaphv1.HelmChartProxySpec{ RepoURL: defaultHelmRepositoryURL, - ChartName: defaultHelmChartName, + ChartName: defaultStorageHelmChartName, ClusterSelector: metav1.LabelSelector{ MatchLabels: map[string]string{clusterv1.ClusterNameLabel: req.Cluster.Name}, }, ReleaseNamespace: req.Cluster.Namespace, - ReleaseName: fmt.Sprintf(defaultHelmReleaseNameTemplate, req.Cluster.Name), - Version: defaultHelmChartVersion, + ReleaseName: fmt.Sprintf(defaultStorageHelmReleaseNameTemplate, req.Cluster.Name), + Version: defaultStorageHelmChartVersion, ValuesTemplate: values, }, } @@ -163,6 +167,41 @@ func (n *NutanixCSI) handleHelmAddonApply( return fmt.Errorf("failed to apply nutanix-csi installation HelmChartProxy: %w", err) } + snapshotChart := &caaphv1.HelmChartProxy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: caaphv1.GroupVersion.String(), + Kind: "HelmChartProxy", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.Cluster.Namespace, + Name: "nutanix-csi-snapshot" + req.Cluster.Name, + }, + Spec: caaphv1.HelmChartProxySpec{ + RepoURL: defaultHelmRepositoryURL, + ChartName: defaultSnapshotHelmChartName, + ClusterSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{clusterv1.ClusterNameLabel: req.Cluster.Name}, + }, + ReleaseNamespace: req.Cluster.Namespace, + ReleaseName: fmt.Sprintf(defaultStorageHelmReleaseNameTemplate, req.Cluster.Name), + Version: defaultSnapshotHelmChartVersion, + }, + } + + if err = controllerutil.SetOwnerReference(&req.Cluster, snapshotChart, n.client.Scheme()); err != nil { + return fmt.Errorf( + "failed to set owner reference on nutanix-csi installation HelmChartProxy: %w", + err, + ) + } + + if err = client.ServerSideApply(ctx, n.client, snapshotChart); err != nil { + return fmt.Errorf( + "failed to apply nutanix-csi-snapshot installation HelmChartProxy: %w", + err, + ) + } + return nil } diff --git a/pkg/handlers/generic/lifecycle/handlers.go b/pkg/handlers/generic/lifecycle/handlers.go index 33d08e130..b3b0d22fb 100644 --- a/pkg/handlers/generic/lifecycle/handlers.go +++ b/pkg/handlers/generic/lifecycle/handlers.go @@ -29,7 +29,7 @@ type Handlers struct { clusterAutoscalerConfig *clusterautoscaler.Config ebsConfig *awsebs.AWSEBSConfig nutnaixCSIConfig *nutanixcsi.NutanixCSIConfig - awsCPIConfig *awscpi.AWSCPIConfig + awsccmConfig *awsccm.AWSCCMConfig } func New(globalOptions *options.GlobalOptions) *Handlers { @@ -70,10 +70,6 @@ func (h *Handlers) AddFlags(flagSet *pflag.FlagSet) { h.calicoCNIConfig.AddFlags("cni.calico", flagSet) h.ciliumCNIConfig.AddFlags("cni.cilium", flagSet) h.ebsConfig.AddFlags("awsebs", pflag.CommandLine) -<<<<<<< HEAD h.awsccmConfig.AddFlags("awsccm", pflag.CommandLine) -======= h.nutnaixCSIConfig.AddFlags("nutanixcsi", flagSet) - h.awsCPIConfig.AddFlags("awscpi", pflag.CommandLine) ->>>>>>> f7ff2e9 (fix: adds proper permissions and api fixes) } From 7665a94b97d763b553123af2358b23e157688b19 Mon Sep 17 00:00:00 2001 From: faiq Date: Mon, 1 Apr 2024 10:43:59 -0600 Subject: [PATCH 31/31] fix: apply suggestions from reviews --- api/v1alpha1/addon_types.go | 32 +++++++++---------- .../nutanix-cluster-class.yaml | 4 +-- .../lifecycle/csi/nutanix-csi/handler.go | 2 +- pkg/handlers/generic/lifecycle/utils/utils.go | 20 ++---------- 4 files changed, 21 insertions(+), 37 deletions(-) diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index a21f1353c..6f524159d 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -5,6 +5,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -12,15 +13,14 @@ import ( ) const ( - AddonStrategyClusterResourceSet AddonStrategy = "ClusterResourceSet" - AddonStrategyHelmAddon AddonStrategy = "HelmAddon" - - VolumeBindingImmediate = "Immmediate" - VolumeBindingWaitForFirstConsumer = "WaitForFirstConsumer" - - VolumeReclaimRecycle = "Recycle" - VolumeReclaimDelete = "Delete" - VolumeReclaimRetain = "Retain" + AddonStrategyClusterResourceSet AddonStrategy = "ClusterResourceSet" + AddonStrategyHelmAddon AddonStrategy = "HelmAddon" + VolumeBindingImmediate = storagev1.VolumeBindingImmediate + VolumeBindingWaitForFirstConsumer = storagev1.VolumeBindingWaitForFirstConsumer + + VolumeReclaimRecycle = corev1.PersistentVolumeReclaimRecycle + VolumeReclaimDelete = corev1.PersistentVolumeReclaimDelete + VolumeReclaimRetain = corev1.PersistentVolumeReclaimRetain ) type Addons struct { @@ -174,10 +174,10 @@ type StorageClassConfig struct { Parameters map[string]string `json:"parameters,omitempty"` // +optional - ReclaimPolicy string `json:"reclaimPolicy,omitempty"` + ReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"reclaimPolicy,omitempty"` // +optional - VolumeBindingMode string `json:"volumeBindingMode,omitempty"` + VolumeBindingMode storagev1.VolumeBindingMode `json:"volumeBindingMode,omitempty"` // +optional AllowExpansion bool `json:"allowExpansion,omitempty"` @@ -185,13 +185,13 @@ type StorageClassConfig struct { func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { supportedReclaimPolicies := []string{ - VolumeReclaimRecycle, - VolumeReclaimDelete, - VolumeReclaimRetain, + string(VolumeReclaimRecycle), + string(VolumeReclaimDelete), + string(VolumeReclaimRetain), } supportedBindingModes := []string{ - VolumeBindingImmediate, - VolumeBindingWaitForFirstConsumer, + string(VolumeBindingImmediate), + string(VolumeBindingWaitForFirstConsumer), } return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ diff --git a/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml b/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml index 58e511c35..9bb5e3ef8 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml @@ -154,9 +154,9 @@ spec: - name: vip_arp value: "true" - name: address - value: "${CONTROL_PLANE_ENDPOINT_IP}" + value: "control_plane_endpoint_ip" - name: port - value: "${CONTROL_PLANE_ENDPOINT_PORT=6443}" + value: "control_plane_endpoint_port" - name: vip_cidr value: "32" - name: cp_enable diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go index 90190bef1..142189b47 100644 --- a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -183,7 +183,7 @@ func (n *NutanixCSI) handleHelmAddonApply( MatchLabels: map[string]string{clusterv1.ClusterNameLabel: req.Cluster.Name}, }, ReleaseNamespace: req.Cluster.Namespace, - ReleaseName: fmt.Sprintf(defaultStorageHelmReleaseNameTemplate, req.Cluster.Name), + ReleaseName: fmt.Sprintf(defaultSnapshotHelmReleaseNameTemplate, req.Cluster.Name), Version: defaultSnapshotHelmChartVersion, }, } diff --git a/pkg/handlers/generic/lifecycle/utils/utils.go b/pkg/handlers/generic/lifecycle/utils/utils.go index 7da8bc5b2..aa6d42b30 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils.go +++ b/pkg/handlers/generic/lifecycle/utils/utils.go @@ -159,22 +159,6 @@ func CreateStorageClass( provisionerName v1alpha1.StorageProvisioner, isDefault bool, ) *storagev1.StorageClass { - var volumeBindingMode *storagev1.VolumeBindingMode - switch storageConfig.VolumeBindingMode { - case v1alpha1.VolumeBindingImmediate: - volumeBindingMode = ptr.To(storagev1.VolumeBindingImmediate) - case v1alpha1.VolumeBindingWaitForFirstConsumer: - volumeBindingMode = ptr.To(storagev1.VolumeBindingWaitForFirstConsumer) - } - var reclaimPolicy *corev1.PersistentVolumeReclaimPolicy - switch storageConfig.ReclaimPolicy { - case v1alpha1.VolumeReclaimRecycle: - reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimRecycle) - case v1alpha1.VolumeReclaimDelete: - reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimDelete) - case v1alpha1.VolumeReclaimRetain: - reclaimPolicy = ptr.To(corev1.PersistentVolumeReclaimRetain) - } var params map[string]string if provisionerName == v1alpha1.AWSEBSProvisioner { params = defaultAWSStorageClassParams @@ -193,8 +177,8 @@ func CreateStorageClass( }, Provisioner: string(provisionerName), Parameters: params, - VolumeBindingMode: volumeBindingMode, - ReclaimPolicy: reclaimPolicy, + VolumeBindingMode: ptr.To(storageConfig.VolumeBindingMode), + ReclaimPolicy: ptr.To(storageConfig.ReclaimPolicy), AllowVolumeExpansion: ptr.To(storageConfig.AllowExpansion), } if isDefault {