diff --git a/api/v1alpha1/addon_types.go b/api/v1alpha1/addon_types.go index c88edf1e7..6f524159d 100644 --- a/api/v1alpha1/addon_types.go +++ b/api/v1alpha1/addon_types.go @@ -4,11 +4,25 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/variables" ) +const ( + AddonStrategyClusterResourceSet AddonStrategy = "ClusterResourceSet" + AddonStrategyHelmAddon AddonStrategy = "HelmAddon" + VolumeBindingImmediate = storagev1.VolumeBindingImmediate + VolumeBindingWaitForFirstConsumer = storagev1.VolumeBindingWaitForFirstConsumer + + VolumeReclaimRecycle = corev1.PersistentVolumeReclaimRecycle + VolumeReclaimDelete = corev1.PersistentVolumeReclaimDelete + VolumeReclaimRetain = corev1.PersistentVolumeReclaimRetain +) + type Addons struct { // +optional CNI *CNI `json:"cni,omitempty"` @@ -23,7 +37,7 @@ type Addons struct { CCM *CCM `json:"ccm,omitempty"` // +optional - CSIProviders *CSIProviders `json:"csi,omitempty"` + CSIProviders *CSI `json:"csi,omitempty"` } func (Addons) VariableSchema() clusterv1.VariableSchema { @@ -35,7 +49,7 @@ func (Addons) VariableSchema() clusterv1.VariableSchema { "cni": CNI{}.VariableSchema().OpenAPIV3Schema, "nfd": NFD{}.VariableSchema().OpenAPIV3Schema, "clusterAutoscaler": ClusterAutoscaler{}.VariableSchema().OpenAPIV3Schema, - "csi": CSIProviders{}.VariableSchema().OpenAPIV3Schema, + "csi": CSI{}.VariableSchema().OpenAPIV3Schema, "ccm": CCM{}.VariableSchema().OpenAPIV3Schema, }, }, @@ -44,11 +58,6 @@ func (Addons) VariableSchema() clusterv1.VariableSchema { type AddonStrategy string -const ( - AddonStrategyClusterResourceSet AddonStrategy = "ClusterResourceSet" - AddonStrategyHelmAddon AddonStrategy = "HelmAddon" -) - // CNI required for providing CNI configuration. type CNI struct { // +optional @@ -134,40 +143,168 @@ func (ClusterAutoscaler) VariableSchema() clusterv1.VariableSchema { } } -type CSIProviders struct { +type DefaultStorage struct { + ProviderName string `json:"providerName"` + StorageClassConfigName string `json:"storageClassConfigName"` +} + +type CSI struct { // +optional Providers []CSIProvider `json:"providers,omitempty"` // +optional - DefaultClassName string `json:"defaultClassName,omitempty"` + DefaultStorage *DefaultStorage `json:"defaultStorage,omitempty"` } type CSIProvider struct { - Name string `json:"name,omitempty"` + Name string `json:"name"` + + // +optional + StorageClassConfig []StorageClassConfig `json:"storageClassConfig,omitempty"` + + Strategy AddonStrategy `json:"strategy"` + + // +optional + Credentials *corev1.SecretReference `json:"credentials,omitempty"` } -func (CSIProviders) VariableSchema() clusterv1.VariableSchema { - supportedCSIProviders := []string{CSIProviderAWSEBS} +type StorageClassConfig struct { + Name string `json:"name"` + + // +optional + Parameters map[string]string `json:"parameters,omitempty"` + + // +optional + ReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"reclaimPolicy,omitempty"` + + // +optional + VolumeBindingMode storagev1.VolumeBindingMode `json:"volumeBindingMode,omitempty"` + + // +optional + AllowExpansion bool `json:"allowExpansion,omitempty"` +} + +func (StorageClassConfig) VariableSchema() clusterv1.VariableSchema { + supportedReclaimPolicies := []string{ + string(VolumeReclaimRecycle), + string(VolumeReclaimDelete), + string(VolumeReclaimRetain), + } + supportedBindingModes := []string{ + string(VolumeBindingImmediate), + string(VolumeBindingWaitForFirstConsumer), + } return clusterv1.VariableSchema{ OpenAPIV3Schema: clusterv1.JSONSchemaProps{ - Type: "object", + Type: "object", + Required: []string{"name"}, Properties: map[string]clusterv1.JSONSchemaProps{ - "providers": { - Type: "array", - Items: &clusterv1.JSONSchemaProps{ - Type: "object", - Properties: map[string]clusterv1.JSONSchemaProps{ - "name": { - Type: "string", - Enum: variables.MustMarshalValuesToEnumJSON( - supportedCSIProviders...), - }, + "name": { + Type: "string", + Description: "Name of storage class config.", + }, + "parameters": { + Type: "object", + Description: "Parameters passed into the storage class object.", + AdditionalProperties: &clusterv1.JSONSchemaProps{ + Type: "string", + }, + }, + "reclaimPolicy": { + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON(supportedReclaimPolicies...), + Default: variables.MustMarshal(VolumeReclaimDelete), + }, + "volumeBindingMode": { + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON(supportedBindingModes...), + Default: variables.MustMarshal(VolumeBindingWaitForFirstConsumer), + }, + "allowExpansion": { + Type: "boolean", + Default: variables.MustMarshal(false), + Description: "If the storage class should allow volume expanding", + }, + }, + }, + } +} + +func (CSIProvider) VariableSchema() clusterv1.VariableSchema { + supportedCSIProviders := []string{CSIProviderAWSEBS, CSIProviderNutanix} + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "object", + Required: []string{"name", "strategy"}, + Properties: map[string]clusterv1.JSONSchemaProps{ + "name": { + Description: "Name of the CSI Provider", + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON( + supportedCSIProviders...), + }, + "strategy": { + Description: "Addon strategy used to deploy the CSI provider to the workload cluster", + Type: "string", + Enum: variables.MustMarshalValuesToEnumJSON( + AddonStrategyClusterResourceSet, + AddonStrategyHelmAddon, + ), + }, + "credentials": { + Type: "object", + Description: "The reference to any secret used by the CSI Provider.", + Properties: map[string]clusterv1.JSONSchemaProps{ + "name": { + Type: "string", + }, + "namespace": { + Type: "string", }, }, }, - "defaultClassName": { - Type: "string", - Enum: variables.MustMarshalValuesToEnumJSON(supportedCSIProviders...), + "storageClassConfig": { + Type: "array", + Items: ptr.To(StorageClassConfig{}.VariableSchema().OpenAPIV3Schema), + }, + }, + }, + } +} + +func (DefaultStorage) VariableSchema() clusterv1.VariableSchema { + supportedCSIProviders := []string{CSIProviderAWSEBS, CSIProviderNutanix} + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "object", + Description: "A tuple of provider name and storage class ", + Required: []string{"providerName", "storageClassConfigName"}, + Properties: map[string]clusterv1.JSONSchemaProps{ + "providerName": { + Type: "string", + Description: "Name of the CSI Provider for the default storage class", + Enum: variables.MustMarshalValuesToEnumJSON( + supportedCSIProviders..., + ), + }, + "storageClassConfigName": { + Type: "string", + Description: "Name of storage class config in any of the provider objects", + }, + }, + }, + } +} + +func (CSI) VariableSchema() clusterv1.VariableSchema { + return clusterv1.VariableSchema{ + OpenAPIV3Schema: clusterv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]clusterv1.JSONSchemaProps{ + "providers": { + Type: "array", + Items: ptr.To(CSIProvider{}.VariableSchema().OpenAPIV3Schema), }, + "defaultStorage": DefaultStorage{}.VariableSchema().OpenAPIV3Schema, }, }, } diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go index 7bbbef0d8..83242d1b9 100644 --- a/api/v1alpha1/clusterconfig_types.go +++ b/api/v1alpha1/clusterconfig_types.go @@ -14,11 +14,16 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/openapi/patterns" ) +type StorageProvisioner string + const ( - CNIProviderCalico = "Calico" - CNIProviderCilium = "Cilium" + CNIProviderCalico = "Calico" + CNIProviderCilium = "Cilium" + AWSEBSProvisioner StorageProvisioner = "ebs.csi.aws.com" + NutanixProvisioner StorageProvisioner = "csi.nutanix.com" - CSIProviderAWSEBS = "aws-ebs" + CSIProviderAWSEBS = "aws-ebs" + CSIProviderNutanix = "nutanix" CCMProviderAWS = "aws" ) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 01692d804..4f33138e6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -9,7 +9,7 @@ package v1alpha1 import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -206,7 +206,7 @@ func (in *Addons) DeepCopyInto(out *Addons) { } if in.CSIProviders != nil { in, out := &in.CSIProviders, &out.CSIProviders - *out = new(CSIProviders) + *out = new(CSI) (*in).DeepCopyInto(*out) } } @@ -252,36 +252,55 @@ func (in *CNI) DeepCopy() *CNI { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CSIProvider) DeepCopyInto(out *CSIProvider) { +func (in *CSI) DeepCopyInto(out *CSI) { *out = *in + if in.Providers != nil { + in, out := &in.Providers, &out.Providers + *out = make([]CSIProvider, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.DefaultStorage != nil { + in, out := &in.DefaultStorage, &out.DefaultStorage + *out = new(DefaultStorage) + **out = **in + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIProvider. -func (in *CSIProvider) DeepCopy() *CSIProvider { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSI. +func (in *CSI) DeepCopy() *CSI { if in == nil { return nil } - out := new(CSIProvider) + out := new(CSI) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CSIProviders) DeepCopyInto(out *CSIProviders) { +func (in *CSIProvider) DeepCopyInto(out *CSIProvider) { *out = *in - if in.Providers != nil { - in, out := &in.Providers, &out.Providers - *out = make([]CSIProvider, len(*in)) - copy(*out, *in) + if in.StorageClassConfig != nil { + in, out := &in.StorageClassConfig, &out.StorageClassConfig + *out = make([]StorageClassConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(v1.SecretReference) + **out = **in } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIProviders. -func (in *CSIProviders) DeepCopy() *CSIProviders { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIProvider. +func (in *CSIProvider) DeepCopy() *CSIProvider { if in == nil { return nil } - out := new(CSIProviders) + out := new(CSIProvider) in.DeepCopyInto(out) return out } @@ -358,6 +377,21 @@ func (in *ClusterConfigSpec) DeepCopy() *ClusterConfigSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DefaultStorage) DeepCopyInto(out *DefaultStorage) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DefaultStorage. +func (in *DefaultStorage) DeepCopy() *DefaultStorage { + if in == nil { + return nil + } + out := new(DefaultStorage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DockerNodeSpec) DeepCopyInto(out *DockerNodeSpec) { *out = *in @@ -730,6 +764,28 @@ func (in *SecurityGroup) DeepCopy() *SecurityGroup { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageClassConfig) DeepCopyInto(out *StorageClassConfig) { + *out = *in + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageClassConfig. +func (in *StorageClassConfig) DeepCopy() *StorageClassConfig { + if in == nil { + return nil + } + out := new(StorageClassConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SubnetSpec) DeepCopyInto(out *SubnetSpec) { *out = *in diff --git a/charts/cluster-api-runtime-extensions-nutanix/README.md b/charts/cluster-api-runtime-extensions-nutanix/README.md index a90733b90..3049ea2a1 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/README.md +++ b/charts/cluster-api-runtime-extensions-nutanix/README.md @@ -49,6 +49,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix | hooks.cni.cilium.crsStrategy.defaultCiliumConfigMap.name | string | `"cilium"` | | | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cilium-cni-helm-values-template"` | | +| hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | +| hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-nutanix-csi-helm-values-template"` | | | hooks.nfd.crsStrategy.defaultInstallationConfigMap.name | string | `"node-feature-discovery"` | | | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-nfd-helm-values-template"` | | diff --git a/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml b/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml index 58e511c35..9bb5e3ef8 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/nutanix-cluster-class.yaml @@ -154,9 +154,9 @@ spec: - name: vip_arp value: "true" - name: address - value: "${CONTROL_PLANE_ENDPOINT_IP}" + value: "control_plane_endpoint_ip" - name: port - value: "${CONTROL_PLANE_ENDPOINT_PORT=6443}" + value: "control_plane_endpoint_port" - name: vip_cidr value: "32" - name: cp_enable diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml index 09e7ce18e..5529a361f 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/aws-ebs/manifests/aws-ebs-csi-configmap.yaml @@ -8,16 +8,6 @@ apiVersion: v1 data: aws-ebs-csi.yaml: | - apiVersion: storage.k8s.io/v1 - kind: StorageClass - metadata: - name: ebs-sc - parameters: - csi.storage.k8s.io/fstype: ext4 - type: gp3 - provisioner: ebs.csi.aws.com - volumeBindingMode: WaitForFirstConsumer - --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/helm-addon-installation.yaml new file mode 100644 index 000000000..b62635034 --- /dev/null +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/helm-addon-installation.yaml @@ -0,0 +1,12 @@ +# Copyright 2023 D2iQ, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: '{{ .Values.hooks.csi.nutanix.helmAddonStrategy.defaultValueTemplateConfigMap.name }}' +data: + values.yaml: |- + createSecret: false +{{- end -}} diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml new file mode 100644 index 000000000..03c2539b2 --- /dev/null +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml @@ -0,0 +1,543 @@ +# Copyright 2023 D2iQ, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +#================================================================= +# DO NOT EDIT THIS FILE +# IT HAS BEEN GENERATED BY /hack/addons/update-nutanix-csi.sh +#================================================================= +apiVersion: v1 +data: + nutanix-csi.yaml: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: nutanix-csi-controller + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: nutanix-csi-node + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: nutanix-csi-controller-role + namespace: nutanix-system + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - update + - patch + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - watch + - update + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots/status + verbs: + - update + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - delete + - update + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: nutanix-csi-node-role + namespace: nutanix-system + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - update + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: nutanix-csi-controller-binding + namespace: nutanix-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nutanix-csi-controller-role + subjects: + - kind: ServiceAccount + name: nutanix-csi-controller + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: nutanix-csi-node-binding + namespace: nutanix-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nutanix-csi-node-role + subjects: + - kind: ServiceAccount + name: nutanix-csi-node + namespace: kube-system + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app: nutanix-csi-metrics + name: nutanix-csi-metrics + namespace: kube-system + spec: + ports: + - name: provisioner + port: 9809 + protocol: TCP + targetPort: 9809 + - name: resizer + port: 9810 + protocol: TCP + targetPort: 9810 + selector: + app: nutanix-csi-controller + type: ClusterIP + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nutanix-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: nutanix-csi-controller + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: nutanix-csi-controller + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app: nutanix-csi-controller + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - --csi-address=$(ADDRESS) + - --timeout=60s + - --worker-threads=16 + - --extra-create-metadata=true + - --default-fstype=ext4 + - --http-endpoint=:9809 + - --v=2 + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.2 + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --v=2 + - --csi-address=$(ADDRESS) + - --timeout=60s + - --leader-election=true + - --handle-volume-inuse-error=false + - --http-endpoint=:9810 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.9.2 + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + requests: + cpu: 5m + memory: 30Mi + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --leader-election=true + - --logtostderr=true + - --timeout=300s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v3.0.3 + imagePullPolicy: IfNotPresent + name: csi-snapshotter + resources: + requests: + cpu: 5m + memory: 30Mi + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --endpoint=$(CSI_ENDPOINT) + - --nodeid=$(NODE_ID) + - --drivername=csi.nutanix.com + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: quay.io/karbon/ntnx-csi:v2.6.6 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: http-endpoint + initialDelaySeconds: 10 + periodSeconds: 2 + timeoutSeconds: 3 + name: nutanix-csi-plugin + ports: + - containerPort: 9807 + name: http-endpoint + protocol: TCP + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: true + privileged: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - mountPath: /host + name: root-dir + - args: + - --csi-address=/csi/csi.sock + - --http-endpoint=:9807 + image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + requests: + cpu: 5m + memory: 20Mi + volumeMounts: + - mountPath: /csi + name: socket-dir + hostNetwork: true + priorityClassName: system-cluster-critical + serviceAccount: nutanix-csi-controller + volumes: + - emptyDir: {} + name: socket-dir + - hostPath: + path: / + type: Directory + name: root-dir + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: nutanix-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: nutanix-csi-node + template: + metadata: + labels: + app: nutanix-csi-node + spec: + containers: + - args: + - --v=2 + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.nutanix.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.1 + imagePullPolicy: IfNotPresent + name: driver-registrar + resources: + requests: + cpu: 100m + memory: 20Mi + volumeMounts: + - mountPath: /csi/ + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --endpoint=$(CSI_ENDPOINT) + - --nodeid=$(NODE_ID) + - --drivername=csi.nutanix.com + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + image: quay.io/karbon/ntnx-csi:v2.6.6 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: http-endpoint + initialDelaySeconds: 10 + periodSeconds: 2 + timeoutSeconds: 3 + name: nutanix-csi-node + ports: + - containerPort: 9808 + name: http-endpoint + protocol: TCP + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: true + privileged: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /etc/iscsi + name: iscsi-dir + - mountPath: /host + mountPropagation: Bidirectional + name: root-dir + - args: + - --csi-address=/csi/csi.sock + - --http-endpoint=:9808 + image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + requests: + cpu: 5m + memory: 20Mi + volumeMounts: + - mountPath: /csi + name: plugin-dir + hostNetwork: true + priorityClassName: system-cluster-critical + serviceAccount: nutanix-csi-node + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/csi.nutanix.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: /dev + name: device-dir + - hostPath: + path: /etc/iscsi + type: Directory + name: iscsi-dir + - hostPath: + path: / + type: Directory + name: root-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + name: csi.nutanix.com + spec: + attachRequired: false + podInfoOnMount: true +kind: ConfigMap +metadata: + creationTimestamp: null + name: nutanix-csi diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml index 9f909c5e3..e3cd174f7 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/role.yaml @@ -69,3 +69,13 @@ rules: - get - list - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - create + - get + - list + - patch + - update diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.yaml b/charts/cluster-api-runtime-extensions-nutanix/values.yaml index 81576e8ba..9a2a4d7ba 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/values.yaml @@ -35,6 +35,12 @@ hooks: defaultValueTemplateConfigMap: create: true name: default-cilium-cni-helm-values-template + csi: + nutanix: + helmAddonStrategy: + defaultValueTemplateConfigMap: + create: true + name: default-nutanix-csi-helm-values-template nfd: crsStrategy: defaultInstallationConfigMap: diff --git a/examples/capi-quick-start/aws-cluster-calico-crs.yaml b/examples/capi-quick-start/aws-cluster-calico-crs.yaml index 847222c10..790b4a640 100644 --- a/examples/capi-quick-start/aws-cluster-calico-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-crs.yaml @@ -28,8 +28,14 @@ spec: provider: Calico strategy: ClusterResourceSet csi: + defaultStorage: + providerName: aws-ebs + storageClassConfigName: aws-ebs providers: - name: aws-ebs + storageClassConfig: + - name: aws-ebs + strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet aws: diff --git a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml index ac26e8051..52bf9648a 100644 --- a/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml @@ -28,8 +28,14 @@ spec: provider: Calico strategy: HelmAddon csi: + defaultStorage: + providerName: aws-ebs + storageClassConfigName: aws-ebs providers: - name: aws-ebs + storageClassConfig: + - name: aws-ebs + strategy: ClusterResourceSet nfd: strategy: HelmAddon aws: diff --git a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml index 2fd77b867..e9541e789 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-crs.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-crs.yaml @@ -28,8 +28,14 @@ spec: provider: Cilium strategy: ClusterResourceSet csi: + defaultStorage: + providerName: aws-ebs + storageClassConfigName: aws-ebs providers: - name: aws-ebs + storageClassConfig: + - name: aws-ebs + strategy: ClusterResourceSet nfd: strategy: ClusterResourceSet aws: diff --git a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml index 4bd929d2f..e803994e8 100644 --- a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml +++ b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml @@ -28,8 +28,14 @@ spec: provider: Cilium strategy: HelmAddon csi: + defaultStorage: + providerName: aws-ebs + storageClassConfigName: aws-ebs providers: - name: aws-ebs + storageClassConfig: + - name: aws-ebs + strategy: ClusterResourceSet nfd: strategy: HelmAddon aws: diff --git a/go.mod b/go.mod index c1df13beb..c55396282 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api v0.0.0-00010101000000-000000000000 github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common v0.0.0-00010101000000-000000000000 github.com/go-logr/logr v1.4.1 + github.com/google/go-cmp v0.6.0 github.com/onsi/ginkgo/v2 v2.16.0 github.com/onsi/gomega v1.31.1 github.com/spf13/pflag v1.0.5 @@ -71,7 +72,6 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/cel-go v0.17.7 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-github/v53 v53.2.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect diff --git a/hack/addons/kustomize/aws-ebs-csi/helm-values.yaml b/hack/addons/kustomize/aws-ebs-csi/helm-values.yaml index d1cf5632d..33bb95823 100644 --- a/hack/addons/kustomize/aws-ebs-csi/helm-values.yaml +++ b/hack/addons/kustomize/aws-ebs-csi/helm-values.yaml @@ -27,10 +27,3 @@ node: sidecars: snapshotter: forceEnable: true -storageClasses: -- metadata: - name: ebs-sc - volumeBindingMode: WaitForFirstConsumer - parameters: - csi.storage.k8s.io/fstype: ext4 - type: gp3 diff --git a/hack/addons/kustomize/nutanix-csi/helm-values.yaml b/hack/addons/kustomize/nutanix-csi/helm-values.yaml new file mode 100644 index 000000000..eebd55d74 --- /dev/null +++ b/hack/addons/kustomize/nutanix-csi/helm-values.yaml @@ -0,0 +1,4 @@ +# Copyright 2023 D2iQ, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +createSecret: false diff --git a/hack/addons/kustomize/nutanix-csi/kustomization.yaml.tmpl b/hack/addons/kustomize/nutanix-csi/kustomization.yaml.tmpl new file mode 100644 index 000000000..bc838e438 --- /dev/null +++ b/hack/addons/kustomize/nutanix-csi/kustomization.yaml.tmpl @@ -0,0 +1,20 @@ +# Copyright 2023 D2iQ, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +metadata: + name: nutanix-csi-kustomize + +namespace: kube-system + +helmCharts: +- name: nutanix-csi-storage + repo: https://nutanix.github.io/helm/ + releaseName: nutanix-csi-storage + version: ${NUTANIX_CSI_CHART_VERSION} + valuesFile: helm-values.yaml + includeCRDs: true + skipTests: true + namespace: nutanix-system diff --git a/hack/addons/update-nutanix-csi.sh b/hack/addons/update-nutanix-csi.sh new file mode 100755 index 000000000..e18414a86 --- /dev/null +++ b/hack/addons/update-nutanix-csi.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR + +# shellcheck source=hack/common.sh +source "${SCRIPT_DIR}/../common.sh" + +if [ -z "${NUTANIX_CSI_CHART_VERSION:-}" ]; then + echo "Missing environment variable: NUTANIX_CSI_CHART_VERSION" + exit 1 +fi + +ASSETS_DIR="$(mktemp -d -p "${TMPDIR:-/tmp}")" +readonly ASSETS_DIR +trap_add "rm -rf ${ASSETS_DIR}" EXIT + +readonly FILE_NAME="nutanix-csi.yaml" + +readonly KUSTOMIZE_BASE_DIR="${SCRIPT_DIR}/kustomize/nutanix-csi" +mkdir -p "${ASSETS_DIR}/nutanix-csi" +envsubst -no-unset <"${KUSTOMIZE_BASE_DIR}/kustomization.yaml.tmpl" >"${ASSETS_DIR}/nutanix-csi/kustomization.yaml" +cp -r "${KUSTOMIZE_BASE_DIR}"/*.yaml "${ASSETS_DIR}/nutanix-csi/" + +kustomize build --enable-helm "${ASSETS_DIR}/nutanix-csi/" >"${ASSETS_DIR}/${FILE_NAME}" + +kubectl create configmap nutanix-csi --dry-run=client --output yaml \ + --from-file "${ASSETS_DIR}/${FILE_NAME}" \ + >"${ASSETS_DIR}/nutanix-csi-configmap.yaml" + +# add warning not to edit file directly +cat <"${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/csi/nutanix/manifests/nutanix-csi-configmap.yaml" +$(cat "${GIT_REPO_ROOT}/hack/license-header.yaml.txt") + +#================================================================= +# DO NOT EDIT THIS FILE +# IT HAS BEEN GENERATED BY /hack/addons/update-nutanix-csi.sh +#================================================================= +$(cat "${ASSETS_DIR}/nutanix-csi-configmap.yaml") +EOF diff --git a/hack/examples/patches/aws/csi.yaml b/hack/examples/patches/aws/csi.yaml index 1d395d192..0bbe4c5bd 100644 --- a/hack/examples/patches/aws/csi.yaml +++ b/hack/examples/patches/aws/csi.yaml @@ -4,5 +4,11 @@ - op: "add" path: "/spec/topology/variables/0/value/addons/csi" value: + defaultStorage: + providerName: aws-ebs + storageClassConfigName: aws-ebs providers: - name: aws-ebs + storageClassConfig: + - name: aws-ebs + strategy: ClusterResourceSet diff --git a/make/addons.mk b/make/addons.mk index 85ad54575..c09890922 100644 --- a/make/addons.mk +++ b/make/addons.mk @@ -7,6 +7,7 @@ export NODE_FEATURE_DISCOVERY_VERSION := $(shell goprintconst -file pkg/handlers export CLUSTER_AUTOSCALER_VERSION := 9.35.0 export AWS_CSI_SNAPSHOT_CONTROLLER_VERSION := v6.3.3 export AWS_EBS_CSI_CHART_VERSION := v2.28.1 +export NUTANIX_CSI_CHART_VERSION := v2.6.6 # a map of AWS CCM versions export AWS_CCM_VERSION_127 := v1.27.1 export AWS_CCM_CHART_VERSION_127 := 0.0.8 @@ -39,3 +40,7 @@ update-addon.aws-ebs-csi: ; $(info $(M) updating aws ebs csi manifests) .PHONY: update-addon.aws-ccm.% update-addon.aws-ccm.%: ; $(info $(M) updating aws ccm $* manifests) ./hack/addons/update-aws-ccm.sh $(AWS_CCM_VERSION_$*) $(AWS_CCM_CHART_VERSION_$*) + +.PHONY: update-addon.nutanix-csi +update-addon.nutanix-csi: ; $(info $(M) updating nutanix csi manifests) + ./hack/addons/update-nutanix-csi.sh diff --git a/pkg/handlers/generic/lifecycle/ccm/handler.go b/pkg/handlers/generic/lifecycle/ccm/handler.go index 43360aa1b..04dd32f85 100644 --- a/pkg/handlers/generic/lifecycle/ccm/handler.go +++ b/pkg/handlers/generic/lifecycle/ccm/handler.go @@ -114,7 +114,7 @@ func (c *CCMHandler) AfterControlPlaneInitialized( ) return } - err = lifecycleutils.EnsureCRSForClusterFromConfigMaps(ctx, cm.Name, c.client, &req.Cluster, cm) + err = lifecycleutils.EnsureCRSForClusterFromObjects(ctx, cm.Name, c.client, &req.Cluster, cm) if err != nil { log.Error( err, diff --git a/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_crs.go b/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_crs.go index 981d4ec01..432efba62 100644 --- a/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_crs.go +++ b/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_crs.go @@ -121,7 +121,7 @@ func (s crsStrategy) apply( ) } - if err = utils.EnsureCRSForClusterFromConfigMaps(ctx, cm.Name, s.client, targetCluster, cm); err != nil { + if err = utils.EnsureCRSForClusterFromObjects(ctx, cm.Name, s.client, targetCluster, cm); err != nil { return fmt.Errorf( "failed to apply cluster-autoscaler installation ClusterResourceSet: %w", err, diff --git a/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go b/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go index 007d19ab4..a22e69d2f 100644 --- a/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go +++ b/pkg/handlers/generic/lifecycle/clusterautoscaler/strategy_helmaddon.go @@ -9,7 +9,6 @@ import ( "github.com/go-logr/logr" "github.com/spf13/pflag" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" @@ -18,6 +17,7 @@ import ( caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" ) const ( @@ -53,7 +53,11 @@ func (s helmAddonStrategy) apply( log logr.Logger, ) error { log.Info("Retrieving cluster-autoscaler installation values template for cluster") - valuesTemplateConfigMap, err := s.retrieveValuesTemplateConfigMap(ctx, defaultsNamespace) + valuesTemplateConfigMap, err := utils.RetrieveValuesTemplateConfigMap( + ctx, + s.client, + s.config.defaultValuesTemplateConfigMapName, + defaultsNamespace) if err != nil { return fmt.Errorf( "failed to retrieve cluster-autoscaler installation values template ConfigMap for cluster: %w", @@ -108,28 +112,3 @@ func (s helmAddonStrategy) apply( return nil } - -func (s helmAddonStrategy) retrieveValuesTemplateConfigMap( - ctx context.Context, - defaultsNamespace string, -) (*corev1.ConfigMap, error) { - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: defaultsNamespace, - Name: s.config.defaultValuesTemplateConfigMapName, - }, - } - configMapObjName := ctrlclient.ObjectKeyFromObject( - configMap, - ) - err := s.client.Get(ctx, configMapObjName, configMap) - if err != nil { - return nil, fmt.Errorf( - "failed to retrieve installation values template ConfigMap %q: %w", - configMapObjName, - err, - ) - } - - return configMap, nil -} diff --git a/pkg/handlers/generic/lifecycle/cni/calico/strategy_crs.go b/pkg/handlers/generic/lifecycle/cni/calico/strategy_crs.go index a08a686e2..27a366137 100644 --- a/pkg/handlers/generic/lifecycle/cni/calico/strategy_crs.go +++ b/pkg/handlers/generic/lifecycle/cni/calico/strategy_crs.go @@ -150,7 +150,7 @@ func (s crsStrategy) ensureCNICRSForCluster( ) } - if err := utils.EnsureCRSForClusterFromConfigMaps(ctx, cm.Name, s.client, cluster, tigeraConfigMap, cm); err != nil { + if err := utils.EnsureCRSForClusterFromObjects(ctx, cm.Name, s.client, cluster, tigeraConfigMap, cm); err != nil { return fmt.Errorf( "failed to apply Calico CNI installation ClusterResourceSet: %w", err, diff --git a/pkg/handlers/generic/lifecycle/cni/cilium/strategy_crs.go b/pkg/handlers/generic/lifecycle/cni/cilium/strategy_crs.go index 28f07a77c..6a9b805ac 100644 --- a/pkg/handlers/generic/lifecycle/cni/cilium/strategy_crs.go +++ b/pkg/handlers/generic/lifecycle/cni/cilium/strategy_crs.go @@ -87,7 +87,7 @@ func (s crsStrategy) apply( ) } - if err := utils.EnsureCRSForClusterFromConfigMaps(ctx, cm.Name, s.client, cluster, cm); err != nil { + if err := utils.EnsureCRSForClusterFromObjects(ctx, cm.Name, s.client, cluster, cm); err != nil { return fmt.Errorf( "failed to apply Cilium CNI installation ClusterResourceSet: %w", err, diff --git a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go index 0d75a2212..90072b3a2 100644 --- a/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/aws-ebs/handler.go @@ -10,10 +10,14 @@ import ( "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" ) @@ -46,10 +50,71 @@ func New( } } -func (a *AWSEBS) EnsureCSIConfigMapForCluster( +func (a *AWSEBS) Apply( ctx context.Context, + provider v1alpha1.CSIProvider, + defaultStorageConfig *v1alpha1.DefaultStorage, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, +) error { + strategy := provider.Strategy + switch strategy { + case v1alpha1.AddonStrategyClusterResourceSet: + err := a.handleCRSApply(ctx, req) + if err != nil { + return err + } + case v1alpha1.AddonStrategyHelmAddon: + default: + return fmt.Errorf("stategy %s not implemented", strategy) + } + return a.createStorageClasses( + ctx, + provider.StorageClassConfig, + &req.Cluster, + defaultStorageConfig, + ) +} + +func (a *AWSEBS) createStorageClasses(ctx context.Context, + configs []v1alpha1.StorageClassConfig, cluster *clusterv1.Cluster, -) (*corev1.ConfigMap, error) { + defaultStorageConfig *v1alpha1.DefaultStorage, +) error { + allStorageClasses := make([]runtime.Object, 0, len(configs)) + for _, c := range configs { + setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && + v1alpha1.CSIProviderAWSEBS == defaultStorageConfig.ProviderName + allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass( + c, + a.config.GlobalOptions.DefaultsNamespace(), + v1alpha1.AWSEBSProvisioner, + setAsDefault, + )) + } + cm, err := lifecycleutils.CreateConfigMapForCRS( + fmt.Sprintf("aws-storageclass-cm-%s", cluster.Name), + a.config.DefaultsNamespace(), + allStorageClasses..., + ) + if err != nil { + return err + } + err = client.ServerSideApply(ctx, a.client, cm) + if err != nil { + return err + } + return lifecycleutils.EnsureCRSForClusterFromObjects( + ctx, + "aws-storageclass-crs", + a.client, + cluster, + cm, + ) +} + +func (a *AWSEBS) handleCRSApply(ctx context.Context, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, +) error { awsEBSCSIConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: a.config.DefaultsNamespace(), @@ -61,22 +126,31 @@ func (a *AWSEBS) EnsureCSIConfigMapForCluster( ) err := a.client.Get(ctx, defaultAWSEBSCSIConfigMapObjName, awsEBSCSIConfigMap) if err != nil { - return nil, fmt.Errorf( + return fmt.Errorf( "failed to retrieve default AWS EBS CSI manifests ConfigMap %q: %w", defaultAWSEBSCSIConfigMapObjName, err, ) } - - awsEBSConfigMap := generateAWSEBSCSIConfigMap(awsEBSCSIConfigMap, cluster) - if err := client.ServerSideApply(ctx, a.client, awsEBSConfigMap); err != nil { - return nil, fmt.Errorf( + cluster := req.Cluster + cm := generateAWSEBSCSIConfigMap(awsEBSCSIConfigMap, &cluster) + if err := client.ServerSideApply(ctx, a.client, cm); err != nil { + return fmt.Errorf( "failed to apply AWS EBS CSI manifests ConfigMap: %w", err, ) } - - return awsEBSConfigMap, nil + err = lifecycleutils.EnsureCRSForClusterFromObjects( + ctx, + cm.Name, + a.client, + &req.Cluster, + cm, + ) + if err != nil { + return err + } + return nil } func generateAWSEBSCSIConfigMap( diff --git a/pkg/handlers/generic/lifecycle/csi/doc.go b/pkg/handlers/generic/lifecycle/csi/doc.go index 9a2cc0ff4..41a048f24 100644 --- a/pkg/handlers/generic/lifecycle/csi/doc.go +++ b/pkg/handlers/generic/lifecycle/csi/doc.go @@ -9,4 +9,5 @@ // // +kubebuilder:rbac:groups=addons.cluster.x-k8s.io,resources=clusterresourcesets,verbs=watch;list;get;create;patch;update;delete // +kubebuilder:rbac:groups="",resources=configmaps,verbs=watch;list;get;create;patch;update;delete +// +kubebuilder:rbac:groups="storage.k8s.io",resources=storageclasses,verbs=list;get;create;patch;update package csi diff --git a/pkg/handlers/generic/lifecycle/csi/handler.go b/pkg/handlers/generic/lifecycle/csi/handler.go index c084c05f1..fd07767c6 100644 --- a/pkg/handlers/generic/lifecycle/csi/handler.go +++ b/pkg/handlers/generic/lifecycle/csi/handler.go @@ -7,11 +7,7 @@ import ( "context" "fmt" - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" - utilyaml "sigs.k8s.io/cluster-api/util/yaml" ctrl "sigs.k8s.io/controller-runtime" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -20,23 +16,19 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/lifecycle" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig" - lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" ) const ( variableRootName = "csi" - kindStorageClass = "StorageClass" -) - -var ( - defualtStorageClassKey = "storageclass.kubernetes.io/is-default-class" - defaultStorageClassMap = map[string]string{ - defualtStorageClassKey: "true", - } ) type CSIProvider interface { - EnsureCSIConfigMapForCluster(context.Context, *clusterv1.Cluster) (*corev1.ConfigMap, error) + Apply( + context.Context, + v1alpha1.CSIProvider, + *v1alpha1.DefaultStorage, + *runtimehooksv1.AfterControlPlaneInitializedRequest, + ) error } type CSIHandler struct { @@ -80,7 +72,7 @@ func (c *CSIHandler) AfterControlPlaneInitialized( ) varMap := variables.ClusterVariablesToVariablesMap(req.Cluster.Spec.Topology.Variables) resp.SetStatus(runtimehooksv1.ResponseStatusSuccess) - csiProviders, found, err := variables.Get[v1alpha1.CSIProviders]( + csiProviders, found, err := variables.Get[v1alpha1.CSI]( varMap, c.variableName, c.variablePath...) @@ -106,84 +98,42 @@ func (c *CSIHandler) AfterControlPlaneInitialized( ) return } + if len(csiProviders.Providers) == 1 && + len(csiProviders.Providers[0].StorageClassConfig) == 1 && + csiProviders.DefaultStorage == nil { + csiProviders.DefaultStorage = &v1alpha1.DefaultStorage{ + ProviderName: csiProviders.Providers[0].Name, + StorageClassConfigName: csiProviders.Providers[0].StorageClassConfig[0].Name, + } + } + for _, provider := range csiProviders.Providers { handler, ok := c.ProviderHandler[provider.Name] if !ok { log.V(4).Info( fmt.Sprintf( - "Skipping CSI handler, for provider given in %q. Provider handler not given ", - provider, + "Skipping CSI handler, for provider given in %s. Provider handler not given.", + provider.Name, ), ) continue } - log.Info(fmt.Sprintf("Creating config map for csi provider %s", provider)) - cm, err := handler.EnsureCSIConfigMapForCluster(ctx, &req.Cluster) + log.Info(fmt.Sprintf("Creating csi provider %s", provider.Name)) + err = handler.Apply( + ctx, + provider, + csiProviders.DefaultStorage, + req, + ) if err != nil { log.Error( err, fmt.Sprintf( - "failed to ensure %s csi driver installation manifests ConfigMap", + "failed to create %s csi driver object.", provider.Name, ), ) resp.SetStatus(runtimehooksv1.ResponseStatusFailure) } - if cm != nil { - if provider.Name == csiProviders.DefaultClassName { - log.Info("Setting default storage class ", provider, csiProviders.DefaultClassName) - err = setDefaultStorageClass(log, cm) - if err != nil { - log.Error(err, "failed to set default storage class") - resp.SetStatus(runtimehooksv1.ResponseStatusFailure) - } - if err := c.client.Update(ctx, cm); err != nil { - log.Error(err, "failed to apply default storage class annotation to configmap") - resp.SetStatus(runtimehooksv1.ResponseStatusFailure) - } - } - err = lifecycleutils.EnsureCRSForClusterFromConfigMaps( - ctx, - cm.Name, - c.client, - &req.Cluster, - cm, - ) - if err != nil { - log.Error( - err, - fmt.Sprintf( - "failed to ensure %s csi driver installation manifests ConfigMap", - provider.Name, - ), - ) - resp.SetStatus(runtimehooksv1.ResponseStatusFailure) - } - } - } -} - -func setDefaultStorageClass( - log logr.Logger, - cm *corev1.ConfigMap, -) error { - for k, contents := range cm.Data { - objs, err := utilyaml.ToUnstructured([]byte(contents)) - if err != nil { - log.Error(err, "failed to parse yaml") - continue - } - for i := range objs { - obj := objs[i] - if obj.GetKind() == kindStorageClass { - obj.SetAnnotations(defaultStorageClassMap) - } - } - rawObjs, err := utilyaml.FromUnstructured(objs) - if err != nil { - return fmt.Errorf("failed to convert unstructured objects back to string %w", err) - } - cm.Data[k] = string(rawObjs) } - return nil } diff --git a/pkg/handlers/generic/lifecycle/csi/handler_test.go b/pkg/handlers/generic/lifecycle/csi/handler_test.go deleted file mode 100644 index de5015895..000000000 --- a/pkg/handlers/generic/lifecycle/csi/handler_test.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2023 D2iQ, Inc. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package csi - -import ( - "strings" - "testing" - - "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -var startAWSConfigMap = ` -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: ebs-sc -parameters: - csi.storage.k8s.io/fstype: ext4 - type: gp3 -provisioner: ebs.csi.aws.com -volumeBindingMode: WaitForFirstConsumer -` - -func Test_setDefaultStorageClass(t *testing.T) { - tests := []struct { - name string - startConfigMap *corev1.ConfigMap - key string - }{ - { - name: "aws config map", - startConfigMap: &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test1", - Namespace: "default", - }, - Data: map[string]string{ - "aws-ebs-csi.yaml": startAWSConfigMap, - }, - }, - key: "aws-ebs-csi.yaml", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := setDefaultStorageClass( - logr.Discard(), - tt.startConfigMap, - ) - if err != nil { - t.Fatal("failed to set default storage class", err) - } - if !strings.Contains(tt.startConfigMap.Data[tt.key], defualtStorageClassKey) { - t.Logf( - "expected %s to containe %s", - tt.startConfigMap.Data[tt.key], - defualtStorageClassKey, - ) - t.Fail() - } - }) - } -} diff --git a/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go new file mode 100644 index 000000000..142189b47 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/csi/nutanix-csi/handler.go @@ -0,0 +1,243 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package nutanix + +import ( + "context" + "fmt" + + "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + caaphv1 "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" + lifecycleutils "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/utils" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" +) + +const ( + defaultHelmRepositoryURL = "https://nutanix.github.io/helm/" + defaultStorageHelmChartVersion = "v2.6.6" + defaultStorageHelmChartName = "nutanix-csi-storage" + defaultStorageHelmReleaseNameTemplate = "nutanix-csi-storage-%s" + + defaultSnapshotHelmChartVersion = "v6.3.2" + defaultSnapshotHelmChartName = "nutanix-csi-snapshot" + defaultSnapshotHelmReleaseNameTemplate = "nutanix-csi-snapshot-%s" +) + +type NutanixCSIConfig struct { + *options.GlobalOptions + defaultValuesTemplateConfigMapName string +} + +func (n *NutanixCSIConfig) AddFlags(prefix string, flags *pflag.FlagSet) { + flags.StringVar( + &n.defaultValuesTemplateConfigMapName, + prefix+".default-values-template-configmap-name", + "default-nutanix-csi-helm-values-template", + "default values ConfigMap name", + ) +} + +type NutanixCSI struct { + client ctrlclient.Client + config *NutanixCSIConfig +} + +func New( + c ctrlclient.Client, + cfg *NutanixCSIConfig, +) *NutanixCSI { + return &NutanixCSI{ + client: c, + config: cfg, + } +} + +func (n *NutanixCSI) Apply( + ctx context.Context, + provider v1alpha1.CSIProvider, + defaultStorageConfig *v1alpha1.DefaultStorage, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, +) error { + strategy := provider.Strategy + switch strategy { + case v1alpha1.AddonStrategyHelmAddon: + err := n.handleHelmAddonApply(ctx, req) + if err != nil { + return err + } + case v1alpha1.AddonStrategyClusterResourceSet: + default: + return fmt.Errorf("stategy %s not implemented", strategy) + } + if provider.Credentials != nil { + sec := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: provider.Credentials.Name, + Name: provider.Credentials.Namespace, + }, + } + err := n.client.Get( + ctx, + ctrlclient.ObjectKeyFromObject(sec), + sec, + ) + if err != nil { + return err + } + err = lifecycleutils.EnsureCRSForClusterFromObjects( + ctx, + fmt.Sprintf("nutanix-csi-credentials-crs-%s", req.Cluster.Name), + n.client, + &req.Cluster, + sec, + ) + if err != nil { + return err + } + } + return n.createStorageClasses( + ctx, + provider.StorageClassConfig, + &req.Cluster, + defaultStorageConfig, + ) +} + +func (n *NutanixCSI) handleHelmAddonApply( + ctx context.Context, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, +) error { + valuesTemplateConfigMap, err := lifecycleutils.RetrieveValuesTemplateConfigMap(ctx, + n.client, + n.config.defaultValuesTemplateConfigMapName, + n.config.DefaultsNamespace()) + if err != nil { + return fmt.Errorf( + "failed to retrieve nutanix csi installation values template ConfigMap for cluster: %w", + err, + ) + } + values := valuesTemplateConfigMap.Data["values.yaml"] + + hcp := &caaphv1.HelmChartProxy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: caaphv1.GroupVersion.String(), + Kind: "HelmChartProxy", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.Cluster.Namespace, + Name: "nutanix-csi-" + req.Cluster.Name, + }, + Spec: caaphv1.HelmChartProxySpec{ + RepoURL: defaultHelmRepositoryURL, + ChartName: defaultStorageHelmChartName, + ClusterSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{clusterv1.ClusterNameLabel: req.Cluster.Name}, + }, + ReleaseNamespace: req.Cluster.Namespace, + ReleaseName: fmt.Sprintf(defaultStorageHelmReleaseNameTemplate, req.Cluster.Name), + Version: defaultStorageHelmChartVersion, + ValuesTemplate: values, + }, + } + + if err = controllerutil.SetOwnerReference(&req.Cluster, hcp, n.client.Scheme()); err != nil { + return fmt.Errorf( + "failed to set owner reference on nutanix-csi installation HelmChartProxy: %w", + err, + ) + } + + if err = client.ServerSideApply(ctx, n.client, hcp); err != nil { + return fmt.Errorf("failed to apply nutanix-csi installation HelmChartProxy: %w", err) + } + + snapshotChart := &caaphv1.HelmChartProxy{ + TypeMeta: metav1.TypeMeta{ + APIVersion: caaphv1.GroupVersion.String(), + Kind: "HelmChartProxy", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.Cluster.Namespace, + Name: "nutanix-csi-snapshot" + req.Cluster.Name, + }, + Spec: caaphv1.HelmChartProxySpec{ + RepoURL: defaultHelmRepositoryURL, + ChartName: defaultSnapshotHelmChartName, + ClusterSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{clusterv1.ClusterNameLabel: req.Cluster.Name}, + }, + ReleaseNamespace: req.Cluster.Namespace, + ReleaseName: fmt.Sprintf(defaultSnapshotHelmReleaseNameTemplate, req.Cluster.Name), + Version: defaultSnapshotHelmChartVersion, + }, + } + + if err = controllerutil.SetOwnerReference(&req.Cluster, snapshotChart, n.client.Scheme()); err != nil { + return fmt.Errorf( + "failed to set owner reference on nutanix-csi installation HelmChartProxy: %w", + err, + ) + } + + if err = client.ServerSideApply(ctx, n.client, snapshotChart); err != nil { + return fmt.Errorf( + "failed to apply nutanix-csi-snapshot installation HelmChartProxy: %w", + err, + ) + } + + return nil +} + +func (n *NutanixCSI) createStorageClasses(ctx context.Context, + configs []v1alpha1.StorageClassConfig, + cluster *clusterv1.Cluster, + defaultStorageConfig *v1alpha1.DefaultStorage, +) error { + allStorageClasses := make([]runtime.Object, 0, len(configs)) + for _, c := range configs { + setAsDefault := c.Name == defaultStorageConfig.StorageClassConfigName && + v1alpha1.CSIProviderNutanix == defaultStorageConfig.ProviderName + allStorageClasses = append(allStorageClasses, lifecycleutils.CreateStorageClass( + c, + n.config.GlobalOptions.DefaultsNamespace(), + v1alpha1.NutanixProvisioner, + setAsDefault, + )) + } + cm, err := lifecycleutils.CreateConfigMapForCRS( + fmt.Sprintf("nutanix-storageclass-cm-%s", cluster.Name), + n.config.DefaultsNamespace(), + allStorageClasses..., + ) + if err != nil { + return err + } + err = client.ServerSideApply(ctx, n.client, cm) + if err != nil { + return err + } + return lifecycleutils.EnsureCRSForClusterFromObjects( + ctx, + "nutanix-storageclass-crs", + n.client, + cluster, + cm, + ) +} diff --git a/pkg/handlers/generic/lifecycle/handlers.go b/pkg/handlers/generic/lifecycle/handlers.go index 366e78196..b3b0d22fb 100644 --- a/pkg/handlers/generic/lifecycle/handlers.go +++ b/pkg/handlers/generic/lifecycle/handlers.go @@ -16,6 +16,7 @@ import ( "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/cni/cilium" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi" awsebs "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/aws-ebs" + nutanixcsi "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/csi/nutanix-csi" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/nfd" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/servicelbgc" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" @@ -27,6 +28,7 @@ type Handlers struct { nfdConfig *nfd.Config clusterAutoscalerConfig *clusterautoscaler.Config ebsConfig *awsebs.AWSEBSConfig + nutnaixCSIConfig *nutanixcsi.NutanixCSIConfig awsccmConfig *awsccm.AWSCCMConfig } @@ -38,12 +40,14 @@ func New(globalOptions *options.GlobalOptions) *Handlers { clusterAutoscalerConfig: &clusterautoscaler.Config{GlobalOptions: globalOptions}, ebsConfig: &awsebs.AWSEBSConfig{GlobalOptions: globalOptions}, awsccmConfig: &awsccm.AWSCCMConfig{GlobalOptions: globalOptions}, + nutnaixCSIConfig: &nutanixcsi.NutanixCSIConfig{GlobalOptions: globalOptions}, } } func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named { csiHandlers := map[string]csi.CSIProvider{ - v1alpha1.CSIProviderAWSEBS: awsebs.New(mgr.GetClient(), h.ebsConfig), + v1alpha1.CSIProviderAWSEBS: awsebs.New(mgr.GetClient(), h.ebsConfig), + v1alpha1.CSIProviderNutanix: nutanixcsi.New(mgr.GetClient(), h.nutnaixCSIConfig), } ccmHandlers := map[string]ccm.CCMProvider{ v1alpha1.CCMProviderAWS: awsccm.New(mgr.GetClient(), h.awsccmConfig), @@ -67,4 +71,5 @@ func (h *Handlers) AddFlags(flagSet *pflag.FlagSet) { h.ciliumCNIConfig.AddFlags("cni.cilium", flagSet) h.ebsConfig.AddFlags("awsebs", pflag.CommandLine) h.awsccmConfig.AddFlags("awsccm", pflag.CommandLine) + h.nutnaixCSIConfig.AddFlags("nutanixcsi", flagSet) } diff --git a/pkg/handlers/generic/lifecycle/nfd/strategy_crs.go b/pkg/handlers/generic/lifecycle/nfd/strategy_crs.go index 461db5318..26498d482 100644 --- a/pkg/handlers/generic/lifecycle/nfd/strategy_crs.go +++ b/pkg/handlers/generic/lifecycle/nfd/strategy_crs.go @@ -87,7 +87,7 @@ func (s crsStrategy) apply( ) } - if err := utils.EnsureCRSForClusterFromConfigMaps(ctx, cm.Name, s.client, cluster, cm); err != nil { + if err := utils.EnsureCRSForClusterFromObjects(ctx, cm.Name, s.client, cluster, cm); err != nil { return fmt.Errorf( "failed to apply NFD installation ClusterResourceSet: %w", err, diff --git a/pkg/handlers/generic/lifecycle/utils/utils.go b/pkg/handlers/generic/lifecycle/utils/utils.go index 00915b7ab..aa6d42b30 100644 --- a/pkg/handlers/generic/lifecycle/utils/utils.go +++ b/pkg/handlers/generic/lifecycle/utils/utils.go @@ -6,29 +6,69 @@ package utils import ( "context" "fmt" + "maps" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" crsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" + utilyaml "sigs.k8s.io/cluster-api/util/yaml" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" ) -func EnsureCRSForClusterFromConfigMaps( +const ( + kindStorageClass = "StorageClass" + defaultCRSConfigMapKey = "custom-resources.yaml" +) + +var ( + defaultStorageClassKey = "storageclass.kubernetes.io/is-default-class" + defaultStorageClassMap = map[string]string{ + defaultStorageClassKey: "true", + } + defaultAWSStorageClassParams = map[string]string{ + "csi.storage.k8s.io/fstype": "ext4", + "type": "gp3", + } +) + +func EnsureCRSForClusterFromObjects( ctx context.Context, crsName string, c ctrlclient.Client, cluster *clusterv1.Cluster, - configMaps ...*corev1.ConfigMap, + objects ...runtime.Object, ) error { - resources := make([]crsv1.ResourceRef, 0, len(configMaps)) - for _, cm := range configMaps { + resources := make([]crsv1.ResourceRef, 0, len(objects)) + for _, obj := range objects { + var name string + var kind crsv1.ClusterResourceSetResourceKind + cm, ok := obj.(*corev1.ConfigMap) + if !ok { + sec, secOk := obj.(*corev1.Secret) + if !secOk { + return fmt.Errorf( + "cannot create ClusterResourceSet with obj %v only secrets and configmaps are supported", + obj, + ) + } + name = sec.Name + kind = crsv1.SecretClusterResourceSetResourceKind + } else { + name = cm.Name + kind = crsv1.ConfigMapClusterResourceSetResourceKind + } resources = append(resources, crsv1.ResourceRef{ - Kind: string(crsv1.ConfigMapClusterResourceSetResourceKind), - Name: cm.Name, + Name: name, + Kind: string(kind), }) } @@ -86,3 +126,97 @@ func EnsureNamespace(ctx context.Context, c ctrlclient.Client, name string) erro return nil } + +func RetrieveValuesTemplateConfigMap( + ctx context.Context, + c ctrlclient.Client, + configMapName, + defaultsNamespace string, +) (*corev1.ConfigMap, error) { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultsNamespace, + Name: configMapName, + }, + } + configMapObjName := ctrlclient.ObjectKeyFromObject( + configMap, + ) + err := c.Get(ctx, configMapObjName, configMap) + if err != nil { + return nil, fmt.Errorf( + "failed to retrieve installation values template ConfigMap %q: %w", + configMapObjName, + err, + ) + } + return configMap, nil +} + +func CreateStorageClass( + storageConfig v1alpha1.StorageClassConfig, + defaultsNamespace string, + provisionerName v1alpha1.StorageProvisioner, + isDefault bool, +) *storagev1.StorageClass { + var params map[string]string + if provisionerName == v1alpha1.AWSEBSProvisioner { + params = defaultAWSStorageClassParams + } + if storageConfig.Parameters != nil { + params = maps.Clone(storageConfig.Parameters) + } + sc := storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: kindStorageClass, + APIVersion: storagev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: storageConfig.Name, + Namespace: defaultsNamespace, + }, + Provisioner: string(provisionerName), + Parameters: params, + VolumeBindingMode: ptr.To(storageConfig.VolumeBindingMode), + ReclaimPolicy: ptr.To(storageConfig.ReclaimPolicy), + AllowVolumeExpansion: ptr.To(storageConfig.AllowExpansion), + } + if isDefault { + sc.ObjectMeta.Annotations = defaultStorageClassMap + } + return &sc +} + +func CreateConfigMapForCRS(configMapName, configMapNamespace string, + objs ...runtime.Object, +) (*corev1.ConfigMap, error) { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: configMapNamespace, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + Data: make(map[string]string), + } + l := make([][]byte, 0, len(objs)) + for _, v := range objs { + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(v) + if err != nil { + return nil, err + } + objYaml, err := utilyaml.FromUnstructured([]unstructured.Unstructured{ + { + Object: obj, + }, + }) + if err != nil { + return nil, err + } + l = append(l, objYaml) + } + cm.Data[defaultCRSConfigMapKey] = string(utilyaml.JoinYaml(l...)) + return cm, nil +} diff --git a/pkg/handlers/generic/lifecycle/utils/utils_test.go b/pkg/handlers/generic/lifecycle/utils/utils_test.go new file mode 100644 index 000000000..420ebe562 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/utils/utils_test.go @@ -0,0 +1,216 @@ +// Copyright 2023 D2iQ, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + + "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/api/v1alpha1" +) + +func TestCreateStorageClass(t *testing.T) { + tests := []struct { + name string + defaultsNamespace string + storageConfig v1alpha1.StorageClassConfig + expectedStorageClass *storagev1.StorageClass + provisioner v1alpha1.StorageProvisioner + isDefault bool + }{ + { + name: "defaulting with AWS", + storageConfig: v1alpha1.StorageClassConfig{ + Name: "aws-ebs", + ReclaimPolicy: v1alpha1.VolumeReclaimDelete, + VolumeBindingMode: v1alpha1.VolumeBindingWaitForFirstConsumer, + Parameters: nil, + AllowExpansion: true, + }, + expectedStorageClass: &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: kindStorageClass, + APIVersion: storagev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "aws-ebs", + Namespace: "default", + }, + Parameters: defaultAWSStorageClassParams, + ReclaimPolicy: ptr.To(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: ptr.To(storagev1.VolumeBindingWaitForFirstConsumer), + Provisioner: string(v1alpha1.AWSEBSProvisioner), + AllowVolumeExpansion: ptr.To(true), + }, + provisioner: v1alpha1.AWSEBSProvisioner, + defaultsNamespace: "default", + }, + { + name: "nutanix for nutanix files", + storageConfig: v1alpha1.StorageClassConfig{ + Name: "nutanix-volumes", + ReclaimPolicy: v1alpha1.VolumeReclaimDelete, + VolumeBindingMode: v1alpha1.VolumeBindingWaitForFirstConsumer, + Parameters: map[string]string{ + "csi.storage.k8s.io/fstype": "ext4", + "flashMode": "ENABLED", + "storageContainer": "storage-container-name", + "chapAuth": "ENABLED", + "storageType": "NutanixVolumes", + "whitelistIPMode": "ENABLED", + "whitelistIPAddr": "1.1.1.1", + }, + }, + expectedStorageClass: &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: kindStorageClass, + APIVersion: storagev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "nutanix-volumes", + Namespace: "default", + }, + Parameters: map[string]string{ + "csi.storage.k8s.io/fstype": "ext4", + "flashMode": "ENABLED", + "storageContainer": "storage-container-name", + "chapAuth": "ENABLED", + "storageType": "NutanixVolumes", + "whitelistIPMode": "ENABLED", + "whitelistIPAddr": "1.1.1.1", + }, + ReclaimPolicy: ptr.To(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: ptr.To(storagev1.VolumeBindingWaitForFirstConsumer), + Provisioner: string(v1alpha1.NutanixProvisioner), + AllowVolumeExpansion: ptr.To(false), + }, + provisioner: v1alpha1.NutanixProvisioner, + defaultsNamespace: "default", + }, + { + name: "nutanix defaults", + storageConfig: v1alpha1.StorageClassConfig{ + Name: "nutanix-volumes", + ReclaimPolicy: v1alpha1.VolumeReclaimDelete, + VolumeBindingMode: v1alpha1.VolumeBindingWaitForFirstConsumer, + }, + expectedStorageClass: &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: kindStorageClass, + APIVersion: storagev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "nutanix-volumes", + Namespace: "default", + }, + ReclaimPolicy: ptr.To(corev1.PersistentVolumeReclaimDelete), + VolumeBindingMode: ptr.To(storagev1.VolumeBindingWaitForFirstConsumer), + Provisioner: string(v1alpha1.NutanixProvisioner), + AllowVolumeExpansion: ptr.To(false), + }, + provisioner: v1alpha1.NutanixProvisioner, + defaultsNamespace: "default", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sc := CreateStorageClass( + tt.storageConfig, + tt.defaultsNamespace, + tt.provisioner, + false, + ) + if diff := cmp.Diff(sc, tt.expectedStorageClass); diff != "" { + t.Errorf("CreateStorageClass() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestCreateConfigMapForCRS(t *testing.T) { + tests := []struct { + name string + testCMName string + testNamespace string + objs []runtime.Object + expectedCM corev1.ConfigMap + }{ + { + name: "multiple storage class objects", + testCMName: "test", + testNamespace: "default", + objs: []runtime.Object{ + &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: kindStorageClass, + APIVersion: storagev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + }, + &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: kindStorageClass, + APIVersion: storagev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-2", + Namespace: "default", + }, + }, + }, + expectedCM: corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + Data: map[string]string{ + defaultCRSConfigMapKey: `apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + creationTimestamp: null + name: test + namespace: default +provisioner: "" +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + creationTimestamp: null + name: test-2 + namespace: default +provisioner: ""`, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cm, err := CreateConfigMapForCRS(tt.testCMName, tt.testNamespace, tt.objs...) + if err != nil { + t.Errorf("failed to create cm with error %v", err) + } + data, ok := cm.Data[defaultCRSConfigMapKey] + if !ok { + t.Errorf("expected %s to exist in cm.Data. got %v", defaultCRSConfigMapKey, cm.Data) + } + expected := tt.expectedCM.Data[defaultCRSConfigMapKey] + if data != expected { + t.Errorf("expected %s \n got %s", expected, data) + } + }) + } +}