[csi-plugins] add support for PVC annotations (#2687)

* [csi-plugins] add support for PVC annotations

* Add support for volume/share name resolve

* Add a timeout to a e2e job, so we can fetch logs

charts/cinder-csi-plugin/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 appVersion: v1.31.0
 description: Cinder CSI Chart for OpenStack
 name: openstack-cinder-csi
-version: 2.31.3
+version: 2.31.4
 home: https://github.com/kubernetes/cloud-provider-openstack
 icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png
 maintainers:

charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml

@@ -175,6 +175,9 @@ spec:
             {{- if .Values.csi.plugin.httpEndpoint.enabled }}
             - "--http-endpoint=:{{ .Values.csi.plugin.httpEndpoint.port }}"
             {{- end }}
+            {{- if .Values.pvcAnnotations }}
+            - "--pvc-annotations"
+            {{- end }}
             {{- if .Values.csi.plugin.extraArgs }}
             {{- with .Values.csi.plugin.extraArgs }}
             {{- tpl . $ | trim | nindent 12 }}

charts/cinder-csi-plugin/values.yaml

@@ -207,6 +207,9 @@ storageClass:
 # to volume metadata in newly provisioned volumes as `cinder.csi.openstack.org/cluster=<cluster ID>`.
 clusterID: "kubernetes"
 
+# Enable PVC annotations support to create PVCs with extra parameters
+pvcAnnotations: false
+
 priorityClassName: ""
 
 imagePullSecrets: []

charts/manila-csi-plugin/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 appVersion: v1.31.0
 description: Manila CSI Chart for OpenStack
 name: openstack-manila-csi
-version: 2.31.0
+version: 2.31.1
 home: http://github.com/kubernetes/cloud-provider-openstack
 icon: https://github.com/kubernetes/kubernetes/blob/master/logo/logo.png
 maintainers:

charts/manila-csi-plugin/templates/controllerplugin-statefulset.yaml

@@ -26,7 +26,7 @@ spec:
             {{- if $.Values.csimanila.topologyAwarenessEnabled }}
             - "--feature-gates=Topology=true"
             {{- end }}
-            {{- if $.Values.controllerplugin.provisioner.extraCreateMetadata }}
+            {{- if or $.Values.controllerplugin.provisioner.extraCreateMetadata $.Values.csimanila.pvcAnnotations }}
             - "--extra-create-metadata"
             {{- end }}
           env:
@@ -101,7 +101,10 @@ spec:
             {{- if .compatibilitySettings }}
             --compatibility-settings={{ .compatibilitySettings }}
             {{- end }}
-            --cluster-id="{{ $.Values.csimanila.clusterID }}"'
+            --cluster-id="{{ $.Values.csimanila.clusterID }}"
+            {{- if $.Values.csimanila.pvcAnnotations }}
+            --pvc-annotations
+            {{- end }}'
           ]
           env:
             - name: DRIVER_NAME

charts/manila-csi-plugin/values.yaml

@@ -43,6 +43,9 @@ csimanila:
   # to share metadata in newly provisioned shares as `manila.csi.openstack.org/cluster=<cluster ID>`.
   clusterID: ""
 
+  # Enable PVC annotations support to create PVCs with extra parameters
+  pvcAnnotations: false
+
   # Image spec
   image:
     repository: registry.k8s.io/provider-os/manila-csi-plugin

cmd/cinder-csi-plugin/main.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+	"k8s.io/cloud-provider-openstack/pkg/csi"
 	"k8s.io/cloud-provider-openstack/pkg/csi/cinder"
 	"k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack"
 	"k8s.io/cloud-provider-openstack/pkg/util/metadata"
@@ -72,6 +73,8 @@ func main() {
 		Version: version.Version,
 	}
 
+	csi.AddPVCFlags(cmd)
+
 	cmd.PersistentFlags().StringVar(&nodeID, "nodeid", "", "node id")
 	if err := cmd.PersistentFlags().MarkDeprecated("nodeid", "This flag would be removed in future. Currently, the value is ignored by the driver"); err != nil {
 		klog.Fatalf("Unable to mark flag nodeid to be deprecated: %v", err)
@@ -103,7 +106,11 @@ func main() {
 
 func handle() {
 	// Initialize cloud
-	d := cinder.NewDriver(&cinder.DriverOpts{Endpoint: endpoint, ClusterID: cluster})
+	d := cinder.NewDriver(&cinder.DriverOpts{
+		Endpoint:  endpoint,
+		ClusterID: cluster,
+		PVCLister: csi.GetPVCLister(),
+	})
 
 	openstack.InitOpenStackProvider(cloudConfig, httpEndpoint)
 

cmd/manila-csi-plugin/main.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/spf13/cobra"
+	"k8s.io/cloud-provider-openstack/pkg/csi"
 	"k8s.io/cloud-provider-openstack/pkg/csi/manila"
 	"k8s.io/cloud-provider-openstack/pkg/csi/manila/csiclient"
 	"k8s.io/cloud-provider-openstack/pkg/csi/manila/manilaclient"
@@ -86,6 +87,7 @@ func main() {
 				ManilaClientBuilder: manilaClientBuilder,
 				CSIClientBuilder:    csiClientBuilder,
 				ClusterID:           clusterID,
+				PVCLister:           csi.GetPVCLister(),
 			}
 
 			if provideNodeService {
@@ -119,6 +121,8 @@ func main() {
 		Version: version.Version,
 	}
 
+	csi.AddPVCFlags(cmd)
+
 	cmd.PersistentFlags().StringVar(&endpoint, "endpoint", "unix://tmp/csi.sock", "CSI endpoint")
 
 	cmd.PersistentFlags().StringVar(&driverName, "drivername", "manila.csi.openstack.org", "name of the driver")

docs/cinder-csi-plugin/examples.md

@@ -10,6 +10,7 @@
   - [Snapshot Create and Restore](#snapshot-create-and-restore)
   - [Use Topology](#use-topology)
   - [Disaster recovery of PV and PVC](#disaster-recovery-of-pv-and-pvc)
+  - [Use scheduler hints annotations](#use-scheduler-hints-annotations)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
@@ -453,3 +454,51 @@ spec:
   storageClassName: sata
   volumeMode: Filesystem
 ```
+
+## Use scheduler hints annotations
+
+Cinder CSI driver supports the use of scheduler hints to influence the
+placement of volumes. Scheduler hints can be specified in the
+PersistentVolumeClaim (PVC) annotations:
+
+* `cinder.csi.openstack.org/affinity`
+* `cinder.csi.openstack.org/anti-affinity`
+
+In order to use scheduler hints, the Cinder CSI controller plugin must be
+started with the `--pvc-annotations` flag. The PVC annotations take effect only
+when the PVC is created. The scheduler hints are not updated when the PVC is
+updated. The following example demonstrates how to use scheduler hints to
+influence the placement of volumes:
+
+```
+$ kubectl apply -f scheduler-hints-pvc.yaml
+```
+
+```
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: csi-pvc-cinderplugin
+  annotations:
+    cinder.csi.openstack.org/affinity: "1b4e28ba-2fa1-11ec-8d3d-0242ac130003"
+    cinder.csi.openstack.org/anti-affinity: "1b4e28ba-2fa1-11ec-8d3d-0242ac130004,pv-k8s--cluster-1b5f47bf-0119-442e-8529-254c36e43644"
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: csi-sc-cinderplugin
+```
+
+where `1b4e28ba-2fa1-11ec-8d3d-0242ac130003`,
+`1b4e28ba-2fa1-11ec-8d3d-0242ac130004` and
+`pv-k8s--cluster-1b5f47bf-0119-442e-8529-254c36e43644` are names or UUIDs of
+the already provisioned volumes in the OpenStack cloud. The scheduler will try
+to place the volume in the same block storage server as the volume with the
+`1b4e28ba-2fa1-11ec-8d3d-0242ac130003` UUID and avoid placing the volume in the
+same block storage server as the volumes with the
+`1b4e28ba-2fa1-11ec-8d3d-0242ac130004` UUID and the
+`pv-k8s--cluster-1b5f47bf-0119-442e-8529-254c36e43644` volume name. If the
+scheduler hints are not satisfied, the volume will not be provisioned with an
+error message in the controller logs.

docs/cinder-csi-plugin/using-cinder-csi-plugin.md

@@ -17,6 +17,7 @@
   - [Supported Features](#supported-features)
   - [Sidecar Compatibility](#sidecar-compatibility)
   - [Supported Parameters](#supported-parameters)
+  - [Supported PVC Annotations](#supported-pvc-annotations)
   - [Local Development](#local-development)
     - [Build](#build)
     - [Testing](#testing)
@@ -110,6 +111,14 @@ In addition to the standard set of klog flags, `cinder-csi-plugin` accepts the f
   If set to true then the CSI driver does provide the node service.
 
   The default is to provide the node service.
+
+  <dt>--pvc-annotations &lt;disabled&gt;</dt>
+  <dd>
+  If set to true then the CSI driver will use PVC annotations to provide volume
+  scheduler hints. See [Supported PVC Annotations](#supported-pvc-annotations)
+  for more information.
+
+  The default is not to provide the PVC annotations support.
   </dd>
 </dl>
 
@@ -273,6 +282,24 @@ helm install --namespace kube-system --name cinder-csi ./charts/cinder-csi-plugi
 | Inline Volume `volumeAttributes`   | `capacity`              | `1Gi`       | volume size for creating inline volumes|
 | Inline Volume `VolumeAttributes`   | `type`              | Empty String  | Name/ID of Volume type. Corresponding volume type should exist in cinder |
 
+## Supported PVC Annotations
+
+The PVC annotations support must be enabled in the Cinder CSI controller with
+the `--pvc-annotations` flag. The PVC annotations take effect only when the PVC
+is created. The scheduler hints are not updated when the PVC is updated. The
+following PVC annotations are supported:
+
+| Annotation Name            | Description      | Example |
+|-------------------------   |-----------------|----------|
+| `cinder.csi.openstack.org/affinity` | Volume affinity to existing volume or volumes names/UUIDs. The value should be a comma-separated list of volume names/UUIDs. | `cinder.csi.openstack.org/affinity: "1b4e28ba-2fa1-11ec-8d3d-0242ac130003"` |
+| `cinder.csi.openstack.org/anti-affinity` | Volume anti-affinity to existing volume or volumes names/UUIDs. The value should be a comma-separated list of volume names/UUIDs. | `cinder.csi.openstack.org/anti-affinity: "1b4e28ba-2fa1-11ec-8d3d-0242ac130004,pv-k8s--cluster-1b5f47bf-0119-442e-8529-254c36e43644"` |
+
+If the PVC annotation is set, the volume will be created according to the
+existing volume names/UUIDs placements, i.e. on the same host as the
+`1b4e28ba-2fa1-11ec-8d3d-0242ac130003` volume and not on the same host as the
+`1b4e28ba-2fa1-11ec-8d3d-0242ac130004` and
+`pv-k8s--cluster-1b5f47bf-0119-442e-8529-254c36e43644` volumes.
+
 ## Local Development
 
 ### Build

docs/manila-csi-plugin/using-manila-csi-plugin.md

@@ -15,6 +15,7 @@
       - [Verifying the deployment](#verifying-the-deployment)
       - [Enabling topology awareness](#enabling-topology-awareness)
   - [Share protocol support matrix](#share-protocol-support-matrix)
+  - [Supported PVC annotations](#supported-pvc-annotations)
   - [For developers](#for-developers)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -42,6 +43,7 @@ Option | Default value | Description
 `--cluster-id` | _none_ | The identifier of the cluster that the plugin is running in. If set then the plugin will add "manila.csi.openstack.org/cluster: \<clusterID\>" to metadata of created shares.
 `--provide-controller-service` | `true` | If set to true then the CSI driver does provide the controller service.
 `--provide-node-service` | `true` | If set to true then the CSI driver does provide the node service.
+`--pvc-annotations` | `false` | If set to true then the CSI driver will use PVC annotations as an additional information when creating shares. See [Supported PVC annotations](#supported-pvc-annotations) for more info.
 
 ### Controller Service volume parameters
 
@@ -53,6 +55,7 @@ Parameter | Required | Description
 `shareNetworkID` | _no_ | Manila [share network ID](https://wiki.openstack.org/wiki/Manila/Concepts#share_network)
 `availability` | _no_ | Manila availability zone of the provisioned share. If none is provided, the default Manila zone will be used. Note that this parameter is opaque to the CO and does not influence placement of workloads that will consume this share, meaning they may be scheduled onto any node of the cluster. If the specified Manila AZ is not equally accessible from all compute nodes of the cluster, use [Topology-aware dynamic provisioning](#topology-aware-dynamic-provisioning).
 `autoTopology` | _no_ | When set to "true" and the `availability` parameter is empty, the Manila CSI controller will map the Manila availability zone to the target compute node availability zone.
+`groupID` | _no_ | The UUID of the share group to which the provisioned share belongs. If not empty, the share will be created in the specified share group. The share group must be created in advance before the PVC is created.
 `appendShareMetadata` | _no_ | Append user-defined metadata to the provisioned share. If not empty, this field must be a string with a valid JSON object. The object must consist of key-value pairs of type string. Example: `"{..., \"key\": \"value\"}"`.
 `cephfs-mounter` | _no_ | Relevant for CephFS Manila shares. Specifies which mounting method to use with the CSI CephFS driver. Available options are `kernel` and `fuse`, defaults to `fuse`. See [CSI CephFS docs](https://github.com/ceph/ceph-csi/blob/csi-v1.0/docs/deploy-cephfs.md#configuration) for further information.
 `cephfs-kernelMountOptions` | _no_ | Relevant for CephFS Manila shares. Specifies mount options for CephFS kernel client. See [CSI CephFS docs](https://github.com/ceph/ceph-csi/blob/csi-v1.0/docs/deploy-cephfs.md#configuration) for further information.
@@ -272,7 +275,30 @@ Manila share protocol | CSI Node Plugin
 `CEPHFS` | [CSI CephFS](https://github.com/ceph/ceph-csi) : v1.0.0
 `NFS` | [CSI NFS](https://github.com/kubernetes-csi/csi-driver-nfs) : v1.0.0
 
+## Supported PVC Annotations
+
+The PVC annotations support must be enabled in the Manila CSI controller with
+the `--pvc-annotations` flag. The PVC annotations take effect only when the PVC
+is created. The scheduler hints are not updated when the PVC is updated. The
+minimum Manila API microversion required for scheduler hints is 2.65. Make sure
+that the Manila API microversion is supported by the Manila backend. The
+following PVC annotations are supported:
+
+| Annotation Name            | Description      | Example |
+|-------------------------   |-----------------|----------|
+| `manila.csi.openstack.org/affinity` | Share affinity to existing share or shares names/UUIDs. The value should be a comma-separated list of share names or UUIDs. | `manila.csi.openstack.org/affinity: "1b4e28ba-2fa1-11ec-8d3d-0242ac130003"` |
+| `manila.csi.openstack.org/anti-affinity` | Share anti-affinity to existing share or shares names/UUIDs. The value should be a comma-separated list of share names/UUIDs. | `manila.csi.openstack.org/anti-affinity: "1b4e28ba-2fa1-11ec-8d3d-0242ac130004,pv-default-50c5a3b3-e0b5-48d6-a163-4e68956aeb54"` |
+| `manila.csi.openstack.org/group-id` | The UUID of the share group to which the provisioned share must belong. The share group must be created before the PVC is created. | `manila.csi.openstack.org/group-id: "1b4e28ba-2fa1-11ec-8d3d-0242ac130006"` |
+
+If the PVC annotation is set, the share will be created according to the
+existing share names/UUIDs placements, i.e. on the same host as the
+`1b4e28ba-2fa1-11ec-8d3d-0242ac130003` share and not on the same host as the
+`1b4e28ba-2fa1-11ec-8d3d-0242ac130004` and
+`pv-default-50c5a3b3-e0b5-48d6-a163-4e68956aeb54` shares.
+
+The `manila.csi.openstack.org/group-id` annotation value overrides the storage
+class `groupID` parameter if both are set.
+
 ## For developers
 
 If you'd like to contribute to CSI Manila, check out `docs/manila-csi-plugin/developers-csi-manila.md` to get you started.
-

go.mod

@@ -5,7 +5,8 @@ go 1.22.0
 require (
 	github.com/container-storage-interface/spec v1.9.0
 	github.com/go-chi/chi/v5 v5.0.8
-	github.com/gophercloud/gophercloud/v2 v2.0.0
+	github.com/google/uuid v1.6.0
+	github.com/gophercloud/gophercloud/v2 v2.1.2-0.20241016132526-1c4dd03733fe
 	github.com/gophercloud/utils/v2 v2.0.0-20240701101423-2401526caee5
 	github.com/hashicorp/go-version v1.6.0
 	github.com/kubernetes-csi/csi-lib-utils v0.13.0
@@ -22,8 +23,8 @@ require (
 	github.com/stretchr/testify v1.9.0
 	go.uber.org/goleak v1.3.0
 	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc
-	golang.org/x/sys v0.21.0
-	golang.org/x/term v0.21.0
+	golang.org/x/sys v0.26.0
+	golang.org/x/term v0.25.0
 	google.golang.org/grpc v1.65.0
 	google.golang.org/protobuf v1.34.2
 	gopkg.in/gcfg.v1 v1.2.3
@@ -91,7 +92,6 @@ require (
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect
-	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
@@ -142,11 +142,11 @@ require (
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect