From 6996623f93cf5fa9a11090250b625ef079e9744d Mon Sep 17 00:00:00 2001 From: taenzeyang Date: Thu, 16 Nov 2023 20:11:56 +0800 Subject: [PATCH] Support for kata direct volume Signed-off-by: tianze.yang@easystack.cn --- .../templates/csi-nfs-node.yaml | 8 +- charts/latest/csi-driver-nfs/values.yaml | 6 +- pkg/nfs/controllerserver.go | 1 + pkg/nfs/nfs.go | 2 + pkg/nfs/nodeserver.go | 14 +++ pkg/utils/utils.go | 100 ++++++++++++++++++ 6 files changed, 127 insertions(+), 4 deletions(-) create mode 100644 pkg/utils/utils.go diff --git a/charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml b/charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml index aac988537..0d94cf032 100644 --- a/charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml +++ b/charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml @@ -16,7 +16,7 @@ spec: template: metadata: {{ include "nfs.labels" . | indent 6 }} - app: {{ .Values.node.name }} + app: {{ .Values.node.name }} spec: {{- if .Values.imagePullSecrets }} imagePullSecrets: @@ -125,6 +125,8 @@ spec: - name: pods-mount-dir mountPath: {{ .Values.kubeletDir }}/pods mountPropagation: "Bidirectional" + - name: kata-containers + mountPath: /run/kata-containers/shared/direct-volumes/ {{- if .Values.feature.propagateHostMountOptions }} - name: host-nfsmount-conf mountPath: /etc/nfsmount.conf @@ -141,6 +143,10 @@ spec: hostPath: path: {{ .Values.kubeletDir }}/pods type: Directory + - name: kata-containers + hostPath: + path: /run/kata-containers/shared/direct-volumes/ + type: Directory - hostPath: path: {{ .Values.kubeletDir }}/plugins_registry type: Directory diff --git a/charts/latest/csi-driver-nfs/values.yaml b/charts/latest/csi-driver-nfs/values.yaml index 9a473993d..986b18103 100755 --- a/charts/latest/csi-driver-nfs/values.yaml +++ b/charts/latest/csi-driver-nfs/values.yaml @@ -1,9 +1,9 @@ customLabels: {} image: nfs: - repository: gcr.io/k8s-staging-sig-storage/nfsplugin - tag: canary - pullPolicy: IfNotPresent + repository: taenyang/nfsplugin + tag: latest + pullPolicy: Always csiProvisioner: repository: registry.k8s.io/sig-storage/csi-provisioner tag: v3.6.1 diff --git a/pkg/nfs/controllerserver.go b/pkg/nfs/controllerserver.go index 86b5b52a2..3d9b4c13c 100644 --- a/pkg/nfs/controllerserver.go +++ b/pkg/nfs/controllerserver.go @@ -137,6 +137,7 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol case pvcNamespaceKey: case pvcNameKey: case pvNameKey: + case directVolume: // no op case mountPermissionsField: if v != "" { diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index e312f05c1..0f56cda21 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -70,6 +70,8 @@ const ( pvcNameMetadata = "${pvc.metadata.name}" pvcNamespaceMetadata = "${pvc.metadata.namespace}" pvNameMetadata = "${pv.metadata.name}" + directVolume = "directvolume" + fsType = "nfs" ) func NewDriver(options *DriverOptions) *Driver { diff --git a/pkg/nfs/nodeserver.go b/pkg/nfs/nodeserver.go index 78e34991b..3ddb4b364 100644 --- a/pkg/nfs/nodeserver.go +++ b/pkg/nfs/nodeserver.go @@ -30,6 +30,8 @@ import ( "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/volume" mount "k8s.io/mount-utils" + + "github.com/kubernetes-csi/csi-driver-nfs/pkg/utils" ) // NodeServer driver @@ -86,9 +88,18 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("invalid mountPermissions %s", v)) } } + case directVolume: + subDirReplaceMap[directVolume] = v } } + if subDirReplaceMap[directVolume] == "true" { + if err := utils.AddDirectVolume(targetPath, server, baseDir, fsType); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return &csi.NodePublishVolumeResponse{}, nil + } + if server == "" { return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("%v is a required parameter", paramServer)) } @@ -156,6 +167,9 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu klog.V(2).Infof("NodeUnpublishVolume: unmounting volume %s on %s", volumeID, targetPath) var err error + if err = utils.Remove(targetPath); err != nil { + klog.V(2).ErrorS(err, "NodeUnpublishVolume Error: unmounting volume %s on %s", volumeID, targetPath) + } extensiveMountPointCheck := true forceUnmounter, ok := ns.mounter.(mount.MounterForceUnmounter) if ok { diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go new file mode 100644 index 000000000..910afa7c5 --- /dev/null +++ b/pkg/utils/utils.go @@ -0,0 +1,100 @@ +package utils + +import ( + b64 "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/klog/v2" +) + +const ( + mountInfoFileName = "mountInfo.json" + nfsDir = "nfs" +) + +// FSGroupChangePolicy holds policies that will be used for applying fsGroup to a volume. +// This type and the allowed values are tracking the PodFSGroupChangePolicy defined in +// https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/api/core/v1/types.go +// It is up to the client using the direct-assigned volume feature (e.g. CSI drivers) to determine +// the optimal setting for this change policy (i.e. from Pod spec or assuming volume ownership +// based on the storage offering). +type FSGroupChangePolicy string + +var kataDirectVolumeRootPath = "/run/kata-containers/shared/direct-volumes" + +// MountInfo contains the information needed by Kata to consume a host block device and mount it as a filesystem inside the guest VM. +type MountInfo struct { + // The type of the volume (ie. block) + VolumeType string `json:"volume-type"` + // The device backing the volume. + Device string `json:"device"` + // The filesystem type to be mounted on the volume. + FsType string `json:"fstype"` + // Additional metadata to pass to the agent regarding this volume. + Metadata map[string]string `json:"metadata,omitempty"` + // Additional mount options. + Options []string `json:"options,omitempty"` +} + +// Add writes the mount info of a direct volume into a filesystem path known to Kata Container. +func Add(volumePath, mountInfo string) error { + volumeDir := filepath.Join(kataDirectVolumeRootPath, nfsDir, b64.URLEncoding.EncodeToString([]byte(volumePath))) + stat, err := os.Stat(volumeDir) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := os.MkdirAll(volumeDir, 0700); err != nil { + return err + } + } + if stat != nil && !stat.IsDir() { + return fmt.Errorf("%s should be a directory", volumeDir) + } + + var deserialized MountInfo + if err := json.Unmarshal([]byte(mountInfo), &deserialized); err != nil { + return err + } + + return os.WriteFile(filepath.Join(volumeDir, mountInfoFileName), []byte(mountInfo), 0600) +} + +// Remove deletes the direct volume path including all the files inside it. +func Remove(volumePath string) error { + return os.RemoveAll(filepath.Join(kataDirectVolumeRootPath, nfsDir, b64.URLEncoding.EncodeToString([]byte(volumePath)))) +} + +func AddDirectVolume(volumePath, server, baseDir, fsType string) error { + mountInfo := struct { + VolumeType string `json:"volume-type"` + Device string `json:"device"` + FsType string `json:"fstype"` + Metadata map[string]string `json:"metadata,omitempty"` + Options []string `json:"options,omitempty"` + }{ + VolumeType: "fs", + FsType: fsType, + Options: []string{server + ":" + baseDir}, + } + + mi, err := json.Marshal(mountInfo) + if err != nil { + klog.Errorf("addDirectVolume - json.Marshal failed: ", err.Error()) + return status.Errorf(codes.Internal, "json.Marshal failed: %s", err.Error()) + } + + if err := Add(volumePath, string(mi)); err != nil { + klog.Errorf("addDirectVolume - add direct volume failed: ", err.Error()) + return status.Errorf(codes.Internal, "add direct volume failed: %s", err.Error()) + } + + klog.Infof("add direct volume done: %s %s", volumePath, string(mi)) + return nil +}