diff --git a/deploy/charts/harvester-node-disk-manager/templates/crds/harvesterhci.io_blockdevices.yaml b/deploy/charts/harvester-node-disk-manager/templates/crds/harvesterhci.io_blockdevices.yaml index 2fa008cf..6e40129a 100644 --- a/deploy/charts/harvester-node-disk-manager/templates/crds/harvesterhci.io_blockdevices.yaml +++ b/deploy/charts/harvester-node-disk-manager/templates/crds/harvesterhci.io_blockdevices.yaml @@ -74,8 +74,9 @@ spec: a string with the partition's mount point, or "" if no mount point was discovered type: string provisioned: - description: a bool indicating whether the filesystem can be provisioned - as a disk for the node to store data. + description: |- + a bool indicating whether the filesystem can be provisioned as a disk for the node to store data. + Deprecated: Replaced by field `spec.provision` type: boolean repaired: description: a bool indicating whether the filesystem is manually @@ -87,6 +88,10 @@ spec: nodeName: description: name of the node to which the block device is attached type: string + provision: + default: false + description: a bool for the device to be provisioned + type: boolean provisioner: properties: longhorn: diff --git a/deploy/charts/harvester-node-disk-manager/templates/crds/harvesterhci.io_lvmvolumegroups.yaml b/deploy/charts/harvester-node-disk-manager/templates/crds/harvesterhci.io_lvmvolumegroups.yaml new file mode 100644 index 00000000..13fc5afc --- /dev/null +++ b/deploy/charts/harvester-node-disk-manager/templates/crds/harvesterhci.io_lvmvolumegroups.yaml @@ -0,0 +1,138 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + {} + name: lvmvolumegroups.harvesterhci.io +spec: + group: harvesterhci.io + names: + kind: LVMVolumeGroup + listKind: LVMVolumeGroupList + plural: lvmvolumegroups + shortNames: + - lvmvg + - lvmvgs + singular: lvmvolumegroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.parameters + name: Parameters + type: string + - jsonPath: .status.vgStatus + name: Status + type: string + - jsonPath: .spec.nodeName + name: Node + type: string + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + desiredState: + description: |- + DesiredState is the desired state of the volume group + enabled means we will keep this vg active, disabled means we will keep this vg inactive + enum: + - Enabled + - Disabled + type: string + devices: + additionalProperties: + type: string + description: |- + The devices of the volume group + format: map[]=devPath" + e.g. map[087fc9702c450bfca5ba56b06ba7d7f2] = /dev/sda + type: object + nodeName: + description: NodeName is the name of the node where the volume group + is created + type: string + parameters: + description: Parameters is the parameters for creating the volume + group *optional* + type: string + vgName: + description: VGName is the name of the volume group + type: string + required: + - desiredState + - nodeName + - vgName + type: object + status: + properties: + conditions: + description: The conditions of the volume group + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + devices: + additionalProperties: + type: string + description: |- + The devices of the volume group + format: map[]=devPath" + type: object + parameters: + description: Parameters is the current parameters of the volume group + type: string + vgStatus: + default: Unknown + description: The status of the volume group + enum: + - Active + - Inactive + - Unknown + type: string + vgTargetType: + description: VGTargetType is the target type of the volume group, + now only support stripe/dm-thin + type: string + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/charts/harvester-node-disk-manager/templates/rbac.yaml b/deploy/charts/harvester-node-disk-manager/templates/rbac.yaml index 61f6cba6..3cbded9c 100644 --- a/deploy/charts/harvester-node-disk-manager/templates/rbac.yaml +++ b/deploy/charts/harvester-node-disk-manager/templates/rbac.yaml @@ -12,7 +12,7 @@ metadata: name: {{ include "harvester-node-disk-manager.name" . }} rules: - apiGroups: [ "harvesterhci.io" ] - resources: [ "blockdevices" ] + resources: [ "blockdevices", "lvmvolumegroups", "lvmvolumegroups/status" ] verbs: [ "*" ] - apiGroups: [ "longhorn.io" ] resources: [ "nodes" ] diff --git a/main.go b/main.go index 9c258a81..b53852f6 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ import ( "github.com/harvester/node-disk-manager/pkg/block" blockdevicev1 "github.com/harvester/node-disk-manager/pkg/controller/blockdevice" nodev1 "github.com/harvester/node-disk-manager/pkg/controller/node" + volumegroupv1 "github.com/harvester/node-disk-manager/pkg/controller/volumegroup" "github.com/harvester/node-disk-manager/pkg/filter" ctldisk "github.com/harvester/node-disk-manager/pkg/generated/controllers/harvesterhci.io" ctllonghorn "github.com/harvester/node-disk-manager/pkg/generated/controllers/longhorn.io" @@ -219,6 +220,7 @@ func run(opt *option.Option) error { locker := &sync.Mutex{} cond := sync.NewCond(locker) bds := disks.Harvesterhci().V1beta1().BlockDevice() + lvmVGs := disks.Harvesterhci().V1beta1().LVMVolumeGroup() nodes := lhs.Longhorn().V1beta2().Node() scanner := blockdevicev1.NewScanner( opt.NodeName, @@ -237,6 +239,7 @@ func run(opt *option.Option) error { ctx, nodes, bds, + lvmVGs, block, opt, scanner, @@ -248,6 +251,10 @@ func run(opt *option.Option) error { logrus.Fatalf("failed to register ndm node controller, %s", err.Error()) } + if err := volumegroupv1.Register(ctx, lvmVGs, opt); err != nil { + logrus.Fatalf("failed to register ndm volume group controller, %s", err.Error()) + } + if err := start.All(ctx, opt.Threadiness, disks, lhs); err != nil { logrus.Fatalf("error starting, %s", err.Error()) } diff --git a/manifests/crds/harvesterhci.io_blockdevices.yaml b/manifests/crds/harvesterhci.io_blockdevices.yaml index 2fa008cf..6e40129a 100644 --- a/manifests/crds/harvesterhci.io_blockdevices.yaml +++ b/manifests/crds/harvesterhci.io_blockdevices.yaml @@ -74,8 +74,9 @@ spec: a string with the partition's mount point, or "" if no mount point was discovered type: string provisioned: - description: a bool indicating whether the filesystem can be provisioned - as a disk for the node to store data. + description: |- + a bool indicating whether the filesystem can be provisioned as a disk for the node to store data. + Deprecated: Replaced by field `spec.provision` type: boolean repaired: description: a bool indicating whether the filesystem is manually @@ -87,6 +88,10 @@ spec: nodeName: description: name of the node to which the block device is attached type: string + provision: + default: false + description: a bool for the device to be provisioned + type: boolean provisioner: properties: longhorn: diff --git a/manifests/crds/harvesterhci.io_lvmvolumegroups.yaml b/manifests/crds/harvesterhci.io_lvmvolumegroups.yaml new file mode 100644 index 00000000..13fc5afc --- /dev/null +++ b/manifests/crds/harvesterhci.io_lvmvolumegroups.yaml @@ -0,0 +1,138 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + {} + name: lvmvolumegroups.harvesterhci.io +spec: + group: harvesterhci.io + names: + kind: LVMVolumeGroup + listKind: LVMVolumeGroupList + plural: lvmvolumegroups + shortNames: + - lvmvg + - lvmvgs + singular: lvmvolumegroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.parameters + name: Parameters + type: string + - jsonPath: .status.vgStatus + name: Status + type: string + - jsonPath: .spec.nodeName + name: Node + type: string + name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + desiredState: + description: |- + DesiredState is the desired state of the volume group + enabled means we will keep this vg active, disabled means we will keep this vg inactive + enum: + - Enabled + - Disabled + type: string + devices: + additionalProperties: + type: string + description: |- + The devices of the volume group + format: map[]=devPath" + e.g. map[087fc9702c450bfca5ba56b06ba7d7f2] = /dev/sda + type: object + nodeName: + description: NodeName is the name of the node where the volume group + is created + type: string + parameters: + description: Parameters is the parameters for creating the volume + group *optional* + type: string + vgName: + description: VGName is the name of the volume group + type: string + required: + - desiredState + - nodeName + - vgName + type: object + status: + properties: + conditions: + description: The conditions of the volume group + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + devices: + additionalProperties: + type: string + description: |- + The devices of the volume group + format: map[]=devPath" + type: object + parameters: + description: Parameters is the current parameters of the volume group + type: string + vgStatus: + default: Unknown + description: The status of the volume group + enum: + - Active + - Inactive + - Unknown + type: string + vgTargetType: + description: VGTargetType is the target type of the volume group, + now only support stripe/dm-thin + type: string + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/harvesterhci.io/v1beta1/types.go b/pkg/apis/harvesterhci.io/v1beta1/blockdevice.go similarity index 98% rename from pkg/apis/harvesterhci.io/v1beta1/types.go rename to pkg/apis/harvesterhci.io/v1beta1/blockdevice.go index ab16ea56..f3696cb0 100644 --- a/pkg/apis/harvesterhci.io/v1beta1/types.go +++ b/pkg/apis/harvesterhci.io/v1beta1/blockdevice.go @@ -44,6 +44,10 @@ type BlockDeviceSpec struct { Tags []string `json:"tags,omitempty"` Provisioner *ProvisionerInfo `json:"provisioner,omitempty"` + + // a bool for the device to be provisioned + // +kubebuilder:default:=false + Provision bool `json:"provision,omitempty"` } type BlockDeviceStatus struct { @@ -102,6 +106,8 @@ type FilesystemInfo struct { ForceFormatted bool `json:"forceFormatted,omitempty"` // a bool indicating whether the filesystem can be provisioned as a disk for the node to store data. + // Deprecated: Replaced by field `spec.provision` + // +optional Provisioned bool `json:"provisioned,omitempty"` // a bool indicating whether the filesystem is manually repaired of not diff --git a/pkg/apis/harvesterhci.io/v1beta1/lvm.go b/pkg/apis/harvesterhci.io/v1beta1/lvm.go new file mode 100644 index 00000000..53f02c92 --- /dev/null +++ b/pkg/apis/harvesterhci.io/v1beta1/lvm.go @@ -0,0 +1,120 @@ +package v1beta1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type ConditionType string +type VGDesireState string +type VGStatus string +type VGType string + +const ( + // VGStatusActive means the volume group is active + VGStatusActive VGStatus = "Active" + // VGStatusInactive means the volume group is inactive + VGStatusInactive VGStatus = "Inactive" + // VGStatusUnknown means the volume group is unknown + VGStatusUnknown VGStatus = "Unknown" + + // VGStateEnabled means the volume group is enabled + VGStateEnabled VGDesireState = "Enabled" + // VGStateDisabled means the volume group is disabled + VGStateDisabled VGDesireState = "Disabled" + // VGStateReconciling means the volume group is reconciling + VGStateReconciling VGDesireState = "Reconciling" + + // VGConditionReady means the volume group is ready + VGConditionReady ConditionType = "Ready" + // VGConditionAddDevice means the volume group is added a device + VGConditionAddDevice ConditionType = "AddDevice" + + // ConditionTypeActive indicates the volume group is active + ConditionTypeActive ConditionType = "Active" + // ConditionTypeInactive indicates the volume group is inactive + ConditionTypeInactive ConditionType = "Inactive" + // ConditionTypeReconciling indicates the new device is added to the volume group + ConditionTypeDeviceAdded ConditionType = "DeviceAdded" + // ConditionTypeEndpointChanged indicates the device is removed from the volume group + ConditionTypeDeviceRemoved ConditionType = "DeviceRemoved" + + // VGTypeStripe indicates the volume group is stripe + VGTypeStripe VGType = "stripe" + // VGTypeDMThin indicates the volume group is dm-thin + VGTypeDMThin VGType = "dm-thin" +) + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:shortName=lvmvg;lvmvgs,scope=Namespaced +// +kubebuilder:printcolumn:name="Parameters",type="string",JSONPath=`.spec.parameters` +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.vgStatus` +// +kubebuilder:printcolumn:name="Node",type="string",JSONPath=`.spec.nodeName` +// +kubebuilder:subresource:status + +type LVMVolumeGroup struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata"` + Spec VolumeGroupSpec `json:"spec"` + Status *VolumeGroupStatus `json:"status,omitempty"` +} + +type VolumeGroupSpec struct { + // VGName is the name of the volume group + // +kubebuilder:validation:Required + VgName string `json:"vgName"` + + // NodeName is the name of the node where the volume group is created + // +kubebuilder:validation:Required + NodeName string `json:"nodeName"` + + // DesiredState is the desired state of the volume group + // enabled means we will keep this vg active, disabled means we will keep this vg inactive + // +kubebuilder:validation:Enum:=Enabled;Disabled + DesiredState VGDesireState `json:"desiredState"` + + // The devices of the volume group + // format: map[]=devPath" + // e.g. map[087fc9702c450bfca5ba56b06ba7d7f2] = /dev/sda + // +kubebuilder:validation:Optional + Devices map[string]string `json:"devices,omitempty"` + + // Parameters is the parameters for creating the volume group *optional* + // +kubebuilder:validation:Optional + Parameters string `json:"parameters,omitempty"` +} + +type VolumeGroupStatus struct { + + // The conditions of the volume group + // +kubebuilder:validation:Optional + // +Kubebuilder:default={} + VGConditions []VolumeGroupCondition `json:"conditions,omitempty"` + + // The devices of the volume group + // format: map[]=devPath" + // +kubebuilder:validation:Optional + Devices map[string]string `json:"devices,omitempty"` + + // Parameters is the current parameters of the volume group + // +kubebuilder:validation:Optional + Parameters string `json:"parameters,omitempty"` + + // VGTargetType is the target type of the volume group, now only support stripe/dm-thin + // +kubebuilder:validation:Eum:=stripe;dm-thin + VGTargetType VGType `json:"vgTargetType,omitempty"` + + // The status of the volume group + // +kubebuilder:validation:Enum:=Active;Inactive;Unknown + // +kubebuilder:default:=Unknown + Status VGStatus `json:"vgStatus,omitempty"` +} + +type VolumeGroupCondition struct { + Type ConditionType `json:"type"` + Status corev1.ConditionStatus `json:"status"` + LastTransitionTime metav1.Time `json:"lastTransitionTime"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` +} diff --git a/pkg/apis/harvesterhci.io/v1beta1/zz_generated_deepcopy.go b/pkg/apis/harvesterhci.io/v1beta1/zz_generated_deepcopy.go index 8dc1590e..f12847e9 100644 --- a/pkg/apis/harvesterhci.io/v1beta1/zz_generated_deepcopy.go +++ b/pkg/apis/harvesterhci.io/v1beta1/zz_generated_deepcopy.go @@ -272,6 +272,71 @@ func (in *LVMProvisionerInfo) DeepCopy() *LVMProvisionerInfo { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LVMVolumeGroup) DeepCopyInto(out *LVMVolumeGroup) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = new(VolumeGroupStatus) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LVMVolumeGroup. +func (in *LVMVolumeGroup) DeepCopy() *LVMVolumeGroup { + if in == nil { + return nil + } + out := new(LVMVolumeGroup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LVMVolumeGroup) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LVMVolumeGroupList) DeepCopyInto(out *LVMVolumeGroupList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LVMVolumeGroup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LVMVolumeGroupList. +func (in *LVMVolumeGroupList) DeepCopy() *LVMVolumeGroupList { + if in == nil { + return nil + } + out := new(LVMVolumeGroupList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LVMVolumeGroupList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LonghornProvisionerInfo) DeepCopyInto(out *LonghornProvisionerInfo) { *out = *in @@ -313,3 +378,73 @@ func (in *ProvisionerInfo) DeepCopy() *ProvisionerInfo { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeGroupCondition) DeepCopyInto(out *VolumeGroupCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupCondition. +func (in *VolumeGroupCondition) DeepCopy() *VolumeGroupCondition { + if in == nil { + return nil + } + out := new(VolumeGroupCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeGroupSpec) DeepCopyInto(out *VolumeGroupSpec) { + *out = *in + if in.Devices != nil { + in, out := &in.Devices, &out.Devices + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSpec. +func (in *VolumeGroupSpec) DeepCopy() *VolumeGroupSpec { + if in == nil { + return nil + } + out := new(VolumeGroupSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeGroupStatus) DeepCopyInto(out *VolumeGroupStatus) { + *out = *in + if in.VGConditions != nil { + in, out := &in.VGConditions, &out.VGConditions + *out = make([]VolumeGroupCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Devices != nil { + in, out := &in.Devices, &out.Devices + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupStatus. +func (in *VolumeGroupStatus) DeepCopy() *VolumeGroupStatus { + if in == nil { + return nil + } + out := new(VolumeGroupStatus) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/apis/harvesterhci.io/v1beta1/zz_generated_list_types.go b/pkg/apis/harvesterhci.io/v1beta1/zz_generated_list_types.go index bf974463..e349311e 100644 --- a/pkg/apis/harvesterhci.io/v1beta1/zz_generated_list_types.go +++ b/pkg/apis/harvesterhci.io/v1beta1/zz_generated_list_types.go @@ -40,3 +40,20 @@ func NewBlockDevice(namespace, name string, obj BlockDevice) *BlockDevice { obj.Namespace = namespace return &obj } + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// LVMVolumeGroupList is a list of LVMVolumeGroup resources +type LVMVolumeGroupList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []LVMVolumeGroup `json:"items"` +} + +func NewLVMVolumeGroup(namespace, name string, obj LVMVolumeGroup) *LVMVolumeGroup { + obj.APIVersion, obj.Kind = SchemeGroupVersion.WithKind("LVMVolumeGroup").ToAPIVersionAndKind() + obj.Name = name + obj.Namespace = namespace + return &obj +} diff --git a/pkg/apis/harvesterhci.io/v1beta1/zz_generated_register.go b/pkg/apis/harvesterhci.io/v1beta1/zz_generated_register.go index 39ab67b6..5e5f6110 100644 --- a/pkg/apis/harvesterhci.io/v1beta1/zz_generated_register.go +++ b/pkg/apis/harvesterhci.io/v1beta1/zz_generated_register.go @@ -28,7 +28,8 @@ import ( ) var ( - BlockDeviceResourceName = "blockdevices" + BlockDeviceResourceName = "blockdevices" + LVMVolumeGroupResourceName = "lvmvolumegroups" ) // SchemeGroupVersion is group version used to register these objects @@ -54,6 +55,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &BlockDevice{}, &BlockDeviceList{}, + &LVMVolumeGroup{}, + &LVMVolumeGroupList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/pkg/codegen/main.go b/pkg/codegen/main.go index 8d5fe5fe..cd8d7f60 100644 --- a/pkg/codegen/main.go +++ b/pkg/codegen/main.go @@ -19,6 +19,7 @@ func main() { "harvesterhci.io": { Types: []interface{}{ diskv1.BlockDevice{}, + diskv1.LVMVolumeGroup{}, }, GenerateTypes: true, GenerateClients: true, diff --git a/pkg/controller/blockdevice/controller.go b/pkg/controller/blockdevice/controller.go index e2b5e864..2e902e47 100644 --- a/pkg/controller/blockdevice/controller.go +++ b/pkg/controller/blockdevice/controller.go @@ -38,6 +38,8 @@ type Controller struct { BlockdeviceCache ctldiskv1.BlockDeviceCache BlockInfo block.Info + LVMVgClient ctldiskv1.LVMVolumeGroupController + scanner *Scanner semaphore *provisioner.Semaphore } @@ -61,6 +63,7 @@ func Register( ctx context.Context, nodes ctllonghornv1.NodeController, bds ctldiskv1.BlockDeviceController, + lvmVGs ctldiskv1.LVMVolumeGroupController, block block.Info, opt *option.Option, scanner *Scanner, @@ -74,6 +77,7 @@ func Register( Nodes: nodes, Blockdevices: bds, BlockdeviceCache: bds.Cache(), + LVMVgClient: lvmVGs, BlockInfo: block, scanner: scanner, semaphore: semaphoreObj, @@ -107,6 +111,10 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) ( logrus.Warnf("Failed to generate provisioner for device %s: %v", device.Name, err) return nil, err } + if provisionerInst == nil { + logrus.Infof("Skip device %s as no provisioner found or not configured", device.Name) + return nil, nil + } // handle remove device no matter inactive or corrupted, we will set `device.Spec.FileSystem.Provisioned` to false if needProvisionerUnprovision(device) { @@ -133,6 +141,7 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) ( return nil, fmt.Errorf("failed to resolve persistent dev path for block device %s", device.Name) } + logrus.Debugf("Checking to format device %s", device.Name) if formatted, requeue, err := provisionerInst.Format(devPath); !formatted { if requeue { c.Blockdevices.EnqueueAfter(c.Namespace, device.Name, jitterEnqueueDelay()) @@ -153,6 +162,7 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) ( * 2. Spec.Filesystem.Provisioned = true, Status.ProvisionPhase = ProvisionPhaseUnprovisioned * -> Provision the device */ + logrus.Debugf("Checking to provision/update device %s", device.Name) if needProvisionerUpdate(device, deviceCpy) { logrus.Infof("Prepare to check the new device tags %v with device: %s", deviceCpy.Spec.Tags, device.Name) requeue, err := provisionerInst.Update() @@ -200,12 +210,26 @@ func (c *Controller) finalizeBlockDevice(oldBd, newBd *diskv1.BlockDevice, devPa } func (c *Controller) generateProvisioner(device *diskv1.BlockDevice) (provisioner.Provisioner, error) { + if device.Spec.Provisioner == nil && device.Status.ProvisionPhase != diskv1.ProvisionPhaseProvisioned { + return nil, nil + } + logrus.Infof("Generate provisioner from device %s, content: %v", device.Name, device.Spec.Provisioner) // set default provisionerType := provisioner.TypeLonghornV1 if device.Spec.Provisioner != nil { + // **TODO**: we should use webhook to validate the provisioner type (and number) + numProvisioner := 0 if device.Spec.Provisioner.Longhorn != nil { + numProvisioner++ provisionerType = device.Spec.Provisioner.Longhorn.EngineVersion } + if device.Spec.Provisioner.LVM != nil { + numProvisioner++ + provisionerType = provisioner.TypeLVM + } + if numProvisioner > 1 { + return nil, fmt.Errorf("multiple provisioner types found for block device %s", device.Name) + } } switch provisionerType { case provisioner.TypeLonghornV1: @@ -220,7 +244,7 @@ func (c *Controller) generateProvisioner(device *diskv1.BlockDevice) (provisione case provisioner.TypeLonghornV2: return nil, fmt.Errorf("TBD type %s", provisionerType) case provisioner.TypeLVM: - return nil, fmt.Errorf("TBD type %s", provisionerType) + return c.generateLVMProvisioner(device) default: return nil, fmt.Errorf("unsupported provisioner type %s", provisionerType) } @@ -237,6 +261,11 @@ func (c *Controller) generateLHv1Provisioner(device *diskv1.BlockDevice) (provis return provisioner.NewLHV1Provisioner(device, c.BlockInfo, node, c.Nodes, c.NodeCache, CacheDiskTags, c.semaphore) } +func (c *Controller) generateLVMProvisioner(device *diskv1.BlockDevice) (provisioner.Provisioner, error) { + vgName := device.Spec.Provisioner.LVM.VgName + return provisioner.NewLVMProvisioner(vgName, c.NodeName, c.LVMVgClient, device, c.BlockInfo) +} + func (c *Controller) updateDeviceStatus(device *diskv1.BlockDevice, devPath string) error { var newStatus diskv1.DeviceStatus var needAutoProvision bool @@ -365,13 +394,16 @@ func canSkipBlockDeviceChange(device *diskv1.BlockDevice, nodeName string) bool } func needProvisionerUnprovision(device *diskv1.BlockDevice) bool { - return !device.Spec.FileSystem.Provisioned && device.Status.ProvisionPhase != diskv1.ProvisionPhaseUnprovisioned + return (!device.Spec.FileSystem.Provisioned && !device.Spec.Provision) && + device.Status.ProvisionPhase != diskv1.ProvisionPhaseUnprovisioned } func needProvisionerUpdate(oldBd, newBd *diskv1.BlockDevice) bool { - return oldBd.Status.ProvisionPhase == diskv1.ProvisionPhaseProvisioned && newBd.Spec.FileSystem.Provisioned + return oldBd.Status.ProvisionPhase == diskv1.ProvisionPhaseProvisioned && + (newBd.Spec.FileSystem.Provisioned || newBd.Spec.Provision) } func needProvisionerProvision(oldBd, newBd *diskv1.BlockDevice) bool { - return oldBd.Status.ProvisionPhase == diskv1.ProvisionPhaseUnprovisioned && newBd.Spec.FileSystem.Provisioned + return oldBd.Status.ProvisionPhase == diskv1.ProvisionPhaseUnprovisioned && + (newBd.Spec.FileSystem.Provisioned || newBd.Spec.Provision) } diff --git a/pkg/controller/volumegroup/controller.go b/pkg/controller/volumegroup/controller.go new file mode 100644 index 00000000..f7e11939 --- /dev/null +++ b/pkg/controller/volumegroup/controller.go @@ -0,0 +1,265 @@ +package volumegroup + +import ( + "context" + "fmt" + "maps" + "reflect" + "strings" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + diskv1 "github.com/harvester/node-disk-manager/pkg/apis/harvesterhci.io/v1beta1" + ctldiskv1 "github.com/harvester/node-disk-manager/pkg/generated/controllers/harvesterhci.io/v1beta1" + "github.com/harvester/node-disk-manager/pkg/lvm" + "github.com/harvester/node-disk-manager/pkg/option" +) + +type Controller struct { + namespace string + nodeName string + + LVMVolumeGroupCache ctldiskv1.LVMVolumeGroupCache + LVMVolumeGroups ctldiskv1.LVMVolumeGroupController +} + +const ( + lvmVGHandlerName = "harvester-lvm-volumegroup-handler" +) + +func Register(ctx context.Context, lvmVGs ctldiskv1.LVMVolumeGroupController, opt *option.Option) error { + + c := &Controller{ + namespace: opt.Namespace, + nodeName: opt.NodeName, + LVMVolumeGroups: lvmVGs, + LVMVolumeGroupCache: lvmVGs.Cache(), + } + + c.LVMVolumeGroups.OnChange(ctx, lvmVGHandlerName, c.OnLVMVGChange) + c.LVMVolumeGroups.OnRemove(ctx, lvmVGHandlerName, c.OnLVMVGRemove) + return nil +} + +func (c *Controller) OnLVMVGChange(_ string, lvmVG *diskv1.LVMVolumeGroup) (*diskv1.LVMVolumeGroup, error) { + if lvmVG == nil || lvmVG.DeletionTimestamp != nil { + logrus.Infof("Skip this round because lvm volume group is deleted or deleting") + return nil, nil + } + + if lvmVG.Spec.NodeName != c.nodeName { + logrus.Infof("Skip this round because lvm volume group is not belong to this node") + return nil, nil + } + + logrus.Infof("Prepare to handle LVMVolumeGroup %s changed: %v", lvmVG.Name, lvmVG) + + switch lvmVG.Spec.DesiredState { + case diskv1.VGStateEnabled: + logrus.Infof("Prepare to enable LVMVolumeGroup %s", lvmVG.Name) + return c.updateEnabledLVMVolumeGroup(lvmVG) + case diskv1.VGStateDisabled: + // should only called manually set the VGState to Disabled + logrus.Infof("Prepare to disable LVMVolumeGroup %s", lvmVG.Name) + return c.disableLVMVolumeGroup(lvmVG) + } + return nil, nil +} + +func (c *Controller) OnLVMVGRemove(_ string, lvmVG *diskv1.LVMVolumeGroup) (*diskv1.LVMVolumeGroup, error) { + if lvmVG == nil || lvmVG.DeletionTimestamp != nil { + // make sure the volume group is already deleted + logrus.Infof("Ensure the lvm volume group is already deleted if the lvmVG CR is nil") + return c.removeLVMVolumeGroup(lvmVG) + } + + return nil, nil +} + +func (c *Controller) updateEnabledLVMVolumeGroup(lvmVG *diskv1.LVMVolumeGroup) (*diskv1.LVMVolumeGroup, error) { + logrus.Infof("Enable LVMVolumeGroup %s", lvmVG.Name) + + pvsResult, err := lvm.GetPVScanResult() + if err != nil { + return nil, fmt.Errorf("failed to get pvscan result. %v", err) + } + logrus.Debugf("pvscan result: %v", pvsResult) + currentDevs := map[string]string{} + if lvmVG.Status != nil && lvmVG.Status.Devices != nil && len(lvmVG.Status.Devices) > 0 { + currentDevs = lvmVG.Status.Devices + } + if maps.Equal(currentDevs, lvmVG.Spec.Devices) { + logrus.Info("Skip this round because the devices are not changed") + return nil, nil + } + lvmVGCpy := lvmVG.DeepCopy() + if lvmVGCpy.Status == nil { + lvmVGCpy.Status = &diskv1.VolumeGroupStatus{} + } + + if lvmVG.Status != nil && len(lvmVG.Status.Devices) == 0 { + logrus.Warnf("No devices found in LVMVolumeGroup %s, skip", lvmVG.Name) + return nil, nil + } + // update devices + toAdd := getToAddDevs(lvmVG.Spec.Devices, currentDevs) + toRemove := getToRemoveDevs(lvmVG.Spec.Devices, currentDevs) + err = updatePVAndVG(lvmVGCpy, toAdd, toRemove, pvsResult) + if err != nil { + return nil, err + } + + vgConds := diskv1.VolumeGroupCondition{ + Type: diskv1.VGConditionReady, + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.Now(), + Reason: "Volume Group is Ready", + Message: fmt.Sprintf("Volume Group is Ready with devices %v", lvmVG.Spec.Devices), + } + newConds := UpdateLVMVGsConds(lvmVGCpy.Status.VGConditions, vgConds) + lvmVGCpy.Status.VGConditions = newConds + lvmVGCpy.Status.Status = diskv1.VGStatusActive + if !reflect.DeepEqual(lvmVG, lvmVGCpy) { + return c.LVMVolumeGroups.UpdateStatus(lvmVGCpy) + } + return nil, nil +} + +func (c *Controller) disableLVMVolumeGroup(lvmVG *diskv1.LVMVolumeGroup) (*diskv1.LVMVolumeGroup, error) { + logrus.Infof("Disable LVMVolumeGroup %s", lvmVG.Spec.VgName) + err := lvm.DoVGDeactive(lvmVG.Spec.VgName) + if err != nil { + if apierrors.IsNotFound(err) { + logrus.Infof("VolumeGroup %s is not found, skip", lvmVG.Spec.VgName) + } else { + return nil, fmt.Errorf("failed to remove VG %s. %v", lvmVG.Spec.VgName, err) + } + } + return nil, nil +} + +func (c *Controller) removeLVMVolumeGroup(lvmVG *diskv1.LVMVolumeGroup) (*diskv1.LVMVolumeGroup, error) { + logrus.Infof("Remove LVMVolumeGroup %s", lvmVG.Name) + err := lvm.DoVGRemove(lvmVG.Spec.VgName) + if err != nil { + if strings.Contains(err.Error(), "not found") { + logrus.Infof("VolumeGroup %s is not found, skip", lvmVG.Spec.VgName) + } else { + return nil, fmt.Errorf("failed to remove VG %s. %v", lvmVG.Spec.VgName, err) + } + } + + return nil, nil +} + +func checkPVAndVG(pvsResult map[string]string, targetPV, targetVG string) (pvFound, vgFound bool, pvCount int) { + pvCount = 0 + for pv, vg := range pvsResult { + if pv == targetPV { + pvFound = true + if vg == targetVG { + pvCount++ + vgFound = true + continue + } + } + if vg == targetVG { + vgFound = true + pvCount++ + } + } + return +} + +func UpdateLVMVGsConds(curConds []diskv1.VolumeGroupCondition, c diskv1.VolumeGroupCondition) []diskv1.VolumeGroupCondition { + found := false + var pod = 0 + logrus.Infof("Prepare to check the coming Type: %s, Status: %s", c.Type, c.Status) + for id, cond := range curConds { + if cond.Type == c.Type { + found = true + pod = id + break + } + } + + if found { + curConds[pod] = c + } else { + curConds = append(curConds, c) + } + return curConds + +} + +func updatePVAndVG(vgCpy *diskv1.LVMVolumeGroup, toAdd, toRemove map[string]string, pvsResult map[string]string) error { + logrus.Infof("Prepare to add devices: %v", toAdd) + for bdName, dev := range toAdd { + pvFound, vgFound, _ := checkPVAndVG(pvsResult, dev, vgCpy.Spec.VgName) + logrus.Infof("pvFound: %v, vgFound: %v", pvFound, vgFound) + if !vgFound { + if err := lvm.DoVGCreate(dev, vgCpy.Spec.VgName); err != nil { + return err + } + } + if !pvFound { + if err := lvm.DoPVCreate(dev); err != nil { + return err + } + if err := lvm.DoVGExtend(dev, vgCpy.Spec.VgName); err != nil { + return err + } + } + if vgCpy.Status.Devices == nil { + vgCpy.Status.Devices = map[string]string{} + } + vgCpy.Status.Devices[bdName] = dev + } + logrus.Infof("Prepare to remove devices: %v", toRemove) + for bdName, dev := range toRemove { + pvFound, vgFound, pvInVGCounts := checkPVAndVG(pvsResult, dev, vgCpy.Spec.VgName) + logrus.Infof("pvFound: %v, vgFound: %v, pvInVGCounts: %v", pvFound, vgFound, pvInVGCounts) + if !pvFound { + logrus.Infof("Block device %s is not in pvs, return directly!", bdName) + return nil + } + // handle if vg is found + if vgFound { + if pvInVGCounts > 1 { + if err := lvm.DoVGReduce(dev, vgCpy.Spec.VgName); err != nil { + return err + } + } else { + if err := lvm.DoVGRemove(vgCpy.Spec.VgName); err != nil { + return err + } + } + } + lvm.DoPVRemove(dev) + delete(vgCpy.Status.Devices, bdName) + } + return nil +} + +func getToAddDevs(specDevs, currentDevs map[string]string) map[string]string { + toAdd := map[string]string{} + for bdName, dev := range specDevs { + if _, found := currentDevs[bdName]; !found { + toAdd[bdName] = dev + } + } + return toAdd +} + +func getToRemoveDevs(specDevs, currentDevs map[string]string) map[string]string { + toRemove := map[string]string{} + for bdName := range currentDevs { + if _, found := specDevs[bdName]; !found { + toRemove[bdName] = currentDevs[bdName] + } + } + return toRemove +} diff --git a/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/fake/fake_harvesterhci.io_client.go b/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/fake/fake_harvesterhci.io_client.go index df70f243..09c2b200 100644 --- a/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/fake/fake_harvesterhci.io_client.go +++ b/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/fake/fake_harvesterhci.io_client.go @@ -32,6 +32,10 @@ func (c *FakeHarvesterhciV1beta1) BlockDevices(namespace string) v1beta1.BlockDe return &FakeBlockDevices{c, namespace} } +func (c *FakeHarvesterhciV1beta1) LVMVolumeGroups(namespace string) v1beta1.LVMVolumeGroupInterface { + return &FakeLVMVolumeGroups{c, namespace} +} + // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *FakeHarvesterhciV1beta1) RESTClient() rest.Interface { diff --git a/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/fake/fake_lvmvolumegroup.go b/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/fake/fake_lvmvolumegroup.go new file mode 100644 index 00000000..e200bba5 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/fake/fake_lvmvolumegroup.go @@ -0,0 +1,141 @@ +/* +Copyright 2024 Rancher Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by main. DO NOT EDIT. + +package fake + +import ( + "context" + + v1beta1 "github.com/harvester/node-disk-manager/pkg/apis/harvesterhci.io/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeLVMVolumeGroups implements LVMVolumeGroupInterface +type FakeLVMVolumeGroups struct { + Fake *FakeHarvesterhciV1beta1 + ns string +} + +var lvmvolumegroupsResource = v1beta1.SchemeGroupVersion.WithResource("lvmvolumegroups") + +var lvmvolumegroupsKind = v1beta1.SchemeGroupVersion.WithKind("LVMVolumeGroup") + +// Get takes name of the lVMVolumeGroup, and returns the corresponding lVMVolumeGroup object, and an error if there is any. +func (c *FakeLVMVolumeGroups) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.LVMVolumeGroup, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(lvmvolumegroupsResource, c.ns, name), &v1beta1.LVMVolumeGroup{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.LVMVolumeGroup), err +} + +// List takes label and field selectors, and returns the list of LVMVolumeGroups that match those selectors. +func (c *FakeLVMVolumeGroups) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.LVMVolumeGroupList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(lvmvolumegroupsResource, lvmvolumegroupsKind, c.ns, opts), &v1beta1.LVMVolumeGroupList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1beta1.LVMVolumeGroupList{ListMeta: obj.(*v1beta1.LVMVolumeGroupList).ListMeta} + for _, item := range obj.(*v1beta1.LVMVolumeGroupList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested lVMVolumeGroups. +func (c *FakeLVMVolumeGroups) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(lvmvolumegroupsResource, c.ns, opts)) + +} + +// Create takes the representation of a lVMVolumeGroup and creates it. Returns the server's representation of the lVMVolumeGroup, and an error, if there is any. +func (c *FakeLVMVolumeGroups) Create(ctx context.Context, lVMVolumeGroup *v1beta1.LVMVolumeGroup, opts v1.CreateOptions) (result *v1beta1.LVMVolumeGroup, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(lvmvolumegroupsResource, c.ns, lVMVolumeGroup), &v1beta1.LVMVolumeGroup{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.LVMVolumeGroup), err +} + +// Update takes the representation of a lVMVolumeGroup and updates it. Returns the server's representation of the lVMVolumeGroup, and an error, if there is any. +func (c *FakeLVMVolumeGroups) Update(ctx context.Context, lVMVolumeGroup *v1beta1.LVMVolumeGroup, opts v1.UpdateOptions) (result *v1beta1.LVMVolumeGroup, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(lvmvolumegroupsResource, c.ns, lVMVolumeGroup), &v1beta1.LVMVolumeGroup{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.LVMVolumeGroup), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeLVMVolumeGroups) UpdateStatus(ctx context.Context, lVMVolumeGroup *v1beta1.LVMVolumeGroup, opts v1.UpdateOptions) (*v1beta1.LVMVolumeGroup, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(lvmvolumegroupsResource, "status", c.ns, lVMVolumeGroup), &v1beta1.LVMVolumeGroup{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.LVMVolumeGroup), err +} + +// Delete takes name of the lVMVolumeGroup and deletes it. Returns an error if one occurs. +func (c *FakeLVMVolumeGroups) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(lvmvolumegroupsResource, c.ns, name, opts), &v1beta1.LVMVolumeGroup{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeLVMVolumeGroups) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(lvmvolumegroupsResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1beta1.LVMVolumeGroupList{}) + return err +} + +// Patch applies the patch and returns the patched lVMVolumeGroup. +func (c *FakeLVMVolumeGroups) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.LVMVolumeGroup, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(lvmvolumegroupsResource, c.ns, name, pt, data, subresources...), &v1beta1.LVMVolumeGroup{}) + + if obj == nil { + return nil, err + } + return obj.(*v1beta1.LVMVolumeGroup), err +} diff --git a/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/generated_expansion.go index 290c44e2..f7bfb724 100644 --- a/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/generated_expansion.go @@ -19,3 +19,5 @@ limitations under the License. package v1beta1 type BlockDeviceExpansion interface{} + +type LVMVolumeGroupExpansion interface{} diff --git a/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/harvesterhci.io_client.go b/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/harvesterhci.io_client.go index fdfeedd0..763a4fc3 100644 --- a/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/harvesterhci.io_client.go +++ b/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/harvesterhci.io_client.go @@ -29,6 +29,7 @@ import ( type HarvesterhciV1beta1Interface interface { RESTClient() rest.Interface BlockDevicesGetter + LVMVolumeGroupsGetter } // HarvesterhciV1beta1Client is used to interact with features provided by the harvesterhci.io group. @@ -40,6 +41,10 @@ func (c *HarvesterhciV1beta1Client) BlockDevices(namespace string) BlockDeviceIn return newBlockDevices(c, namespace) } +func (c *HarvesterhciV1beta1Client) LVMVolumeGroups(namespace string) LVMVolumeGroupInterface { + return newLVMVolumeGroups(c, namespace) +} + // NewForConfig creates a new HarvesterhciV1beta1Client for the given config. // NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), // where httpClient was generated with rest.HTTPClientFor(c). diff --git a/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/lvmvolumegroup.go b/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/lvmvolumegroup.go new file mode 100644 index 00000000..0fea1095 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/harvesterhci.io/v1beta1/lvmvolumegroup.go @@ -0,0 +1,195 @@ +/* +Copyright 2024 Rancher Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by main. DO NOT EDIT. + +package v1beta1 + +import ( + "context" + "time" + + v1beta1 "github.com/harvester/node-disk-manager/pkg/apis/harvesterhci.io/v1beta1" + scheme "github.com/harvester/node-disk-manager/pkg/generated/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// LVMVolumeGroupsGetter has a method to return a LVMVolumeGroupInterface. +// A group's client should implement this interface. +type LVMVolumeGroupsGetter interface { + LVMVolumeGroups(namespace string) LVMVolumeGroupInterface +} + +// LVMVolumeGroupInterface has methods to work with LVMVolumeGroup resources. +type LVMVolumeGroupInterface interface { + Create(ctx context.Context, lVMVolumeGroup *v1beta1.LVMVolumeGroup, opts v1.CreateOptions) (*v1beta1.LVMVolumeGroup, error) + Update(ctx context.Context, lVMVolumeGroup *v1beta1.LVMVolumeGroup, opts v1.UpdateOptions) (*v1beta1.LVMVolumeGroup, error) + UpdateStatus(ctx context.Context, lVMVolumeGroup *v1beta1.LVMVolumeGroup, opts v1.UpdateOptions) (*v1beta1.LVMVolumeGroup, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1beta1.LVMVolumeGroup, error) + List(ctx context.Context, opts v1.ListOptions) (*v1beta1.LVMVolumeGroupList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.LVMVolumeGroup, err error) + LVMVolumeGroupExpansion +} + +// lVMVolumeGroups implements LVMVolumeGroupInterface +type lVMVolumeGroups struct { + client rest.Interface + ns string +} + +// newLVMVolumeGroups returns a LVMVolumeGroups +func newLVMVolumeGroups(c *HarvesterhciV1beta1Client, namespace string) *lVMVolumeGroups { + return &lVMVolumeGroups{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the lVMVolumeGroup, and returns the corresponding lVMVolumeGroup object, and an error if there is any. +func (c *lVMVolumeGroups) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.LVMVolumeGroup, err error) { + result = &v1beta1.LVMVolumeGroup{} + err = c.client.Get(). + Namespace(c.ns). + Resource("lvmvolumegroups"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of LVMVolumeGroups that match those selectors. +func (c *lVMVolumeGroups) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.LVMVolumeGroupList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1beta1.LVMVolumeGroupList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("lvmvolumegroups"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested lVMVolumeGroups. +func (c *lVMVolumeGroups) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("lvmvolumegroups"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a lVMVolumeGroup and creates it. Returns the server's representation of the lVMVolumeGroup, and an error, if there is any. +func (c *lVMVolumeGroups) Create(ctx context.Context, lVMVolumeGroup *v1beta1.LVMVolumeGroup, opts v1.CreateOptions) (result *v1beta1.LVMVolumeGroup, err error) { + result = &v1beta1.LVMVolumeGroup{} + err = c.client.Post(). + Namespace(c.ns). + Resource("lvmvolumegroups"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(lVMVolumeGroup). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a lVMVolumeGroup and updates it. Returns the server's representation of the lVMVolumeGroup, and an error, if there is any. +func (c *lVMVolumeGroups) Update(ctx context.Context, lVMVolumeGroup *v1beta1.LVMVolumeGroup, opts v1.UpdateOptions) (result *v1beta1.LVMVolumeGroup, err error) { + result = &v1beta1.LVMVolumeGroup{} + err = c.client.Put(). + Namespace(c.ns). + Resource("lvmvolumegroups"). + Name(lVMVolumeGroup.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(lVMVolumeGroup). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *lVMVolumeGroups) UpdateStatus(ctx context.Context, lVMVolumeGroup *v1beta1.LVMVolumeGroup, opts v1.UpdateOptions) (result *v1beta1.LVMVolumeGroup, err error) { + result = &v1beta1.LVMVolumeGroup{} + err = c.client.Put(). + Namespace(c.ns). + Resource("lvmvolumegroups"). + Name(lVMVolumeGroup.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(lVMVolumeGroup). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the lVMVolumeGroup and deletes it. Returns an error if one occurs. +func (c *lVMVolumeGroups) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("lvmvolumegroups"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *lVMVolumeGroups) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("lvmvolumegroups"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched lVMVolumeGroup. +func (c *lVMVolumeGroups) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.LVMVolumeGroup, err error) { + result = &v1beta1.LVMVolumeGroup{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("lvmvolumegroups"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/generated/controllers/harvesterhci.io/v1beta1/interface.go b/pkg/generated/controllers/harvesterhci.io/v1beta1/interface.go index 077b7d73..fc50bea1 100644 --- a/pkg/generated/controllers/harvesterhci.io/v1beta1/interface.go +++ b/pkg/generated/controllers/harvesterhci.io/v1beta1/interface.go @@ -32,6 +32,7 @@ func init() { type Interface interface { BlockDevice() BlockDeviceController + LVMVolumeGroup() LVMVolumeGroupController } func New(controllerFactory controller.SharedControllerFactory) Interface { @@ -47,3 +48,7 @@ type version struct { func (v *version) BlockDevice() BlockDeviceController { return generic.NewController[*v1beta1.BlockDevice, *v1beta1.BlockDeviceList](schema.GroupVersionKind{Group: "harvesterhci.io", Version: "v1beta1", Kind: "BlockDevice"}, "blockdevices", true, v.controllerFactory) } + +func (v *version) LVMVolumeGroup() LVMVolumeGroupController { + return generic.NewController[*v1beta1.LVMVolumeGroup, *v1beta1.LVMVolumeGroupList](schema.GroupVersionKind{Group: "harvesterhci.io", Version: "v1beta1", Kind: "LVMVolumeGroup"}, "lvmvolumegroups", true, v.controllerFactory) +} diff --git a/pkg/generated/controllers/harvesterhci.io/v1beta1/lvmvolumegroup.go b/pkg/generated/controllers/harvesterhci.io/v1beta1/lvmvolumegroup.go new file mode 100644 index 00000000..bf13f44d --- /dev/null +++ b/pkg/generated/controllers/harvesterhci.io/v1beta1/lvmvolumegroup.go @@ -0,0 +1,39 @@ +/* +Copyright 2024 Rancher Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by main. DO NOT EDIT. + +package v1beta1 + +import ( + v1beta1 "github.com/harvester/node-disk-manager/pkg/apis/harvesterhci.io/v1beta1" + "github.com/rancher/wrangler/v3/pkg/generic" +) + +// LVMVolumeGroupController interface for managing LVMVolumeGroup resources. +type LVMVolumeGroupController interface { + generic.ControllerInterface[*v1beta1.LVMVolumeGroup, *v1beta1.LVMVolumeGroupList] +} + +// LVMVolumeGroupClient interface for managing LVMVolumeGroup resources in Kubernetes. +type LVMVolumeGroupClient interface { + generic.ClientInterface[*v1beta1.LVMVolumeGroup, *v1beta1.LVMVolumeGroupList] +} + +// LVMVolumeGroupCache interface for retrieving LVMVolumeGroup resources in memory. +type LVMVolumeGroupCache interface { + generic.CacheInterface[*v1beta1.LVMVolumeGroup] +} diff --git a/pkg/lvm/lvm.go b/pkg/lvm/lvm.go new file mode 100644 index 00000000..46467a95 --- /dev/null +++ b/pkg/lvm/lvm.go @@ -0,0 +1,101 @@ +package lvm + +import ( + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + + "github.com/harvester/node-disk-manager/pkg/utils" +) + +const ( + LVMTopoKeyNode = "topology.lvm.csi/node" +) + +func GetPVScanResult() (map[string]string, error) { + ns := utils.GetHostNamespacePath(utils.HostProcPath) + executor, err := utils.NewExecutorWithNS(ns) + if err != nil { + return nil, fmt.Errorf("generate executor failed. %v", err) + } + + args := []string{"--noheadings", "-o", "pv_name,vg_name"} + output, err := executor.Execute("pvs", args) + if err != nil { + return nil, fmt.Errorf("failed to execute 'pvs' command: %v", err) + } + lines := strings.Split(output, "\n") + pvScanResult := make(map[string]string) + for _, line := range lines { + if line == "" { + continue + } + fields := strings.Fields(line) + // Format should be like: /dev/sda vg01 + pv := fields[0] + vg := "" + if len(fields) >= 2 { + vg = fields[1] + } + pvScanResult[pv] = vg + } + return pvScanResult, nil +} + +func executeCommandWithNS(cmd string, args []string) error { + ns := utils.GetHostNamespacePath(utils.HostProcPath) + executor, err := utils.NewExecutorWithNS(ns) + if err != nil { + return fmt.Errorf("generate executor failed: %v", err) + } + + _, err = executor.Execute(cmd, args) + if err != nil { + return fmt.Errorf("execute command '%s' with args '%v' failed: %v", cmd, args, err) + } + return nil +} + +func DoPVCreate(devPath string) error { + return executeCommandWithNS("pvcreate", []string{devPath}) +} + +func DoVGCreate(devPath, vgName string) error { + return executeCommandWithNS("vgcreate", []string{vgName, devPath}) +} + +func DoVGExtend(devPath, vgName string) error { + return executeCommandWithNS("vgextend", []string{vgName, devPath}) +} + +func DoVGReduce(devPath, vgName string) error { + return executeCommandWithNS("vgreduce", []string{vgName, devPath}) +} + +func DoVGRemove(vgName string) error { + return executeCommandWithNS("vgremove", []string{vgName}) +} + +func DoPVRemove(devPath string) error { + return executeCommandWithNS("pvremove", []string{devPath}) +} + +func DoVGActivate(vgName string) error { + return executeCommandWithNS("vgchange", []string{"--activate", "y", vgName}) +} + +func DoVGDeactive(vgName string) error { + return executeCommandWithNS("vgchange", []string{"--activate", "n", vgName}) +} + +func GenerateSelector(nodeName string) (labels.Selector, error) { + nodeReq, err := labels.NewRequirement(LVMTopoKeyNode, selection.Equals, []string{nodeName}) + if err != nil { + return nil, fmt.Errorf("error creating requirement: %v", err) + } + lvmVGSelector := labels.NewSelector() + lvmVGSelector = lvmVGSelector.Add(*nodeReq) + return lvmVGSelector, nil +} diff --git a/pkg/provisioner/lvm.go b/pkg/provisioner/lvm.go new file mode 100644 index 00000000..387db25a --- /dev/null +++ b/pkg/provisioner/lvm.go @@ -0,0 +1,236 @@ +package provisioner + +import ( + "fmt" + "reflect" + + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + diskv1 "github.com/harvester/node-disk-manager/pkg/apis/harvesterhci.io/v1beta1" + "github.com/harvester/node-disk-manager/pkg/block" + ctldiskv1 "github.com/harvester/node-disk-manager/pkg/generated/controllers/harvesterhci.io/v1beta1" + "github.com/harvester/node-disk-manager/pkg/lvm" + "github.com/harvester/node-disk-manager/pkg/utils" +) + +type LVMProvisioner struct { + *provisioner + vgName string + nodeName string + vgClient ctldiskv1.LVMVolumeGroupController +} + +func NewLVMProvisioner(vgName, nodeName string, lvmVGs ctldiskv1.LVMVolumeGroupController, device *diskv1.BlockDevice, blockInfo block.Info) (Provisioner, error) { + baseProvisioner := &provisioner{ + name: TypeLVM, + blockInfo: blockInfo, + device: device, + } + return &LVMProvisioner{ + provisioner: baseProvisioner, + vgName: vgName, + vgClient: lvmVGs, + nodeName: nodeName, + }, nil +} + +func (l *LVMProvisioner) GetProvisionerName() string { + return l.name +} + +func (l *LVMProvisioner) Format(string) (bool, bool, error) { + // LVM provisioner does not need format + return true, false, nil +} + +func (l *LVMProvisioner) UnFormat() (bool, error) { + // LVM provisioner does not need unformat + return false, nil +} + +// Provision creates (if needed) a LVMVolumeGroup CRD and update the corresponding fields. +func (l *LVMProvisioner) Provision() (bool, error) { + logrus.Infof("Provisioning block device %s to vg: %s", l.device.Name, l.vgName) + found := true + lvmvg, err := l.getTargetLVMVG() + if err != nil { + if !errors.IsNotFound(err) { + return true, err + } + found = false + } + requeue, err := l.addDevOrCreateLVMVgCRD(lvmvg, found) + if err != nil { + return requeue, err + } + + // first round the lvmvg must be nil, so we need to check it. + if lvmvg != nil && lvmvg.Status != nil && lvmvg.Status.Status == diskv1.VGStatusActive { + setCondDiskAddedToNodeTrue(l.device, fmt.Sprintf("Added disk %s to volume group %s ", l.device.Name, l.vgName), diskv1.ProvisionPhaseProvisioned) + return false, nil + } + return true, nil +} + +// UnProvision update the LVMVolumeGroup CRD and remove the LVMVolumeGroup CRD if the device is the last one in the VG. +func (l *LVMProvisioner) UnProvision() (bool, error) { + logrus.Infof("Unprovisioning block device %s from vg: %s", l.device.Name, l.vgName) + lvmvg, err := l.getTargetLVMVG() + if err != nil { + if errors.IsNotFound(err) { + // do nothing if the LVMVolumeGroup CRD is not found + logrus.Warn("CR LVMVolumeGroup is not found, skip UnProvision") + msg := fmt.Sprintf("Removed disk %s from volume group %s ", l.device.Name, l.vgName) + setCondDiskAddedToNodeFalse(l.device, msg, diskv1.ProvisionPhaseUnprovisioned) + return false, nil + } + return true, err + } + logrus.Infof("%s unprovisioning block device %s from vg: %s", l.name, l.device.Name, l.vgName) + requeue, err := l.removeDevFromLVMVgCRD(lvmvg, l.device.Name) + if err != nil { + return requeue, err + } + if lvmvg.Status != nil { + if _, found := lvmvg.Status.Devices[l.device.Name]; !found { + msg := fmt.Sprintf("Removed disk %s from volume group %s ", l.device.Name, l.vgName) + setCondDiskAddedToNodeFalse(l.device, msg, diskv1.ProvisionPhaseUnprovisioned) + return false, nil + } + } + // waiting the device removed from the LVMVolumeGroup CRD + logrus.Infof("Waiting for the device %s removed from the LVMVolumeGroup CRD %v", l.device.Name, lvmvg) + return true, nil +} + +func (l *LVMProvisioner) Update() (requeue bool, err error) { + // Update DesiredState to Reconciling + logrus.Infof("Prepare to Update LVMVolumeGroup %s", l.vgName) + lvmvg, err := l.getTargetLVMVG() + if err != nil { + if errors.IsNotFound(err) { + return true, fmt.Errorf("failed to get LVMVolumeGroup %s, err: %v", l.vgName, err) + } + return true, err + } + + if lvmvg.Spec.DesiredState == diskv1.VGStateEnabled { + // make sure the volume group is active + err := lvm.DoVGActivate(lvmvg.Spec.VgName) + if err != nil { + return true, fmt.Errorf("failed to activate volume group %s, err: %v", l.vgName, err) + } + } else if lvmvg.Spec.DesiredState == diskv1.VGStateDisabled { + // make sure the volume group is inactive + logrus.Infof("Should not go here, because the LVMVolumeGroup %s should not be disabled", l.vgName) + } + return +} + +func (l *LVMProvisioner) addDevOrCreateLVMVgCRD(lvmVG *diskv1.LVMVolumeGroup, found bool) (requeue bool, err error) { + logrus.Infof("addDevOrCreateLVMVgCRD: %v, found: %v", lvmVG, found) + requeue = false + err = nil + if !found { + lvmVG = &diskv1.LVMVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s-", l.vgName), + Namespace: utils.HarvesterNS, + Labels: map[string]string{ + lvm.LVMTopoKeyNode: l.nodeName, + }, + }, + Spec: diskv1.VolumeGroupSpec{ + NodeName: l.nodeName, + VgName: l.vgName, + DesiredState: diskv1.VGStateEnabled, + Devices: map[string]string{l.device.Name: l.device.Status.DeviceStatus.DevPath}, + }, + } + if _, err = l.vgClient.Create(lvmVG); err != nil { + requeue = true + logrus.Infof("[DEBUG]: error: %v", err) + err = fmt.Errorf("failed to create LVMVolumeGroup %s. %v", l.vgName, err) + return + } + logrus.Infof("Created LVMVolumeGroup %s, content: %v", l.vgName, lvmVG) + return + } + if lvmVG == nil { + requeue = true + err = fmt.Errorf("failed to get LVMVolumeGroup %s, but notFound is False", l.vgName) + return + } + if _, found := lvmVG.Spec.Devices[l.device.Name]; found { + logrus.Infof("Skip this round because the devices are not changed") + return + } + lvmVGCpy := lvmVG.DeepCopy() + lvmVGCpy.Spec.Devices[l.device.Name] = l.device.Status.DeviceStatus.DevPath + if !reflect.DeepEqual(lvmVG, lvmVGCpy) { + if _, err = l.vgClient.Update(lvmVGCpy); err != nil { + requeue = true + err = fmt.Errorf("failed to update LVMVolumeGroup %s. %v", l.vgName, err) + return + } + logrus.Infof("Updated LVMVolumeGroup %s, content: %v", l.vgName, lvmVGCpy) + } + return +} + +func (l *LVMProvisioner) removeDevFromLVMVgCRD(lvmVG *diskv1.LVMVolumeGroup, targetDevice string) (requeue bool, err error) { + logrus.Infof("removeDevFromLVMVG %s, devices before remove: %v", lvmVG.Spec.VgName, lvmVG.Spec.Devices) + requeue = false + err = nil + + lvmVGCpy := lvmVG.DeepCopy() + delete(lvmVGCpy.Spec.Devices, targetDevice) + logrus.Debugf("New devices (after remove %v): %v", targetDevice, lvmVGCpy.Spec.Devices) + if len(lvmVGCpy.Status.Devices) == 0 { + if err = l.vgClient.Delete(lvmVGCpy.Namespace, lvmVGCpy.Name, &metav1.DeleteOptions{}); err != nil { + requeue = true + err = fmt.Errorf("failed to delete LVMVolumeGroup %s. %v", l.vgName, err) + return + } + logrus.Infof("Deleted LVMVolumeGroup %s", l.vgName) + return + } + // we need to wait the device + if !reflect.DeepEqual(lvmVG, lvmVGCpy) { + if _, err = l.vgClient.Update(lvmVGCpy); err != nil { + requeue = true + err = fmt.Errorf("failed to update LVMVolumeGroup %s. %v", l.vgName, err) + return + } + } + logrus.Infof("Updated LVMVolumeGroup %s, content: %v", l.vgName, lvmVGCpy) + return +} + +func (l *LVMProvisioner) getTargetLVMVG() (target *diskv1.LVMVolumeGroup, err error) { + found := false + // check if the LVMVolumeGroup CRD is already provisioned + selector, err := lvm.GenerateSelector(l.nodeName) + if err != nil { + err = fmt.Errorf("failed to generate selector: %v", err) + return + } + lvmvgs, err := l.vgClient.List(utils.HarvesterNS, metav1.ListOptions{LabelSelector: selector.String()}) + if err != nil { + err = fmt.Errorf("failed to list LVMVolumeGroup %s. %v", l.vgName, err) + return + } + for _, lvmvg := range lvmvgs.Items { + if lvmvg.Spec.NodeName == l.nodeName && lvmvg.Spec.VgName == l.vgName { + target = lvmvg.DeepCopy() + found = true + break + } + } + if !found { + err = errors.NewNotFound(diskv1.Resource("lvmvolumegroups"), l.vgName) + } + return +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 79085f3f..0b2f2e77 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -19,6 +19,8 @@ const ( HostProcPath = "/host/proc" // DiskRemoveTag indicates a Longhorn is pending to remove. DiskRemoveTag = "harvester-ndm-disk-remove" + // Harvester Namespace + HarvesterNS = "harvester-system" ) var CmdTimeoutError error