diff --git a/main.go b/main.go index 9c258a81..b53852f6 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ import ( "github.com/harvester/node-disk-manager/pkg/block" blockdevicev1 "github.com/harvester/node-disk-manager/pkg/controller/blockdevice" nodev1 "github.com/harvester/node-disk-manager/pkg/controller/node" + volumegroupv1 "github.com/harvester/node-disk-manager/pkg/controller/volumegroup" "github.com/harvester/node-disk-manager/pkg/filter" ctldisk "github.com/harvester/node-disk-manager/pkg/generated/controllers/harvesterhci.io" ctllonghorn "github.com/harvester/node-disk-manager/pkg/generated/controllers/longhorn.io" @@ -219,6 +220,7 @@ func run(opt *option.Option) error { locker := &sync.Mutex{} cond := sync.NewCond(locker) bds := disks.Harvesterhci().V1beta1().BlockDevice() + lvmVGs := disks.Harvesterhci().V1beta1().LVMVolumeGroup() nodes := lhs.Longhorn().V1beta2().Node() scanner := blockdevicev1.NewScanner( opt.NodeName, @@ -237,6 +239,7 @@ func run(opt *option.Option) error { ctx, nodes, bds, + lvmVGs, block, opt, scanner, @@ -248,6 +251,10 @@ func run(opt *option.Option) error { logrus.Fatalf("failed to register ndm node controller, %s", err.Error()) } + if err := volumegroupv1.Register(ctx, lvmVGs, opt); err != nil { + logrus.Fatalf("failed to register ndm volume group controller, %s", err.Error()) + } + if err := start.All(ctx, opt.Threadiness, disks, lhs); err != nil { logrus.Fatalf("error starting, %s", err.Error()) } diff --git a/pkg/controller/blockdevice/controller.go b/pkg/controller/blockdevice/controller.go index e2b5e864..2e902e47 100644 --- a/pkg/controller/blockdevice/controller.go +++ b/pkg/controller/blockdevice/controller.go @@ -38,6 +38,8 @@ type Controller struct { BlockdeviceCache ctldiskv1.BlockDeviceCache BlockInfo block.Info + LVMVgClient ctldiskv1.LVMVolumeGroupController + scanner *Scanner semaphore *provisioner.Semaphore } @@ -61,6 +63,7 @@ func Register( ctx context.Context, nodes ctllonghornv1.NodeController, bds ctldiskv1.BlockDeviceController, + lvmVGs ctldiskv1.LVMVolumeGroupController, block block.Info, opt *option.Option, scanner *Scanner, @@ -74,6 +77,7 @@ func Register( Nodes: nodes, Blockdevices: bds, BlockdeviceCache: bds.Cache(), + LVMVgClient: lvmVGs, BlockInfo: block, scanner: scanner, semaphore: semaphoreObj, @@ -107,6 +111,10 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) ( logrus.Warnf("Failed to generate provisioner for device %s: %v", device.Name, err) return nil, err } + if provisionerInst == nil { + logrus.Infof("Skip device %s as no provisioner found or not configured", device.Name) + return nil, nil + } // handle remove device no matter inactive or corrupted, we will set `device.Spec.FileSystem.Provisioned` to false if needProvisionerUnprovision(device) { @@ -133,6 +141,7 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) ( return nil, fmt.Errorf("failed to resolve persistent dev path for block device %s", device.Name) } + logrus.Debugf("Checking to format device %s", device.Name) if formatted, requeue, err := provisionerInst.Format(devPath); !formatted { if requeue { c.Blockdevices.EnqueueAfter(c.Namespace, device.Name, jitterEnqueueDelay()) @@ -153,6 +162,7 @@ func (c *Controller) OnBlockDeviceChange(_ string, device *diskv1.BlockDevice) ( * 2. Spec.Filesystem.Provisioned = true, Status.ProvisionPhase = ProvisionPhaseUnprovisioned * -> Provision the device */ + logrus.Debugf("Checking to provision/update device %s", device.Name) if needProvisionerUpdate(device, deviceCpy) { logrus.Infof("Prepare to check the new device tags %v with device: %s", deviceCpy.Spec.Tags, device.Name) requeue, err := provisionerInst.Update() @@ -200,12 +210,26 @@ func (c *Controller) finalizeBlockDevice(oldBd, newBd *diskv1.BlockDevice, devPa } func (c *Controller) generateProvisioner(device *diskv1.BlockDevice) (provisioner.Provisioner, error) { + if device.Spec.Provisioner == nil && device.Status.ProvisionPhase != diskv1.ProvisionPhaseProvisioned { + return nil, nil + } + logrus.Infof("Generate provisioner from device %s, content: %v", device.Name, device.Spec.Provisioner) // set default provisionerType := provisioner.TypeLonghornV1 if device.Spec.Provisioner != nil { + // **TODO**: we should use webhook to validate the provisioner type (and number) + numProvisioner := 0 if device.Spec.Provisioner.Longhorn != nil { + numProvisioner++ provisionerType = device.Spec.Provisioner.Longhorn.EngineVersion } + if device.Spec.Provisioner.LVM != nil { + numProvisioner++ + provisionerType = provisioner.TypeLVM + } + if numProvisioner > 1 { + return nil, fmt.Errorf("multiple provisioner types found for block device %s", device.Name) + } } switch provisionerType { case provisioner.TypeLonghornV1: @@ -220,7 +244,7 @@ func (c *Controller) generateProvisioner(device *diskv1.BlockDevice) (provisione case provisioner.TypeLonghornV2: return nil, fmt.Errorf("TBD type %s", provisionerType) case provisioner.TypeLVM: - return nil, fmt.Errorf("TBD type %s", provisionerType) + return c.generateLVMProvisioner(device) default: return nil, fmt.Errorf("unsupported provisioner type %s", provisionerType) } @@ -237,6 +261,11 @@ func (c *Controller) generateLHv1Provisioner(device *diskv1.BlockDevice) (provis return provisioner.NewLHV1Provisioner(device, c.BlockInfo, node, c.Nodes, c.NodeCache, CacheDiskTags, c.semaphore) } +func (c *Controller) generateLVMProvisioner(device *diskv1.BlockDevice) (provisioner.Provisioner, error) { + vgName := device.Spec.Provisioner.LVM.VgName + return provisioner.NewLVMProvisioner(vgName, c.NodeName, c.LVMVgClient, device, c.BlockInfo) +} + func (c *Controller) updateDeviceStatus(device *diskv1.BlockDevice, devPath string) error { var newStatus diskv1.DeviceStatus var needAutoProvision bool @@ -365,13 +394,16 @@ func canSkipBlockDeviceChange(device *diskv1.BlockDevice, nodeName string) bool } func needProvisionerUnprovision(device *diskv1.BlockDevice) bool { - return !device.Spec.FileSystem.Provisioned && device.Status.ProvisionPhase != diskv1.ProvisionPhaseUnprovisioned + return (!device.Spec.FileSystem.Provisioned && !device.Spec.Provision) && + device.Status.ProvisionPhase != diskv1.ProvisionPhaseUnprovisioned } func needProvisionerUpdate(oldBd, newBd *diskv1.BlockDevice) bool { - return oldBd.Status.ProvisionPhase == diskv1.ProvisionPhaseProvisioned && newBd.Spec.FileSystem.Provisioned + return oldBd.Status.ProvisionPhase == diskv1.ProvisionPhaseProvisioned && + (newBd.Spec.FileSystem.Provisioned || newBd.Spec.Provision) } func needProvisionerProvision(oldBd, newBd *diskv1.BlockDevice) bool { - return oldBd.Status.ProvisionPhase == diskv1.ProvisionPhaseUnprovisioned && newBd.Spec.FileSystem.Provisioned + return oldBd.Status.ProvisionPhase == diskv1.ProvisionPhaseUnprovisioned && + (newBd.Spec.FileSystem.Provisioned || newBd.Spec.Provision) } diff --git a/pkg/controller/volumegroup/controller.go b/pkg/controller/volumegroup/controller.go new file mode 100644 index 00000000..baaa5274 --- /dev/null +++ b/pkg/controller/volumegroup/controller.go @@ -0,0 +1,261 @@ +package volumegroup + +import ( + "context" + "fmt" + "maps" + "reflect" + "strings" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + diskv1 "github.com/harvester/node-disk-manager/pkg/apis/harvesterhci.io/v1beta1" + ctldiskv1 "github.com/harvester/node-disk-manager/pkg/generated/controllers/harvesterhci.io/v1beta1" + "github.com/harvester/node-disk-manager/pkg/lvm" + "github.com/harvester/node-disk-manager/pkg/option" +) + +type Controller struct { + namespace string + nodeName string + + LVMVolumeGroupCache ctldiskv1.LVMVolumeGroupCache + LVMVolumeGroups ctldiskv1.LVMVolumeGroupController +} + +const ( + lvmVGHandlerName = "harvester-lvm-volumegroup-handler" +) + +func Register(ctx context.Context, lvmVGs ctldiskv1.LVMVolumeGroupController, opt *option.Option) error { + + c := &Controller{ + namespace: opt.Namespace, + nodeName: opt.NodeName, + LVMVolumeGroups: lvmVGs, + LVMVolumeGroupCache: lvmVGs.Cache(), + } + + c.LVMVolumeGroups.OnChange(ctx, lvmVGHandlerName, c.OnLVMVGChange) + c.LVMVolumeGroups.OnRemove(ctx, lvmVGHandlerName, c.OnLVMVGRemove) + return nil +} + +func (c *Controller) OnLVMVGChange(_ string, lvmVG *diskv1.LVMVolumeGroup) (*diskv1.LVMVolumeGroup, error) { + if lvmVG == nil || lvmVG.DeletionTimestamp != nil { + logrus.Infof("Skip this round because lvm volume group is deleted or deleting") + return nil, nil + } + + if lvmVG.Spec.NodeName != c.nodeName { + logrus.Infof("Skip this round because lvm volume group is not belong to this node") + return nil, nil + } + + logrus.Infof("Prepare to handle LVMVolumeGroup %s changed: %v", lvmVG.Name, lvmVG) + + switch lvmVG.Spec.DesiredState { + case diskv1.VGStateEnabled: + logrus.Infof("Prepare to enable LVMVolumeGroup %s", lvmVG.Name) + return c.updateEnabledLVMVolumeGroup(lvmVG) + case diskv1.VGStateDisabled: + // should only called manually set the VGState to Disabled + logrus.Infof("Prepare to disable LVMVolumeGroup %s", lvmVG.Name) + return c.disableLVMVolumeGroup(lvmVG) + } + return nil, nil +} + +func (c *Controller) OnLVMVGRemove(_ string, lvmVG *diskv1.LVMVolumeGroup) (*diskv1.LVMVolumeGroup, error) { + if lvmVG == nil || lvmVG.DeletionTimestamp != nil { + // make sure the volume group is already deleted + logrus.Infof("Ensure the lvm volume group is already deleted if the lvmVG CR is nil") + return c.removeLVMVolumeGroup(lvmVG) + } + + return nil, nil +} + +func (c *Controller) updateEnabledLVMVolumeGroup(lvmVG *diskv1.LVMVolumeGroup) (*diskv1.LVMVolumeGroup, error) { + logrus.Infof("Enable LVMVolumeGroup %s", lvmVG.Name) + + pvsResult, err := lvm.GetPVScanResult() + if err != nil { + return nil, fmt.Errorf("failed to get pvscan result. %v", err) + } + logrus.Debugf("pvscan result: %v", pvsResult) + currentDevs := map[string]string{} + if lvmVG.Status != nil && lvmVG.Status.Devices != nil && len(lvmVG.Status.Devices) > 0 { + currentDevs = lvmVG.Status.Devices + } + if maps.Equal(currentDevs, lvmVG.Spec.Devices) { + logrus.Info("Skip this round because the devices are not changed") + return nil, nil + } + lvmVGCpy := lvmVG.DeepCopy() + if lvmVGCpy.Status == nil { + lvmVGCpy.Status = &diskv1.VolumeGroupStatus{} + } + + if lvmVG.Status != nil && len(lvmVG.Status.Devices) == 0 { + logrus.Warnf("No devices found in LVMVolumeGroup %s, skip", lvmVG.Name) + return nil, nil + } + // update devices + toAdd := getToAddDevs(lvmVG.Spec.Devices, currentDevs) + toRemove := getToRemoveDevs(lvmVG.Spec.Devices, currentDevs) + err = updatePVAndVG(lvmVGCpy, toAdd, toRemove, pvsResult) + if err != nil { + return nil, err + } + + vgConds := diskv1.VolumeGroupCondition{ + Type: diskv1.VGConditionReady, + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.Now(), + Reason: "Volume Group is Ready", + Message: fmt.Sprintf("Volume Group is Ready with devices %v", lvmVG.Spec.Devices), + } + newConds := UpdateLVMVGsConds(lvmVGCpy.Status.VGConditions, vgConds) + lvmVGCpy.Status.VGConditions = newConds + lvmVGCpy.Status.Status = diskv1.VGStatusActive + if !reflect.DeepEqual(lvmVG, lvmVGCpy) { + return c.LVMVolumeGroups.UpdateStatus(lvmVGCpy) + } + return nil, nil +} + +func (c *Controller) disableLVMVolumeGroup(lvmVG *diskv1.LVMVolumeGroup) (*diskv1.LVMVolumeGroup, error) { + logrus.Infof("Disable LVMVolumeGroup %s", lvmVG.Spec.VgName) + err := lvm.DoVGDeactive(lvmVG.Spec.VgName) + if err != nil { + if apierrors.IsNotFound(err) { + logrus.Infof("VolumeGroup %s is not found, skip", lvmVG.Spec.VgName) + } else { + return nil, fmt.Errorf("failed to remove VG %s. %v", lvmVG.Spec.VgName, err) + } + } + return nil, nil +} + +func (c *Controller) removeLVMVolumeGroup(lvmVG *diskv1.LVMVolumeGroup) (*diskv1.LVMVolumeGroup, error) { + logrus.Infof("Remove LVMVolumeGroup %s", lvmVG.Name) + err := lvm.DoVGRemove(lvmVG.Spec.VgName) + if err != nil { + if strings.Contains(err.Error(), "not found") { + logrus.Infof("VolumeGroup %s is not found, skip", lvmVG.Spec.VgName) + } else { + return nil, fmt.Errorf("failed to remove VG %s. %v", lvmVG.Spec.VgName, err) + } + } + + return nil, nil +} + +func checkPVAndVG(pvsResult map[string]string, targetPV, targetVG string) (pvFound, vgFound bool, pvCount int) { + pvCount = 0 + for pv, vg := range pvsResult { + if pv == targetPV { + pvFound = true + if vg == targetVG { + pvCount++ + vgFound = true + } + } + if vg == targetVG { + vgFound = true + pvCount++ + } + } + return +} + +func UpdateLVMVGsConds(curConds []diskv1.VolumeGroupCondition, c diskv1.VolumeGroupCondition) []diskv1.VolumeGroupCondition { + found := false + var pod = 0 + logrus.Infof("Prepare to check the coming Type: %s, Status: %s", c.Type, c.Status) + for id, cond := range curConds { + if cond.Type == c.Type { + found = true + pod = id + break + } + } + + if found { + curConds[pod] = c + } else { + curConds = append(curConds, c) + } + return curConds + +} + +func updatePVAndVG(vgCpy *diskv1.LVMVolumeGroup, toAdd, toRemove map[string]string, pvsResult map[string]string) error { + logrus.Infof("Prepare to add devices: %v", toAdd) + for bdName, dev := range toAdd { + pvFound, vgFound, _ := checkPVAndVG(pvsResult, dev, vgCpy.Spec.VgName) + logrus.Infof("pvFound: %v, vgFound: %v", pvFound, vgFound) + if !vgFound { + if err := lvm.DoVGCreate(dev, vgCpy.Spec.VgName); err != nil { + return err + } + } + if !pvFound { + if err := lvm.DoPVCreate(dev); err != nil { + return err + } + if err := lvm.DoVGExtend(dev, vgCpy.Spec.VgName); err != nil { + return err + } + } + if vgCpy.Status.Devices == nil { + vgCpy.Status.Devices = map[string]string{} + } + vgCpy.Status.Devices[bdName] = dev + } + logrus.Infof("Prepare to remove devices: %v", toRemove) + for bdName, dev := range toRemove { + pvFound, vgFound, pvInVGCounts := checkPVAndVG(pvsResult, dev, vgCpy.Spec.VgName) + logrus.Infof("pvFound: %v, vgFound: %v, pvInVGCounts: %v", pvFound, vgFound, pvInVGCounts) + if !pvFound { + logrus.Infof("Block device %s is not in pvs, return directly!", bdName) + return nil + } + if vgFound && pvInVGCounts > 1 { + if err := lvm.DoVGReduce(dev, vgCpy.Spec.VgName); err != nil { + return err + } + } else { + if err := lvm.DoVGRemove(vgCpy.Spec.VgName); err != nil { + return err + } + } + lvm.DoPVRemove(dev) + delete(vgCpy.Status.Devices, bdName) + } + return nil +} + +func getToAddDevs(specDevs, currentDevs map[string]string) map[string]string { + toAdd := map[string]string{} + for bdName, dev := range specDevs { + if _, found := currentDevs[bdName]; !found { + toAdd[bdName] = dev + } + } + return toAdd +} + +func getToRemoveDevs(specDevs, currentDevs map[string]string) map[string]string { + toRemove := map[string]string{} + for bdName := range currentDevs { + if _, found := specDevs[bdName]; !found { + toRemove[bdName] = currentDevs[bdName] + } + } + return toRemove +} diff --git a/pkg/lvm/lvm.go b/pkg/lvm/lvm.go new file mode 100644 index 00000000..232e11fb --- /dev/null +++ b/pkg/lvm/lvm.go @@ -0,0 +1,84 @@ +package lvm + +import ( + "fmt" + "strings" + + "github.com/harvester/node-disk-manager/pkg/utils" +) + +func GetPVScanResult() (map[string]string, error) { + ns := utils.GetHostNamespacePath(utils.HostProcPath) + executor, err := utils.NewExecutorWithNS(ns) + if err != nil { + return nil, fmt.Errorf("generate executor failed. %v", err) + } + + args := []string{"--noheadings", "-o", "pv_name,vg_name"} + output, err := executor.Execute("pvs", args) + if err != nil { + return nil, fmt.Errorf("failed to execute 'pvs' command: %v", err) + } + lines := strings.Split(output, "\n") + pvScanResult := make(map[string]string) + for _, line := range lines { + if line == "" { + continue + } + fields := strings.Fields(line) + // Format should be like: /dev/sda vg01 + pv := fields[0] + vg := "" + if len(fields) >= 2 { + vg = fields[1] + } + pvScanResult[pv] = vg + } + return pvScanResult, nil +} + +func executeCommandWithNS(cmd string, args []string) error { + ns := utils.GetHostNamespacePath(utils.HostProcPath) + executor, err := utils.NewExecutorWithNS(ns) + if err != nil { + return fmt.Errorf("generate executor failed: %v", err) + } + + _, err = executor.Execute(cmd, args) + if err != nil { + return fmt.Errorf("execute command '%s' with args '%v' failed: %v", cmd, args, err) + } + return nil +} + +func DoPVCreate(devPath string) error { + return executeCommandWithNS("pvcreate", []string{devPath}) +} + +func DoVGCreate(devPath, vgName string) error { + return executeCommandWithNS("vgcreate", []string{vgName, devPath}) +} + +func DoVGExtend(devPath, vgName string) error { + return executeCommandWithNS("vgextend", []string{vgName, devPath}) +} + +func DoVGReduce(devPath, vgName string) error { + return executeCommandWithNS("vgreduce", []string{vgName, devPath}) +} + +func DoVGRemove(vgName string) error { + return executeCommandWithNS("vgremove", []string{vgName}) +} + +func DoPVRemove(devPath string) error { + return executeCommandWithNS("pvremove", []string{devPath}) +} + +func DoVGActivate(vgName string) error { + return executeCommandWithNS("vgchange", []string{"--activate", "y", vgName}) +} + +func DoVGDeactive(vgName string) error { + return executeCommandWithNS("vgchange", []string{"--activate", "n", vgName}) +} diff --git a/pkg/provisioner/lvm.go b/pkg/provisioner/lvm.go new file mode 100644 index 00000000..6977653e --- /dev/null +++ b/pkg/provisioner/lvm.go @@ -0,0 +1,222 @@ +package provisioner + +import ( + "encoding/hex" + "fmt" + "reflect" + + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + diskv1 "github.com/harvester/node-disk-manager/pkg/apis/harvesterhci.io/v1beta1" + "github.com/harvester/node-disk-manager/pkg/block" + ctldiskv1 "github.com/harvester/node-disk-manager/pkg/generated/controllers/harvesterhci.io/v1beta1" + "github.com/harvester/node-disk-manager/pkg/lvm" + "github.com/harvester/node-disk-manager/pkg/utils" +) + +type LVMProvisioner struct { + *provisioner + vgName string + nodeName string + vgClient ctldiskv1.LVMVolumeGroupController +} + +func NewLVMProvisioner(vgName, nodeName string, lvmVGs ctldiskv1.LVMVolumeGroupController, device *diskv1.BlockDevice, blockInfo block.Info) (Provisioner, error) { + baseProvisioner := &provisioner{ + name: TypeLVM, + blockInfo: blockInfo, + device: device, + } + return &LVMProvisioner{ + provisioner: baseProvisioner, + vgName: vgName, + vgClient: lvmVGs, + nodeName: nodeName, + }, nil +} + +func (l *LVMProvisioner) GetProvisionerName() string { + return l.name +} + +func (l *LVMProvisioner) Format(string) (bool, bool, error) { + // LVM provisioner does not need format + return true, false, nil +} + +func (l *LVMProvisioner) UnFormat() (bool, error) { + // LVM provisioner does not need unformat + return false, nil +} + +// Provision creates (if needed) a LVMVolumeGroup CRD and update the corresponding fields. +func (l *LVMProvisioner) Provision() (bool, error) { + logrus.Infof("Provisioning block device %s to vg: %s", l.device.Name, l.vgName) + notFound := false + + // check if the LVMVolumeGroup CRD is already provisioned + lvmVGName := genLVMVGName(l.nodeName, l.vgName) + lvmvg, err := l.vgClient.Get(utils.HarvesterNS, lvmVGName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + lvmvg = nil + notFound = true + } else { + return true, fmt.Errorf("failed to get LVMVolumeGroup %s. %v", l.vgName, err) + } + } + requeue, err := l.addDevOrCreateLVMVgCRD(lvmvg, notFound) + if err != nil { + return requeue, err + } + + // first round the lvmvg must be nil, so we need to check it. + if lvmvg != nil && lvmvg.Status != nil && lvmvg.Status.Status == diskv1.VGStatusActive { + setCondDiskAddedToNodeTrue(l.device, fmt.Sprintf("Added disk %s to volume group %s ", l.device.Name, l.vgName), diskv1.ProvisionPhaseProvisioned) + return false, nil + } + return true, nil +} + +// UnProvision update the LVMVolumeGroup CRD and remove the LVMVolumeGroup CRD if the device is the last one in the VG. +func (l *LVMProvisioner) UnProvision() (bool, error) { + // check if the LVMVolumeGroup CRD is already provisioned + lvmVGName := genLVMVGName(l.nodeName, l.vgName) + lvmvg, err := l.vgClient.Get(utils.HarvesterNS, lvmVGName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + // do nothing if the LVMVolumeGroup CRD is not found + logrus.Warnf("CR LVMVolumeGroup %s is not found, skip UnProvision", lvmVGName) + msg := fmt.Sprintf("Removed disk %s from volume group %s ", l.device.Name, l.vgName) + setCondDiskAddedToNodeFalse(l.device, msg, diskv1.ProvisionPhaseUnprovisioned) + return false, nil + } + return true, fmt.Errorf("failed to get LVMVolumeGroup %s. %v", l.vgName, err) + } + logrus.Infof("%s unprovisioning block device %s from vg: %s", l.name, l.device.Name, l.vgName) + requeue, err := l.removeDevFromLVMVgCRD(lvmvg, l.device.Name) + if err != nil { + return requeue, err + } + if lvmvg.Status != nil { + if _, found := lvmvg.Status.Devices[l.device.Name]; !found { + msg := fmt.Sprintf("Removed disk %s from volume group %s ", l.device.Name, l.vgName) + setCondDiskAddedToNodeFalse(l.device, msg, diskv1.ProvisionPhaseUnprovisioned) + return false, nil + } + } + // waiting the device removed from the LVMVolumeGroup CRD + logrus.Infof("Waiting for the device %s removed from the LVMVolumeGroup CRD %v", l.device.Name, lvmvg) + return true, nil +} + +func (l *LVMProvisioner) Update() (requeue bool, err error) { + // Update DesiredState to Reconciling + logrus.Infof("Prepare to Update LVMVolumeGroup %s", l.vgName) + lvmVGName := genLVMVGName(l.nodeName, l.vgName) + lvmvg, err := l.vgClient.Get(utils.HarvesterNS, lvmVGName, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return true, fmt.Errorf("failed to get LVMVolumeGroup %s, err: %v", lvmVGName, err) + } + + if lvmvg.Spec.DesiredState == diskv1.VGStateEnabled { + // make sure the volume group is active + err := lvm.DoVGActivate(lvmvg.Spec.VgName) + if err != nil { + return true, fmt.Errorf("failed to activate volume group %s, err: %v", l.vgName, err) + } + } else if lvmvg.Spec.DesiredState == diskv1.VGStateDisabled { + // make sure the volume group is inactive + logrus.Infof("Should not go here, because the LVMVolumeGroup %s should not be disabled", l.vgName) + } + return +} + +func (l *LVMProvisioner) addDevOrCreateLVMVgCRD(lvmVG *diskv1.LVMVolumeGroup, notFound bool) (requeue bool, err error) { + logrus.Infof("addDevOrCreateLVMVgCRD: %v, notFound: %v", lvmVG, notFound) + requeue = false + err = nil + if notFound { + lvmVGName := genLVMVGName(l.nodeName, l.vgName) + logrus.Infof("[DEBUG]: lvmVGName: %s", lvmVGName) + lvmVG = &diskv1.LVMVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: lvmVGName, + Namespace: utils.HarvesterNS, + }, + Spec: diskv1.VolumeGroupSpec{ + NodeName: l.nodeName, + VgName: l.vgName, + DesiredState: diskv1.VGStateEnabled, + Devices: map[string]string{l.device.Name: l.device.Status.DeviceStatus.DevPath}, + }, + } + if _, err = l.vgClient.Create(lvmVG); err != nil { + requeue = true + logrus.Infof("[DEBUG]: error: %v", err) + err = fmt.Errorf("failed to create LVMVolumeGroup %s. %v", l.vgName, err) + return + } + logrus.Infof("Created LVMVolumeGroup %s, content: %v", l.vgName, lvmVG) + return + } + if lvmVG == nil { + requeue = true + err = fmt.Errorf("failed to get LVMVolumeGroup %s, but notFound is False", l.vgName) + return + } + if _, found := lvmVG.Spec.Devices[l.device.Name]; found { + logrus.Infof("Skip this round because the devices are not changed") + return + } + lvmVGCpy := lvmVG.DeepCopy() + lvmVGCpy.Spec.Devices[l.device.Name] = l.device.Status.DeviceStatus.DevPath + if !reflect.DeepEqual(lvmVG, lvmVGCpy) { + if _, err = l.vgClient.Update(lvmVGCpy); err != nil { + requeue = true + err = fmt.Errorf("failed to update LVMVolumeGroup %s. %v", l.vgName, err) + return + } + logrus.Infof("Updated LVMVolumeGroup %s, content: %v", l.vgName, lvmVGCpy) + } + return +} + +func (l *LVMProvisioner) removeDevFromLVMVgCRD(lvmVG *diskv1.LVMVolumeGroup, targetDevice string) (requeue bool, err error) { + logrus.Infof("removeDevFromLVMVG %s, devices before remove: %v", lvmVG.Spec.VgName, lvmVG.Spec.Devices) + requeue = false + err = nil + + lvmVGCpy := lvmVG.DeepCopy() + delete(lvmVGCpy.Spec.Devices, targetDevice) + logrus.Debugf("New devices (after remove %v): %v", targetDevice, lvmVGCpy.Spec.Devices) + if len(lvmVGCpy.Status.Devices) == 0 { + if err = l.vgClient.Delete(lvmVGCpy.Namespace, lvmVGCpy.Name, &metav1.DeleteOptions{}); err != nil { + requeue = true + err = fmt.Errorf("failed to delete LVMVolumeGroup %s. %v", l.vgName, err) + return + } + logrus.Infof("Deleted LVMVolumeGroup %s", l.vgName) + return + } + // we need to wait the device + if !reflect.DeepEqual(lvmVG, lvmVGCpy) { + if _, err = l.vgClient.Update(lvmVGCpy); err != nil { + requeue = true + err = fmt.Errorf("failed to update LVMVolumeGroup %s. %v", l.vgName, err) + return + } + } + logrus.Infof("Updated LVMVolumeGroup %s, content: %v", l.vgName, lvmVGCpy) + return +} + +// genLVMVGName generates the name of the LVMVolumeGroup CRD +func genLVMVGName(nodeName, vgName string) string { + //format N+nodeName+-+V+vgName + nHex := hex.EncodeToString([]byte("N")) + vHex := hex.EncodeToString([]byte("V")) + return fmt.Sprintf(nHex + nodeName + "-" + vHex + vgName) +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 79085f3f..0b2f2e77 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -19,6 +19,8 @@ const ( HostProcPath = "/host/proc" // DiskRemoveTag indicates a Longhorn is pending to remove. DiskRemoveTag = "harvester-ndm-disk-remove" + // Harvester Namespace + HarvesterNS = "harvester-system" ) var CmdTimeoutError error