Skip to content

Commit

Permalink
Add or improve read-only BackingImage related get/list functions for …
Browse files Browse the repository at this point in the history 
…controller usage

Signed-off-by: Shuo Wu <shuo.wu@suse.com>
  • Loading branch information
@shuo-wu
shuo-wu committed Oct 24, 2023
1 parent 8bc1d71 commit 414be6a
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 75 deletions.
108 changes: 44 additions & 64 deletions controller/backing_image_manager_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,14 +378,14 @@ func (c *BackingImageManagerController) updateForUnknownBackingImageManager(bim
if _, ok := bim.Status.BackingImageFileMap[biName]; ok {
continue
}
bi, err := c.ds.GetBackingImage(biName)
biRO, err := c.ds.GetBackingImageRO(biName)
if err != nil {
log.Warnf("Failed to get backing image %v before marking the empty file record in an unavailable disk as unknown", biName)
continue
}
info := longhorn.BackingImageFileInfo{
Name: bi.Name,
UUID: bi.Status.UUID,
Name: biRO.Name,
UUID: biRO.Status.UUID,
State: longhorn.BackingImageStateUnknown,
}
bim.Status.BackingImageFileMap[biName] = info
Expand Down Expand Up @@ -572,15 +572,15 @@ func (c *BackingImageManagerController) deleteInvalidBackingImages(bim *longhorn
for biName, biFileInfo := range bim.Status.BackingImageFileMap {
deleteRequired := false

bi, err := c.ds.GetBackingImage(biName)
biRO, err := c.ds.GetBackingImageRO(biName)
if err != nil {
if !apierrors.IsNotFound(err) {
return err
}
deleteRequired = true
log.Warnf("Failed to find backing image %v during invalid backing image cleanup, will skip it", biName)
}
if bi != nil && bi.Status.UUID == "" {
if biRO != nil && biRO.Status.UUID == "" {
continue
}

Expand All @@ -590,18 +590,18 @@ func (c *BackingImageManagerController) deleteInvalidBackingImages(bim *longhorn
// 2. The status record does not match the current backing image.
// 3. The file state recorded in the current backing image is failed
// and there are available files in other backing image managers.
deleteRequired = deleteRequired || (bi != nil && bim.Spec.BackingImages[biName] != bi.Status.UUID)
deleteRequired = deleteRequired || (bi != nil && biFileInfo.UUID != "" && biFileInfo.UUID != bi.Status.UUID)
if !deleteRequired && bi != nil {
deleteRequired = deleteRequired || (biRO != nil && bim.Spec.BackingImages[biName] != biRO.Status.UUID)
deleteRequired = deleteRequired || (biRO != nil && biFileInfo.UUID != "" && biFileInfo.UUID != biRO.Status.UUID)
if !deleteRequired && biRO != nil {
// Prefer to check the file state in BackingImage.Status,
// which is synced from BackingImageManager.Status with some
// adjustments.
fileState := biFileInfo.State
if bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID] != nil {
fileState = bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID].State
if biRO.Status.DiskFileStatusMap[bim.Spec.DiskUUID] != nil {
fileState = biRO.Status.DiskFileStatusMap[bim.Spec.DiskUUID].State
}
if fileState == longhorn.BackingImageStateFailed {
for _, biFileInfo := range bi.Status.DiskFileStatusMap {
for _, biFileInfo := range biRO.Status.DiskFileStatusMap {
if biFileInfo.State == longhorn.BackingImageStateFailed {
continue
}
Expand Down Expand Up @@ -631,14 +631,14 @@ func (c *BackingImageManagerController) prepareBackingImageFiles(currentBIM *lon
err = errors.Wrap(err, "failed to prepare backing image files")
}()

bims, err := c.ds.ListBackingImageManagers()
bimsRO, err := c.ds.ListBackingImageManagersRO()
if err != nil {
return err
}
for biName := range currentBIM.Spec.BackingImages {
log := bimLog.WithFields(logrus.Fields{"backingImage": biName})

bi, err := c.ds.GetBackingImage(biName)
biRO, err := c.ds.GetBackingImageRO(biName)
if err != nil {
if !apierrors.IsNotFound(err) {
log.WithError(err).Warn("Failed to get backing image before preparing files, will skip handling this backing image")
Expand Down Expand Up @@ -667,7 +667,7 @@ func (c *BackingImageManagerController) prepareBackingImageFiles(currentBIM *lon
// If bids is failed and not transferred, orphan tmp file might be left on the host.
// Clean up and set the state to failed-and-cleanup
if bids.Status.CurrentState == longhorn.BackingImageStateFailed {
if err := cli.Delete(bi.Name, bi.Status.UUID); err != nil {
if err := cli.Delete(biRO.Name, biRO.Status.UUID); err != nil {
return err
}
bids.Status.CurrentState = longhorn.BackingImageStateFailedAndCleanUp
Expand All @@ -688,29 +688,29 @@ func (c *BackingImageManagerController) prepareBackingImageFiles(currentBIM *lon
continue
}
log.Infof("Starting to fetch the data source file from the backing image data source work directory %v", bimtypes.DataSourceDirectoryName)
if _, err := cli.Fetch(bi.Name, bi.Status.UUID, bids.Status.Checksum, fmt.Sprintf("%s:%d", bids.Status.StorageIP, engineapi.BackingImageDataSourceDefaultPort), bids.Status.Size); err != nil {
if _, err := cli.Fetch(biRO.Name, biRO.Status.UUID, bids.Status.Checksum, fmt.Sprintf("%s:%d", bids.Status.StorageIP, engineapi.BackingImageDataSourceDefaultPort), bids.Status.Size); err != nil {
if types.ErrorAlreadyExists(err) {
continue
}
return err
}
// No backoff when fetching the 1st file.
c.eventRecorder.Eventf(currentBIM, corev1.EventTypeNormal, constant.EventReasonFetching, "Fetched the first file for backing image %v in disk %v on node %v", bi.Name, currentBIM.Spec.DiskUUID, currentBIM.Spec.NodeID)
c.eventRecorder.Eventf(currentBIM, corev1.EventTypeNormal, constant.EventReasonFetching, "Fetched the first file for backing image %v in disk %v on node %v", biRO.Name, currentBIM.Spec.DiskUUID, currentBIM.Spec.NodeID)
continue
}

if backoff.IsInBackOffSinceUpdate(bi.Name, time.Now()) {
log.Debugf("Failed to re-fetch or re-sync backing image file %v immediately since it is still in the backoff window", bi.Name)
if backoff.IsInBackOffSinceUpdate(biRO.Name, time.Now()) {
log.Debugf("Failed to re-fetch or re-sync backing image file %v immediately since it is still in the backoff window", biRO.Name)
continue
}

noReadyFile := true
var senderCandidate *longhorn.BackingImageManager
for _, bim := range bims {
if bim.Status.CurrentState != longhorn.BackingImageManagerStateRunning || bim.Spec.Image != c.bimImageName {
var senderCandidateRO *longhorn.BackingImageManager
for _, bimRO := range bimsRO {
if bimRO.Status.CurrentState != longhorn.BackingImageManagerStateRunning || bimRO.Spec.Image != c.bimImageName {
continue
}
info, exists := bim.Status.BackingImageFileMap[biName]
info, exists := bimRO.Status.BackingImageFileMap[biName]
if !exists {
continue
}
Expand All @@ -721,44 +721,44 @@ func (c *BackingImageManagerController) prepareBackingImageFiles(currentBIM *lon
if info.SendingReference >= bimtypes.SendingLimit {
continue
}
senderCandidate = bim
senderCandidateRO = bimRO
break
}

// Due to cases like upgrade, there is no ready record among all default backing image manager.
// Then Longhorn will ask managers to check then reuse existing files.
if noReadyFile {
size := bi.Status.Size
size := biRO.Status.Size
if size == 0 {
size = bids.Status.Size
}
// Empty source file name means trying to find and reuse the file in the work directory.
if _, err := cli.Fetch(bi.Name, bi.Status.UUID, bi.Status.Checksum, "", size); err != nil {
if _, err := cli.Fetch(biRO.Name, biRO.Status.UUID, biRO.Status.Checksum, "", size); err != nil {
if types.ErrorAlreadyExists(err) {
log.Warn("Backing image already exists, no need to check and reuse file")
continue
}
backoff.Next(bi.Name, time.Now())
backoff.Next(biRO.Name, time.Now())
return err
}
backoff.Next(bi.Name, time.Now())
backoff.Next(biRO.Name, time.Now())
log.Info("Reusing the existing file in the work directory")
c.eventRecorder.Eventf(currentBIM, corev1.EventTypeNormal, constant.EventReasonFetching, "Reuse the existing file for backing image %v in disk %v on node %v", bi.Name, currentBIM.Spec.DiskUUID, currentBIM.Spec.NodeID)
c.eventRecorder.Eventf(currentBIM, corev1.EventTypeNormal, constant.EventReasonFetching, "Reuse the existing file for backing image %v in disk %v on node %v", biRO.Name, currentBIM.Spec.DiskUUID, currentBIM.Spec.NodeID)
continue
}

if senderCandidate != nil {
log.WithFields(logrus.Fields{"fromHost": senderCandidate.Status.StorageIP, "size": bi.Status.Size}).Info("Requesting syncing backing image")
if _, err := cli.Sync(biName, bi.Status.UUID, bi.Status.Checksum, senderCandidate.Status.StorageIP, bi.Status.Size); err != nil {
if senderCandidateRO != nil {
log.WithFields(logrus.Fields{"fromHost": senderCandidateRO.Status.StorageIP, "size": biRO.Status.Size}).Info("Requesting syncing backing image")
if _, err := cli.Sync(biName, biRO.Status.UUID, biRO.Status.Checksum, senderCandidateRO.Status.StorageIP, biRO.Status.Size); err != nil {
if types.ErrorAlreadyExists(err) {
log.WithFields(logrus.Fields{"fromHost": senderCandidate.Status.StorageIP, "size": bi.Status.Size}).Warn("Backing image already exists, no need to sync from others")
log.WithFields(logrus.Fields{"fromHost": senderCandidateRO.Status.StorageIP, "size": biRO.Status.Size}).Warn("Backing image already exists, no need to sync from others")
continue
}
backoff.Next(bi.Name, time.Now())
backoff.Next(biRO.Name, time.Now())
return err
}
backoff.Next(bi.Name, time.Now())
c.eventRecorder.Eventf(currentBIM, corev1.EventTypeNormal, constant.EventReasonSyncing, "Syncing backing image %v in disk %v on node %v from %v(%v)", bi.Name, currentBIM.Spec.DiskUUID, currentBIM.Spec.NodeID, senderCandidate.Name, senderCandidate.Status.StorageIP)
backoff.Next(biRO.Name, time.Now())
c.eventRecorder.Eventf(currentBIM, corev1.EventTypeNormal, constant.EventReasonSyncing, "Syncing backing image %v in disk %v on node %v from %v(%v)", biRO.Name, currentBIM.Spec.DiskUUID, currentBIM.Spec.NodeID, senderCandidateRO.Name, senderCandidateRO.Status.StorageIP)
continue
}
}
Expand Down Expand Up @@ -953,16 +953,7 @@ func (c *BackingImageManagerController) enqueueForBackingImage(obj interface{})
}
}

backingImage, err := c.ds.GetBackingImage(backingImage.Name)
if err != nil {
if apierrors.IsNotFound(err) {
return
}
utilruntime.HandleError(fmt.Errorf("failed to get backing image %v: %v ", backingImage.Name, err))
return
}

bims, err := c.ds.ListBackingImageManagers()
bimsRO, err := c.ds.ListBackingImageManagersRO()
if err != nil {
if apierrors.IsNotFound(err) {
c.logger.WithField("backingImage", backingImage.Name).Warn("Failed to list backing image managers for a backing image, may be deleted")
Expand All @@ -972,9 +963,9 @@ func (c *BackingImageManagerController) enqueueForBackingImage(obj interface{})
return
}

for _, bim := range bims {
if _, exists := bim.Spec.BackingImages[backingImage.Name]; exists {
c.enqueueBackingImageManager(bim)
for _, bimRO := range bimsRO {
if _, exists := bimRO.Spec.BackingImages[backingImage.Name]; exists {
c.enqueueBackingImageManager(bimRO)
}
}
}
Expand All @@ -996,18 +987,7 @@ func (c *BackingImageManagerController) enqueueForLonghornNode(obj interface{})
}
}

node, err := c.ds.GetNode(node.Name)
if err != nil {
if apierrors.IsNotFound(err) {
// there is no Longhorn node created for the Kubernetes
// node (e.g. controller/etcd node). Skip it
return
}
utilruntime.HandleError(fmt.Errorf("failed to get node %v: %v ", node.Name, err))
return
}

bims, err := c.ds.ListBackingImageManagersByNode(node.Name)
bimsRO, err := c.ds.ListBackingImageManagersByNodeRO(node.Name)
if err != nil {
if apierrors.IsNotFound(err) {
c.logger.WithField("node", node.Name).Warn("Failed to list backing image managers for a node, may be deleted")
Expand All @@ -1017,8 +997,8 @@ func (c *BackingImageManagerController) enqueueForLonghornNode(obj interface{})
return
}

for _, bim := range bims {
c.enqueueBackingImageManager(bim)
for _, bimRO := range bimsRO {
c.enqueueBackingImageManager(bimRO)
}
}

Expand All @@ -1039,7 +1019,7 @@ func (c *BackingImageManagerController) enqueueForBackingImageManagerPod(obj int
}
}

bim, err := c.ds.GetBackingImageManager(pod.Name)
bimRO, err := c.ds.GetBackingImageManagerRO(pod.Name)
if err != nil {
if apierrors.IsNotFound(err) {
c.logger.WithField("pod", pod.Name).Warn("Failed to find backing image manager for pod, may be deleted")
Expand All @@ -1048,7 +1028,7 @@ func (c *BackingImageManagerController) enqueueForBackingImageManagerPod(obj int
utilruntime.HandleError(fmt.Errorf("couldn't get backing image manager: %v", err))
return
}
c.enqueueBackingImageManager(bim)
c.enqueueBackingImageManager(bimRO)
}

func (c *BackingImageManagerController) startMonitoring(bim *longhorn.BackingImageManager, backoff *flowcontrol.Backoff) {
Expand Down
38 changes: 27 additions & 11 deletions datastore/longhorn.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1646,7 +1646,7 @@ func (s *DataStore) CreateBackingImage(backingImage *longhorn.BackingImage) (*lo
}

obj, err := verifyCreation(ret.Name, "backing image", func(name string) (runtime.Object, error) {
return s.getBackingImageRO(name)
return s.GetBackingImageRO(name)
})
if err != nil {
return nil, err
Expand All @@ -1666,7 +1666,7 @@ func (s *DataStore) UpdateBackingImage(backingImage *longhorn.BackingImage) (*lo
return nil, err
}
verifyUpdate(backingImage.Name, obj, func(name string) (runtime.Object, error) {
return s.getBackingImageRO(name)
return s.GetBackingImageRO(name)
})
return obj, nil
}
Expand All @@ -1679,7 +1679,7 @@ func (s *DataStore) UpdateBackingImageStatus(backingImage *longhorn.BackingImage
return nil, err
}
verifyUpdate(backingImage.Name, obj, func(name string) (runtime.Object, error) {
return s.getBackingImageRO(name)
return s.GetBackingImageRO(name)
})
return obj, nil
}
Expand Down Expand Up @@ -1711,14 +1711,14 @@ func (s *DataStore) RemoveFinalizerForBackingImage(obj *longhorn.BackingImage) e
return nil
}

func (s *DataStore) getBackingImageRO(name string) (*longhorn.BackingImage, error) {
func (s *DataStore) GetBackingImageRO(name string) (*longhorn.BackingImage, error) {
return s.backingImageLister.BackingImages(s.namespace).Get(name)
}

// GetBackingImage returns a new BackingImage object for the given name and
// namespace
func (s *DataStore) GetBackingImage(name string) (*longhorn.BackingImage, error) {
resultRO, err := s.getBackingImageRO(name)
resultRO, err := s.GetBackingImageRO(name)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -1781,7 +1781,7 @@ func (s *DataStore) CreateBackingImageManager(backingImageManager *longhorn.Back
}

obj, err := verifyCreation(ret.Name, "backing image manager", func(name string) (runtime.Object, error) {
return s.getBackingImageManagerRO(name)
return s.GetBackingImageManagerRO(name)
})
if err != nil {
return nil, err
Expand Down Expand Up @@ -1815,7 +1815,7 @@ func (s *DataStore) UpdateBackingImageManager(backingImageManager *longhorn.Back
return nil, err
}
verifyUpdate(backingImageManager.Name, obj, func(name string) (runtime.Object, error) {
return s.getBackingImageManagerRO(name)
return s.GetBackingImageManagerRO(name)
})
return obj, nil
}
Expand All @@ -1828,7 +1828,7 @@ func (s *DataStore) UpdateBackingImageManagerStatus(backingImageManager *longhor
return nil, err
}
verifyUpdate(backingImageManager.Name, obj, func(name string) (runtime.Object, error) {
return s.getBackingImageManagerRO(name)
return s.GetBackingImageManagerRO(name)
})
return obj, nil
}
Expand Down Expand Up @@ -1859,25 +1859,29 @@ func (s *DataStore) RemoveFinalizerForBackingImageManager(obj *longhorn.BackingI
return nil
}

func (s *DataStore) getBackingImageManagerRO(name string) (*longhorn.BackingImageManager, error) {
func (s *DataStore) GetBackingImageManagerRO(name string) (*longhorn.BackingImageManager, error) {
return s.backingImageManagerLister.BackingImageManagers(s.namespace).Get(name)
}

// GetBackingImageManager returns a new BackingImageManager object for the given name and
// namespace
func (s *DataStore) GetBackingImageManager(name string) (*longhorn.BackingImageManager, error) {
resultRO, err := s.getBackingImageManagerRO(name)
resultRO, err := s.GetBackingImageManagerRO(name)
if err != nil {
return nil, err
}
// Cannot use cached object from lister
return resultRO.DeepCopy(), nil
}

func (s *DataStore) ListBackingImageManagersRO() ([]*longhorn.BackingImageManager, error) {
return s.listBackingImageManagersRO(labels.Everything())
}

func (s *DataStore) listBackingImageManagers(selector labels.Selector) (map[string]*longhorn.BackingImageManager, error) {
itemMap := map[string]*longhorn.BackingImageManager{}

list, err := s.backingImageManagerLister.BackingImageManagers(s.namespace).List(selector)
list, err := s.listBackingImageManagersRO(selector)
if err != nil {
return nil, err
}
Expand All @@ -1888,6 +1892,10 @@ func (s *DataStore) listBackingImageManagers(selector labels.Selector) (map[stri
return itemMap, nil
}

func (s *DataStore) listBackingImageManagersRO(selector labels.Selector) ([]*longhorn.BackingImageManager, error) {
return s.backingImageManagerLister.BackingImageManagers(s.namespace).List(selector)
}

// ListBackingImageManagers returns object includes all BackingImageManager in namespace
func (s *DataStore) ListBackingImageManagers() (map[string]*longhorn.BackingImageManager, error) {
return s.listBackingImageManagers(labels.Everything())
Expand All @@ -1903,6 +1911,14 @@ func (s *DataStore) ListBackingImageManagersByNode(nodeName string) (map[string]
return s.listBackingImageManagers(nodeSelector)
}

func (s *DataStore) ListBackingImageManagersByNodeRO(nodeName string) ([]*longhorn.BackingImageManager, error) {
nodeSelector, err := getNodeSelector(nodeName)
if err != nil {
return nil, err
}
return s.listBackingImageManagersRO(nodeSelector)
}

// ListBackingImageManagersByDiskUUID gets a list of BackingImageManager
// in a specific disk with the given namespace.
func (s *DataStore) ListBackingImageManagersByDiskUUID(diskUUID string) (map[string]*longhorn.BackingImageManager, error) {
Expand Down

0 comments on commit 414be6a

Please sign in to comment.