Skip to content

Commit

Permalink
feat(backingimage): add new diskFileSpecMap to backingimage
Browse files Browse the repository at this point in the history 
ref: longhorn/longhorn 2856

Signed-off-by: Jack Lin <jack.lin@suse.com>
  • Loading branch information
@ChanYiLin @derekbit
ChanYiLin authored and derekbit committed Jul 2, 2024
1 parent ae5c434 commit 9710ade
Show file tree
Hide file tree
Showing 14 changed files with 157 additions and 64 deletions.
4 changes: 2 additions & 2 deletions api/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1992,8 +1992,8 @@ func toBackingImageResource(bi *longhorn.BackingImage, apiContext *api.ApiContex
diskFileStatusMap[diskUUID] = *diskStatus
}
}
if bi.Spec.Disks != nil {
for diskUUID := range bi.Spec.Disks {
if bi.Spec.DiskFileSpecMap != nil {
for diskUUID := range bi.Spec.DiskFileSpecMap {
if _, exists := bi.Status.DiskFileStatusMap[diskUUID]; !exists {
diskFileStatusMap[diskUUID] = longhorn.BackingImageDiskFileStatus{
Message: "File processing is not started",
Expand Down
52 changes: 28 additions & 24 deletions controller/backing_image_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,8 +287,8 @@ func (bic *BackingImageController) syncBackingImage(key string) (err error) {
return err
}

// We cannot continue without `Spec.Disks`. The backing image data source controller can update it.
if backingImage.Spec.Disks == nil {
// We cannot continue without `Spec.DiskFileSpecMap`. The backing image data source controller can update it.
if backingImage.Spec.DiskFileSpecMap == nil {
return nil
}

Expand Down Expand Up @@ -338,7 +338,7 @@ func (bic *BackingImageController) replenishBackingImageCopies(bi *longhorn.Back

nonFailedCopies := 0
usedDisks := map[string]bool{}
for diskUUID := range bi.Spec.Disks {
for diskUUID := range bi.Spec.DiskFileSpecMap {
fileStatus, exists := bi.Status.DiskFileStatusMap[diskUUID]
if !exists || (fileStatus.State != longhorn.BackingImageStateFailed &&
fileStatus.State != longhorn.BackingImageStateFailedAndCleanUp &&
Expand Down Expand Up @@ -366,7 +366,7 @@ func (bic *BackingImageController) replenishBackingImageCopies(bi *longhorn.Back
return nil
}
// BackingImageManager will then sync the BackingImage to the disk
bi.Spec.Disks[readyNode.Status.DiskStatus[readyDiskName].DiskUUID] = ""
bi.Spec.DiskFileSpecMap[readyNode.Status.DiskStatus[readyDiskName].DiskUUID] = &longhorn.BackingImageDiskFileSpec{}
}

return nil
Expand All @@ -377,8 +377,8 @@ func (bic *BackingImageController) handleBackingImageCopiesEvictions(nonFailedCo
log := getLoggerForBackingImage(bic.logger, bi)
NonEvictingCount := nonFailedCopies

for _, fileStatus := range bi.Status.DiskFileStatusMap {
if fileStatus.EvictionRequested {
for _, fileSpec := range bi.Spec.DiskFileSpecMap {
if fileSpec.EvictionRequested {
NonEvictingCount--
}
}
Expand All @@ -391,7 +391,7 @@ func (bic *BackingImageController) handleBackingImageCopiesEvictions(nonFailedCo
return nil
}
// BackingImageManager will then sync the BackingImage to the disk
bi.Spec.Disks[readyNode.Status.DiskStatus[readyDiskName].DiskUUID] = ""
bi.Spec.DiskFileSpecMap[readyNode.Status.DiskStatus[readyDiskName].DiskUUID] = &longhorn.BackingImageDiskFileSpec{}
}

return nil
Expand All @@ -404,19 +404,23 @@ func (bic *BackingImageController) cleanupEvictionRequestedBackingImageCopies(bi
// Longhorn should retain one evicting healthy backing image copy for replenishing.
hasNonEvictingHealthyBackingImageCopy := false
evictingHealthyBackingImageCopyDiskUUID := ""
for diskUUID, fileStatus := range bi.Status.DiskFileStatusMap {
if fileStatus.State != longhorn.BackingImageStateReady {
continue
for diskUUID, fileSpec := range bi.Spec.DiskFileSpecMap {
fileStatus, exists := bi.Status.DiskFileStatusMap[diskUUID]
if exists && fileStatus != nil {
if fileStatus.State != longhorn.BackingImageStateReady {
continue
}
}
if !fileStatus.EvictionRequested {

if !fileSpec.EvictionRequested {
hasNonEvictingHealthyBackingImageCopy = true
break
}
evictingHealthyBackingImageCopyDiskUUID = diskUUID
}

for diskUUID, fileStatus := range bi.Status.DiskFileStatusMap {
if !fileStatus.EvictionRequested {
for diskUUID, fileSpec := range bi.Spec.DiskFileSpecMap {
if !fileSpec.EvictionRequested {
continue
}
if !hasNonEvictingHealthyBackingImageCopy && diskUUID == evictingHealthyBackingImageCopyDiskUUID {
Expand All @@ -425,7 +429,7 @@ func (bic *BackingImageController) cleanupEvictionRequestedBackingImageCopies(bi
bic.eventRecorder.Event(bi, corev1.EventTypeNormal, constant.EventReasonFailedDeleting, msg)
continue
}
delete(bi.Spec.Disks, diskUUID)
delete(bi.Spec.DiskFileSpecMap, diskUUID)
log.Infof("Evicted backing image copy on disk %v", diskUUID)
}
}
Expand Down Expand Up @@ -477,7 +481,7 @@ func (bic *BackingImageController) cleanupBackingImageManagers(bi *longhorn.Back
continue
}
// The entry in the backing image manager matches the current backing image.
if _, isStillRequiredByCurrentBI := bi.Spec.Disks[bim.Spec.DiskUUID]; isStillRequiredByCurrentBI && bi.DeletionTimestamp == nil {
if _, isStillRequiredByCurrentBI := bi.Spec.DiskFileSpecMap[bim.Spec.DiskUUID]; isStillRequiredByCurrentBI && bi.DeletionTimestamp == nil {
if bim.Spec.BackingImages[bi.Name] == bi.Status.UUID {
continue
}
Expand Down Expand Up @@ -512,8 +516,8 @@ func (bic *BackingImageController) handleBackingImageDataSource(bi *longhorn.Bac
var readyDiskUUID, readyDiskPath, readyNodeID string
isReadyFile := false
foundReadyDisk := false
if bi.Spec.Disks != nil {
for diskUUID := range bi.Spec.Disks {
if bi.Spec.DiskFileSpecMap != nil {
for diskUUID := range bi.Spec.DiskFileSpecMap {
node, diskName, err := bic.ds.GetReadyDiskNode(diskUUID)
if err != nil {
if !types.ErrorIsNotFound(err) {
Expand Down Expand Up @@ -590,8 +594,8 @@ func (bic *BackingImageController) handleBackingImageDataSource(bi *longhorn.Bac

// If all files in Spec.Disk becomes unavailable and there is no extra ready files.
allFilesUnavailable := true
if bi.Spec.Disks != nil {
for diskUUID := range bi.Spec.Disks {
if bi.Spec.DiskFileSpecMap != nil {
for diskUUID := range bi.Spec.DiskFileSpecMap {
fileStatus, ok := bi.Status.DiskFileStatusMap[diskUUID]
if !ok {
allFilesUnavailable = false
Expand All @@ -616,9 +620,9 @@ func (bic *BackingImageController) handleBackingImageDataSource(bi *longhorn.Bac
}
}
if allFilesUnavailable {
// Check if there are extra available files outside of Spec.Disks
// Check if there are extra available files outside of Spec.DiskFileSpecMap
for diskUUID, fileStatus := range bi.Status.DiskFileStatusMap {
if _, exists := bi.Spec.Disks[diskUUID]; exists {
if _, exists := bi.Spec.DiskFileSpecMap[diskUUID]; exists {
continue
}
if fileStatus.State != longhorn.BackingImageStateFailed {
Expand Down Expand Up @@ -695,7 +699,7 @@ func (bic *BackingImageController) handleBackingImageManagers(bi *longhorn.Backi
return err
}

for diskUUID := range bi.Spec.Disks {
for diskUUID := range bi.Spec.DiskFileSpecMap {
noDefaultBIM := true
requiredBIs := map[string]string{}

Expand Down Expand Up @@ -894,11 +898,11 @@ func (bic *BackingImageController) updateDiskLastReferenceMap(bi *longhorn.Backi
delete(bi.Status.DiskLastRefAtMap, replica.Spec.DiskID)
}
for diskUUID := range bi.Status.DiskLastRefAtMap {
if _, exists := bi.Spec.Disks[diskUUID]; !exists {
if _, exists := bi.Spec.DiskFileSpecMap[diskUUID]; !exists {
delete(bi.Status.DiskLastRefAtMap, diskUUID)
}
}
for diskUUID := range bi.Spec.Disks {
for diskUUID := range bi.Spec.DiskFileSpecMap {
_, isActiveFile := filesInUse[diskUUID]
_, isRecordedHistoricFile := bi.Status.DiskLastRefAtMap[diskUUID]
if !isActiveFile && !isRecordedHistoricFile {
Expand Down
8 changes: 4 additions & 4 deletions controller/backing_image_data_source_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,13 +395,13 @@ func (c *BackingImageDataSourceController) syncBackingImage(bids *longhorn.Backi
}
}()

if bi.Spec.Disks == nil {
bi.Spec.Disks = map[string]string{}
if bi.Spec.DiskFileSpecMap == nil {
bi.Spec.DiskFileSpecMap = map[string]*longhorn.BackingImageDiskFileSpec{}
}

if !bids.Spec.FileTransferred {
if _, exists := bi.Spec.Disks[bids.Spec.DiskUUID]; !exists {
bi.Spec.Disks[bids.Spec.DiskUUID] = ""
if _, exists := bi.Spec.DiskFileSpecMap[bids.Spec.DiskUUID]; !exists {
bi.Spec.DiskFileSpecMap[bids.Spec.DiskUUID] = &longhorn.BackingImageDiskFileSpec{}
}
}

Expand Down
2 changes: 1 addition & 1 deletion controller/backup_backing_image_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,7 @@ func (bc *BackupBackingImageController) checkMonitor(bbi *longhorn.BackupBacking

// pick one backing image manager to do backup
var targetBim *longhorn.BackingImageManager
for diskUUID := range backingImage.Spec.Disks {
for diskUUID := range backingImage.Spec.DiskFileSpecMap {
bimMap, err := bc.ds.ListBackingImageManagersByDiskUUID(diskUUID)
if err != nil {
return nil, err
Expand Down
34 changes: 17 additions & 17 deletions controller/independent_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,10 @@ func (s *TestSuite) TestBackingImageCleanup(c *C) {
Namespace: TestNamespace,
},
Spec: longhorn.BackingImageSpec{
Disks: map[string]string{
TestDiskID1: "",
TestDiskID2: "",
TestDiskID3: "",
DiskFileSpecMap: map[string]*longhorn.BackingImageDiskFileSpec{
TestDiskID1: {},
TestDiskID2: {},
TestDiskID3: {},
},
},
Status: longhorn.BackingImageStatus{
Expand Down Expand Up @@ -114,12 +114,12 @@ func (s *TestSuite) TestBackingImageCleanup(c *C) {
// Test case 1: clean up unused ready disk file when there are enough ready files
bi := biTemplate.DeepCopy()
expectedBI := bi.DeepCopy()
expectedBI.Spec.Disks = map[string]string{
TestDiskID1: "",
TestDiskID2: "",
expectedBI.Spec.DiskFileSpecMap = map[string]*longhorn.BackingImageDiskFileSpec{
TestDiskID1: {},
TestDiskID2: {},
}
BackingImageDiskFileCleanup(node, bi, bidsTemplate, time.Duration(0), 2)
c.Assert(bi.Spec.Disks, DeepEquals, expectedBI.Spec.Disks)
c.Assert(bi.Spec.DiskFileSpecMap, DeepEquals, expectedBI.Spec.DiskFileSpecMap)

// Test case 2: cannot delete the unused ready disk file if there are no enough ready files
bi = biTemplate.DeepCopy()
Expand All @@ -130,7 +130,7 @@ func (s *TestSuite) TestBackingImageCleanup(c *C) {
}
expectedBI = bi.DeepCopy()
BackingImageDiskFileCleanup(node, bi, bidsTemplate, time.Duration(0), 1)
c.Assert(bi.Spec.Disks, DeepEquals, expectedBI.Spec.Disks)
c.Assert(bi.Spec.DiskFileSpecMap, DeepEquals, expectedBI.Spec.DiskFileSpecMap)

// Test case 2: clean up all unused files when there are enough ready files
bi = biTemplate.DeepCopy()
Expand All @@ -144,11 +144,11 @@ func (s *TestSuite) TestBackingImageCleanup(c *C) {
TestDiskID2: util.Now(),
}
expectedBI = bi.DeepCopy()
expectedBI.Spec.Disks = map[string]string{
TestDiskID3: "",
expectedBI.Spec.DiskFileSpecMap = map[string]*longhorn.BackingImageDiskFileSpec{
TestDiskID3: {},
}
BackingImageDiskFileCleanup(node, bi, bidsTemplate, time.Duration(0), 1)
c.Assert(bi.Spec.Disks, DeepEquals, expectedBI.Spec.Disks)
c.Assert(bi.Spec.DiskFileSpecMap, DeepEquals, expectedBI.Spec.DiskFileSpecMap)

// Test case 3: retain (some) unused handling files if there are no enough ready files.
bi = biTemplate.DeepCopy()
Expand All @@ -162,12 +162,12 @@ func (s *TestSuite) TestBackingImageCleanup(c *C) {
TestDiskID2: util.Now(),
}
expectedBI = bi.DeepCopy()
expectedBI.Spec.Disks = map[string]string{
TestDiskID2: "",
TestDiskID3: "",
expectedBI.Spec.DiskFileSpecMap = map[string]*longhorn.BackingImageDiskFileSpec{
TestDiskID2: {},
TestDiskID3: {},
}
BackingImageDiskFileCleanup(node, bi, bidsTemplate, time.Duration(0), 2)
c.Assert(bi.Spec.Disks, DeepEquals, expectedBI.Spec.Disks)
c.Assert(bi.Spec.DiskFileSpecMap, DeepEquals, expectedBI.Spec.DiskFileSpecMap)

// Test case 3: retain all files if there are no enough files.
bi = biTemplate.DeepCopy()
Expand All @@ -183,5 +183,5 @@ func (s *TestSuite) TestBackingImageCleanup(c *C) {
}
expectedBI = bi.DeepCopy()
BackingImageDiskFileCleanup(node, bi, bidsTemplate, time.Duration(0), 3)
c.Assert(bi.Spec.Disks, DeepEquals, expectedBI.Spec.Disks)
c.Assert(bi.Spec.DiskFileSpecMap, DeepEquals, expectedBI.Spec.DiskFileSpecMap)
}
14 changes: 7 additions & 7 deletions controller/node_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1206,7 +1206,7 @@ func (nc *NodeController) cleanUpBackingImagesInDisks(node *longhorn.Node) error
}

func BackingImageDiskFileCleanup(node *longhorn.Node, bi *longhorn.BackingImage, bids *longhorn.BackingImageDataSource, waitInterval time.Duration, minNumberOfCopies int) {
if bi.Spec.Disks == nil || bi.Status.DiskLastRefAtMap == nil || !bids.Spec.FileTransferred {
if bi.Spec.DiskFileSpecMap == nil || bi.Status.DiskLastRefAtMap == nil || !bids.Spec.FileTransferred {
return
}

Expand All @@ -1215,7 +1215,7 @@ func BackingImageDiskFileCleanup(node *longhorn.Node, bi *longhorn.BackingImage,
}

var readyDiskFileCount, handlingDiskFileCount, failedDiskFileCount int
for diskUUID := range bi.Spec.Disks {
for diskUUID := range bi.Spec.DiskFileSpecMap {
// Consider non-existing files as pending/handling backing image files.
fileStatus, exists := bi.Status.DiskFileStatusMap[diskUUID]
if !exists {
Expand All @@ -1234,7 +1234,7 @@ func BackingImageDiskFileCleanup(node *longhorn.Node, bi *longhorn.BackingImage,

for _, diskStatus := range node.Status.DiskStatus {
diskUUID := diskStatus.DiskUUID
if _, exists := bi.Spec.Disks[diskUUID]; !exists {
if _, exists := bi.Spec.DiskFileSpecMap[diskUUID]; !exists {
continue
}
lastRefAtStr, exists := bi.Status.DiskLastRefAtMap[diskUUID]
Expand Down Expand Up @@ -1280,7 +1280,7 @@ func BackingImageDiskFileCleanup(node *longhorn.Node, bi *longhorn.BackingImage,
}

logrus.Infof("Cleaning up the unused file in disk %v for backing image %v", diskUUID, bi.Name)
delete(bi.Spec.Disks, diskUUID)
delete(bi.Spec.DiskFileSpecMap, diskUUID)
}
}

Expand Down Expand Up @@ -1610,14 +1610,14 @@ func (nc *NodeController) syncBackingImageEvictionRequested(node *longhorn.Node)
if diskSpec.EvictionRequested || node.Spec.EvictionRequested {
for _, backingImage := range diskBackingImageMap[diskUUID] {
// trigger eviction request
backingImage.Status.DiskFileStatusMap[diskUUID].EvictionRequested = true
backingImage.Spec.DiskFileSpecMap[diskUUID].EvictionRequested = true
backingImagesToSync = append(backingImagesToSync, backingImageToSync{backingImage, diskUUID, true})
}
} else {
for _, backingImage := range diskBackingImageMap[diskUUID] {
if backingImage.Status.DiskFileStatusMap[diskUUID].EvictionRequested {
if backingImage.Spec.DiskFileSpecMap[diskUUID].EvictionRequested {
// if it is previously set to true, cancel the eviction request
backingImage.Status.DiskFileStatusMap[diskUUID].EvictionRequested = false
backingImage.Spec.DiskFileSpecMap[diskUUID].EvictionRequested = false
backingImagesToSync = append(backingImagesToSync, backingImageToSync{backingImage, diskUUID, false})
}
}
Expand Down
6 changes: 3 additions & 3 deletions controller/replica_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -412,11 +412,11 @@ func (rc *ReplicaController) GetBackingImagePathForReplicaStarting(r *longhorn.R
log.Warnf("The requested backing image %v has not been initialized, UUID is empty", bi.Name)
return "", nil
}
if bi.Spec.Disks == nil {
if bi.Spec.DiskFileSpecMap == nil {
log.Warnf("The requested backing image %v has not started disk file handling", bi.Name)
return "", nil
}
if _, exists := bi.Spec.Disks[r.Spec.DiskID]; !exists {
if _, exists := bi.Spec.DiskFileSpecMap[r.Spec.DiskID]; !exists {
res, err := rc.ds.CanPutBackingImageOnDisk(bi, r.Spec.DiskID)
if err != nil {
log.Warnf("Failed to check if backing image %v can be put on disk %v", bi.Name, r.Spec.DiskID)
Expand All @@ -425,7 +425,7 @@ func (rc *ReplicaController) GetBackingImagePathForReplicaStarting(r *longhorn.R
log.Warnf("The backing image %v can not be put on the disk %v", bi.Name, r.Spec.DiskID)
return "", nil
}
bi.Spec.Disks[r.Spec.DiskID] = ""
bi.Spec.DiskFileSpecMap[r.Spec.DiskID] = &longhorn.BackingImageDiskFileSpec{}
log.Infof("Asking backing image %v to download file to node %v disk %v", bi.Name, r.Spec.NodeID, r.Spec.DiskID)
if _, err := rc.ds.UpdateBackingImage(bi); err != nil {
return "", err
Expand Down
10 changes: 8 additions & 2 deletions k8s/crds.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -441,13 +441,21 @@ spec:
properties:
checksum:
type: string
diskFileSpecMap:
additionalProperties:
properties:
evictionRequested:
type: boolean
type: object
type: object
diskSelector:
items:
type: string
type: array
disks:
additionalProperties:
type: string
description: Deprecated. We are now using DiskFileSpecMap to assign different spec to the file on different disks.
type: object
minNumberOfCopies:
type: integer
Expand Down Expand Up @@ -475,8 +483,6 @@ spec:
diskFileStatusMap:
additionalProperties:
properties:
evictionRequested:
type: boolean
lastStateTransitionTime:
type: string
message:
Expand Down
1 change: 1 addition & 0 deletions k8s/pkg/apis/longhorn/v1beta1/backingimage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ func (bi *BackingImage) ConvertTo(dst conversion.Hub) error {
biV1beta2.Spec.Disks = make(map[string]string)
for name := range bi.Spec.Disks {
biV1beta2.Spec.Disks[name] = ""
biV1beta2.Spec.DiskFileSpecMap[name] = &v1beta2.BackingImageDiskFileSpec{}
}
return nil
default:
Expand Down
Loading

0 comments on commit 9710ade

Please sign in to comment.