From 1d932c480f97dfb908594edc02a5aff1540018d7 Mon Sep 17 00:00:00 2001 From: Jack Lin Date: Mon, 26 Aug 2024 18:52:28 +0800 Subject: [PATCH] feat: add real size to backing image ref: longhorn/longhonr 8757 Signed-off-by: Jack Lin --- controller/backing_image_controller.go | 15 +++++++++++++++ controller/node_controller.go | 14 ++++++++++++++ engineapi/backing_image_manager.go | 1 + go.mod | 2 +- go.sum | 4 ++-- k8s/crds.yaml | 14 ++++++++++++++ k8s/pkg/apis/longhorn/v1beta2/backingimage.go | 3 +++ .../apis/longhorn/v1beta2/backingimagemanager.go | 2 ++ k8s/pkg/apis/longhorn/v1beta2/node.go | 3 +++ .../longhorn/v1beta2/zz_generated.deepcopy.go | 7 +++++++ .../longhorn/backing-image-manager/api/types.go | 3 +++ .../backing-image-manager/pkg/types/types.go | 3 ++- .../backing-image-manager/pkg/util/util.go | 16 ++++++++++++++++ vendor/modules.txt | 2 +- 14 files changed, 84 insertions(+), 5 deletions(-) diff --git a/controller/backing_image_controller.go b/controller/backing_image_controller.go index b2b3e9d6f1..1ebc32c9c1 100644 --- a/controller/backing_image_controller.go +++ b/controller/backing_image_controller.go @@ -840,6 +840,21 @@ func (bic *BackingImageController) syncBackingImageFileInfo(bi *longhorn.Backing } } } + if info.RealSize > 0 { + if bi.Status.RealSize == 0 { + bi.Status.RealSize = info.RealSize + bic.eventRecorder.Eventf(bi, corev1.EventTypeNormal, constant.EventReasonUpdate, "Set realSize to %v", bi.Status.RealSize) + } + if bi.Status.RealSize != info.RealSize { + if bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID].State != longhorn.BackingImageStateFailed { + msg := fmt.Sprintf("found mismatching realSize %v reported by backing image manager %v in disk %v, the realSoze recorded in status is %v", + info.RealSize, bim.Name, bim.Spec.DiskUUID, bi.Status.RealSize) + log.Error(msg) + bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID].State = longhorn.BackingImageStateFailed + bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID].Message = msg + } + } + } } for diskUUID := range bi.Status.DiskFileStatusMap { diff --git a/controller/node_controller.go b/controller/node_controller.go index 7a131bd72f..1d68843bf6 100644 --- a/controller/node_controller.go +++ b/controller/node_controller.go @@ -806,6 +806,10 @@ func (nc *NodeController) updateDiskStatusSchedulableCondition(node *longhorn.No if err != nil { return err } + backingImages, err := nc.ds.ListBackingImagesRO() + if err != nil { + return err + } for diskName, disk := range node.Spec.Disks { diskStatus := diskStatusMap[diskName] @@ -843,6 +847,7 @@ func (nc *NodeController) updateDiskStatusSchedulableCondition(node *longhorn.No return err } scheduledReplica := map[string]int64{} + scheduledBackingImage := map[string]int64{} storageScheduled := int64(0) for _, replica := range replicas { if replica.Spec.NodeID != node.Name || replica.Spec.DiskPath != disk.Path { @@ -857,8 +862,17 @@ func (nc *NodeController) updateDiskStatusSchedulableCondition(node *longhorn.No storageScheduled += replica.Spec.VolumeSize scheduledReplica[replica.Name] = replica.Spec.VolumeSize } + + for _, backingImage := range backingImages { + if _, exists := backingImage.Spec.DiskFileSpecMap[diskStatus.DiskUUID]; exists { + storageScheduled += backingImage.Status.RealSize + scheduledBackingImage[backingImage.Name] = backingImage.Status.RealSize + } + } + diskStatus.StorageScheduled = storageScheduled diskStatus.ScheduledReplica = scheduledReplica + diskStatus.ScheduledBackingImage = scheduledBackingImage // check disk pressure info, err := nc.scheduler.GetDiskSchedulingInfo(disk, diskStatus) if err != nil { diff --git a/engineapi/backing_image_manager.go b/engineapi/backing_image_manager.go index 9d7df41ce9..874a7c42ea 100644 --- a/engineapi/backing_image_manager.go +++ b/engineapi/backing_image_manager.go @@ -58,6 +58,7 @@ func (c *BackingImageManagerClient) parseBackingImageFileInfo(bi *bimapi.Backing UUID: bi.UUID, Size: bi.Size, VirtualSize: bi.VirtualSize, + RealSize: bi.RealSize, State: longhorn.BackingImageState(bi.Status.State), CurrentChecksum: bi.Status.CurrentChecksum, diff --git a/go.mod b/go.mod index eb5550b253..e21d3b3c0c 100644 --- a/go.mod +++ b/go.mod @@ -60,7 +60,7 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/jinzhu/copier v0.4.0 github.com/kubernetes-csi/csi-lib-utils v0.19.0 - github.com/longhorn/backing-image-manager v1.8.0-dev-20240922 + github.com/longhorn/backing-image-manager v1.8.0-dev-20240922.0.20240923074219-9aff32760277 github.com/longhorn/backupstore v0.0.0-20240922062439-e33cb1230db9 github.com/longhorn/go-common-libs v0.0.0-20240921050101-797b589b669d github.com/longhorn/go-iscsi-helper v0.0.0-20240922062410-c17a624e50c1 diff --git a/go.sum b/go.sum index 3feec1d01c..dc616f1331 100644 --- a/go.sum +++ b/go.sum @@ -157,8 +157,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/longhorn/backing-image-manager v1.8.0-dev-20240922 h1:Ikx3UIgHD90DHr1CBfu9IUTdyt+NzM313IHau5DMzJQ= -github.com/longhorn/backing-image-manager v1.8.0-dev-20240922/go.mod h1:raIdWG9r1P0YlnVRCqsCpavnEHd3TsqREQE2skKNQEA= +github.com/longhorn/backing-image-manager v1.8.0-dev-20240922.0.20240923074219-9aff32760277 h1:b0IhPPH+F2ZOC701kzLDExSAAKN9FLVTHikhGaKYJWo= +github.com/longhorn/backing-image-manager v1.8.0-dev-20240922.0.20240923074219-9aff32760277/go.mod h1:rLnxTIs4uEY0FUkHalr//1IKePxlqKXBhv+NRQ968yg= github.com/longhorn/backupstore v0.0.0-20240922062439-e33cb1230db9 h1:ua4PCfArX2TyPVnldF+m7I9c2eGi+TcP8pcPPx2wp0w= github.com/longhorn/backupstore v0.0.0-20240922062439-e33cb1230db9/go.mod h1:UTONYTgRryrw0RgDZbxwU+r1s4PeKQ+v0Z8Pb6jUWn4= github.com/longhorn/go-common-libs v0.0.0-20240921050101-797b589b669d h1:MXJlzyXLptspJEc1UC7ee2eBIYksTl0RT2bXUAXU+8Q= diff --git a/k8s/crds.yaml b/k8s/crds.yaml index 783cd2ddd9..92bb90189f 100644 --- a/k8s/crds.yaml +++ b/k8s/crds.yaml @@ -343,6 +343,9 @@ spec: type: string progress: type: integer + realSize: + format: int64 + type: integer senderManagerAddress: type: string sendingReference: @@ -560,6 +563,11 @@ spec: type: object ownerID: type: string + realSize: + description: Real size of image, which may be smaller than the size + when the file is a sparse file. Will be zero until known (e.g. while a backing image is uploading) + format: int64 + type: integer size: format: int64 type: integer @@ -2507,6 +2515,12 @@ spec: type: string instanceManagerName: type: string + scheduledBackingImage: + additionalProperties: + format: int64 + type: integer + nullable: true + type: object scheduledReplica: additionalProperties: format: int64 diff --git a/k8s/pkg/apis/longhorn/v1beta2/backingimage.go b/k8s/pkg/apis/longhorn/v1beta2/backingimage.go index 741bc38052..e43982cce8 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/backingimage.go +++ b/k8s/pkg/apis/longhorn/v1beta2/backingimage.go @@ -80,6 +80,9 @@ type BackingImageStatus struct { // Virtual size of image, which may be larger than physical size. Will be zero until known (e.g. while a backing image is uploading) // +optional VirtualSize int64 `json:"virtualSize"` + // Real size of image, which may be smaller than the size when the file is a sparse file. Will be zero until known (e.g. while a backing image is uploading) + // +optional + RealSize int64 `json:"realSize"` // +optional Checksum string `json:"checksum"` // +optional diff --git a/k8s/pkg/apis/longhorn/v1beta2/backingimagemanager.go b/k8s/pkg/apis/longhorn/v1beta2/backingimagemanager.go index 303b2f81ae..7d0ed0aaeb 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/backingimagemanager.go +++ b/k8s/pkg/apis/longhorn/v1beta2/backingimagemanager.go @@ -22,6 +22,8 @@ type BackingImageFileInfo struct { // +optional VirtualSize int64 `json:"virtualSize"` // +optional + RealSize int64 `json:"realSize"` + // +optional State BackingImageState `json:"state"` // +optional CurrentChecksum string `json:"currentChecksum"` diff --git a/k8s/pkg/apis/longhorn/v1beta2/node.go b/k8s/pkg/apis/longhorn/v1beta2/node.go index 7943543bbf..c1f3cae7b3 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/node.go +++ b/k8s/pkg/apis/longhorn/v1beta2/node.go @@ -111,6 +111,9 @@ type DiskStatus struct { // +nullable ScheduledReplica map[string]int64 `json:"scheduledReplica"` // +optional + // +nullable + ScheduledBackingImage map[string]int64 `json:"scheduledBackingImage"` + // +optional DiskUUID string `json:"diskUUID"` // +optional DiskName string `json:"diskName"` diff --git a/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go b/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go index a906205877..83ae61fbdb 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go +++ b/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go @@ -1014,6 +1014,13 @@ func (in *DiskStatus) DeepCopyInto(out *DiskStatus) { (*out)[key] = val } } + if in.ScheduledBackingImage != nil { + in, out := &in.ScheduledBackingImage, &out.ScheduledBackingImage + *out = make(map[string]int64, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } diff --git a/vendor/github.com/longhorn/backing-image-manager/api/types.go b/vendor/github.com/longhorn/backing-image-manager/api/types.go index 03b55cba35..3b5b1c012e 100644 --- a/vendor/github.com/longhorn/backing-image-manager/api/types.go +++ b/vendor/github.com/longhorn/backing-image-manager/api/types.go @@ -13,6 +13,7 @@ type BackingImage struct { UUID string `json:"uuid"` Size int64 `json:"size"` VirtualSize int64 `json:"virtualSize"` + RealSize int64 `json:"realSize"` ExpectedChecksum string `json:"expectedChecksum"` Status BackingImageStatus `json:"status"` @@ -33,6 +34,7 @@ func RPCToBackingImage(obj *rpc.BackingImageResponse) *BackingImage { UUID: obj.Spec.Uuid, Size: obj.Spec.Size, VirtualSize: obj.Spec.VirtualSize, + RealSize: obj.Spec.RealSize, ExpectedChecksum: obj.Spec.Checksum, Status: BackingImageStatus{ @@ -114,6 +116,7 @@ type FileInfo struct { UUID string `json:"uuid"` Size int64 `json:"size"` VirtualSize int64 `json:"virtualSize"` + RealSize int64 `json:"realSize"` State string `json:"state"` Progress int `json:"progress"` ProcessedSize int64 `json:"processedSize"` diff --git a/vendor/github.com/longhorn/backing-image-manager/pkg/types/types.go b/vendor/github.com/longhorn/backing-image-manager/pkg/types/types.go index 1d17abd84c..22e589bd69 100644 --- a/vendor/github.com/longhorn/backing-image-manager/pkg/types/types.go +++ b/vendor/github.com/longhorn/backing-image-manager/pkg/types/types.go @@ -13,7 +13,8 @@ const ( DiskPathInContainer = "/data/" DataSourceDirectoryName = "/tmp/" - DefaultSectorSize = 512 + DefaultSectorSize = 512 + DefaultLinuxBlcokSize = 512 DefaultManagerPort = 8000 DefaultDataSourceServerPort = 8000 diff --git a/vendor/github.com/longhorn/backing-image-manager/pkg/util/util.go b/vendor/github.com/longhorn/backing-image-manager/pkg/util/util.go index 2bcd23cbcc..dd5f5f5597 100644 --- a/vendor/github.com/longhorn/backing-image-manager/pkg/util/util.go +++ b/vendor/github.com/longhorn/backing-image-manager/pkg/util/util.go @@ -14,6 +14,7 @@ import ( "os/exec" "path/filepath" "regexp" + "syscall" "time" "github.com/pkg/errors" @@ -23,6 +24,8 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" + + "github.com/longhorn/backing-image-manager/pkg/types" ) const ( @@ -151,6 +154,7 @@ type SyncingFileConfig struct { UUID string `json:"uuid"` Size int64 `json:"size"` VirtualSize int64 `json:"virtualSize"` + RealSize int64 `json:"realSize"` ExpectedChecksum string `json:"expectedChecksum"` CurrentChecksum string `json:"currentChecksum"` ModificationTime string `json:"modificationTime"` @@ -313,6 +317,18 @@ func ConvertFromQcow2ToRaw(sourcePath, targetPath string) error { return nil } +func GetFileRealSize(filePath string) (int64, error) { + var stat syscall.Stat_t + err := syscall.Stat(filePath, &stat) + if err != nil { + return 0, err + } + fmt.Printf("stat.Blksize: %v\n", stat.Blksize) + + // 512 is defined in the Linux kernel and remains consistent across all distributions. + return stat.Blocks * types.DefaultLinuxBlcokSize, nil +} + func FileModificationTime(filePath string) string { fi, err := os.Stat(filePath) if err != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index b839d2be68..c6e2b3b478 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -232,7 +232,7 @@ github.com/kylelemons/godebug/diff # github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de ## explicit github.com/liggitt/tabwriter -# github.com/longhorn/backing-image-manager v1.8.0-dev-20240922 +# github.com/longhorn/backing-image-manager v1.8.0-dev-20240922.0.20240923074219-9aff32760277 ## explicit; go 1.22.2 github.com/longhorn/backing-image-manager/api github.com/longhorn/backing-image-manager/pkg/client