From 20bda41c2f1bd2d3d0e4f23dba0a189d12d23b55 Mon Sep 17 00:00:00 2001 From: Jack Lin Date: Mon, 26 Aug 2024 18:21:09 +0800 Subject: [PATCH] feat: add real size to backing image ref: longhorn/longhorn 8757 Signed-off-by: Jack Lin --- api/types.go | 3 +++ pkg/manager/service.go | 1 + pkg/sync/server_test.go | 1 + pkg/sync/sync_file.go | 20 +++++++++++++++++++- pkg/types/types.go | 3 ++- pkg/util/util.go | 16 ++++++++++++++++ 6 files changed, 42 insertions(+), 2 deletions(-) diff --git a/api/types.go b/api/types.go index 03b55cba..3b5b1c01 100644 --- a/api/types.go +++ b/api/types.go @@ -13,6 +13,7 @@ type BackingImage struct { UUID string `json:"uuid"` Size int64 `json:"size"` VirtualSize int64 `json:"virtualSize"` + RealSize int64 `json:"realSize"` ExpectedChecksum string `json:"expectedChecksum"` Status BackingImageStatus `json:"status"` @@ -33,6 +34,7 @@ func RPCToBackingImage(obj *rpc.BackingImageResponse) *BackingImage { UUID: obj.Spec.Uuid, Size: obj.Spec.Size, VirtualSize: obj.Spec.VirtualSize, + RealSize: obj.Spec.RealSize, ExpectedChecksum: obj.Spec.Checksum, Status: BackingImageStatus{ @@ -114,6 +116,7 @@ type FileInfo struct { UUID string `json:"uuid"` Size int64 `json:"size"` VirtualSize int64 `json:"virtualSize"` + RealSize int64 `json:"realSize"` State string `json:"state"` Progress int `json:"progress"` ProcessedSize int64 `json:"processedSize"` diff --git a/pkg/manager/service.go b/pkg/manager/service.go index 870e6a03..71167328 100644 --- a/pkg/manager/service.go +++ b/pkg/manager/service.go @@ -300,6 +300,7 @@ func backingImageResponse(fInfo *api.FileInfo) *rpc.BackingImageResponse { Uuid: fInfo.UUID, Size: fInfo.Size, VirtualSize: fInfo.VirtualSize, + RealSize: fInfo.RealSize, Checksum: fInfo.ExpectedChecksum, }, Status: &rpc.BackingImageStatus{ diff --git a/pkg/sync/server_test.go b/pkg/sync/server_test.go index dbf610af..94192944 100644 --- a/pkg/sync/server_test.go +++ b/pkg/sync/server_test.go @@ -770,6 +770,7 @@ func getAndWaitFileState(cli *client.SyncClient, curPath, desireState string, wa UUID: fInfo.UUID, Size: fInfo.Size, VirtualSize: fInfo.VirtualSize, + RealSize: fInfo.RealSize, ExpectedChecksum: fInfo.ExpectedChecksum, CurrentChecksum: fInfo.CurrentChecksum, ModificationTime: fInfo.ModificationTime, diff --git a/pkg/sync/sync_file.go b/pkg/sync/sync_file.go index a43679c2..921f25a2 100644 --- a/pkg/sync/sync_file.go +++ b/pkg/sync/sync_file.go @@ -85,6 +85,7 @@ type SyncingFile struct { diskUUID string size int64 virtualSize int64 + realSize int64 state types.State progress int processedSize int64 @@ -258,6 +259,7 @@ func (sf *SyncingFile) checkAndReuseFile() (err error) { sf.modificationTime = info.ModTime().UTC().String() sf.updateSyncReadyNoLock() sf.updateVirtualSizeNoLock(sf.filePath) + sf.updateRealSizeNoLock(sf.filePath) sf.writeConfigNoLock() sf.lock.Unlock() @@ -359,6 +361,7 @@ func (sf *SyncingFile) getNoLock() api.FileInfo { UUID: sf.uuid, Size: sf.size, VirtualSize: sf.virtualSize, + RealSize: sf.realSize, State: string(sf.state), Progress: sf.progress, ProcessedSize: sf.processedSize, @@ -917,6 +920,7 @@ func (sf *SyncingFile) finishProcessing(err error) (finalErr error) { sf.currentChecksum = config.CurrentChecksum sf.updateSyncReadyNoLock() sf.updateVirtualSizeNoLock(sf.tmpFilePath) + sf.updateRealSizeNoLock(sf.tmpFilePath) sf.writeConfigNoLock() // Renaming won't change the file modification time. @@ -966,6 +970,7 @@ func (sf *SyncingFile) postProcessSyncFile() { } sf.updateSyncReadyNoLock() sf.updateVirtualSizeNoLock(sf.tmpFilePath) + sf.updateRealSizeNoLock(sf.tmpFilePath) sf.writeConfigNoLock() // Renaming won't change the file modification time. @@ -992,7 +997,7 @@ func (sf *SyncingFile) updateVirtualSizeNoLock(filePath string) { // with sf.tmpFilePath, sometimes with sf.filePath :-/ imgInfo, err := util.GetQemuImgInfo(filePath) if err != nil { - sf.log.Warnf("SyncingFile: failed to get backing image virtual size: %v", err) + sf.log.WithError(err).Warnf("SyncingFile: failed to get backing image virtual size") } // This will be zero when there is an error, which allows components // further up the stack to know that the virtual size somehow isn't @@ -1000,6 +1005,18 @@ func (sf *SyncingFile) updateVirtualSizeNoLock(filePath string) { sf.virtualSize = imgInfo.VirtualSize } +func (sf *SyncingFile) updateRealSizeNoLock(filePath string) { + realSize, err := util.GetFileRealSize(filePath) + if err != nil { + sf.log.WithError(err).Warnf("SyncingFile: failed to get backing image virtual size") + } + + // This will be zero when there is an error, which allows components + // further up the stack to know that the real size somehow isn't + // available yet. + sf.realSize = realSize +} + func (sf *SyncingFile) handleFailureNoLock(err error) { if err == nil { return @@ -1029,6 +1046,7 @@ func (sf *SyncingFile) writeConfigNoLock() { UUID: sf.uuid, Size: sf.size, VirtualSize: sf.virtualSize, + RealSize: sf.realSize, ExpectedChecksum: sf.expectedChecksum, CurrentChecksum: sf.currentChecksum, ModificationTime: sf.modificationTime, diff --git a/pkg/types/types.go b/pkg/types/types.go index 1d17abd8..22e589bd 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -13,7 +13,8 @@ const ( DiskPathInContainer = "/data/" DataSourceDirectoryName = "/tmp/" - DefaultSectorSize = 512 + DefaultSectorSize = 512 + DefaultLinuxBlcokSize = 512 DefaultManagerPort = 8000 DefaultDataSourceServerPort = 8000 diff --git a/pkg/util/util.go b/pkg/util/util.go index 2bcd23cb..dd5f5f55 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -14,6 +14,7 @@ import ( "os/exec" "path/filepath" "regexp" + "syscall" "time" "github.com/pkg/errors" @@ -23,6 +24,8 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" + + "github.com/longhorn/backing-image-manager/pkg/types" ) const ( @@ -151,6 +154,7 @@ type SyncingFileConfig struct { UUID string `json:"uuid"` Size int64 `json:"size"` VirtualSize int64 `json:"virtualSize"` + RealSize int64 `json:"realSize"` ExpectedChecksum string `json:"expectedChecksum"` CurrentChecksum string `json:"currentChecksum"` ModificationTime string `json:"modificationTime"` @@ -313,6 +317,18 @@ func ConvertFromQcow2ToRaw(sourcePath, targetPath string) error { return nil } +func GetFileRealSize(filePath string) (int64, error) { + var stat syscall.Stat_t + err := syscall.Stat(filePath, &stat) + if err != nil { + return 0, err + } + fmt.Printf("stat.Blksize: %v\n", stat.Blksize) + + // 512 is defined in the Linux kernel and remains consistent across all distributions. + return stat.Blocks * types.DefaultLinuxBlcokSize, nil +} + func FileModificationTime(filePath string) string { fi, err := os.Stat(filePath) if err != nil {