Expose virtual size of backing images (backport #2680)

controller/backing_image_controller.go

@@ -663,7 +663,28 @@ func (bic *BackingImageController) syncBackingImageFileInfo(bi *longhorn.Backing
 				bic.eventRecorder.Eventf(bi, corev1.EventTypeNormal, constant.EventReasonUpdate, "Set size to %v", bi.Status.Size)
 			}
 			if bi.Status.Size != info.Size {
-				return fmt.Errorf("found mismatching size %v reported by backing image manager %v in disk %v, the size recorded in status is %v", info.Size, bim.Name, bim.Spec.DiskUUID, bi.Status.Size)
+				if bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID].State != longhorn.BackingImageStateFailed {
+					msg := fmt.Sprintf("found mismatching size %v reported by backing image manager %v in disk %v, the size recorded in status is %v",
+						info.Size, bim.Name, bim.Spec.DiskUUID, bi.Status.Size)
+					log.Error(msg)
+					bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID].State = longhorn.BackingImageStateFailed
+					bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID].Message = msg
+				}
+			}
+		}
+		if info.VirtualSize > 0 {
+			if bi.Status.VirtualSize == 0 {
+				bi.Status.VirtualSize = info.VirtualSize
+				bic.eventRecorder.Eventf(bi, corev1.EventTypeNormal, constant.EventReasonUpdate, "Set virtualSize to %v", bi.Status.VirtualSize)
+			}
+			if bi.Status.VirtualSize != info.VirtualSize {
+				if bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID].State != longhorn.BackingImageStateFailed {
+					msg := fmt.Sprintf("found mismatching virtualSize %v reported by backing image manager %v in disk %v, the virtualSize recorded in status is %v",
+						info.VirtualSize, bim.Name, bim.Spec.DiskUUID, bi.Status.VirtualSize)
+					log.Error(msg)
+					bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID].State = longhorn.BackingImageStateFailed
+					bi.Status.DiskFileStatusMap[bim.Spec.DiskUUID].Message = msg
+				}
 			}
 		}
 	}

engineapi/backing_image_manager.go

@@ -54,9 +54,10 @@ func (c *BackingImageManagerClient) parseBackingImageFileInfo(bi *bimapi.Backing
 		return nil
 	}
 	return &longhorn.BackingImageFileInfo{
-		Name: bi.Name,
-		UUID: bi.UUID,
-		Size: bi.Size,
+		Name:        bi.Name,
+		UUID:        bi.UUID,
+		Size:        bi.Size,
+		VirtualSize: bi.VirtualSize,
 
 		State:                longhorn.BackingImageState(bi.Status.State),
 		CurrentChecksum:      bi.Status.CurrentChecksum,

go.mod

@@ -44,7 +44,11 @@ require (
 	github.com/gorilla/websocket v1.5.1
 	github.com/jinzhu/copier v0.3.5
 	github.com/kubernetes-csi/csi-lib-utils v0.6.1
+<<<<<<< HEAD
 	github.com/longhorn/backing-image-manager v1.5.4
+=======
+	github.com/longhorn/backing-image-manager v1.7.0-dev.0.20240326182459-c5288d745f4a
+>>>>>>> ffe9a837 (vendor: Update longhorn/backing-image-manager to get VirtualSize)
 	github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77
 	github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc
 	github.com/longhorn/go-iscsi-helper v0.0.0-20240308033847-bc3aab599425
@@ -98,7 +102,11 @@ require (
 require (
 	github.com/Microsoft/go-winio v0.6.0 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
+<<<<<<< HEAD
 	github.com/RoaringBitmap/roaring v1.2.3 // indirect
+=======
+	github.com/RoaringBitmap/roaring v1.9.1 // indirect
+>>>>>>> ffe9a837 (vendor: Update longhorn/backing-image-manager to get VirtualSize)
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/beorn7/perks v1.0.1 // indirect

go.sum

@@ -611,8 +611,15 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+<<<<<<< HEAD
 github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY=
 github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
+=======
+github.com/RoaringBitmap/roaring v1.9.0 h1:lwKhr90/j0jVXJyh5X+vQN1VVn77rQFfYnh6RDRGCcE=
+github.com/RoaringBitmap/roaring v1.9.0/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
+github.com/RoaringBitmap/roaring v1.9.1 h1:LXcSqGGGMKm+KAzUyWn7ZeREqoOkoMX+KwLOK1thc4I=
+github.com/RoaringBitmap/roaring v1.9.1/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
+>>>>>>> ffe9a837 (vendor: Update longhorn/backing-image-manager to get VirtualSize)
 github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
 github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
@@ -1034,8 +1041,15 @@ github.com/kubernetes-csi/csi-lib-utils v0.6.1 h1:+AZ58SRSRWh2vmMoWAAGcv7x6fIyBM
 github.com/kubernetes-csi/csi-lib-utils v0.6.1/go.mod h1:GVmlUmxZ+SUjVLXicRFjqWUUvWez0g0Y78zNV9t7KfQ=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
+<<<<<<< HEAD
 github.com/longhorn/backing-image-manager v1.5.4 h1:z5eQ3r39q3VyuCSNtX2S++2tPaorsnOopKf3cjjvNzE=
 github.com/longhorn/backing-image-manager v1.5.4/go.mod h1:M28iC1bpVgefAt8E72ozeE9aSwxXcoXmmIClj5zC7bo=
+=======
+github.com/longhorn/backing-image-manager v1.6.0 h1:Jmlc8+W63l0VZoVhPwNLniAk+eBC4CNaadoqpqA51KE=
+github.com/longhorn/backing-image-manager v1.6.0/go.mod h1:IH0mgbK+Dr13xkY+LhDaufyd9YIpiKqYo1AeRLFYGrk=
+github.com/longhorn/backing-image-manager v1.7.0-dev.0.20240326182459-c5288d745f4a h1:Z/vdszQazSZJ51IlJvtEuDViNn6AShJx6xGcOwwa9Ho=
+github.com/longhorn/backing-image-manager v1.7.0-dev.0.20240326182459-c5288d745f4a/go.mod h1:LuPH4YQ7+phuzSXEPT8b1/iBUsOvmnsCaogU4EImy+k=
+>>>>>>> ffe9a837 (vendor: Update longhorn/backing-image-manager to get VirtualSize)
 github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77 h1:iJRq59kA22f9HIjFtY/lz5rKCorZJrrYXju70XoWdmE=
 github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77/go.mod h1:4cbJWtlrD2cGTQxQLtdlPTYopiJiusXH7CpOBrn/s3k=
 github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc h1:Eh9Npc5yBcVD8E4zVQIGUtC62HcfqevrHjQ2kh7fJ/E=

k8s/crds.yaml

@@ -314,6 +314,9 @@ spec:
                       type: string
                     uuid:
                       type: string
+                    virtualSize:
+                      format: int64
+                      type: integer
                   type: object
                 nullable: true
                 type: object
@@ -412,6 +415,10 @@ spec:
       jsonPath: .status.size
       name: Size
       type: string
+    - description: The virtual size of the image (may be larger than file size)
+      jsonPath: .status.virtualSize
+      name: VirtualSize
+      type: string
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
@@ -479,6 +486,10 @@ spec:
                 type: integer
               uuid:
                 type: string
+              virtualSize:
+                description: Virtual size of image, which may be larger than physical size. Will be zero until known (e.g. while a backing image is uploading)
+                format: int64
+                type: integer
             type: object
         type: object
     served: true

k8s/pkg/apis/longhorn/v1beta2/backingimage.go

@@ -56,6 +56,9 @@ type BackingImageStatus struct {
 	UUID string `json:"uuid"`
 	// +optional
 	Size int64 `json:"size"`
+	// Virtual size of image, which may be larger than physical size. Will be zero until known (e.g. while a backing image is uploading)
+	// +optional
+	VirtualSize int64 `json:"virtualSize"`
 	// +optional
 	Checksum string `json:"checksum"`
 	// +optional
@@ -74,6 +77,7 @@ type BackingImageStatus struct {
 // +kubebuilder:printcolumn:name="UUID",type=string,JSONPath=`.status.uuid`,description="The system generated UUID"
 // +kubebuilder:printcolumn:name="SourceType",type=string,JSONPath=`.spec.sourceType`,description="The source of the backing image file data"
 // +kubebuilder:printcolumn:name="Size",type=string,JSONPath=`.status.size`,description="The backing image file size in each disk"
+// +kubebuilder:printcolumn:name="VirtualSize",type=string,JSONPath=`.status.virtualSize`,description="The virtual size of the image (may be larger than file size)"
 // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
 
 // BackingImage is where Longhorn stores backing image object.

k8s/pkg/apis/longhorn/v1beta2/backingimagemanager.go

@@ -20,6 +20,8 @@ type BackingImageFileInfo struct {
 	// +optional
 	Size int64 `json:"size"`
 	// +optional
+	VirtualSize int64 `json:"virtualSize"`
+	// +optional
 	State BackingImageState `json:"state"`
 	// +optional
 	CurrentChecksum string `json:"currentChecksum"`

scripts/validate

@@ -11,12 +11,31 @@ PACKAGES="$(find . -name '*.go' | grep -Ev '.pb.go' | xargs -I{} dirname {} | \
 echo Packages: ${PACKAGES}
 echo Running: go vet
 go vet ${PACKAGES}
+<<<<<<< HEAD
 echo Running: golint
 for i in ${PACKAGES}; do
     if [ -n "$(golint $i | grep -v 'should have comment.*or be unexported' | grep -v 'just return error instead.' | tee /dev/stderr)" ]; then
         failed=true
     fi
 done
 test -z "$failed"
+=======
+
+if [ ! -z "${DRONE_REPO}" ] && [ ! -z "${DRONE_PULL_REQUEST}" ]; then
+	wget https://github.com/$DRONE_REPO/pull/$DRONE_PULL_REQUEST.patch
+	echo "Running: golangci-lint run --new-from-patch=${DRONE_PULL_REQUEST}.patch"
+	golangci-lint run --new-from-patch="${DRONE_PULL_REQUEST}.patch"
+	rm "${DRONE_PULL_REQUEST}.patch"
+elif [ ! -z "${DRONE_COMMIT_REF}" ]; then
+	echo "Running: golangci-lint run --new-from-rev=${DRONE_COMMIT_REF}"
+	golangci-lint run --new-from-rev=${DRONE_COMMIT_REF}
+else
+	git symbolic-ref -q HEAD && REV="HEAD" || REV="HEAD^"
+	headSHA=$(git rev-parse --short=12 ${REV})
+	echo "Running: golangci-lint run --new-from-rev=${headSHA}"
+	golangci-lint run --new-from-rev=${headSHA}
+fi
+
+>>>>>>> 9707ff03 (scripts/validate: Fix `git rev-parse` when doing local builds)
 echo Running: go fmt
 test -z "$(go fmt ${PACKAGES} | tee /dev/stderr)"