Expose virtual size of qcow2 backing images

api/types.go

@@ -13,6 +13,7 @@ type BackingImage struct {
 	Name             string `json:"name"`
 	UUID             string `json:"uuid"`
 	Size             int64  `json:"size"`
+	VirtualSize      int64  `json:"virtualSize"`
 	ExpectedChecksum string `json:"expectedChecksum"`
 
 	Status BackingImageStatus `json:"status"`
@@ -32,6 +33,7 @@ func RPCToBackingImage(obj *rpc.BackingImageResponse) *BackingImage {
 		Name:             obj.Spec.Name,
 		UUID:             obj.Spec.Uuid,
 		Size:             obj.Spec.Size,
+		VirtualSize:      obj.Spec.VirtualSize,
 		ExpectedChecksum: obj.Spec.Checksum,
 
 		Status: BackingImageStatus{
@@ -112,6 +114,7 @@ type FileInfo struct {
 	FilePath         string `json:"filePath"`
 	UUID             string `json:"uuid"`
 	Size             int64  `json:"size"`
+	VirtualSize      int64  `json:"virtualSize"`
 	State            string `json:"state"`
 	Progress         int    `json:"progress"`
 	ProcessedSize    int64  `json:"processedSize"`

pkg/backingimage/backingimage.go

@@ -183,13 +183,13 @@ func OpenBackingImage(path string) (*BackingImage, error) {
 		return nil, err
 	}
 
-	format, err := util.DetectFileFormat(file)
+	imgInfo, err := util.GetQemuImgInfo(file)
 	if err != nil {
 		return nil, err
 	}
 
 	var f enginetypes.DiffDisk
-	switch format {
+	switch imgInfo.Format {
 	case "qcow2":
 		// This is only used when doing backup.
 		// We open qcow2 like raw file and simply backup all the blocks of the qcow2
@@ -202,7 +202,7 @@ func OpenBackingImage(path string) (*BackingImage, error) {
 			return nil, err
 		}
 	default:
-		return nil, fmt.Errorf("format %v of the backing file %v is not supported", format, file)
+		return nil, fmt.Errorf("format %v of the backing file %v is not supported", imgInfo.Format, file)
 	}
 
 	size, err := f.Size()
@@ -224,7 +224,7 @@ func OpenBackingImage(path string) (*BackingImage, error) {
 		SectorSize: diskutil.BackingImageSectorSize,
 		Path:       file,
 		Disk:       f,
-		Format:     format,
+		Format:     imgInfo.Format,
 
 		Location: location,
 	}, nil

pkg/manager/service.go

@@ -289,10 +289,11 @@ func (m *Manager) listAndUpdate() (biFileInfoMap map[string]*api.FileInfo, err e
 func backingImageResponse(fInfo *api.FileInfo) *rpc.BackingImageResponse {
 	return &rpc.BackingImageResponse{
 		Spec: &rpc.BackingImageSpec{
-			Name:     types.GetBackingImageNameFromFilePath(fInfo.FilePath, fInfo.UUID),
-			Uuid:     fInfo.UUID,
-			Size:     fInfo.Size,
-			Checksum: fInfo.ExpectedChecksum,
+			Name:        types.GetBackingImageNameFromFilePath(fInfo.FilePath, fInfo.UUID),
+			Uuid:        fInfo.UUID,
+			Size:        fInfo.Size,
+			VirtualSize: fInfo.VirtualSize,
+			Checksum:    fInfo.ExpectedChecksum,
 		},
 		Status: &rpc.BackingImageStatus{
 			State:            fInfo.State,

pkg/rpc/rpc.proto

@@ -26,6 +26,7 @@ message BackingImageSpec {
 	string uuid = 2;
 	int64 size = 3;
 	string checksum = 4;
+	int64 virtualSize = 5;
 }
 
 message BackingImageStatus {

pkg/sync/server_test.go

@@ -501,9 +501,9 @@ func (s *SyncTestSuite) TestDownloadToDst(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloadChecksum, Equals, checksum)
 	// Downloaded file can be identified as a qcow2 file as well.
-	downloadFileFormat, err := util.DetectFileFormat(unzipDownloadFilePath)
+	downloadFileInfo, err := util.GetQemuImgInfo(unzipDownloadFilePath)
 	c.Assert(err, IsNil)
-	c.Assert(downloadFileFormat, Equals, "qcow2")
+	c.Assert(downloadFileInfo.Format, Equals, "qcow2")
 }
 
 func (s *SyncTestSuite) TestDuplicateCalls(c *C) {
@@ -650,6 +650,96 @@ func (s *SyncTestSuite) TestReadyFileValidation(c *C) {
 	c.Assert(err, IsNil)
 }
 
+func (s *SyncTestSuite) TestVirtualSizeQcow2(c *C) {
+	logrus.Debugf("Testing sync server: TestVirtualSizeQcow2")
+
+	fileName := "sync-virtual-size-qcow2"
+	curPath := filepath.Join(s.dir, fileName)
+	sizeInGB := int64(20)
+
+	logrus.Debugf("creating %dG qcow2 file %v", sizeInGB, curPath)
+	err := createQcow2File(curPath, sizeInGB)
+	c.Assert(err, IsNil)
+
+	checksum, err := util.GetFileChecksum(curPath)
+	c.Assert(err, IsNil)
+	c.Assert(checksum, Not(Equals), "")
+
+	stat, err := os.Stat(curPath)
+	c.Assert(err, IsNil)
+	fileSize := stat.Size()
+
+	go func() {
+		_ = NewServer(s.ctx, s.addr, &MockHandler{})
+	}()
+	isRunning := util.DetectHTTPServerAvailability(s.httpAddr, 5, true)
+	c.Assert(isRunning, Equals, true)
+
+	cli := &client.SyncClient{
+		Remote: s.addr,
+	}
+
+	err = cli.Fetch(curPath, curPath, TestSyncingFileUUID, TestDiskUUID, checksum, fileSize)
+	c.Assert(err, IsNil)
+	fInfo, err := getAndWaitFileState(cli, curPath, string(types.StateReady), 1)
+	c.Assert(err, IsNil)
+
+	// With a qcow2 file, the actual file size should be less
+	// than the virtual size, and both file size and virtual
+	// size should be correctly reported in the FileInfo struct
+	c.Assert(fInfo.Size < fInfo.VirtualSize, Equals, true)
+	c.Assert(fInfo.Size, Equals, fileSize)
+	c.Assert(fInfo.VirtualSize, Equals, sizeInGB*1024*1024*1024)
+	c.Assert(fInfo.Size, Not(Equals), fInfo.VirtualSize)
+
+	err = cli.Delete(curPath)
+	c.Assert(err, IsNil)
+}
+
+func (s *SyncTestSuite) TestVirtualSizeRaw(c *C) {
+	logrus.Debugf("Testing sync server: TestVirtualSizeRaw")
+
+	fileName := "sync-virtual-size-raw"
+	curPath := filepath.Join(s.dir, fileName)
+	sizeInMB := int64(1)
+
+	logrus.Debugf("creating %dM raw file %v", sizeInMB, curPath)
+	err := generateRandomDataFile(curPath, strconv.FormatInt(sizeInMB, 10))
+	c.Assert(err, IsNil)
+
+	checksum, err := util.GetFileChecksum(curPath)
+	c.Assert(err, IsNil)
+	c.Assert(checksum, Not(Equals), "")
+
+	stat, err := os.Stat(curPath)
+	c.Assert(err, IsNil)
+	fileSize := stat.Size()
+
+	go func() {
+		_ = NewServer(s.ctx, s.addr, &MockHandler{})
+	}()
+	isRunning := util.DetectHTTPServerAvailability(s.httpAddr, 5, true)
+	c.Assert(isRunning, Equals, true)
+
+	cli := &client.SyncClient{
+		Remote: s.addr,
+	}
+
+	err = cli.Fetch(curPath, curPath, TestSyncingFileUUID, TestDiskUUID, checksum, fileSize)
+	c.Assert(err, IsNil)
+	fInfo, err := getAndWaitFileState(cli, curPath, string(types.StateReady), 1)
+	c.Assert(err, IsNil)
+
+	// With a raw file, the actual file size and the
+	// virtual size should be equal, and both should
+	// be correctly reported in the FileInfo struct
+	c.Assert(fInfo.Size, Equals, fileSize)
+	c.Assert(fInfo.VirtualSize, Equals, fileSize)
+
+	err = cli.Delete(curPath)
+	c.Assert(err, IsNil)
+}
+
 func getAndWaitFileState(cli *client.SyncClient, curPath, desireState string, waitIntervalInSecond int) (fInfo *api.FileInfo, err error) {
 	endTime := time.Now().Add(time.Duration(waitIntervalInSecond) * time.Second)
 
@@ -679,6 +769,7 @@ func getAndWaitFileState(cli *client.SyncClient, curPath, desireState string, wa
 				FilePath:         fInfo.FilePath,
 				UUID:             fInfo.UUID,
 				Size:             fInfo.Size,
+				VirtualSize:      fInfo.VirtualSize,
 				ExpectedChecksum: fInfo.ExpectedChecksum,
 				CurrentChecksum:  fInfo.CurrentChecksum,
 				ModificationTime: fInfo.ModificationTime,
@@ -731,3 +822,26 @@ func generateRandomDataFile(filePath, sizeInMB string) error {
 
 	return exec.Command("dd", "if=/dev/urandom", "of="+filePath, "bs=1M", "count="+sizeInMB).Run()
 }
+
+func createQcow2File(filePath string, sizeInGB int64) error {
+	// A simple call to `qemu-img create -f qcow2 $filePath 20G` gives
+	// us a file that's only 196928 bytes on disk.  Unfortunately, this is
+	// *not* an even multiple of 512 bytes (196928 / 512 == 384.625), which
+	// means when we use it as sync file we hit an error: "the file size
+	// 196928 should be a multiple of 512 bytes since Longhorn uses directIO
+	// by default". The least worst workaround for this I could think of
+	// is using `dd` to stick an extra 512 byte block of zeros on the end
+	// if the file size isn't evenly divisible by 512.
+	err := exec.Command(util.QemuImgBinary, "create", "-f", "qcow2", filePath, strconv.FormatInt(sizeInGB, 10)+"G").Run()
+	if err != nil {
+		return err
+	}
+	stat, err := os.Stat(filePath)
+	if err != nil {
+		return err
+	}
+	if stat.Size()%512 == 0 {
+		return nil
+	}
+	return exec.Command("dd", "if=/dev/zero", "of="+filePath, "bs=512", "count=1", "seek="+strconv.FormatInt(stat.Size()/512+1, 10)).Run()
+}

pkg/sync/sync_file.go

@@ -80,6 +80,7 @@ type SyncingFile struct {
 	uuid             string
 	diskUUID         string
 	size             int64
+	virtualSize      int64
 	state            types.State
 	progress         int
 	processedSize    int64
@@ -252,6 +253,7 @@ func (sf *SyncingFile) checkAndReuseFile() (err error) {
 	sf.processedSize = info.Size()
 	sf.modificationTime = info.ModTime().UTC().String()
 	sf.updateSyncReadyNoLock()
+	sf.updateVirtualSizeNoLock(sf.filePath)
 	sf.writeConfigNoLock()
 	sf.lock.Unlock()
 
@@ -356,6 +358,7 @@ func (sf *SyncingFile) getNoLock() api.FileInfo {
 		FilePath:         sf.filePath,
 		UUID:             sf.uuid,
 		Size:             sf.size,
+		VirtualSize:      sf.virtualSize,
 		State:            string(sf.state),
 		Progress:         sf.progress,
 		ProcessedSize:    sf.processedSize,
@@ -743,6 +746,7 @@ func (sf *SyncingFile) finishProcessing(err error) (finalErr error) {
 		logrus.Debugf("SyncingFile: directly get the checksum from the valid config during processing wrap-up: %v", config.CurrentChecksum)
 		sf.currentChecksum = config.CurrentChecksum
 		sf.updateSyncReadyNoLock()
+		sf.updateVirtualSizeNoLock(sf.tmpFilePath)
 		sf.writeConfigNoLock()
 
 		// Renaming won't change the file modification time.
@@ -791,6 +795,7 @@ func (sf *SyncingFile) postProcessSyncFile() {
 		return
 	}
 	sf.updateSyncReadyNoLock()
+	sf.updateVirtualSizeNoLock(sf.tmpFilePath)
 	sf.writeConfigNoLock()
 
 	// Renaming won't change the file modification time.
@@ -812,6 +817,19 @@ func (sf *SyncingFile) updateSyncReadyNoLock() {
 	})
 }
 
+func (sf *SyncingFile) updateVirtualSizeNoLock(filePath string) {
+	// This only works if filePath is valid - sometimes we need to call it
+	// with sf.tmpFilePath, sometimes with sf.filePath :-/
+	imgInfo, err := util.GetQemuImgInfo(filePath)
+	if err != nil {
+		sf.log.Warnf("SyncingFile: failed to get backing image virtual size: %v", err)
+	}
+	// This will be zero when there is an error, which allows components
+	// further up the stack to know that the virtual size somehow isn't
+	// available yet.
+	sf.virtualSize = imgInfo.VirtualSize
+}
+
 func (sf *SyncingFile) handleFailureNoLock(err error) {
 	if err == nil {
 		return
@@ -840,6 +858,7 @@ func (sf *SyncingFile) writeConfigNoLock() {
 		FilePath:         sf.filePath,
 		UUID:             sf.uuid,
 		Size:             sf.size,
+		VirtualSize:      sf.virtualSize,
 		ExpectedChecksum: sf.expectedChecksum,
 		CurrentChecksum:  sf.currentChecksum,
 		ModificationTime: sf.modificationTime,