manager: Implement the backing image download to local

Longhorn 3155

Signed-off-by: Shuo Wu <shuo.wu@suse.com>

app/cmd/backing_image.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"fmt"
 	"strconv"
 
 	"github.com/sirupsen/logrus"
@@ -27,6 +28,7 @@ func BackingImageCmd() cli.Command {
 			GetCmd(),
 			ListCmd(),
 			FetchCmd(),
+			PrepareDownloadCmd(),
 		},
 	}
 }
@@ -214,3 +216,34 @@ func fetch(c *cli.Context) error {
 	}
 	return util.PrintJSON(bi)
 }
+
+func PrepareDownloadCmd() cli.Command {
+	return cli.Command{
+		Name: "prepare-download",
+		Flags: []cli.Flag{
+			cli.StringFlag{
+				Name: "name",
+			},
+			cli.StringFlag{
+				Name: "uuid",
+			},
+		},
+		Action: func(c *cli.Context) {
+			if err := prepareDownload(c); err != nil {
+				logrus.Fatalf("Error running backing image prepare download command: %v.", err)
+			}
+		},
+	}
+}
+
+func prepareDownload(c *cli.Context) error {
+	url := c.GlobalString("url")
+	bimClient := client.NewBackingImageManagerClient(url)
+	srcFilePath, address, err := bimClient.PrepareDownload(c.String("name"), c.String("uuid"))
+	if err != nil {
+		return err
+	}
+	fmt.Println("Source file path:", srcFilePath)
+	fmt.Println("Download server address:", address)
+	return nil
+}

pkg/client/manager_client.go

@@ -172,6 +172,31 @@ func (cli *BackingImageManagerClient) Fetch(name, uuid, checksum, dataSourceAddr
 	return api.RPCToBackingImage(resp), nil
 }
 
+func (cli *BackingImageManagerClient) PrepareDownload(name, uuid string) (string, string, error) {
+	if name == "" || uuid == "" {
+		return "", "", fmt.Errorf("failed to get backing image: missing required parameter")
+	}
+
+	conn, err := grpc.Dial(cli.Address, grpc.WithInsecure())
+	if err != nil {
+		return "", "", fmt.Errorf("cannot connect backing image manager service to %v: %v", cli.Address, err)
+	}
+	defer conn.Close()
+
+	client := rpc.NewBackingImageManagerServiceClient(conn)
+	ctx, cancel := context.WithTimeout(context.Background(), types.GRPCServiceTimeout)
+	defer cancel()
+
+	resp, err := client.PrepareDownload(ctx, &rpc.PrepareDownloadRequest{
+		Name: name,
+		Uuid: uuid,
+	})
+	if err != nil {
+		return "", "", err
+	}
+	return resp.SrcFilePath, resp.Address, nil
+}
+
 func (cli *BackingImageManagerClient) VersionGet() (*meta.VersionOutput, error) {
 	conn, err := grpc.Dial(cli.Address, grpc.WithInsecure())
 	if err != nil {

pkg/manager/server_test.go

@@ -380,6 +380,60 @@ func (s *TestSuite) TestSingleBackingImageSync(c *C) {
 	}
 }
 
+func (s *TestSuite) TestBackingImageDownloadToLocal(c *C) {
+	biName := "test-download-src-sync-file"
+	biUUID := TestBackingImageUUID
+	biFilePath1 := types.GetBackingImageFilePath(s.testDiskPath1, biName, biUUID)
+
+	biFileDir1 := filepath.Dir(biFilePath1)
+	err := os.MkdirAll(biFileDir1, 0777)
+	c.Assert(err, IsNil)
+
+	sizeInMB := 1024
+	size := int64(sizeInMB * MB)
+	logrus.Debugf("preparing the original file %v for the test, this may be time-consuming", biFilePath1)
+	err = generateRandomDataFile(biFilePath1, sizeInMB)
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(biFilePath1)
+	checksum, err := util.GetFileChecksum(biFilePath1)
+	c.Assert(err, IsNil)
+	logrus.Debugf("the original file %v for the test is ready", biFilePath1)
+
+	cli1 := client.NewBackingImageManagerClient(s.addr1)
+
+	// The 1st manager directly reuses the file.
+	bi, err := cli1.Fetch(biName, biUUID, checksum, "", size)
+	c.Assert(err, IsNil)
+	c.Assert(bi.Status.State, Not(Equals), string(types.StateFailed))
+	bi, err = getAndWaitFileState(cli1, biName, biUUID, string(types.StateReady), 30)
+	c.Assert(err, IsNil)
+	c.Assert(bi.Status.CurrentChecksum, Equals, checksum)
+	_, err = os.Stat(biFilePath1)
+	c.Assert(err, IsNil)
+	defer s.deleteBackingImage(c, s.addr1, s.testDiskPath1, biName, biUUID)
+
+	// Prepare the download
+	downloadFilePath := filepath.Join(s.testBIMDir1, "test-download-dst-sync-file")
+	srcFilePath, addr, err := cli1.PrepareDownload(biName, biUUID)
+	c.Assert(err, IsNil)
+	c.Assert(addr, Equals, s.syncAddr1)
+	c.Assert(srcFilePath, Equals, biFilePath1)
+
+	// Start the actual download
+	syncCli1 := &client.SyncClient{
+		Remote: addr,
+	}
+	err = syncCli1.DownloadToDst(srcFilePath, downloadFilePath)
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(downloadFilePath)
+
+	downloadChecksum, err := util.GetFileChecksum(downloadFilePath)
+	c.Assert(err, IsNil)
+	c.Assert(downloadChecksum, Equals, checksum)
+
+	s.deleteBackingImage(c, s.addr1, s.testDiskPath1, biName, biUUID)
+}
+
 func (s *TestSuite) TestDuplicateCalls(c *C) {
 	biName := "test-duplicate-calls-file"
 	biUUID := TestBackingImageUUID

pkg/manager/service.go

@@ -414,6 +414,43 @@ func (m *Manager) Fetch(ctx context.Context, req *rpc.FetchRequest) (resp *rpc.B
 	return m.getAndUpdate(req.Spec.Name, req.Spec.Uuid)
 }
 
+func (m *Manager) PrepareDownload(ctx context.Context, req *rpc.PrepareDownloadRequest) (resp *rpc.PrepareDownloadResponse, err error) {
+	log := m.log.WithFields(logrus.Fields{"biName": req.Name, "biUUID": req.Uuid})
+	log.Infof("Backing Image Manager: start to make preparation for backing image download")
+
+	defer func() {
+		if err != nil {
+			log.WithError(err).Error("Backing Image Manager: failed to make preparation for backing image download")
+		}
+	}()
+
+	if req.Name == "" || req.Uuid == "" {
+		return nil, status.Errorf(codes.InvalidArgument, "missing required argument")
+	}
+
+	bi, err := m.getAndUpdate(req.Name, req.Uuid)
+	if err != nil {
+		return nil, err
+	}
+
+	if bi.Status.State != string(types.StateReady) {
+		return nil, status.Errorf(codes.FailedPrecondition, "invalid backing image state %v for the download", bi.Status.State)
+	}
+
+	address, err := util.ConvertToStorageAddress(m.syncAddress)
+	if err != nil {
+		return nil, status.Errorf(codes.Unknown, "failed to detect the storage address of the sync server for the download: %v", err)
+	}
+	srcFilePath := types.GetBackingImageFilePath(m.diskPath, req.Name, req.Uuid)
+
+	log.Infof("Backing Image Manager: prepared for backing image download, file path %v, address %v", srcFilePath, address)
+
+	return &rpc.PrepareDownloadResponse{
+		SrcFilePath: srcFilePath,
+		Address:     address,
+	}, nil
+}
+
 func (m *Manager) allocatePorts(portCount int32) (int32, int32, error) {
 	if portCount < 0 {
 		return 0, 0, fmt.Errorf("invalid port count %v", portCount)