Merge pull request #376 from zong-zhe/unify-pull-api

feat: add unified api for CLI pull command

Author: Peefy

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/distribution/reference v0.6.0
 	github.com/dominikbraun/graph v0.23.0
 	github.com/elliotchance/orderedmap/v2 v2.2.0
+	github.com/google/uuid v1.6.0
 	github.com/opencontainers/image-spec v1.1.0
 	github.com/otiai10/copy v1.14.0
 	github.com/sirupsen/logrus v1.9.3
@@ -74,7 +75,6 @@ require (
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
-	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.2 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
@@ -145,7 +145,6 @@ require (
 
 require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/elliotchance/orderedmap v1.6.0
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/gofrs/flock v0.8.1
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3

go.sum

@@ -314,8 +314,6 @@ github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucV
 github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
-github.com/elliotchance/orderedmap v1.6.0 h1:xjn+kbbKXeDq6v9RVE+WYwRbYfAZKvlWfcJNxM8pvEw=
-github.com/elliotchance/orderedmap v1.6.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
 github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk=
 github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q=
 github.com/emicklei/proto v1.13.2 h1:z/etSFO3uyXeuEsVPzfl56WNgzcvIr42aQazXaQmFZY=

pkg/api/kpm_run_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/elliotchance/orderedmap/v2"
 	"github.com/stretchr/testify/assert"
 	"kcl-lang.io/kcl-go/pkg/kcl"
+	"kcl-lang.io/kpm/pkg/downloader"
 	"kcl-lang.io/kpm/pkg/opt"
 	pkg "kcl-lang.io/kpm/pkg/package"
 	"kcl-lang.io/kpm/pkg/utils"
@@ -266,8 +267,8 @@ func TestStoreModAndModLockFile(t *testing.T) {
 		Version:       "0.0.1",
 		Sum:           "sLr3e6W4RPrXYyswdOSiKqkHes1QHX2tk6SwxAPDqqo=",
 		LocalFullPath: filepath.Join(testPath, "dep1_0.0.1"),
-		Source: pkg.Source{
-			Oci: &pkg.Oci{
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
 				Reg:  "ghcr.io",
 				Repo: "kcl-lang/dep1",
 				Tag:  "0.0.1",

pkg/client/client.go

@@ -952,7 +952,7 @@ func (c *KpmClient) FillDependenciesInfo(modFile *pkg.ModFile) error {
 }
 
 // AcquireTheLatestOciVersion will acquire the latest version of the OCI reference.
-func (c *KpmClient) AcquireTheLatestOciVersion(ociSource pkg.Oci) (string, error) {
+func (c *KpmClient) AcquireTheLatestOciVersion(ociSource downloader.Oci) (string, error) {
 	ociClient, err := oci.NewOciClient(ociSource.Reg, ociSource.Repo, &c.settings)
 	if err != nil {
 		return "", err
@@ -961,6 +961,64 @@ func (c *KpmClient) AcquireTheLatestOciVersion(ociSource pkg.Oci) (string, error
 	return ociClient.TheLatestTag()
 }
 
+func (c *KpmClient) downloadPkg(options ...downloader.Option) (*pkg.KclPkg, error) {
+	opts := downloader.DownloadOptions{}
+	for _, option := range options {
+		option(&opts)
+	}
+
+	localPath := opts.LocalPath
+	tmpDir, err := os.MkdirTemp("", "")
+	if err != nil {
+		return nil, err
+	}
+	tmpDir = filepath.Join(tmpDir, constants.GitScheme)
+	// clean the temp dir.
+	defer os.RemoveAll(tmpDir)
+	err = c.DepDownloader.Download(*downloader.NewDownloadOptions(
+		downloader.WithLocalPath(tmpDir),
+		downloader.WithSource(opts.Source),
+		downloader.WithLogWriter(c.GetLogWriter()),
+		downloader.WithSettings(*c.GetSettings()),
+	))
+
+	if err != nil {
+		return nil, err
+	}
+
+	if utils.DirExists(localPath) {
+		err := os.RemoveAll(localPath)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	destDir := filepath.Dir(localPath)
+	if !utils.DirExists(destDir) {
+		err = os.MkdirAll(destDir, os.ModePerm)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	err = utils.MoveFile(tmpDir, localPath)
+	if err != nil {
+		return nil, err
+	}
+
+	localPath, err = filepath.Abs(localPath)
+	if err != nil {
+		return nil, err
+	}
+
+	pkg, err := c.LoadPkgFromPath(localPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return pkg, nil
+}
+
 // Download will download the dependency to the local path.
 func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*pkg.Dependency, error) {
 	if dep.Source.Git != nil {
@@ -992,7 +1050,7 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*
 	}
 
 	if dep.Source.Oci != nil || dep.Source.Registry != nil {
-		var ociSource *pkg.Oci
+		var ociSource *downloader.Oci
 		if dep.Source.Oci != nil {
 			ociSource = dep.Source.Oci
 		} else if dep.Source.Registry != nil {
@@ -1135,7 +1193,7 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*
 }
 
 // DownloadFromGit will download the dependency from the git repository.
-func (c *KpmClient) DownloadFromGit(dep *pkg.Git, localPath string) (string, error) {
+func (c *KpmClient) DownloadFromGit(dep *downloader.Git, localPath string) (string, error) {
 	var msg string
 	if len(dep.Tag) != 0 {
 		msg = fmt.Sprintf("with tag '%s'", dep.Tag)
@@ -1217,7 +1275,7 @@ func (c *KpmClient) ParseKclModFile(kclPkg *pkg.KclPkg) (map[string]map[string]s
 }
 
 // LoadPkgFromOci will download the kcl package from the oci repository and return an `KclPkg`.
-func (c *KpmClient) DownloadPkgFromOci(dep *pkg.Oci, localPath string) (*pkg.KclPkg, error) {
+func (c *KpmClient) DownloadPkgFromOci(dep *downloader.Oci, localPath string) (*pkg.KclPkg, error) {
 	ociClient, err := oci.NewOciClient(dep.Reg, dep.Repo, &c.settings)
 	if err != nil {
 		return nil, err
@@ -1263,7 +1321,7 @@ func (c *KpmClient) DownloadPkgFromOci(dep *pkg.Oci, localPath string) (*pkg.Kcl
 
 // DownloadFromOci will download the dependency from the oci repository.
 // Deprecated: Use the DownloadPkgFromOci instead.
-func (c *KpmClient) DownloadFromOci(dep *pkg.Oci, localPath string) (string, error) {
+func (c *KpmClient) DownloadFromOci(dep *downloader.Oci, localPath string) (string, error) {
 	ociClient, err := oci.NewOciClient(dep.Reg, dep.Repo, &c.settings)
 	if err != nil {
 		return "", err

pkg/client/client_test.go

@@ -67,8 +67,8 @@ func TestDownloadOci(t *testing.T) {
 	depFromOci := pkg.Dependency{
 		Name:    "helloworld",
 		Version: "0.1.2",
-		Source: pkg.Source{
-			Oci: &pkg.Oci{
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
 				Reg:  "ghcr.io",
 				Repo: "kcl-lang/helloworld",
 				Tag:  "0.1.2",
@@ -110,8 +110,8 @@ func TestDownloadLatestOci(t *testing.T) {
 	depFromOci := pkg.Dependency{
 		Name:    "helloworld",
 		Version: "",
-		Source: pkg.Source{
-			Oci: &pkg.Oci{
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
 				Reg:  "ghcr.io",
 				Repo: "kcl-lang/helloworld",
 				Tag:  "",
@@ -284,8 +284,8 @@ func TestUpdateKclModAndLock(t *testing.T) {
 		FullName: "test_version",
 		Version:  "test_version",
 		Sum:      "test_sum",
-		Source: pkg.Source{
-			Git: &pkg.Git{
+		Source: downloader.Source{
+			Git: &downloader.Git{
 				Url: "test_url",
 				Tag: "test_tag",
 			},
@@ -297,8 +297,8 @@ func TestUpdateKclModAndLock(t *testing.T) {
 		FullName: "test_version",
 		Version:  "test_version",
 		Sum:      "test_sum",
-		Source: pkg.Source{
-			Oci: &pkg.Oci{
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
 				Reg:  "test_reg",
 				Repo: "test_repo",
 				Tag:  "test_tag",
@@ -1305,7 +1305,7 @@ func TestLoadPkgFormOci(t *testing.T) {
 			assert.Equal(t, err, nil)
 		}()
 
-		kclpkg, err := cli.DownloadPkgFromOci(&pkg.Oci{
+		kclpkg, err := cli.DownloadPkgFromOci(&downloader.Oci{
 			Reg:  tc.Reg,
 			Repo: tc.Repo,
 			Tag:  tc.Tag,
@@ -1455,8 +1455,8 @@ func testAddWithOciDownloader(t *testing.T) {
 	dep := pkg.Dependency{
 		Name:     "helloworld",
 		FullName: "helloworld_0.0.3",
-		Source: pkg.Source{
-			Oci: &pkg.Oci{
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
 				Reg:  "ghcr.io",
 				Repo: "zong-zhe/helloworld",
 				Tag:  "0.0.3",

pkg/client/pull.go

@@ -0,0 +1,102 @@
+package client
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+
+	"kcl-lang.io/kpm/pkg/downloader"
+	pkg "kcl-lang.io/kpm/pkg/package"
+	"kcl-lang.io/kpm/pkg/reporter"
+)
+
+// The PullOptions struct contains the options for pulling a package from the registry.
+type PullOptions struct {
+	// Source is the source of the package to be pulled.
+	// Including git, oci, local.
+	Source *downloader.Source
+	// LocalPath is the local path to download the package.
+	LocalPath string
+}
+
+type PullOption func(*PullOptions) error
+
+func WithPullSourceUrl(sourceUrl string) PullOption {
+	return func(opts *PullOptions) error {
+		source, err := downloader.NewSourceFromStr(sourceUrl)
+		if err != nil {
+			return err
+		}
+		opts.Source = source
+		return nil
+	}
+}
+
+func WithPullSource(source *downloader.Source) PullOption {
+	return func(opts *PullOptions) error {
+		if source == nil {
+			return errors.New("source cannot be nil")
+		}
+		opts.Source = source
+		return nil
+	}
+}
+
+func WithLocalPath(path string) PullOption {
+	return func(opts *PullOptions) error {
+		opts.LocalPath = path
+		return nil
+	}
+}
+
+func NewPullOptions(opts ...PullOption) *PullOptions {
+	do := &PullOptions{}
+	for _, opt := range opts {
+		opt(do)
+	}
+	return do
+}
+
+func (c *KpmClient) Pull(options ...PullOption) (*pkg.KclPkg, error) {
+	opts := &PullOptions{}
+	for _, option := range options {
+		if err := option(opts); err != nil {
+			return nil, err
+		}
+	}
+
+	sourceFilePath, err := opts.Source.ToFilePath()
+	if err != nil {
+		return nil, err
+	}
+
+	sourceStr, err := opts.Source.ToString()
+	if err != nil {
+		return nil, err
+	}
+	reporter.ReportMsgTo(
+		fmt.Sprintf("start to pull %s", sourceStr),
+		c.GetLogWriter(),
+	)
+
+	kPkg, err := c.downloadPkg(
+		// The package pulled will be stored in the 'opts.LocalPath/sourceFilePath'
+		// 'opts.LocalPath' is the local path input by the user.
+		// 'sourceFilePath' is generated by the source.
+		// For example,
+		// kcl package from 'https://github.com/kcl-lang/kcl' will be stored in '$LOCAL_PATH/git/github.com/kcl-lang/kcl'
+		// kcl package from 'oci://ghcr.io/kcl-lang/kcl' will be stored in '$LOCAL_PATH/oci/ghcr.io/kcl-lang/kcl'
+		downloader.WithLocalPath(filepath.Join(opts.LocalPath, sourceFilePath)),
+		downloader.WithSource(*opts.Source),
+	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	reporter.ReportMsgTo(
+		fmt.Sprintf("pulled %s %s successfully", kPkg.GetPkgName(), kPkg.GetPkgVersion()),
+		c.GetLogWriter(),
+	)
+	return kPkg, nil
+}

pkg/client/pull_test.go

@@ -0,0 +1,53 @@
+package client
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"gotest.tools/v3/assert"
+	"kcl-lang.io/kpm/pkg/downloader"
+)
+
+func TestPull(t *testing.T) {
+	pulledPath := getTestDir("test_pull")
+
+	kpmcli, err := NewKpmClient()
+	assert.NilError(t, err)
+
+	var buf bytes.Buffer
+	kpmcli.SetLogWriter(&buf)
+
+	kPkg, err := kpmcli.Pull(
+		WithLocalPath(pulledPath),
+		WithPullSource(&downloader.Source{
+			Oci: &downloader.Oci{
+				Reg:  "ghcr.io",
+				Repo: "kcl-lang/helloworld",
+				Tag:  "0.0.1",
+			},
+		}),
+	)
+
+	pkgPath := filepath.Join(pulledPath, "oci", "ghcr.io", "kcl-lang", "helloworld", "0.0.1")
+	assert.NilError(t, err)
+	assert.Equal(t, kPkg.GetPkgName(), "helloworld")
+	assert.Equal(t, kPkg.GetPkgVersion(), "0.0.1")
+	assert.Equal(t, kPkg.HomePath, pkgPath)
+
+	kPkg, err = kpmcli.Pull(
+		WithLocalPath(pulledPath),
+		WithPullSourceUrl("oci://ghcr.io/kcl-lang/helloworld?tag=0.1.0"),
+	)
+	pkgPath = filepath.Join(pulledPath, "oci", "ghcr.io", "kcl-lang", "helloworld", "0.1.0")
+	assert.NilError(t, err)
+	assert.Equal(t, kPkg.GetPkgName(), "helloworld")
+	assert.Equal(t, kPkg.GetPkgVersion(), "0.1.0")
+	assert.Equal(t, kPkg.HomePath, pkgPath)
+
+	defer func() {
+		err = os.RemoveAll(filepath.Join(pulledPath, "oci"))
+		assert.NilError(t, err)
+	}()
+}

pkg/client/test_data/test_pull/.gitkeep

@@ -180,7 +180,7 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli
 		)
 	}
 
-	ociOpts.Annotations, err = oci.GenOciManifestFromPkg(kclPkg)
+	ociOpts.Annotations, err = kclPkg.GenOciManifestFromPkg()
 	if err != nil {
 		return err
 	}