feat: add marshall for oci url in kcl.mod

Signed-off-by: zongz <zongzhe1024@163.com>

.gitignore

@@ -12,10 +12,11 @@
 *.out
 
 # Dependency directories (remove the comment below to include it)
-# vendor/
+vendor/
 build/
 .vscode/
 .kclvm/
+.idea/
 
 # kpm binary
 kpm

pkg/client/client.go

@@ -15,6 +15,8 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/otiai10/copy"
 	"kcl-lang.io/kcl-go/pkg/kcl"
+	"oras.land/oras-go/v2"
+
 	"kcl-lang.io/kpm/pkg/constants"
 	"kcl-lang.io/kpm/pkg/env"
 	"kcl-lang.io/kpm/pkg/errors"
@@ -26,7 +28,6 @@ import (
 	"kcl-lang.io/kpm/pkg/runner"
 	"kcl-lang.io/kpm/pkg/settings"
 	"kcl-lang.io/kpm/pkg/utils"
-	"oras.land/oras-go/v2"
 )
 
 // KpmClient is the client of kpm.
@@ -787,20 +788,20 @@ func (c *KpmClient) Download(dep *pkg.Dependency, localPath string) (*pkg.Depend
 	}
 
 	if dep.Source.Oci != nil {
-		localPath, err := c.DownloadFromOci(dep.Source.Oci, localPath)
+		pkg, err := c.DownloadPkgFromOci(dep.Source.Oci, localPath)
 		if err != nil {
 			return nil, err
 		}
-		dep.Version = dep.Source.Oci.Tag
-		dep.LocalFullPath = localPath
+		dep.Version = pkg.GetPkgVersion()
+		dep.LocalFullPath = pkg.HomePath
 		// Creating symbolic links in a global cache is not an optimal solution.
 		// This allows kclvm to locate the package by default.
 		// This feature is unstable and will be removed soon.
 		err = createDepRef(dep.LocalFullPath, filepath.Join(filepath.Dir(localPath), dep.Name))
 		if err != nil {
 			return nil, err
 		}
-		dep.FullName = dep.GenDepFullName()
+		dep.FullName = pkg.GetPkgFullName()
 	}
 
 	if dep.Source.Local != nil {
@@ -898,7 +899,53 @@ func (c *KpmClient) ParseKclModFile(kclPkg *pkg.KclPkg) (map[string]map[string]s
 	return dependencies, nil
 }
 
+// LoadPkgFromOci will download the kcl package from the oci repository and return an `KclPkg`.
+func (c *KpmClient) DownloadPkgFromOci(dep *pkg.Oci, localPath string) (*pkg.KclPkg, error) {
+	ociClient, err := oci.NewOciClient(dep.Reg, dep.Repo, &c.settings)
+	if err != nil {
+		return nil, err
+	}
+	ociClient.SetLogWriter(c.logWriter)
+	// Select the latest tag, if the tag, the user inputed, is empty.
+	var tagSelected string
+	if len(dep.Tag) == 0 {
+		tagSelected, err = ociClient.TheLatestTag()
+		if err != nil {
+			return nil, err
+		}
+
+		reporter.ReportMsgTo(
+			fmt.Sprintf("the lastest version '%s' will be added", tagSelected),
+			c.logWriter,
+		)
+
+		dep.Tag = tagSelected
+		localPath = localPath + dep.Tag
+	} else {
+		tagSelected = dep.Tag
+	}
+
+	reporter.ReportMsgTo(
+		fmt.Sprintf("downloading '%s:%s' from '%s/%s:%s'", dep.Repo, tagSelected, dep.Reg, dep.Repo, tagSelected),
+		c.logWriter,
+	)
+
+	// Pull the package with the tag.
+	err = ociClient.Pull(localPath, tagSelected)
+	if err != nil {
+		return nil, err
+	}
+
+	pkg, err := pkg.FindFirstKclPkgFrom(localPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return pkg, nil
+}
+
 // DownloadFromOci will download the dependency from the oci repository.
+// Deprecated: Use the DownloadPkgFromOci instead.
 func (c *KpmClient) DownloadFromOci(dep *pkg.Oci, localPath string) (string, error) {
 	ociClient, err := oci.NewOciClient(dep.Reg, dep.Repo, &c.settings)
 	if err != nil {
@@ -935,29 +982,29 @@ func (c *KpmClient) DownloadFromOci(dep *pkg.Oci, localPath string) (string, err
 		return "", err
 	}
 
-	matches, finderr := filepath.Glob(filepath.Join(localPath, "*.tar"))
-	if finderr != nil || len(matches) != 1 {
-		if finderr == nil {
-			err = reporter.NewErrorEvent(
+	matches, _ := filepath.Glob(filepath.Join(localPath, "*.tar"))
+	if matches == nil || len(matches) != 1 {
+		// then try to glob tgz file
+		matches, _ = filepath.Glob(filepath.Join(localPath, "*.tgz"))
+		if matches == nil || len(matches) != 1 {
+			return "", reporter.NewErrorEvent(
 				reporter.InvalidKclPkg,
 				err,
 				fmt.Sprintf("failed to find the kcl package tar from '%s'.", localPath),
 			)
 		}
-
-		return "", reporter.NewErrorEvent(
-			reporter.InvalidKclPkg,
-			err,
-			fmt.Sprintf("failed to find the kcl package tar from '%s'.", localPath),
-		)
 	}
 
 	tarPath := matches[0]
-	untarErr := utils.UnTarDir(tarPath, localPath)
-	if untarErr != nil {
+	if utils.IsTar(tarPath) {
+		err = utils.UnTarDir(tarPath, localPath)
+	} else {
+		err = utils.ExtractTarball(tarPath, localPath)
+	}
+	if err != nil {
 		return "", reporter.NewErrorEvent(
 			reporter.FailedUntarKclPkg,
-			untarErr,
+			err,
 			fmt.Sprintf("failed to untar the kcl package tar from '%s' into '%s'.", tarPath, localPath),
 		)
 	}

pkg/client/client_test.go

@@ -185,7 +185,6 @@ func TestDependencyGraph(t *testing.T) {
 		Weight:     0,
 		Data:       nil,
 	}
-
 	assert.Equal(t, adjMap,
 		map[string]map[string]graph.Edge[string]{
 			"dependency_graph@0.0.1": {
@@ -277,7 +276,6 @@ func TestParseKclModFile(t *testing.T) {
 
 	assert.Equal(t, expectedDependencies, dependencies, "parsed dependencies do not match expected dependencies")
 }
-
 func TestInitEmptyPkg(t *testing.T) {
 	testDir := initTestDir("test_init_empty_mod")
 	kclPkg := pkg.NewKclPkg(&opt.InitOptions{Name: "test_name", InitPath: testDir})
@@ -617,8 +615,8 @@ func TestPackageCurrentPkgPath(t *testing.T) {
 	assert.Equal(t, err, nil)
 	assert.Equal(t, kclPkg.GetPkgTag(), "0.0.1")
 	assert.Equal(t, kclPkg.GetPkgName(), "test_tar")
-	assert.Equal(t, kclPkg.GetPkgFullName(), "test_tar-0.0.1")
-	assert.Equal(t, kclPkg.GetPkgTarName(), "test_tar-0.0.1.tar")
+	assert.Equal(t, kclPkg.GetPkgFullName(), "test_tar_0.0.1")
+	assert.Equal(t, kclPkg.GetPkgTarName(), "test_tar_0.0.1.tar")
 
 	assert.Equal(t, utils.DirExists(filepath.Join(testDir, kclPkg.GetPkgTarName())), false)
 
@@ -1270,3 +1268,50 @@ func TestAddWithGitCommit(t *testing.T) {
 		_ = os.Remove(testPkgPathModLock)
 	}()
 }
+
+func TestLoadPkgFormOci(t *testing.T) {
+	type testCase struct {
+		Reg  string
+		Repo string
+		Tag  string
+		Name string
+	}
+
+	testCases := []testCase{
+		{
+			Reg:  "ghcr.io",
+			Repo: "kusionstack/opsrule",
+			Tag:  "0.0.9",
+			Name: "opsrule",
+		},
+		{
+			Reg:  "ghcr.io",
+			Repo: "kcl-lang/helloworld",
+			Tag:  "0.1.1",
+			Name: "helloworld",
+		},
+	}
+
+	cli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+	pkgPath := getTestDir("test_load_pkg_from_oci")
+
+	for _, tc := range testCases {
+		localpath := filepath.Join(pkgPath, tc.Name)
+
+		err = os.MkdirAll(localpath, 0755)
+		assert.Equal(t, err, nil)
+		defer func() {
+			err := os.RemoveAll(localpath)
+			assert.Equal(t, err, nil)
+		}()
+
+		kclpkg, err := cli.DownloadPkgFromOci(&pkg.Oci{
+			Reg:  tc.Reg,
+			Repo: tc.Repo,
+			Tag:  tc.Tag,
+		}, localpath)
+		assert.Equal(t, err, nil)
+		assert.Equal(t, kclpkg.GetPkgName(), tc.Name)
+	}
+}

pkg/client/test_data/expected/kcl.mod

@@ -4,5 +4,5 @@ edition = "v0.8.0"
 version = "0.0.1"
 
 [dependencies]
-oci_name = "test_tag"
+oci_name = { oci = "oci://test_reg/test_repo", tag = "test_tag" }
 name = { git = "test_url", tag = "test_tag" }

pkg/client/test_data/expected/kcl.reverse.mod

@@ -5,4 +5,4 @@ version = "0.0.1"
 
 [dependencies]
 name = { git = "test_url", tag = "test_tag" }
-oci_name = "test_tag"
+oci_name = { oci = "oci://test_reg/test_repo", tag = "test_tag" }

pkg/client/test_data/test_add_diff_version/no_sum_check/kcl.mod.expect

@@ -4,4 +4,4 @@ edition = "0.0.1"
 version = "0.0.1"
 
 [dependencies]
-helloworld = "0.1.1"
+helloworld = { oci = "oci://ghcr.io/kcl-lang/helloworld", tag = "0.1.1" }

pkg/client/test_data/test_add_diff_version/with_sum_check/kcl.mod.expect

@@ -4,4 +4,4 @@ edition = "0.0.1"
 version = "0.0.1"
 
 [dependencies]
-helloworld = "0.1.1"
+helloworld = { oci = "oci://ghcr.io/kcl-lang/helloworld", tag = "0.1.1" }

pkg/constants/constants.go

@@ -1,16 +1,17 @@
 package constants
 
 const (
-	KFilePathSuffix                      = ".k"
-	TarPathSuffix                        = ".tar"
-	GitPathSuffix                        = ".git"
-	OciScheme                            = "oci"
-	FileEntry                            = "file"
-	FileWithKclModEntry                  = "file_with_kcl_mod"
-	UrlEntry                             = "url"
-	RefEntry                             = "ref"
-	TarEntry                             = "tar"
-	GitEntry                             = "git"
+	KFilePathSuffix     = ".k"
+	TarPathSuffix       = ".tar"
+	GitPathSuffix       = ".git"
+	OciScheme           = "oci"
+	FileEntry           = "file"
+	FileWithKclModEntry = "file_with_kcl_mod"
+	UrlEntry            = "url"
+	RefEntry            = "ref"
+	TarEntry            = "tar"
+	GitEntry            = "git"
+
 	KCL_MOD                              = "kcl.mod"
 	OCI_SEPARATOR                        = ":"
 	KCL_PKG_TAR                          = "*.tar"
@@ -21,6 +22,7 @@ const (
 	DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION = "org.kcllang.package.description"
 	DEFAULT_KCL_OCI_MANIFEST_SUM         = "org.kcllang.package.sum"
 	DEFAULT_CREATE_OCI_MANIFEST_TIME     = "org.opencontainers.image.created"
+	URL_PATH_SEPARATOR                   = "/"
 
 	// The pattern of the external package argument.
 	EXTERNAL_PKGS_ARG_PATTERN = "%s=%s"

pkg/oci/oci.go

@@ -9,16 +9,17 @@ import (
 
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/thoas/go-funk"
+	"oras.land/oras-go/pkg/auth"
+	dockerauth "oras.land/oras-go/pkg/auth/docker"
+	remoteauth "oras.land/oras-go/v2/registry/remote/auth"
+
 	"kcl-lang.io/kpm/pkg/constants"
 	"kcl-lang.io/kpm/pkg/opt"
 	pkg "kcl-lang.io/kpm/pkg/package"
 	"kcl-lang.io/kpm/pkg/reporter"
 	"kcl-lang.io/kpm/pkg/semver"
 	"kcl-lang.io/kpm/pkg/settings"
 	"kcl-lang.io/kpm/pkg/utils"
-	"oras.land/oras-go/pkg/auth"
-	dockerauth "oras.land/oras-go/pkg/auth/docker"
-	remoteauth "oras.land/oras-go/v2/registry/remote/auth"
 
 	"oras.land/oras-go/v2"
 	"oras.land/oras-go/v2/content/file"
@@ -137,17 +138,25 @@ func NewOciClient(regName, repoName string, settings *settings.Settings) (*OciCl
 	}, nil
 }
 
+// The default limit of the store size is 64 MiB.
+const DEFAULT_LIMIT_STORE_SIZE = 64 * 1024 * 1024
+
 // Pull will pull the oci artifacts from oci registry to local path.
 func (ociClient *OciClient) Pull(localPath, tag string) error {
 	// Create a file store
-	fs, err := file.New(localPath)
+	fs, err := file.NewWithFallbackLimit(localPath, DEFAULT_LIMIT_STORE_SIZE)
 	if err != nil {
 		return reporter.NewErrorEvent(reporter.FailedCreateStorePath, err, "Failed to create store path ", localPath)
 	}
 	defer fs.Close()
 
 	// Copy from the remote repository to the file store
-	_, err = oras.Copy(*ociClient.ctx, ociClient.repo, tag, fs, tag, oras.DefaultCopyOptions)
+	customCopyOptions := oras.CopyOptions{
+		CopyGraphOptions: oras.CopyGraphOptions{
+			MaxMetadataBytes: DEFAULT_LIMIT_STORE_SIZE, // default is 64 MiB
+		},
+	}
+	_, err = oras.Copy(*ociClient.ctx, ociClient.repo, tag, fs, tag, customCopyOptions)
 	if err != nil {
 		return reporter.NewErrorEvent(
 			reporter.FailedGetPkg,

pkg/oci/oci_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
 	"kcl-lang.io/kpm/pkg/settings"
 	"kcl-lang.io/kpm/pkg/utils"
 )
@@ -40,3 +41,38 @@ func TestLogin(t *testing.T) {
 	err := Login(hostName, userName, userPwd, &settings)
 	assert.Equal(t, err.Error(), "failed to login 'ghcr.io', please check registry, username and password is valid\nGet \"https://ghcr.io/v2/\": denied: denied\n")
 }
+
+func TestPull(t *testing.T) {
+	type TestCase struct {
+		Registry string
+		Image    string
+		Tag      string
+	}
+
+	testCases := []TestCase{
+		{"ghcr.io", "kusionstack/opsrule", "0.0.9"},
+		{"ghcr.io", "kcl-lang/helloworld", "0.1.1"},
+	}
+
+	defer func() {
+		err := os.RemoveAll(getTestDir("test_pull"))
+		assert.Equal(t, err, nil)
+	}()
+
+	for _, tc := range testCases {
+		client, err := NewOciClient(tc.Registry, tc.Image, settings.GetSettings())
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+
+		tmpPath := filepath.Join(getTestDir("test_pull"), tc.Tag)
+
+		err = os.MkdirAll(tmpPath, 0755)
+		assert.Equal(t, err, nil)
+
+		err = client.Pull(tmpPath, tc.Tag)
+		if err != nil {
+			t.Errorf(err.Error())
+		}
+	}
+}