diff --git a/.gitignore b/.gitignore index a8399f49..881f1c61 100644 --- a/.gitignore +++ b/.gitignore @@ -12,10 +12,11 @@ *.out # Dependency directories (remove the comment below to include it) -# vendor/ +vendor/ build/ .vscode/ .kclvm/ +.idea/ # kpm binary kpm diff --git a/pkg/client/client.go b/pkg/client/client.go index 084cff01..b762fcc0 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -15,6 +15,8 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/otiai10/copy" "kcl-lang.io/kcl-go/pkg/kcl" + "oras.land/oras-go/v2" + "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/env" "kcl-lang.io/kpm/pkg/errors" @@ -26,7 +28,6 @@ import ( "kcl-lang.io/kpm/pkg/runner" "kcl-lang.io/kpm/pkg/settings" "kcl-lang.io/kpm/pkg/utils" - "oras.land/oras-go/v2" ) // KpmClient is the client of kpm. @@ -787,12 +788,12 @@ func (c *KpmClient) Download(dep *pkg.Dependency, localPath string) (*pkg.Depend } if dep.Source.Oci != nil { - localPath, err := c.DownloadFromOci(dep.Source.Oci, localPath) + pkg, err := c.DownloadPkgFromOci(dep.Source.Oci, localPath) if err != nil { return nil, err } - dep.Version = dep.Source.Oci.Tag - dep.LocalFullPath = localPath + dep.Version = pkg.GetPkgVersion() + dep.LocalFullPath = pkg.HomePath // Creating symbolic links in a global cache is not an optimal solution. // This allows kclvm to locate the package by default. // This feature is unstable and will be removed soon. @@ -800,7 +801,7 @@ func (c *KpmClient) Download(dep *pkg.Dependency, localPath string) (*pkg.Depend if err != nil { return nil, err } - dep.FullName = dep.GenDepFullName() + dep.FullName = pkg.GetPkgFullName() } if dep.Source.Local != nil { @@ -898,7 +899,53 @@ func (c *KpmClient) ParseKclModFile(kclPkg *pkg.KclPkg) (map[string]map[string]s return dependencies, nil } +// LoadPkgFromOci will download the kcl package from the oci repository and return an `KclPkg`. +func (c *KpmClient) DownloadPkgFromOci(dep *pkg.Oci, localPath string) (*pkg.KclPkg, error) { + ociClient, err := oci.NewOciClient(dep.Reg, dep.Repo, &c.settings) + if err != nil { + return nil, err + } + ociClient.SetLogWriter(c.logWriter) + // Select the latest tag, if the tag, the user inputed, is empty. + var tagSelected string + if len(dep.Tag) == 0 { + tagSelected, err = ociClient.TheLatestTag() + if err != nil { + return nil, err + } + + reporter.ReportMsgTo( + fmt.Sprintf("the lastest version '%s' will be added", tagSelected), + c.logWriter, + ) + + dep.Tag = tagSelected + localPath = localPath + dep.Tag + } else { + tagSelected = dep.Tag + } + + reporter.ReportMsgTo( + fmt.Sprintf("downloading '%s:%s' from '%s/%s:%s'", dep.Repo, tagSelected, dep.Reg, dep.Repo, tagSelected), + c.logWriter, + ) + + // Pull the package with the tag. + err = ociClient.Pull(localPath, tagSelected) + if err != nil { + return nil, err + } + + pkg, err := pkg.FindFirstKclPkgFrom(localPath) + if err != nil { + return nil, err + } + + return pkg, nil +} + // DownloadFromOci will download the dependency from the oci repository. +// Deprecated: Use the DownloadPkgFromOci instead. func (c *KpmClient) DownloadFromOci(dep *pkg.Oci, localPath string) (string, error) { ociClient, err := oci.NewOciClient(dep.Reg, dep.Repo, &c.settings) if err != nil { @@ -935,29 +982,29 @@ func (c *KpmClient) DownloadFromOci(dep *pkg.Oci, localPath string) (string, err return "", err } - matches, finderr := filepath.Glob(filepath.Join(localPath, "*.tar")) - if finderr != nil || len(matches) != 1 { - if finderr == nil { - err = reporter.NewErrorEvent( + matches, _ := filepath.Glob(filepath.Join(localPath, "*.tar")) + if matches == nil || len(matches) != 1 { + // then try to glob tgz file + matches, _ = filepath.Glob(filepath.Join(localPath, "*.tgz")) + if matches == nil || len(matches) != 1 { + return "", reporter.NewErrorEvent( reporter.InvalidKclPkg, err, fmt.Sprintf("failed to find the kcl package tar from '%s'.", localPath), ) } - - return "", reporter.NewErrorEvent( - reporter.InvalidKclPkg, - err, - fmt.Sprintf("failed to find the kcl package tar from '%s'.", localPath), - ) } tarPath := matches[0] - untarErr := utils.UnTarDir(tarPath, localPath) - if untarErr != nil { + if utils.IsTar(tarPath) { + err = utils.UnTarDir(tarPath, localPath) + } else { + err = utils.ExtractTarball(tarPath, localPath) + } + if err != nil { return "", reporter.NewErrorEvent( reporter.FailedUntarKclPkg, - untarErr, + err, fmt.Sprintf("failed to untar the kcl package tar from '%s' into '%s'.", tarPath, localPath), ) } diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index ac54c5b7..0f759e36 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -185,7 +185,6 @@ func TestDependencyGraph(t *testing.T) { Weight: 0, Data: nil, } - assert.Equal(t, adjMap, map[string]map[string]graph.Edge[string]{ "dependency_graph@0.0.1": { @@ -277,7 +276,6 @@ func TestParseKclModFile(t *testing.T) { assert.Equal(t, expectedDependencies, dependencies, "parsed dependencies do not match expected dependencies") } - func TestInitEmptyPkg(t *testing.T) { testDir := initTestDir("test_init_empty_mod") kclPkg := pkg.NewKclPkg(&opt.InitOptions{Name: "test_name", InitPath: testDir}) @@ -617,8 +615,8 @@ func TestPackageCurrentPkgPath(t *testing.T) { assert.Equal(t, err, nil) assert.Equal(t, kclPkg.GetPkgTag(), "0.0.1") assert.Equal(t, kclPkg.GetPkgName(), "test_tar") - assert.Equal(t, kclPkg.GetPkgFullName(), "test_tar-0.0.1") - assert.Equal(t, kclPkg.GetPkgTarName(), "test_tar-0.0.1.tar") + assert.Equal(t, kclPkg.GetPkgFullName(), "test_tar_0.0.1") + assert.Equal(t, kclPkg.GetPkgTarName(), "test_tar_0.0.1.tar") assert.Equal(t, utils.DirExists(filepath.Join(testDir, kclPkg.GetPkgTarName())), false) @@ -1270,3 +1268,50 @@ func TestAddWithGitCommit(t *testing.T) { _ = os.Remove(testPkgPathModLock) }() } + +func TestLoadPkgFormOci(t *testing.T) { + type testCase struct { + Reg string + Repo string + Tag string + Name string + } + + testCases := []testCase{ + { + Reg: "ghcr.io", + Repo: "kusionstack/opsrule", + Tag: "0.0.9", + Name: "opsrule", + }, + { + Reg: "ghcr.io", + Repo: "kcl-lang/helloworld", + Tag: "0.1.1", + Name: "helloworld", + }, + } + + cli, err := NewKpmClient() + assert.Equal(t, err, nil) + pkgPath := getTestDir("test_load_pkg_from_oci") + + for _, tc := range testCases { + localpath := filepath.Join(pkgPath, tc.Name) + + err = os.MkdirAll(localpath, 0755) + assert.Equal(t, err, nil) + defer func() { + err := os.RemoveAll(localpath) + assert.Equal(t, err, nil) + }() + + kclpkg, err := cli.DownloadPkgFromOci(&pkg.Oci{ + Reg: tc.Reg, + Repo: tc.Repo, + Tag: tc.Tag, + }, localpath) + assert.Equal(t, err, nil) + assert.Equal(t, kclpkg.GetPkgName(), tc.Name) + } +} diff --git a/pkg/client/test_data/expected/kcl.mod b/pkg/client/test_data/expected/kcl.mod index 3b53b8aa..2c66884a 100644 --- a/pkg/client/test_data/expected/kcl.mod +++ b/pkg/client/test_data/expected/kcl.mod @@ -4,5 +4,5 @@ edition = "v0.8.0" version = "0.0.1" [dependencies] -oci_name = "test_tag" +oci_name = { oci = "oci://test_reg/test_repo", tag = "test_tag" } name = { git = "test_url", tag = "test_tag" } diff --git a/pkg/client/test_data/expected/kcl.reverse.mod b/pkg/client/test_data/expected/kcl.reverse.mod index c6a1e623..95bbbce3 100644 --- a/pkg/client/test_data/expected/kcl.reverse.mod +++ b/pkg/client/test_data/expected/kcl.reverse.mod @@ -5,4 +5,4 @@ version = "0.0.1" [dependencies] name = { git = "test_url", tag = "test_tag" } -oci_name = "test_tag" +oci_name = { oci = "oci://test_reg/test_repo", tag = "test_tag" } diff --git a/pkg/client/test_data/test_add_diff_version/no_sum_check/kcl.mod.expect b/pkg/client/test_data/test_add_diff_version/no_sum_check/kcl.mod.expect index f77d425c..6a23e353 100644 --- a/pkg/client/test_data/test_add_diff_version/no_sum_check/kcl.mod.expect +++ b/pkg/client/test_data/test_add_diff_version/no_sum_check/kcl.mod.expect @@ -4,4 +4,4 @@ edition = "0.0.1" version = "0.0.1" [dependencies] -helloworld = "0.1.1" +helloworld = { oci = "oci://ghcr.io/kcl-lang/helloworld", tag = "0.1.1" } diff --git a/pkg/client/test_data/test_add_diff_version/with_sum_check/kcl.mod.expect b/pkg/client/test_data/test_add_diff_version/with_sum_check/kcl.mod.expect index 36ec88e0..9ed0fba7 100644 --- a/pkg/client/test_data/test_add_diff_version/with_sum_check/kcl.mod.expect +++ b/pkg/client/test_data/test_add_diff_version/with_sum_check/kcl.mod.expect @@ -4,4 +4,4 @@ edition = "0.0.1" version = "0.0.1" [dependencies] -helloworld = "0.1.1" +helloworld = { oci = "oci://ghcr.io/kcl-lang/helloworld", tag = "0.1.1" } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 1a9e9938..ef15ab2e 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -1,16 +1,17 @@ package constants const ( - KFilePathSuffix = ".k" - TarPathSuffix = ".tar" - GitPathSuffix = ".git" - OciScheme = "oci" - FileEntry = "file" - FileWithKclModEntry = "file_with_kcl_mod" - UrlEntry = "url" - RefEntry = "ref" - TarEntry = "tar" - GitEntry = "git" + KFilePathSuffix = ".k" + TarPathSuffix = ".tar" + GitPathSuffix = ".git" + OciScheme = "oci" + FileEntry = "file" + FileWithKclModEntry = "file_with_kcl_mod" + UrlEntry = "url" + RefEntry = "ref" + TarEntry = "tar" + GitEntry = "git" + KCL_MOD = "kcl.mod" OCI_SEPARATOR = ":" KCL_PKG_TAR = "*.tar" @@ -21,6 +22,7 @@ const ( DEFAULT_KCL_OCI_MANIFEST_DESCRIPTION = "org.kcllang.package.description" DEFAULT_KCL_OCI_MANIFEST_SUM = "org.kcllang.package.sum" DEFAULT_CREATE_OCI_MANIFEST_TIME = "org.opencontainers.image.created" + URL_PATH_SEPARATOR = "/" // The pattern of the external package argument. EXTERNAL_PKGS_ARG_PATTERN = "%s=%s" diff --git a/pkg/oci/oci.go b/pkg/oci/oci.go index f2767f56..989ac88c 100644 --- a/pkg/oci/oci.go +++ b/pkg/oci/oci.go @@ -9,6 +9,10 @@ import ( v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/thoas/go-funk" + "oras.land/oras-go/pkg/auth" + dockerauth "oras.land/oras-go/pkg/auth/docker" + remoteauth "oras.land/oras-go/v2/registry/remote/auth" + "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/opt" pkg "kcl-lang.io/kpm/pkg/package" @@ -16,9 +20,6 @@ import ( "kcl-lang.io/kpm/pkg/semver" "kcl-lang.io/kpm/pkg/settings" "kcl-lang.io/kpm/pkg/utils" - "oras.land/oras-go/pkg/auth" - dockerauth "oras.land/oras-go/pkg/auth/docker" - remoteauth "oras.land/oras-go/v2/registry/remote/auth" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content/file" @@ -137,17 +138,25 @@ func NewOciClient(regName, repoName string, settings *settings.Settings) (*OciCl }, nil } +// The default limit of the store size is 64 MiB. +const DEFAULT_LIMIT_STORE_SIZE = 64 * 1024 * 1024 + // Pull will pull the oci artifacts from oci registry to local path. func (ociClient *OciClient) Pull(localPath, tag string) error { // Create a file store - fs, err := file.New(localPath) + fs, err := file.NewWithFallbackLimit(localPath, DEFAULT_LIMIT_STORE_SIZE) if err != nil { return reporter.NewErrorEvent(reporter.FailedCreateStorePath, err, "Failed to create store path ", localPath) } defer fs.Close() // Copy from the remote repository to the file store - _, err = oras.Copy(*ociClient.ctx, ociClient.repo, tag, fs, tag, oras.DefaultCopyOptions) + customCopyOptions := oras.CopyOptions{ + CopyGraphOptions: oras.CopyGraphOptions{ + MaxMetadataBytes: DEFAULT_LIMIT_STORE_SIZE, // default is 64 MiB + }, + } + _, err = oras.Copy(*ociClient.ctx, ociClient.repo, tag, fs, tag, customCopyOptions) if err != nil { return reporter.NewErrorEvent( reporter.FailedGetPkg, diff --git a/pkg/oci/oci_test.go b/pkg/oci/oci_test.go index 8b28b9b3..b1386b5c 100644 --- a/pkg/oci/oci_test.go +++ b/pkg/oci/oci_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "kcl-lang.io/kpm/pkg/settings" "kcl-lang.io/kpm/pkg/utils" ) @@ -40,3 +41,38 @@ func TestLogin(t *testing.T) { err := Login(hostName, userName, userPwd, &settings) assert.Equal(t, err.Error(), "failed to login 'ghcr.io', please check registry, username and password is valid\nGet \"https://ghcr.io/v2/\": denied: denied\n") } + +func TestPull(t *testing.T) { + type TestCase struct { + Registry string + Image string + Tag string + } + + testCases := []TestCase{ + {"ghcr.io", "kusionstack/opsrule", "0.0.9"}, + {"ghcr.io", "kcl-lang/helloworld", "0.1.1"}, + } + + defer func() { + err := os.RemoveAll(getTestDir("test_pull")) + assert.Equal(t, err, nil) + }() + + for _, tc := range testCases { + client, err := NewOciClient(tc.Registry, tc.Image, settings.GetSettings()) + if err != nil { + t.Fatalf(err.Error()) + } + + tmpPath := filepath.Join(getTestDir("test_pull"), tc.Tag) + + err = os.MkdirAll(tmpPath, 0755) + assert.Equal(t, err, nil) + + err = client.Pull(tmpPath, tc.Tag) + if err != nil { + t.Errorf(err.Error()) + } + } +} diff --git a/pkg/package/modfile.go b/pkg/package/modfile.go index 1593b8f8..0509e486 100644 --- a/pkg/package/modfile.go +++ b/pkg/package/modfile.go @@ -4,12 +4,16 @@ package pkg import ( "errors" "fmt" + "net/url" "os" "path/filepath" "strings" "github.com/BurntSushi/toml" "kcl-lang.io/kcl-go/pkg/kcl" + "oras.land/oras-go/v2/registry" + + "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/opt" "kcl-lang.io/kpm/pkg/reporter" "kcl-lang.io/kpm/pkg/runner" @@ -207,9 +211,14 @@ func (dep *Dependency) FillDepInfo() error { if settings.ErrorEvent != nil { return settings.ErrorEvent } - dep.Source.Oci.Reg = settings.DefaultOciRegistry() - urlpath := utils.JoinPath(settings.DefaultOciRepo(), dep.Name) - dep.Source.Oci.Repo = urlpath + if dep.Source.Oci.Reg == "" { + dep.Source.Oci.Reg = settings.DefaultOciRegistry() + } + + if dep.Source.Oci.Repo == "" { + urlpath := utils.JoinPath(settings.DefaultOciRepo(), dep.Name) + dep.Source.Oci.Repo = urlpath + } } return nil } @@ -238,6 +247,41 @@ type Oci struct { Tag string `toml:"oci_tag,omitempty"` } +func (oci *Oci) IntoOciUrl() string { + if oci != nil { + u := &url.URL{ + Scheme: constants.OciScheme, + Host: oci.Reg, + Path: oci.Repo, + } + + return u.String() + } + return "" +} + +func (oci *Oci) FromString(ociUrl string) (*Oci, error) { + u, err := url.Parse(ociUrl) + if err != nil { + return nil, err + } + + if u.Scheme != constants.OciScheme { + return nil, fmt.Errorf("invalid oci url with schema: %s", u.Scheme) + } + + ref, err := registry.ParseReference(u.Host + u.Path) + if err != nil { + return nil, fmt.Errorf("'%s' invalid URL format: %w", ociUrl, err) + } + + oci.Reg = ref.Registry + oci.Repo = ref.Repository + oci.Tag = ref.ReferenceOrDefault() + + return oci, nil +} + // Git is the package source from git registry. type Git struct { Url string `toml:"url,omitempty"` diff --git a/pkg/package/modfile_test.go b/pkg/package/modfile_test.go index de89f2a1..f3f05e2e 100644 --- a/pkg/package/modfile_test.go +++ b/pkg/package/modfile_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "kcl-lang.io/kpm/pkg/opt" "kcl-lang.io/kpm/pkg/runner" "kcl-lang.io/kpm/pkg/utils" @@ -169,7 +170,7 @@ func TestLoadModFile(t *testing.T) { assert.Equal(t, modFile.Pkg.Version, "0.0.1") assert.Equal(t, modFile.Pkg.Edition, "0.0.1") - assert.Equal(t, len(modFile.Dependencies.Deps), 2) + assert.Equal(t, len(modFile.Dependencies.Deps), 3) assert.Equal(t, modFile.Dependencies.Deps["name"].Name, "name") assert.Equal(t, modFile.Dependencies.Deps["name"].Source.Git.Url, "test_url") assert.Equal(t, modFile.Dependencies.Deps["name"].Source.Git.Tag, "test_tag") @@ -179,6 +180,11 @@ func TestLoadModFile(t *testing.T) { assert.Equal(t, modFile.Dependencies.Deps["oci_name"].Version, "oci_tag") assert.Equal(t, modFile.Dependencies.Deps["oci_name"].Source.Oci.Tag, "oci_tag") assert.Equal(t, err, nil) + + assert.Equal(t, modFile.Dependencies.Deps["helloworld"].Name, "helloworld") + assert.Equal(t, modFile.Dependencies.Deps["helloworld"].Version, "0.1.1") + assert.Equal(t, modFile.Dependencies.Deps["helloworld"].Source.Oci.Tag, "0.1.1") + assert.Equal(t, err, nil) } func TestLoadLockDeps(t *testing.T) { diff --git a/pkg/package/package.go b/pkg/package/package.go index 1134e5f5..01058e25 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -2,6 +2,7 @@ package pkg import ( "fmt" + "os" "path/filepath" "strings" @@ -55,6 +56,65 @@ func LoadKclPkg(pkgPath string) (*KclPkg, error) { }, nil } +func FindFirstKclPkgFrom(path string) (*KclPkg, error) { + matches, _ := filepath.Glob(filepath.Join(path, "*.tar")) + if matches == nil || len(matches) != 1 { + // then try to glob tgz file + matches, _ = filepath.Glob(filepath.Join(path, "*.tgz")) + if matches == nil || len(matches) != 1 { + pkg, err := LoadKclPkg(path) + if err != nil { + return nil, reporter.NewErrorEvent( + reporter.InvalidKclPkg, + err, + fmt.Sprintf("failed to find the kcl package tar from '%s'.", path), + ) + } + + return pkg, nil + } + } + + tarPath := matches[0] + unTarPath := filepath.Dir(tarPath) + var err error + if utils.IsTar(tarPath) { + err = utils.UnTarDir(tarPath, unTarPath) + } else { + err = utils.ExtractTarball(tarPath, unTarPath) + } + if err != nil { + return nil, reporter.NewErrorEvent( + reporter.FailedUntarKclPkg, + err, + fmt.Sprintf("failed to untar the kcl package tar from '%s' into '%s'.", tarPath, unTarPath), + ) + } + + // After untar the downloaded kcl package tar file, remove the tar file. + if utils.DirExists(tarPath) { + rmErr := os.Remove(tarPath) + if rmErr != nil { + return nil, reporter.NewErrorEvent( + reporter.FailedUntarKclPkg, + err, + fmt.Sprintf("failed to untar the kcl package tar from '%s' into '%s'.", tarPath, unTarPath), + ) + } + } + + pkg, err := LoadKclPkg(unTarPath) + if err != nil { + return nil, reporter.NewErrorEvent( + reporter.InvalidKclPkg, + err, + fmt.Sprintf("failed to find the kcl package tar from '%s'.", path), + ) + } + + return pkg, nil +} + func LoadKclPkgFromTar(pkgTarPath string) (*KclPkg, error) { destDir := strings.TrimSuffix(pkgTarPath, filepath.Ext(pkgTarPath)) err := utils.UnTarDir(pkgTarPath, destDir) @@ -166,7 +226,7 @@ func (kclPkg *KclPkg) ValidateKpmHome(kpmHome string) *reporter.KpmEvent { // is the name of package. // is the version of package func (kclPkg *KclPkg) GetPkgFullName() string { - return kclPkg.ModFile.Pkg.Name + "-" + kclPkg.ModFile.Pkg.Version + return fmt.Sprintf(PKG_NAME_PATTERN, kclPkg.ModFile.Pkg.Name, kclPkg.ModFile.Pkg.Version) } // GetPkgName returns name of package. diff --git a/pkg/package/package_test.go b/pkg/package/package_test.go index f4d7d41d..c68bbe57 100644 --- a/pkg/package/package_test.go +++ b/pkg/package/package_test.go @@ -130,8 +130,8 @@ func TestLoadKclPkgFromTar(t *testing.T) { assert.Equal(t, kclPkg.GetPkgTag(), "0.0.3") assert.Equal(t, kclPkg.GetPkgName(), "kcl1") - assert.Equal(t, kclPkg.GetPkgFullName(), "kcl1-0.0.3") - assert.Equal(t, kclPkg.GetPkgTarName(), "kcl1-0.0.3.tar") + assert.Equal(t, kclPkg.GetPkgFullName(), "kcl1_0.0.3") + assert.Equal(t, kclPkg.GetPkgTarName(), "kcl1_0.0.3.tar") assert.Equal(t, utils.DirExists(filepath.Join(testDir, "kcl1-v0.0.3")), true) err = os.RemoveAll(filepath.Join(testDir, "kcl1-v0.0.3")) diff --git a/pkg/package/test_data/load_mod_file/kcl.mod b/pkg/package/test_data/load_mod_file/kcl.mod index 27215014..1c893dfc 100644 --- a/pkg/package/test_data/load_mod_file/kcl.mod +++ b/pkg/package/test_data/load_mod_file/kcl.mod @@ -5,4 +5,5 @@ version = "0.0.1" [dependencies] name = { git = "test_url", tag = "test_tag" } -oci_name = "oci_tag" \ No newline at end of file +oci_name = "oci_tag" +helloworld = { oci = "oci://ghcr.io/kcl-lang/helloworld", tag = "0.1.1" } \ No newline at end of file diff --git a/pkg/package/test_data/test_oci_url/marshal_0/kcl_mod_bk/kcl.mod b/pkg/package/test_data/test_oci_url/marshal_0/kcl_mod_bk/kcl.mod new file mode 100644 index 00000000..ad1966b1 --- /dev/null +++ b/pkg/package/test_data/test_oci_url/marshal_0/kcl_mod_bk/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "marshal_0" +edition = "v0.8.0" +version = "0.0.1" + +[dependencies] +oci_pkg = { oci = "oci://ghcr.io/kcl-lang/oci_pkg", tag = "0.0.1" } diff --git a/pkg/package/test_data/test_oci_url/marshal_1/expect.mod b/pkg/package/test_data/test_oci_url/marshal_1/expect.mod new file mode 100644 index 00000000..ad1966b1 --- /dev/null +++ b/pkg/package/test_data/test_oci_url/marshal_1/expect.mod @@ -0,0 +1,7 @@ +[package] +name = "marshal_0" +edition = "v0.8.0" +version = "0.0.1" + +[dependencies] +oci_pkg = { oci = "oci://ghcr.io/kcl-lang/oci_pkg", tag = "0.0.1" } diff --git a/pkg/package/test_data/test_oci_url/marshal_1/kcl.mod b/pkg/package/test_data/test_oci_url/marshal_1/kcl.mod new file mode 100644 index 00000000..ee1d950e --- /dev/null +++ b/pkg/package/test_data/test_oci_url/marshal_1/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "marshal_0" +edition = "v0.8.0" +version = "0.0.1" + +[dependencies] +oci_pkg = { oci="oci://ghcr.io/kcl-lang/oci_pkg", tag="0.0.1" } diff --git a/pkg/package/test_data/test_oci_url/marshal_2/expect.mod b/pkg/package/test_data/test_oci_url/marshal_2/expect.mod new file mode 100644 index 00000000..ad1966b1 --- /dev/null +++ b/pkg/package/test_data/test_oci_url/marshal_2/expect.mod @@ -0,0 +1,7 @@ +[package] +name = "marshal_0" +edition = "v0.8.0" +version = "0.0.1" + +[dependencies] +oci_pkg = { oci = "oci://ghcr.io/kcl-lang/oci_pkg", tag = "0.0.1" } diff --git a/pkg/package/test_data/test_oci_url/marshal_2/kcl.mod b/pkg/package/test_data/test_oci_url/marshal_2/kcl.mod new file mode 100644 index 00000000..8f71ecb9 --- /dev/null +++ b/pkg/package/test_data/test_oci_url/marshal_2/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "marshal_0" +edition = "v0.8.0" +version = "0.0.1" + +[dependencies] +oci_pkg = "0.0.1" diff --git a/pkg/package/test_data/test_oci_url/marshal_3/expect.mod b/pkg/package/test_data/test_oci_url/marshal_3/expect.mod new file mode 100644 index 00000000..d4d8b2c0 --- /dev/null +++ b/pkg/package/test_data/test_oci_url/marshal_3/expect.mod @@ -0,0 +1,7 @@ +[package] +name = "marshal_0" +edition = "v0.8.0" +version = "0.0.1" + +[dependencies] +oci_pkg = { oci = "oci://localhost:5001/test/oci_pkg", tag = "0.0.1" } diff --git a/pkg/package/test_data/test_oci_url/marshal_3/kcl.mod b/pkg/package/test_data/test_oci_url/marshal_3/kcl.mod new file mode 100644 index 00000000..d4d8b2c0 --- /dev/null +++ b/pkg/package/test_data/test_oci_url/marshal_3/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "marshal_0" +edition = "v0.8.0" +version = "0.0.1" + +[dependencies] +oci_pkg = { oci = "oci://localhost:5001/test/oci_pkg", tag = "0.0.1" } diff --git a/pkg/package/test_data/test_oci_url/unmarshal_0/kcl.mod b/pkg/package/test_data/test_oci_url/unmarshal_0/kcl.mod new file mode 100644 index 00000000..a5698545 --- /dev/null +++ b/pkg/package/test_data/test_oci_url/unmarshal_0/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "marshal_0" +edition = "v0.8.0" +version = "0.0.1" + +[dependencies] +oci_pkg_name = { oci="oci://ghcr.io/test/helloworld", tag="0.0.1" } \ No newline at end of file diff --git a/pkg/package/test_data/test_oci_url/unmarshal_1/kcl.mod b/pkg/package/test_data/test_oci_url/unmarshal_1/kcl.mod new file mode 100644 index 00000000..35a270d6 --- /dev/null +++ b/pkg/package/test_data/test_oci_url/unmarshal_1/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "marshal_1" +edition = "v0.8.0" +version = "0.0.1" + +[dependencies] +oci_pkg_name = { oci="oci://localhost:5001/test/helloworld", tag="0.0.1" } \ No newline at end of file diff --git a/pkg/package/toml.go b/pkg/package/toml.go index c033a87f..6bc8a34e 100644 --- a/pkg/package/toml.go +++ b/pkg/package/toml.go @@ -26,6 +26,8 @@ import ( "strings" "github.com/BurntSushi/toml" + + "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/reporter" ) @@ -95,7 +97,11 @@ func (source *Source) MarshalTOML() string { if source.Oci != nil { ociToml := source.Oci.MarshalTOML() if len(ociToml) != 0 { - sb.WriteString(ociToml) + if len(source.Oci.Reg) != 0 && len(source.Oci.Repo) != 0 { + sb.WriteString(fmt.Sprintf(SOURCE_PATTERN, ociToml)) + } else { + sb.WriteString(ociToml) + } } } @@ -110,7 +116,7 @@ func (source *Source) MarshalTOML() string { } const GIT_URL_PATTERN = "git = \"%s\"" -const GIT_TAG_PATTERN = "tag = \"%s\"" +const TAG_PATTERN = "tag = \"%s\"" const GIT_COMMIT_PATTERN = "commit = \"%s\"" const SEPARATOR = ", " @@ -121,7 +127,7 @@ func (git *Git) MarshalTOML() string { } if len(git.Tag) != 0 { sb.WriteString(SEPARATOR) - sb.WriteString(fmt.Sprintf(GIT_TAG_PATTERN, git.Tag)) + sb.WriteString(fmt.Sprintf(TAG_PATTERN, git.Tag)) } if len(git.Commit) != 0 { sb.WriteString(SEPARATOR) @@ -130,11 +136,20 @@ func (git *Git) MarshalTOML() string { return sb.String() } +const OCI_URL_PATTERN = "oci = \"%s\"" + func (oci *Oci) MarshalTOML() string { var sb strings.Builder - if len(oci.Tag) != 0 { + if len(oci.Reg) != 0 && len(oci.Repo) != 0 { + sb.WriteString(fmt.Sprintf(OCI_URL_PATTERN, oci.IntoOciUrl())) + if len(oci.Tag) != 0 { + sb.WriteString(SEPARATOR) + sb.WriteString(fmt.Sprintf(TAG_PATTERN, oci.Tag)) + } + } else if len(oci.Reg) == 0 && len(oci.Repo) == 0 && len(oci.Tag) != 0 { sb.WriteString(fmt.Sprintf(`"%s"`, oci.Tag)) } + return sb.String() } @@ -295,13 +310,20 @@ func (source *Source) UnmarshalModTOML(data interface{}) error { return err } source.Local = &localPath - } else { + } else if _, ok := meta["git"]; ok { git := Git{} err := git.UnmarshalModTOML(data) if err != nil { return err } source.Git = &git + } else { + oci := Oci{} + err := oci.UnmarshalModTOML(data) + if err != nil { + return err + } + source.Oci = &oci } } @@ -319,7 +341,7 @@ func (source *Source) UnmarshalModTOML(data interface{}) error { } const GIT_URL_FLAG = "git" -const GIT_TAG_FLAG = "tag" +const TAG_FLAG = "tag" const GIT_COMMIT_FLAG = "commit" func (git *Git) UnmarshalModTOML(data interface{}) error { @@ -332,7 +354,7 @@ func (git *Git) UnmarshalModTOML(data interface{}) error { git.Url = v } - if v, ok := meta[GIT_TAG_FLAG].(string); ok { + if v, ok := meta[TAG_FLAG].(string); ok { git.Tag = v } @@ -344,12 +366,23 @@ func (git *Git) UnmarshalModTOML(data interface{}) error { } func (oci *Oci) UnmarshalModTOML(data interface{}) error { - meta, ok := data.(string) - if !ok { - return fmt.Errorf("expected string, got %T", data) - } + tag, ok := data.(string) + if ok { + oci.Tag = tag + } else if meta, ok := data.(map[string]interface{}); ok { + if v, ok := meta[constants.OciScheme].(string); ok { + _, err := oci.FromString(v) + if err != nil { + return err + } + } - oci.Tag = meta + if v, ok := meta[TAG_FLAG].(string); ok { + oci.Tag = v + } + } else { + return fmt.Errorf("unexpected data %T", data) + } return nil } diff --git a/pkg/package/toml_test.go b/pkg/package/toml_test.go index b9ef5f95..0b0e78c3 100644 --- a/pkg/package/toml_test.go +++ b/pkg/package/toml_test.go @@ -8,6 +8,7 @@ import ( "github.com/BurntSushi/toml" "github.com/stretchr/testify/assert" + "kcl-lang.io/kpm/pkg/utils" ) @@ -169,3 +170,114 @@ func TestUnMarshalTOMLWithProfile(t *testing.T) { assert.Equal(t, modfile.Pkg.Edition, "0.0.1") assert.Equal(t, *modfile.Profiles.Entries, []string{"main.k", "xxx/xxx/dir", "test.yaml"}) } + +func TestUnMarshalOciUrl(t *testing.T) { + testDataDir := getTestDir("test_oci_url") + + testCases := []struct { + Name string + DepName string + DepFullName string + DepVersion string + DepSourceReg string + DepSourceRepo string + DepSourceTag string + }{ + {"unmarshal_0", "oci_pkg_name", "oci_pkg_name_0.0.1", "0.0.1", "ghcr.io", "test/helloworld", "0.0.1"}, + {"unmarshal_1", "oci_pkg_name", "oci_pkg_name_0.0.1", "0.0.1", "localhost:5001", "test/helloworld", "0.0.1"}, + } + + for _, tc := range testCases { + modfile, err := LoadModFile(filepath.Join(testDataDir, tc.Name)) + assert.Equal(t, err, nil) + assert.Equal(t, len(modfile.Dependencies.Deps), 1) + assert.Equal(t, modfile.Dependencies.Deps["oci_pkg_name"].Name, tc.DepName) + assert.Equal(t, modfile.Dependencies.Deps["oci_pkg_name"].FullName, tc.DepFullName) + assert.Equal(t, modfile.Dependencies.Deps["oci_pkg_name"].Version, tc.DepVersion) + assert.Equal(t, modfile.Dependencies.Deps["oci_pkg_name"].Source.Oci.Reg, tc.DepSourceReg) + assert.Equal(t, modfile.Dependencies.Deps["oci_pkg_name"].Source.Oci.Repo, tc.DepSourceRepo) + assert.Equal(t, modfile.Dependencies.Deps["oci_pkg_name"].Source.Oci.Tag, tc.DepVersion) + } +} + +func TestMarshalOciUrl(t *testing.T) { + testDataDir := getTestDir("test_oci_url") + + expectPkgPath := filepath.Join(testDataDir, "marshal_0", "kcl_mod_bk") + gotPkgPath := filepath.Join(testDataDir, "marshal_0", "kcl_mod_tmp") + + expect, err := LoadModFile(expectPkgPath) + assert.Equal(t, err, nil) + + err = os.MkdirAll(gotPkgPath, 0755) + assert.Equal(t, err, nil) + gotFile, _ := os.Create(filepath.Join(gotPkgPath, "kcl.mod")) + + defer func() { + err = gotFile.Close() + assert.Equal(t, err, nil) + err = os.RemoveAll(gotPkgPath) + assert.Equal(t, err, nil) + }() + + modfile := ModFile{ + Pkg: Package{ + Name: "marshal_0", + Edition: "v0.8.0", + Version: "0.0.1", + }, + Dependencies: Dependencies{ + make(map[string]Dependency), + }, + } + + ociDep := Dependency{ + Name: "oci_pkg", + FullName: "oci_pkg_0.0.1", + Version: "0.0.1", + Source: Source{ + Oci: &Oci{ + Tag: "0.0.1", + }, + }, + } + + modfile.Dependencies.Deps["oci_pkg_0.0.1"] = ociDep + + got_data := modfile.MarshalTOML() + _, err = gotFile.WriteString(got_data) + assert.Equal(t, err, nil) + + got, err := LoadModFile(gotPkgPath) + assert.Equal(t, err, nil) + + assert.Equal(t, expect.Pkg.Name, got.Pkg.Name) + assert.Equal(t, expect.Pkg.Edition, got.Pkg.Edition) + assert.Equal(t, expect.Pkg.Version, got.Pkg.Version) + assert.Equal(t, len(expect.Dependencies.Deps), len(got.Dependencies.Deps)) + assert.Equal(t, expect.Dependencies.Deps["oci_pkg"].Name, got.Dependencies.Deps["oci_pkg"].Name) + assert.Equal(t, expect.Dependencies.Deps["oci_pkg"].FullName, got.Dependencies.Deps["oci_pkg"].FullName) + assert.Equal(t, expect.Dependencies.Deps["oci_pkg"].Source.Oci.Reg, got.Dependencies.Deps["oci_pkg"].Source.Oci.Reg) + assert.Equal(t, expect.Dependencies.Deps["oci_pkg"].Source.Oci.Repo, got.Dependencies.Deps["oci_pkg"].Source.Oci.Repo) + assert.Equal(t, expect.Dependencies.Deps["oci_pkg"].Source.Oci.Tag, got.Dependencies.Deps["oci_pkg"].Source.Oci.Tag) + assert.Equal(t, expect.Dependencies.Deps["oci_pkg"].Source.IntoOciUrl(), got.Dependencies.Deps["oci_pkg"].Source.IntoOciUrl()) +} + +func TestMarshalOciUrlIntoFile(t *testing.T) { + testDataDir := getTestDir("test_oci_url") + + testCases := []string{"marshal_1", "marshal_2", "marshal_3"} + + for _, tc := range testCases { + readKclModPath := filepath.Join(testDataDir, tc) + expectPath := filepath.Join(readKclModPath, "expect.mod") + + readKclModFile, err := LoadModFile(readKclModPath) + assert.Equal(t, err, nil) + writeKclModFileContents := readKclModFile.MarshalTOML() + expectKclModFileContents, err := os.ReadFile(expectPath) + assert.Equal(t, err, nil) + + assert.Equal(t, utils.RmNewline(string(expectKclModFileContents)), utils.RmNewline(writeKclModFileContents)) + } +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 083fba28..ca0c82a1 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -3,8 +3,10 @@ package utils import ( "archive/tar" "bufio" + "compress/gzip" "crypto/sha256" "encoding/base64" + goerrors "errors" "fmt" "io" "log" @@ -14,10 +16,9 @@ import ( "regexp" "strings" - goerrors "errors" - "github.com/docker/distribution/reference" "github.com/moby/term" + "kcl-lang.io/kpm/pkg/constants" "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/reporter" @@ -226,6 +227,57 @@ func UnTarDir(tarPath string, destDir string) error { return nil } +// ExtractTarball support extracting tarball with '.tgz' format. +func ExtractTarball(tarPath, destDir string) error { + f, err := os.Open(tarPath) + if err != nil { + return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) + } + defer f.Close() + + zip, err := gzip.NewReader(f) + if err != nil { + return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) + } + tarReader := tar.NewReader(zip) + + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) + } + + destFilePath := filepath.Join(destDir, header.Name) + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(destFilePath, 0755); err != nil { + return errors.FailedUnTarKclPackage + } + case tar.TypeReg: + err := os.MkdirAll(filepath.Dir(destFilePath), 0755) + if err != nil { + return err + } + outFile, err := os.Create(destFilePath) + if err != nil { + return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) + } + defer outFile.Close() + + if _, err := io.Copy(outFile, tarReader); err != nil { + return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) + } + default: + return errors.UnknownTarFormat + } + } + + return nil +} + // DirExists will check whether the directory 'path' exists. func DirExists(path string) bool { _, err := os.Stat(path)