diff --git a/pkg/checker/checker_test.go b/pkg/checker/checker_test.go index 56e51281..0ead1534 100644 --- a/pkg/checker/checker_test.go +++ b/pkg/checker/checker_test.go @@ -12,6 +12,7 @@ import ( pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" "kcl-lang.io/kpm/pkg/settings" + "kcl-lang.io/kpm/pkg/test" ) func TestModCheckerCheck(t *testing.T) { @@ -145,126 +146,130 @@ func getTestSettings() (*settings.Settings, error) { } func TestModCheckerCheck_WithTrustedSum(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Skipping TestModCheckerCheck_WithTrustedSum test on Windows") - } + testFunc := func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping TestModCheckerCheck_WithTrustedSum test on Windows") + } - // Start the local Docker registry required for testing - err := mock.StartDockerRegistry() - assert.Equal(t, err, nil) + // Start the local Docker registry required for testing + err := mock.StartDockerRegistry() + assert.Equal(t, err, nil) - // Push the test package to the local OCI registry - err = mock.PushTestPkgToRegistry() - assert.Equal(t, err, nil) + // Push the test package to the local OCI registry + err = mock.PushTestPkgToRegistry() + assert.Equal(t, err, nil) - // Initialize settings for use with the ModChecker - settings, err := getTestSettings() - assert.Equal(t, err, nil) + // Initialize settings for use with the ModChecker + settings, err := getTestSettings() + assert.Equal(t, err, nil) - // Initialize the ModChecker with required checkers - ModChecker := NewModChecker(WithCheckers(NewIdentChecker(), NewVersionChecker(), NewSumChecker(WithSettings(*settings)))) + // Initialize the ModChecker with required checkers + ModChecker := NewModChecker(WithCheckers(NewIdentChecker(), NewVersionChecker(), NewSumChecker(WithSettings(*settings)))) - deps1 := orderedmap.NewOrderedMap[string, pkg.Dependency]() - deps1.Set("kcl1", pkg.Dependency{ - Name: "test_data", - FullName: "test_data", - Version: "0.0.1", - Sum: "RpZZIvrXwfn5dpt6LqBR8+FlPE9Y+BEou47L3qaCCqk=", - Source: downloader.Source{ - Oci: &downloader.Oci{ - Reg: "localhost:5001", - Repo: "test", - Tag: "0.0.1", + deps1 := orderedmap.NewOrderedMap[string, pkg.Dependency]() + deps1.Set("kcl1", pkg.Dependency{ + Name: "test_data", + FullName: "test_data", + Version: "0.0.1", + Sum: "RpZZIvrXwfn5dpt6LqBR8+FlPE9Y+BEou47L3qaCCqk=", + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "localhost:5001", + Repo: "test/test_data", + Tag: "0.0.1", + }, }, - }, - }) + }) - deps2 := orderedmap.NewOrderedMap[string, pkg.Dependency]() - deps2.Set("kcl1", pkg.Dependency{ - Name: "test_data", - FullName: "test_data", - Version: "0.0.1", - Sum: "Invalid-sum", - Source: downloader.Source{ - Oci: &downloader.Oci{ - Reg: "localhost:5001", - Repo: "test", - Tag: "0.0.1", + deps2 := orderedmap.NewOrderedMap[string, pkg.Dependency]() + deps2.Set("kcl1", pkg.Dependency{ + Name: "test_data", + FullName: "test_data", + Version: "0.0.1", + Sum: "Invalid-sum", + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "localhost:5001", + Repo: "test/test_data", + Tag: "0.0.1", + }, }, - }, - }) + }) - tests := []struct { - name string - KclPkg pkg.KclPkg - wantErr bool - }{ - { - name: "valid kcl package - with sum check", - KclPkg: pkg.KclPkg{ - ModFile: pkg.ModFile{ - Pkg: pkg.Package{ - Name: "testmod", - Version: "0.0.1", + tests := []struct { + name string + KclPkg pkg.KclPkg + wantErr bool + }{ + { + name: "valid kcl package - with sum check", + KclPkg: pkg.KclPkg{ + ModFile: pkg.ModFile{ + Pkg: pkg.Package{ + Name: "testmod", + Version: "0.0.1", + }, + HomePath: "path/to/modfile", }, - HomePath: "path/to/modfile", - }, - HomePath: "path/to/kcl/pkg", - Dependencies: pkg.Dependencies{ - Deps: deps1, + HomePath: "path/to/kcl/pkg", + Dependencies: pkg.Dependencies{ + Deps: deps1, + }, + NoSumCheck: false, }, - NoSumCheck: false, + wantErr: false, }, - wantErr: false, - }, - { - name: "valid kcl package - with no sum check enabled", - KclPkg: pkg.KclPkg{ - ModFile: pkg.ModFile{ - Pkg: pkg.Package{ - Name: "testmod", - Version: "0.0.1", + { + name: "valid kcl package - with no sum check enabled", + KclPkg: pkg.KclPkg{ + ModFile: pkg.ModFile{ + Pkg: pkg.Package{ + Name: "testmod", + Version: "0.0.1", + }, + HomePath: "path/to/modfile", }, - HomePath: "path/to/modfile", - }, - HomePath: "path/to/kcl/pkg", - Dependencies: pkg.Dependencies{ - Deps: deps2, + HomePath: "path/to/kcl/pkg", + Dependencies: pkg.Dependencies{ + Deps: deps2, + }, + NoSumCheck: true, }, - NoSumCheck: true, + wantErr: false, }, - wantErr: false, - }, - { - name: "Invalid kcl package - with no sum check disabled - checksum mismatches", - KclPkg: pkg.KclPkg{ - ModFile: pkg.ModFile{ - Pkg: pkg.Package{ - Name: "testmod", - Version: "0.0.1", + { + name: "Invalid kcl package - with no sum check disabled - checksum mismatches", + KclPkg: pkg.KclPkg{ + ModFile: pkg.ModFile{ + Pkg: pkg.Package{ + Name: "testmod", + Version: "0.0.1", + }, + HomePath: "path/to/modfile", }, - HomePath: "path/to/modfile", - }, - HomePath: "path/to/kcl/pkg", - Dependencies: pkg.Dependencies{ - Deps: deps2, + HomePath: "path/to/kcl/pkg", + Dependencies: pkg.Dependencies{ + Deps: deps2, + }, + NoSumCheck: false, }, - NoSumCheck: false, + wantErr: true, }, - wantErr: true, - }, - } + } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotErr := ModChecker.Check(tt.KclPkg) - if (gotErr != nil) != tt.wantErr { - t.Errorf("ModChecker.Check(%v) = %v, want error %v", tt.KclPkg, gotErr, tt.wantErr) - } - }) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotErr := ModChecker.Check(tt.KclPkg) + if (gotErr != nil) != tt.wantErr { + t.Errorf("ModChecker.Check(%v) = %v, want error %v", tt.KclPkg, gotErr, tt.wantErr) + } + }) + } + + // Clean the environment after all tests have been run + err = mock.CleanTestEnv() + assert.Equal(t, err, nil) } - // Clean the environment after all tests have been run - err = mock.CleanTestEnv() - assert.Equal(t, err, nil) + test.RunTestWithGlobalLock(t, "TestModCheckerCheck_WithTrustedSum", testFunc) } diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 2f2944cc..977fcb04 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -1029,7 +1029,7 @@ func TestParseOciOptionFromString(t *testing.T) { assert.Equal(t, err, nil) assert.Equal(t, ociOption.Ref, "") assert.Equal(t, ociOption.Reg, "test_reg") - assert.Equal(t, ociOption.Repo, "/test_oci_repo") + assert.Equal(t, ociOption.Repo, "test_oci_repo") assert.Equal(t, ociOption.Tag, "test_tag") } @@ -2255,12 +2255,12 @@ func testPushWithInsecureSkipTLSverify(t *testing.T) { kpmcli, err := NewKpmClient() assert.Equal(t, err, nil) - _ = kpmcli.PushToOci("test", ociOpts) + _ = kpmcli.pushToOci("test", ociOpts) assert.Equal(t, buf.String(), "") kpmcli.SetInsecureSkipTLSverify(true) - _ = kpmcli.PushToOci("test", ociOpts) + _ = kpmcli.pushToOci("test", ociOpts) assert.Equal(t, buf.String(), "Called Success\n") } diff --git a/pkg/client/deperated.go b/pkg/client/deperated.go index 9cc72974..8a6d6385 100644 --- a/pkg/client/deperated.go +++ b/pkg/client/deperated.go @@ -198,7 +198,7 @@ func (c *KpmClient) CompileOciPkg(ociSource, version string, opts *opt.CompileOp source := downloader.Source{ Oci: &downloader.Oci{ Reg: ociOpts.Reg, - Repo: ociOpts.Repo, + Repo: utils.JoinPath(ociOpts.Repo, ociOpts.Ref), Tag: ociOpts.Tag, }, } @@ -1338,7 +1338,7 @@ func (c *KpmClient) pullTarFromOci(localPath string, ociOpts *opt.OciOptions) er return reporter.NewErrorEvent(reporter.Bug, err) } - repoPath := utils.JoinPath(ociOpts.Reg, ociOpts.Repo) + repoPath := utils.JoinPath(ociOpts.Reg, ociOpts.Repo, ociOpts.Ref) cred, err := c.GetCredentials(ociOpts.Reg) if err != nil { return err @@ -1373,9 +1373,10 @@ func (c *KpmClient) pullTarFromOci(localPath string, ociOpts *opt.OciOptions) er ociOpts.Tag = tagSelected - full_repo := utils.JoinPath(ociOpts.Reg, ociOpts.Repo) + full_repo := utils.JoinPath(ociOpts.Reg, ociOpts.Repo, ociOpts.Ref) + full_ref := utils.JoinPath(ociOpts.Repo, ociOpts.Ref) reporter.ReportMsgTo( - fmt.Sprintf("pulling '%s:%s' from '%s'", ociOpts.Repo, tagSelected, full_repo), + fmt.Sprintf("pulling '%s:%s' from '%s'", full_ref, tagSelected, full_repo), c.logWriter, ) diff --git a/pkg/client/issues_test.go b/pkg/client/issues_test.go index 247bdef5..19dba65f 100644 --- a/pkg/client/issues_test.go +++ b/pkg/client/issues_test.go @@ -13,7 +13,9 @@ import ( "github.com/stretchr/testify/assert" "kcl-lang.io/kpm/pkg/downloader" "kcl-lang.io/kpm/pkg/features" + "kcl-lang.io/kpm/pkg/mock" pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/reporter" "kcl-lang.io/kpm/pkg/utils" ) @@ -430,3 +432,125 @@ func TestKpmIssue226(t *testing.T) { RunTestWithGlobalLockAndKpmCli(t, []TestSuite{{Name: "update_with_git_commit", TestFunc: test_update_with_git_commit}}) RunTestWithGlobalLockAndKpmCli(t, []TestSuite{{Name: "update_with_git_commit_invalid", TestFunc: test_update_with_git_commit_invalid}}) } + +func TestKclIssue1768(t *testing.T) { + testPath := "github.com/kcl-lang/kcl/issues/1768" + test_push_with_tag := func(t *testing.T, kpmcli *KpmClient) { + if runtime.GOOS == "windows" { + t.Skip("Skipping test on Windows") + } + err := mock.StartDockerRegistry() + if err != nil { + t.Errorf("Error starting docker registry: %v", err) + } + + defer func() { + err = mock.CleanTestEnv() + if err != nil { + t.Errorf("Error stopping docker registry: %v", err) + } + }() + + kpmcli.SetInsecureSkipTLSverify(true) + err = kpmcli.LoginOci("localhost:5001", "test", "1234") + if err != nil { + t.Errorf("Error logging in to docker registry: %v", err) + } + + rootPath := getTestDir("issues") + pushedModPath := filepath.Join(rootPath, testPath, "pushed_mod") + + modPath := filepath.Join(rootPath, testPath, "depends_on_pushed_mod") + modFileBk := filepath.Join(modPath, "kcl.mod.bk") + LockFileBk := filepath.Join(modPath, "kcl.mod.lock.bk") + modFile := filepath.Join(modPath, "kcl.mod") + LockFile := filepath.Join(modPath, "kcl.mod.lock") + modFileExpect := filepath.Join(modPath, "kcl.mod.expect") + LockFileExpect := filepath.Join(modPath, "kcl.mod.lock.expect") + + defer func() { + _ = os.RemoveAll(modFile) + _ = os.RemoveAll(LockFile) + }() + + err = copy.Copy(modFileBk, modFile) + if err != nil { + t.Fatal(err) + } + err = copy.Copy(LockFileBk, LockFile) + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + kpmcli.SetLogWriter(&buf) + + err = kpmcli.Push( + WithPushModPath(pushedModPath), + WithPushSource(downloader.Source{ + Oci: &downloader.Oci{ + Reg: "localhost:5001", + Repo: "test/oci_pushed_mod", + Tag: "v9.9.9", + }, + }), + ) + + if err != (*reporter.KpmEvent)(nil) { + t.Errorf("Error pushing kcl package: %v", err) + } + + assert.Contains(t, buf.String(), "package 'pushed_mod' will be pushed") + assert.Contains(t, buf.String(), "pushed [registry] localhost:5001/test/oci_pushed_mod") + assert.Contains(t, buf.String(), "digest: sha256:") + + kmod, err := pkg.LoadKclPkgWithOpts( + pkg.WithPath(modPath), + ) + + err = kpmcli.Add( + WithAddKclPkg(kmod), + WithAddSource( + &downloader.Source{ + Oci: &downloader.Oci{ + Reg: "localhost:5001", + Repo: "test/oci_pushed_mod", + Tag: "v9.9.9", + }}, + ), + WithAddModSpec( + &downloader.ModSpec{ + Name: "pushed_mod", + Version: "0.0.1", + }, + ), + ) + + if err != nil { + t.Errorf("Error adding dependency: %v", err) + } + + modFileContent, err := os.ReadFile(modFile) + if err != nil { + t.Fatal(err) + } + lockFileContent, err := os.ReadFile(LockFile) + if err != nil { + t.Fatal(err) + } + + modFileExpectContent, err := os.ReadFile(modFileExpect) + if err != nil { + t.Fatal(err) + } + + lockFileExpectContent, err := os.ReadFile(LockFileExpect) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, utils.RmNewline(string(modFileContent)), utils.RmNewline(string(modFileExpectContent))) + assert.Equal(t, utils.RmNewline(string(lockFileContent)), utils.RmNewline(string(lockFileExpectContent))) + } + RunTestWithGlobalLockAndKpmCli(t, []TestSuite{{Name: "test_push_with_tag", TestFunc: test_push_with_tag}}) +} diff --git a/pkg/client/push.go b/pkg/client/push.go index aa540ff7..a48da46e 100644 --- a/pkg/client/push.go +++ b/pkg/client/push.go @@ -2,16 +2,140 @@ package client import ( "fmt" + "os" + "kcl-lang.io/kpm/pkg/downloader" + "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/oci" "kcl-lang.io/kpm/pkg/opt" + pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" "kcl-lang.io/kpm/pkg/utils" ) +// PushOptions contains the options for the Push method. +type PushOptions struct { + // By default, the registry, repository, reference and tag are set to the default values + // registry is set to the default registry `ghcr.io` in the settings + // repository is set to the default repository `kcl-lang` in the settings + // reference is set to the package name in `kcl.mod` + // tag is set to the package version in `kcl.mod` + Source downloader.Source + ModPath string + VendorMode bool +} + +type PushOption func(*PushOptions) error + +// WithPushSource sets the source for the Push method. +func WithPushSource(source downloader.Source) PushOption { + return func(opts *PushOptions) error { + opts.Source = source + return nil + } +} + +// WithPushVendorMode sets the vendor mode for the Push method. +func WithPushVendorMode(vendorMode bool) PushOption { + return func(opts *PushOptions) error { + opts.VendorMode = vendorMode + return nil + } +} + +// WithPushModPath sets the modPath for the Push method. +func WithPushModPath(modPath string) PushOption { + return func(opts *PushOptions) error { + if modPath == "" { + return fmt.Errorf("modPath cannot be empty") + } + opts.ModPath = modPath + return nil + } +} + +// fillDefaultPushOptions will fill the default values for the PushOptions. +func (c *KpmClient) fillDefaultPushOptions(ociOpt *opt.OciOptions, kMod *pkg.KclPkg) { + if ociOpt.Reg == "" { + ociOpt.Reg = c.GetSettings().DefaultOciRegistry() + } + + if ociOpt.Repo == "" { + ociOpt.Repo = c.GetSettings().DefaultOciRepo() + } + + if ociOpt.Tag == "" { + ociOpt.Tag = kMod.ModFile.Pkg.Version + } +} + +// Push will push a kcl package to a registry. +func (c *KpmClient) Push(opts ...PushOption) error { + pushOpts := &PushOptions{} + for _, opt := range opts { + if err := opt(pushOpts); err != nil { + return err + } + } + + kMod, err := pkg.LoadKclPkgWithOpts( + pkg.WithPath(pushOpts.ModPath), + pkg.WithSettings(c.GetSettings()), + ) + + if err != nil { + return err + } + + source := pushOpts.Source + ociUrl, err := source.ToString() + if err != nil { + return err + } + if source.Oci == nil { + return fmt.Errorf("'%s' is not an oci source, only support oci source", ociUrl) + } + + var tag string + if source.Oci.Tag == "" { + tag = kMod.ModFile.Pkg.Version + } else { + tag = source.Oci.Tag + } + + ociOpts, err := c.ParseOciOptionFromString(ociUrl, tag) + if err != nil { + return err + } + c.fillDefaultPushOptions(ociOpts, kMod) + + ociOpts.Annotations, err = kMod.GenOciManifestFromPkg() + if err != nil { + return err + } + + tarPath, err := c.PackagePkg(kMod, pushOpts.VendorMode) + if err != nil { + return err + } + + // clean the tar path. + defer func() { + if kMod != nil && utils.DirExists(tarPath) { + err = os.RemoveAll(tarPath) + if err != nil { + err = errors.InternalBug + } + } + }() + + reporter.ReportMsgTo(fmt.Sprintf("package '%s' will be pushed", kMod.GetPkgName()), c.GetLogWriter()) + return c.pushToOci(tarPath, ociOpts) +} + // PushToOci will push a kcl package to oci registry. -func (c *KpmClient) PushToOci(localPath string, ociOpts *opt.OciOptions) error { - repoPath := utils.JoinPath(ociOpts.Reg, ociOpts.Repo) +func (c *KpmClient) pushToOci(localPath string, ociOpts *opt.OciOptions) error { + repoPath := utils.JoinPath(ociOpts.Reg, ociOpts.Repo, ociOpts.Ref) cred, err := c.GetCredentials(ociOpts.Reg) if err != nil { return err diff --git a/pkg/client/push_test.go b/pkg/client/push_test.go new file mode 100644 index 00000000..2aa2509e --- /dev/null +++ b/pkg/client/push_test.go @@ -0,0 +1,93 @@ +package client + +import ( + "bytes" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "kcl-lang.io/kpm/pkg/downloader" + "kcl-lang.io/kpm/pkg/mock" + pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/reporter" +) + +func TestPush(t *testing.T) { + testFunc := func(t *testing.T, kpmcli *KpmClient) { + if runtime.GOOS == "windows" { + t.Skip("Skipping test on Windows") + } + err := mock.StartDockerRegistry() + if err != nil { + t.Errorf("Error starting docker registry: %v", err) + } + + defer func() { + err = mock.CleanTestEnv() + if err != nil { + t.Errorf("Error stopping docker registry: %v", err) + } + }() + + kpmcli.SetInsecureSkipTLSverify(true) + err = kpmcli.LoginOci("localhost:5001", "test", "1234") + if err != nil { + t.Errorf("Error logging in to docker registry: %v", err) + } + + var buf bytes.Buffer + kpmcli.SetLogWriter(&buf) + + testDir := getTestDir("test_push") + pushedModPath := filepath.Join(testDir, "push_0") + + err = kpmcli.Push( + WithPushModPath(pushedModPath), + WithPushSource( + downloader.Source{ + Oci: &downloader.Oci{ + Reg: "localhost:5001", + Repo: "test/push_0", + }, + }, + ), + ) + + if err != (*reporter.KpmEvent)(nil) { + t.Errorf("Error pushing kcl package: %v", err) + } + + assert.Contains(t, buf.String(), "package 'push_0' will be pushed") + assert.Contains(t, buf.String(), "pushed [registry] localhost:5001/test/push_0") + assert.Contains(t, buf.String(), "digest: sha256:") + + testPushModPath := filepath.Join(testDir, "test_pushed_mod") + + err = kpmcli.Init( + WithInitModPath(testPushModPath), + ) + if err != nil { + t.Errorf("Error initializing kcl package: %v", err) + } + + testMod, err := pkg.LoadKclPkgWithOpts( + pkg.WithPath(testPushModPath), + ) + + err = kpmcli.Add( + WithAddKclPkg(testMod), + WithAddSourceUrl("oci://localhost:5001/test/push_0"), + WithAddModSpec(&downloader.ModSpec{ + Name: "push_0", + Version: "0.0.1", + }), + ) + + if err != nil { + t.Errorf("Error adding kcl package: %v", err) + } + + } + RunTestWithGlobalLockAndKpmCli(t, []TestSuite{{Name: "TestPush", TestFunc: testFunc}}) +} diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/README.md b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/README.md new file mode 100644 index 00000000..05756ead --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/README.md @@ -0,0 +1,22 @@ +## Bug Report + +https://github.com/kcl-lang/kcl/issues/1768 + +The [documentation](https://www.kcl-lang.io/docs/user_docs/guides/package-management/how-to/kpm_oci#kcl-mod-push-to-upload-a-kcl-package) mentions a `--tag` flag for `kcl mod push` to overwrite the version written in `kcl.mod`. + +However, it appears this flag doesn't exist. + +### 1. Minimal reproduce step (Required) + +Run `kcl mod push --tag 0.0.42` + +### 2. What did you expect to see? (Required) + +### 3. What did you see instead (Required) + +``` +unknown flag: --tag +``` +### 4. What is your KCL components version? (Required) + +kcl version: 0.10.10-linux-amd64 diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.bk b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.bk new file mode 100644 index 00000000..f2e4fb50 --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.bk @@ -0,0 +1,4 @@ +[package] +name = "depends_on_pushed_mod" +edition = "v0.10.0" +version = "0.0.1" diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.expect b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.expect new file mode 100644 index 00000000..8dd01d70 --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.expect @@ -0,0 +1,7 @@ +[package] +name = "depends_on_pushed_mod" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +pushed_mod = { oci = "oci://localhost:5001/test/oci_pushed_mod", tag = "v9.9.9", version = "0.0.1" } diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.lock.bk b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.lock.bk new file mode 100644 index 00000000..e69de29b diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.lock.expect b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.lock.expect new file mode 100644 index 00000000..0478255a --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/kcl.mod.lock.expect @@ -0,0 +1,9 @@ +[dependencies] + [dependencies.pushed_mod] + name = "pushed_mod" + full_name = "pushed_mod_0.0.1" + version = "0.0.1" + sum = "7reGjELgXlmS2kU3VeRzvlLv2gauyGmeUNyJmV4R2Kw=" + reg = "localhost:5001" + repo = "test/oci_pushed_mod" + oci_tag = "v9.9.9" diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/main.k b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/depends_on_pushed_mod/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/kcl.mod b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/kcl.mod new file mode 100644 index 00000000..e562f6ba --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/kcl.mod @@ -0,0 +1,4 @@ +[package] +name = "pushed_mod" +edition = "v0.10.0" +version = "0.0.1" diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/kcl.mod.lock b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/kcl.mod.lock new file mode 100644 index 00000000..e69de29b diff --git a/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/main.k b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/issues/github.com/kcl-lang/kcl/issues/1768/pushed_mod/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_push/push_0/kcl.mod b/pkg/client/test_data/test_push/push_0/kcl.mod new file mode 100644 index 00000000..0dbc0c9f --- /dev/null +++ b/pkg/client/test_data/test_push/push_0/kcl.mod @@ -0,0 +1,4 @@ +[package] +name = "push_0" +edition = "v0.10.0" +version = "0.0.1" diff --git a/pkg/client/test_data/test_push/push_0/kcl.mod.lock b/pkg/client/test_data/test_push/push_0/kcl.mod.lock new file mode 100644 index 00000000..e69de29b diff --git a/pkg/client/test_data/test_push/push_0/main.k b/pkg/client/test_data/test_push/push_0/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_push/push_0/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_push/test_pushed_mod/kcl.mod b/pkg/client/test_data/test_push/test_pushed_mod/kcl.mod new file mode 100644 index 00000000..9c637545 --- /dev/null +++ b/pkg/client/test_data/test_push/test_pushed_mod/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "test_pushed_mod" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +push_0 = { oci = "oci://localhost:5001/test/push_0", tag = "0.0.1", version = "0.0.1" } diff --git a/pkg/client/test_data/test_push/test_pushed_mod/kcl.mod.lock b/pkg/client/test_data/test_push/test_pushed_mod/kcl.mod.lock new file mode 100644 index 00000000..524ea52b --- /dev/null +++ b/pkg/client/test_data/test_push/test_pushed_mod/kcl.mod.lock @@ -0,0 +1,9 @@ +[dependencies] + [dependencies.push_0] + name = "push_0" + full_name = "push_0_0.0.1" + version = "0.0.1" + sum = "m9g+EU6NLk5sbcV2C26/ikgrkcNF7Myk8751xkfUwIk=" + reg = "localhost:5001" + repo = "test/push_0" + oci_tag = "0.0.1" diff --git a/pkg/client/test_data/test_push/test_pushed_mod/main.k b/pkg/client/test_data/test_push/test_pushed_mod/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_push/test_pushed_mod/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/cmd/cmd_push.go b/pkg/cmd/cmd_push.go index 3e6dca8b..51aa7b2e 100644 --- a/pkg/cmd/cmd_push.go +++ b/pkg/cmd/cmd_push.go @@ -11,6 +11,7 @@ import ( "github.com/urfave/cli/v2" "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/downloader" "kcl-lang.io/kpm/pkg/errors" "kcl-lang.io/kpm/pkg/oci" "kcl-lang.io/kpm/pkg/opt" @@ -142,23 +143,8 @@ func pushTarPackage(ociUrl, localTarPath string, vendorMode bool, kpmcli *client // 3. Generate the OCI options from oci url and the version of current kcl package. // 4. Push the package to the oci registry. func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *client.KpmClient) error { - - tarPath, err := kpmcli.PackagePkg(kclPkg, vendorMode) - if err != nil { - return err - } - - // clean the tar path. - defer func() { - if kclPkg != nil && utils.DirExists(tarPath) { - err = os.RemoveAll(tarPath) - if err != nil { - err = reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, failed to clean the temp dir.") - } - } - }() - - // 2. If the oci url is not specified, generate the default oci url from the current package. + // If the oci url is not specified, generate the default oci url from the current package. + var err error if len(ociUrl) == 0 { ociUrl, err = genDefaultOciUrlForKclPkg(kclPkg, kpmcli) if err != nil || len(ociUrl) == 0 { @@ -170,7 +156,7 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli } } - // 3. Generate the OCI options from oci url and the version of current kcl package. + // Generate the OCI options from oci url and the version of current kcl package. ociOpts, err := opt.ParseOciOptionFromOciUrl(ociUrl, kclPkg.GetPkgTag()) if err != (*reporter.KpmEvent)(nil) { return reporter.NewErrorEvent( @@ -180,14 +166,20 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli ) } - ociOpts.Annotations, err = kclPkg.GenOciManifestFromPkg() - if err != nil { - return err - } - - reporter.ReportMsgTo(fmt.Sprintf("package '%s' will be pushed", kclPkg.GetPkgName()), kpmcli.GetLogWriter()) - // 4. Push it. - err = kpmcli.PushToOci(tarPath, ociOpts) + // Push it. + err = kpmcli.Push( + client.WithPushModPath(kclPkg.HomePath), + client.WithPushSource( + downloader.Source{ + Oci: &downloader.Oci{ + Reg: ociOpts.Reg, + Repo: utils.JoinPath(ociOpts.Repo, ociOpts.Ref), + Tag: ociOpts.Tag, + }, + }, + ), + client.WithPushVendorMode(vendorMode), + ) if err != (*reporter.KpmEvent)(nil) { return err } diff --git a/pkg/mock/test_script/push_pkg.sh b/pkg/mock/test_script/push_pkg.sh index 64a41530..b6280ff0 100755 --- a/pkg/mock/test_script/push_pkg.sh +++ b/pkg/mock/test_script/push_pkg.sh @@ -29,4 +29,4 @@ echo $current_dir # Push the test_data package to the registry cd "$SCRIPT_DIR/../test_data" -"$current_dir/bin/kpm" push oci://$KPM_REG/$KPM_REPO +"$current_dir/bin/kpm" push oci://$KPM_REG/$KPM_REPO/test_data diff --git a/pkg/opt/opt.go b/pkg/opt/opt.go index 78e34e9f..31d184c6 100644 --- a/pkg/opt/opt.go +++ b/pkg/opt/opt.go @@ -7,6 +7,7 @@ import ( "io" "net/url" "os" + gpath "path" "path/filepath" "strings" @@ -467,9 +468,18 @@ func ParseOciUrl(ociUrl string) (*OciOptions, *reporter.KpmEvent) { return nil, reporter.NewEvent(reporter.UrlSchemeNotOci) } + repo, ref := gpath.Split(u.Path) + repo = strings.TrimPrefix(strings.TrimSuffix(repo, "/"), "/") + + if repo == "" { + repo = strings.TrimPrefix(strings.TrimSuffix(u.Path, "/"), "/") + ref = "" + } + return &OciOptions{ Reg: u.Host, - Repo: u.Path, + Repo: repo, + Ref: ref, }, nil } @@ -491,7 +501,7 @@ func (oci *OciOptions) AddStoragePathSuffix(pathPrefix string) string { // SanitizePathSuffix will take 'Registry/Repo/Tag' as a path suffix and sanitize it. func (oci *OciOptions) SanitizePathWithSuffix(pathPrefix string) string { - return path.SanitizePath(filepath.Join(filepath.Join(filepath.Join(pathPrefix, oci.Reg), oci.Repo), oci.Tag)) + return path.SanitizePath(filepath.Join(pathPrefix, oci.Reg, oci.Repo, oci.Ref, oci.Tag)) } type OciManifestOptions struct { diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 61508ba9..50e6d803 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -552,10 +552,21 @@ func RmNewline(s string) string { } // JoinPath will join the 'elem' to the 'base' with '/'. -func JoinPath(base, elem string) string { - base = strings.TrimSuffix(base, "/") - elem = strings.TrimPrefix(elem, "/") - return base + "/" + elem +func JoinPath(elems ...string) string { + var result string + for i, elem := range elems { + if elem == "" { + continue + } + elem = strings.TrimPrefix(elem, "/") + elem = strings.TrimSuffix(elem, "/") + if i == 0 || result == "" { + result = elem + } else { + result = result + "/" + elem + } + } + return result } // IsUrl will check whether the string 'str' is a url. diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index dbf87bd5..129390cd 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -173,10 +173,10 @@ func TestJoinPath(t *testing.T) { assert.Equal(t, JoinPath("base", "elem"), "base/elem") assert.Equal(t, JoinPath("base/", "elem"), "base/elem") assert.Equal(t, JoinPath("base", "/elem"), "base/elem") - assert.Equal(t, JoinPath("", "/elem"), "/elem") - assert.Equal(t, JoinPath("", "elem"), "/elem") - assert.Equal(t, JoinPath("base/", ""), "base/") - assert.Equal(t, JoinPath("base", ""), "base/") + assert.Equal(t, JoinPath("", "elem"), "elem") + assert.Equal(t, JoinPath("", "elem"), "elem") + assert.Equal(t, JoinPath("base/", ""), "base") + assert.Equal(t, JoinPath("base", ""), "base") } func TestIsUrl(t *testing.T) { diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout index b6cf454b..2cc0b57c 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url/test_suite.stdout @@ -1,4 +1,4 @@ start to pull 'oci://localhost:5001/test/k8s' the latest version '1.31.2' will be pulled -pulling '/test/k8s:1.31.2' from 'localhost:5001/test/k8s' +pulling 'test/k8s:1.31.2' from 'localhost:5001/test/k8s' pulled 'oci://localhost:5001/test/k8s' in '/localhost:5001/test/k8s/1.31.2' successfully \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url_tag/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url_tag/test_suite.stdout index 3656dc86..369fe05d 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url_tag/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/kpm_pull_with_oci_url_tag/test_suite.stdout @@ -1,3 +1,3 @@ start to pull 'oci://localhost:5001/test/k8s' with tag '1.14' -pulling '/test/k8s:1.14' from 'localhost:5001/test/k8s' +pulling 'test/k8s:1.14' from 'localhost:5001/test/k8s' pulled 'oci://localhost:5001/test/k8s' in '/localhost:5001/test/k8s/1.14' successfully \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/pull_private_pkg/test_suite.stdout b/test/e2e/test_suites/kpm/exec_outside_pkg/pull_private_pkg/test_suite.stdout index 2d980a93..46493f6e 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/pull_private_pkg/test_suite.stdout +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/pull_private_pkg/test_suite.stdout @@ -1,4 +1,4 @@ start to pull 'oci://localhost:5001/test/helloworld' the latest version '0.1.2' will be pulled -pulling '/test/helloworld:0.1.2' from 'localhost:5001/test/helloworld' +pulling 'test/helloworld:0.1.2' from 'localhost:5001/test/helloworld' pulled 'oci://localhost:5001/test/helloworld' in '/localhost:5001/test/helloworld/0.1.2' successfully \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url/test_suite.stderr b/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url/test_suite.stderr index 7dfb5555..545b0fd2 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url/test_suite.stderr +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url/test_suite.stderr @@ -1 +1 @@ -repository 'invalid_url/' not found \ No newline at end of file +repository 'invalid_url' not found \ No newline at end of file diff --git a/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url_without_tag/test_suite.stderr b/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url_without_tag/test_suite.stderr index affa8704..de1ddacb 100644 --- a/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url_without_tag/test_suite.stderr +++ b/test/e2e/test_suites/kpm/exec_outside_pkg/run_oci_with_invalid_url_without_tag/test_suite.stderr @@ -1 +1 @@ -repository 'invalid_url/' not found +repository 'invalid_url' not found