diff --git a/pkg/3rdparty/mvs/mvs.go b/pkg/3rdparty/mvs/mvs.go index 4376c9d1..84542347 100644 --- a/pkg/3rdparty/mvs/mvs.go +++ b/pkg/3rdparty/mvs/mvs.go @@ -114,7 +114,7 @@ func buildList(targets []module.Version, reqs Reqs, upgrade func(module.Version) for _, target := range targets { work.Add(target) } - work.Do(10, func(m module.Version) { + work.Do(1, func(m module.Version) { var required []module.Version var err error diff --git a/pkg/client/client.go b/pkg/client/client.go index 0732784e..331c2863 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -313,6 +313,23 @@ func (c *KpmClient) ResolvePkgDepsMetadata(kclPkg *pkg.KclPkg, update bool) erro return nil } +func GetReleasesFromSource(sourceType, uri string) ([]string, error) { + var releases []string + var err error + + switch sourceType { + case pkg.GIT: + releases, err = git.GetAllGithubReleases(uri) + case pkg.OCI: + releases, err = oci.GetAllImageTags(uri) + } + if err != nil { + return nil, err + } + + return releases, nil +} + // UpdateDeps will update the dependencies. func (c *KpmClient) UpdateDeps(kclPkg *pkg.KclPkg) error { _, err := c.ResolveDepsMetadataInJsonStr(kclPkg, true) @@ -320,6 +337,12 @@ func (c *KpmClient) UpdateDeps(kclPkg *pkg.KclPkg) error { return err } + // update kcl.mod + err = kclPkg.ModFile.StoreModFile() + if err != nil { + return err + } + // Generate file kcl.mod.lock. if !kclPkg.NoSumCheck { err := kclPkg.LockDepsVersion() diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 79ee1665..6fb53403 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -10,10 +10,12 @@ import ( "os" "path/filepath" "runtime" + "sort" "strings" "testing" "github.com/dominikbraun/graph" + "github.com/hashicorp/go-version" "github.com/otiai10/copy" "github.com/stretchr/testify/assert" "golang.org/x/mod/module" @@ -815,6 +817,43 @@ func TestParseOciOptionFromString(t *testing.T) { assert.Equal(t, ociOption.Tag, "test_tag") } +func TestGetReleasesFromSource(t *testing.T) { + sortVersions := func(versions []string) ([]string, error) { + var vers []*version.Version + for _, raw := range versions { + v, err := version.NewVersion(raw) + if err != nil { + return nil, err + } + vers = append(vers, v) + } + sort.Slice(vers, func(i, j int) bool { + return vers[i].LessThan(vers[j]) + }) + var res []string + for _, v := range vers { + res = append(res, v.Original()) + } + return res, nil + } + + releases, err := GetReleasesFromSource(pkg.GIT, "https://github.com/kcl-lang/kpm") + assert.Equal(t, err, nil) + length := len(releases) + assert.True(t, length >= 5) + releasesVersions, err := sortVersions(releases) + assert.Equal(t, err, nil) + assert.Equal(t, releasesVersions[:5], []string{"v0.1.0", "v0.2.0", "v0.2.1", "v0.2.2", "v0.2.3"}) + + releases, err = GetReleasesFromSource(pkg.OCI, "oci://ghcr.io/kcl-lang/k8s") + assert.Equal(t, err, nil) + length = len(releases) + assert.True(t, length >= 5) + releasesVersions, err = sortVersions(releases) + assert.Equal(t, err, nil) + assert.Equal(t, releasesVersions[:5], []string{"1.14", "1.15", "1.16", "1.17", "1.18"}) +} + func TestUpdateWithKclMod(t *testing.T) { kpmcli, err := NewKpmClient() assert.Equal(t, err, nil) @@ -1469,4 +1508,3 @@ func testRunWithOciDownloader(t *testing.T) { assert.Equal(t, buf.String(), "downloading 'zong-zhe/helloworld:0.0.3' from 'ghcr.io/zong-zhe/helloworld:0.0.3'\n") assert.Equal(t, res.GetRawYamlResult(), "The_first_kcl_program: Hello World!") } - diff --git a/pkg/cmd/cmd_add.go b/pkg/cmd/cmd_add.go index ec311e7d..95338de1 100644 --- a/pkg/cmd/cmd_add.go +++ b/pkg/cmd/cmd_add.go @@ -208,7 +208,7 @@ func parseGitRegistryOptions(c *cli.Context) (*opt.RegistryOptions, *reporter.Kp // parseOciRegistryOptions will parse the oci registry information from user cli inputs. func parseOciRegistryOptions(c *cli.Context, kpmcli *client.KpmClient) (*opt.RegistryOptions, error) { ociPkgRef := c.Args().First() - name, version, err := parseOciPkgNameAndVersion(ociPkgRef) + name, version, err := ParseOciPkgNameAndVersion(ociPkgRef) if err != nil { return nil, err } @@ -243,7 +243,7 @@ func parseLocalPathOptions(c *cli.Context) (*opt.RegistryOptions, *reporter.KpmE // parseOciPkgNameAndVersion will parse package name and version // from string ":". -func parseOciPkgNameAndVersion(s string) (string, string, error) { +func ParseOciPkgNameAndVersion(s string) (string, string, error) { parts := strings.Split(s, ":") if len(parts) == 1 { return parts[0], "", nil diff --git a/pkg/cmd/cmd_update.go b/pkg/cmd/cmd_update.go index 330d3678..32a2acc0 100644 --- a/pkg/cmd/cmd_update.go +++ b/pkg/cmd/cmd_update.go @@ -5,12 +5,20 @@ package cmd import ( + "fmt" "os" + "slices" + "strings" + "github.com/dominikbraun/graph" "github.com/urfave/cli/v2" + "golang.org/x/mod/module" "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/env" + "kcl-lang.io/kpm/pkg/mvs" + pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" + "kcl-lang.io/kpm/pkg/semver" ) // NewUpdateCmd new a Command for `kpm update`. @@ -48,39 +56,144 @@ func KpmUpdate(c *cli.Context, kpmcli *client.KpmClient) error { } }() - input_paths := c.Args().Slice() + pkgInfos := c.Args().Slice() - pkg_paths := []string{} - if len(input_paths) == 0 { - pwd, err := os.Getwd() + pwd, err := os.Getwd() + if err != nil { + return reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, please contact us to fix it.") + } + + kclPkg, err := kpmcli.LoadPkgFromPath(pwd) + if err != nil { + return err + } + + globalPkgPath, err := env.GetAbsPkgPath() + if err != nil { + return err + } + + err = kclPkg.ValidateKpmHome(globalPkgPath) + if err != (*reporter.KpmEvent)(nil) { + return err + } + + var ( + modulesToUpgrade []module.Version + modulesToDowngrade []module.Version + ) + + for _, pkgInfo := range pkgInfos { + err = GetModulesToUpdate(kclPkg, modulesToUpgrade, modulesToDowngrade, pkgInfo) if err != nil { - return reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, please contact us to fix it") + reporter.Report(err) } - pkg_paths = append(pkg_paths, pwd) - } else { - pkg_paths = input_paths } - for _, pkg_path := range pkg_paths { - kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + if err != nil { + return err + } + + reqs := mvs.ReqsGraph{ + Graph: depGraph, + KpmClient: kpmcli, + KpmPkg: kclPkg, + } + + target := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} + buildList, err := mvs.UpdateBuildList(target, modulesToUpgrade, modulesToDowngrade, &reqs) + if err != nil { + return reporter.NewErrorEvent(reporter.FailedUpdatingBuildList, err, "failed to update build list") + } + + // get all the vertices in the graph + modules, err := graph.TopologicalSort(depGraph) + if err != nil { + return reporter.NewErrorEvent(reporter.FailedTopologicalSort, err, "failed to sort the dependencies") + } + + kclPkg.ModFile.Dependencies.Deps = make(map[string]pkg.Dependency) + + for _, module := range modules { + err = InsertModuleToDeps(kclPkg, module, target, buildList, reqs) if err != nil { return err } + } + + err = kpmcli.UpdateDeps(kclPkg) + if err != nil { + return err + } + return nil +} + +// GetModulesToUpdate validates if the packages is present in kcl.mod file and +// find the latest version if version is not specified. Depending on the value of pkgVersion, +// modulesToUpgrade or modulesToDowngrade will be updated. +func GetModulesToUpdate(kclPkg *pkg.KclPkg, modulesToUpgrade []module.Version, modulesToDowngrade []module.Version, pkgInfo string) error { + pkgInfo = strings.TrimSpace(pkgInfo) + pkgName, pkgVersion, err := ParseOciPkgNameAndVersion(pkgInfo) + if err != nil { + return err + } + + var dep pkg.Dependency + var ok bool + if dep, ok = kclPkg.Deps[pkgName]; !ok { + return err + } - globalPkgPath, err := env.GetAbsPkgPath() + if pkgVersion == "" { + var releases []string + releases, err = client.GetReleasesFromSource(dep.GetSourceType(), dep.GetDownloadPath()) if err != nil { - return err + return reporter.NewErrorEvent( + reporter.FailedGetReleases, + err, + fmt.Sprintf("failed to get releases for %s", pkgName), + ) } - - err = kclPkg.ValidateKpmHome(globalPkgPath) - if err != (*reporter.KpmEvent)(nil) { - return err + pkgVersion, err = semver.LatestCompatibleVersion(releases, dep.Version) + if err != nil { + return reporter.NewErrorEvent( + reporter.FailedSelectLatestCompatibleVersion, + err, + fmt.Sprintf("failed to find the latest version for %s", pkgName), + ) } + } + if pkgVersion < dep.Version { + modulesToDowngrade = append(modulesToDowngrade, module.Version{Path: pkgName, Version: pkgVersion}) + } else if pkgVersion > dep.Version { + modulesToUpgrade = append(modulesToUpgrade, module.Version{Path: pkgName, Version: pkgVersion}) + } + return nil +} - err = kpmcli.UpdateDeps(kclPkg) +// InsertModuleToDeps checks whether module is present in the buildList and it is not the same as the target module, +// and inserts it to the dependencies of kclPkg +func InsertModuleToDeps(kclPkg *pkg.KclPkg, module module.Version, target module.Version, buildList []module.Version, reqs mvs.ReqsGraph) (error) { + if module.Path == target.Path || !slices.Contains(buildList, module) { + return nil + } + d := pkg.Dependency{ + Name: module.Path, + Version: module.Version, + } + d.FullName = d.GenDepFullName() + _, properties, err := reqs.VertexWithProperties(module) + if err != nil { + return reporter.NewErrorEvent(reporter.FailedGetVertexProperties, err, "failed to get vertex with properties") + } + // there must be one property depending on the download source type + for sourceType, uri := range properties.Attributes { + d.Source, err = pkg.GenSource(sourceType, uri, module.Version) if err != nil { - return err + return reporter.NewErrorEvent(reporter.FailedGenerateSource, err, "failed to generate source") } } + kclPkg.ModFile.Dependencies.Deps[module.Path] = d return nil -} +} \ No newline at end of file diff --git a/pkg/mvs/mvs.go b/pkg/mvs/mvs.go index fbd817c7..52f414cf 100644 --- a/pkg/mvs/mvs.go +++ b/pkg/mvs/mvs.go @@ -6,10 +6,9 @@ import ( "github.com/dominikbraun/graph" "github.com/hashicorp/go-version" "golang.org/x/mod/module" + "kcl-lang.io/kpm/pkg/3rdparty/mvs" "kcl-lang.io/kpm/pkg/client" errInt "kcl-lang.io/kpm/pkg/errors" - "kcl-lang.io/kpm/pkg/git" - "kcl-lang.io/kpm/pkg/oci" pkg "kcl-lang.io/kpm/pkg/package" "kcl-lang.io/kpm/pkg/reporter" "kcl-lang.io/kpm/pkg/semver" @@ -17,8 +16,8 @@ import ( type ReqsGraph struct { graph.Graph[module.Version, module.Version] - kpmClient *client.KpmClient - kpmPkg *pkg.KclPkg + KpmClient *client.KpmClient + KpmPkg *pkg.KclPkg } func (r ReqsGraph) Max(path, v1, v2 string) string { @@ -50,9 +49,17 @@ func (r ReqsGraph) Upgrade(m module.Version) (module.Version, error) { return module.Version{}, err } - releases, err := getReleasesFromSource(properties) - if err != nil { - return module.Version{}, err + // there must be only one property depending on the download source type + if len(properties.Attributes) != 1 { + return module.Version{}, errInt.MultipleSources + } + + var releases []string + for sourceType, uri := range properties.Attributes { + releases, err = client.GetReleasesFromSource(sourceType, uri) + if err != nil { + return module.Version{}, err + } } if releases == nil { @@ -84,7 +91,7 @@ func (r ReqsGraph) Upgrade(m module.Version) (module.Version, error) { lockDeps := pkg.Dependencies{ Deps: make(map[string]pkg.Dependency), } - _, err = r.kpmClient.DownloadDeps(&deps, &lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) + _, err = r.KpmClient.DownloadDeps(&deps, &lockDeps, r.Graph, r.KpmPkg.HomePath, module.Version{}) if err != nil { return module.Version{}, err } @@ -98,9 +105,17 @@ func (r ReqsGraph) Previous(m module.Version) (module.Version, error) { return module.Version{}, err } - releases, err := getReleasesFromSource(properties) - if err != nil { - return module.Version{}, err + // there must be only one property depending on the download source type + if len(properties.Attributes) != 1 { + return module.Version{}, errInt.MultipleSources + } + + var releases []string + for sourceType, uri := range properties.Attributes { + releases, err = client.GetReleasesFromSource(sourceType, uri) + if err != nil { + return module.Version{}, err + } } if releases == nil { @@ -140,7 +155,7 @@ func (r ReqsGraph) Previous(m module.Version) (module.Version, error) { lockDeps := pkg.Dependencies{ Deps: make(map[string]pkg.Dependency), } - _, err = r.kpmClient.DownloadDeps(&deps, &lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) + _, err = r.KpmClient.DownloadDeps(&deps, &lockDeps, r.Graph, r.KpmPkg.HomePath, module.Version{}) if err != nil { return module.Version{}, err } @@ -160,26 +175,32 @@ func (r ReqsGraph) Required(m module.Version) ([]module.Version, error) { return reqs, nil } -func getReleasesFromSource(properties graph.VertexProperties) ([]string, error) { - var releases []string - var err error - - // there must be only one property depending on the download source type - if len(properties.Attributes) != 1 { - return nil, errInt.MultipleSources +// UpdateBuildList decides whether to upgrade or downgrade based on modulesToUpgrade and modulesToDowngrade. +// if modulesToUpgrade is empty, upgrade all dependencies. if modulesToUpgrade is not empty, upgrade the dependencies. +// if modulesToDowngrade is not empty, downgrade the dependencies. +// if modulesToUpgrade and modulesToDowngrade are both empty, first apply upgrade operation and +// then downgrade the build list returned from previous operation. +func UpdateBuildList(target module.Version, modulesToUpgrade []module.Version, modulesToDowngrade []module.Version, reqs *ReqsGraph) ([]module.Version, error) { + var ( + UpdBuildLists []module.Version + err error + ) + + if len(modulesToUpgrade) == 0 { + UpdBuildLists, err = mvs.UpgradeAll(target, reqs) + } else { + UpdBuildLists, err = mvs.Upgrade(target, reqs, modulesToUpgrade...) + } + if err != nil { + return []module.Version{}, err } - for k, v := range properties.Attributes { - switch k { - case pkg.GIT: - releases, err = git.GetAllGithubReleases(v) - case pkg.OCI: - releases, err = oci.GetAllImageTags(v) - } - if err != nil { - return nil, err - } + if len(modulesToDowngrade) != 0 { + UpdBuildLists, err = mvs.Downgrade(target, reqs, modulesToDowngrade...) + } + if err != nil { + return []module.Version{}, err } - return releases, nil + return UpdBuildLists, nil } diff --git a/pkg/package/modfile.go b/pkg/package/modfile.go index 38946a04..fff7dbac 100644 --- a/pkg/package/modfile.go +++ b/pkg/package/modfile.go @@ -283,6 +283,11 @@ func GenSource(sourceType string, uri string, tagName string) (Source, error) { } source.Oci = &oci } + if sourceType == LOCAL { + source.Local = &Local{ + Path: uri, + } + } return source, nil } diff --git a/pkg/reporter/reporter.go b/pkg/reporter/reporter.go index 29e0ccf9..27af7912 100644 --- a/pkg/reporter/reporter.go +++ b/pkg/reporter/reporter.go @@ -55,6 +55,11 @@ const ( FailedLoadCredential FailedCreateOciClient FailedSelectLatestVersion + FailedSelectLatestCompatibleVersion + FailedGetReleases + FailedTopologicalSort + FailedGetVertexProperties + FailedGenerateSource FailedGetPackageVersions FailedCreateStorePath FailedPush @@ -81,6 +86,7 @@ const ( WithoutGitTag FailedCloneFromGit FailedHashPkg + FailedUpdatingBuildList Bug // normal event type means the event is a normal event. diff --git a/test/e2e/test_suites/kpm/kpm_update/test_update_with_diff_version/test_suite.stdout b/test/e2e/test_suites/kpm/kpm_update/test_update_with_diff_version/test_suite.stdout index 0c666431..f37f0853 100644 --- a/test/e2e/test_suites/kpm/kpm_update/test_update_with_diff_version/test_suite.stdout +++ b/test/e2e/test_suites/kpm/kpm_update/test_update_with_diff_version/test_suite.stdout @@ -1 +1 @@ -adding 'k8s' with version '1.14' \ No newline at end of file +downloading 'test/k8s:1.14' from 'localhost:5001/test/k8s:1.14' \ No newline at end of file