diff --git a/go.mod b/go.mod index b5de1d7e..ee5dd98d 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/jsonv v1.1.3 // indirect github.com/chai2010/protorpc v1.1.4 // indirect + github.com/chainguard-dev/git-urls v1.0.2 // indirect github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/containerd v1.7.11 // indirect github.com/containerd/log v0.1.0 // indirect @@ -48,10 +49,12 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fatih/color v1.10.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/getkin/kin-openapi v0.123.0 // indirect + github.com/ghodss/yaml v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-logr/logr v1.3.0 // indirect @@ -131,10 +134,13 @@ require ( google.golang.org/grpc v1.62.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect kcl-lang.io/lib v0.8.0 // indirect ) require ( + github.com/containers/image v3.0.2+incompatible github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/go-git/go-git/v5 v5.11.0 github.com/gofrs/flock v0.8.1 @@ -142,6 +148,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-getter v1.7.4 github.com/hashicorp/go-version v1.6.0 + github.com/kubescape/go-git-url v0.0.30 github.com/moby/term v0.0.0-20221205130635-1aeaba878587 github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 diff --git a/go.sum b/go.sum index f2d6ed7a..9a7993f4 100644 --- a/go.sum +++ b/go.sum @@ -246,6 +246,8 @@ github.com/chai2010/jsonv v1.1.3 h1:gBIHXn/5mdEPTuWZfjC54fn/yUSRR8OGobXobcc6now= github.com/chai2010/jsonv v1.1.3/go.mod h1:mEoT1dQ9qVF4oP9peVTl0UymTmJwXoTDOh+sNA6+XII= github.com/chai2010/protorpc v1.1.4 h1:CTtFUhzXRoeuR7FtgQ2b2vdT/KgWVpCM+sIus8zJjHs= github.com/chai2010/protorpc v1.1.4/go.mod h1:/wO0kiyVdu7ug8dCMrA2yDr2vLfyhsLEuzLa9J2HJ+I= +github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiwNkJrVcKQ= +github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -267,12 +269,16 @@ github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/P github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/containerd/containerd v1.7.0 h1:G/ZQr3gMZs6ZT0qPUZ15znx5QSdQdASW11nXTLTM2Pg= +github.com/containerd/containerd v1.7.0/go.mod h1:QfR7Efgb/6X2BDpTPJRvPTYDE9rsF0FsXX9J8sIs/sc= github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw= github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE= -github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= -github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= +github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containers/image v3.0.2+incompatible h1:B1lqAE8MUPCrsBLE86J0gnXleeRq8zJnQryhiiGQNyE= +github.com/containers/image v3.0.2+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -327,6 +333,7 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/getkin/kin-openapi v0.123.0 h1:zIik0mRwFNLyvtXK274Q6ut+dPh6nlxBp0x7mNrPhs8= github.com/getkin/kin-openapi v0.123.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= @@ -547,6 +554,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubescape/go-git-url v0.0.30 h1:PIbg86ae0ftee/p/Tu/6CA1ju6VoJ51G3sQWNHOm6wg= +github.com/kubescape/go-git-url v0.0.30/go.mod h1:3ddc1HEflms1vMhD9owt/3FBES070UaYTUarcjx8jDk= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -1311,6 +1320,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kcl-lang.io/kcl-go v0.8.0 h1:ep+r4QMiAOeTkOJl9then58D+W8OpX5WHKuL+Tf8+po= kcl-lang.io/kcl-go v0.8.0/go.mod h1:Z+bJWXe5X1Xra7AUOvDpCb4WBcCGNo0sqQG/bNjz6+k= kcl-lang.io/lib v0.8.0 h1:bzMzPpaXaAxWO9JP0B7eI2ZFOYfojdEYUMtNGlUrPx4= diff --git a/pkg/3rdparty/gover/gover.go b/pkg/3rdparty/gover/gover.go new file mode 100644 index 00000000..06c29459 --- /dev/null +++ b/pkg/3rdparty/gover/gover.go @@ -0,0 +1,75 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gover implements support for Go toolchain versions like 1.21.0 and 1.21rc1. +// (For historical reasons, Go does not use semver for its toolchains.) +// This package provides the same basic analysis that golang.org/x/mod/semver does for semver. +// It also provides some helpers for extracting versions from go.mod files +// and for dealing with module.Versions that may use Go versions or semver +// depending on the module path. +package gover + +import ( + "kcl-lang.io/kpm/pkg/3rdparty/gover/internal" +) + +// Compare returns -1, 0, or +1 depending on whether +// x < y, x == y, or x > y, interpreted as toolchain versions. +// The versions x and y must not begin with a "go" prefix: just "1.21" not "go1.21". +// Malformed versions compare less than well-formed versions and equal to each other. +// The language version "1.21" compares less than the release candidate and eventual releases "1.21rc1" and "1.21.0". +func Compare(x, y string) int { + return gover.Compare(x, y) +} + +// Max returns the maximum of x and y interpreted as toolchain versions, +// compared using Compare. +// If x and y compare equal, Max returns x. +func Max(x, y string) string { + return gover.Max(x, y) +} + +// IsLang reports whether v denotes the overall Go language version +// and not a specific release. Starting with the Go 1.21 release, "1.x" denotes +// the overall language version; the first release is "1.x.0". +// The distinction is important because the relative ordering is +// +// 1.21 < 1.21rc1 < 1.21.0 +// +// meaning that Go 1.21rc1 and Go 1.21.0 will both handle go.mod files that +// say "go 1.21", but Go 1.21rc1 will not handle files that say "go 1.21.0". +func IsLang(x string) bool { + return gover.IsLang(x) +} + +// Lang returns the Go language version. For example, Lang("1.2.3") == "1.2". +func Lang(x string) string { + return gover.Lang(x) +} + +// IsPrerelease reports whether v denotes a Go prerelease version. +func IsPrerelease(x string) bool { + return gover.Parse(x).Kind != "" +} + +// Prev returns the Go major release immediately preceding v, +// or v itself if v is the first Go major release (1.0) or not a supported +// Go version. +// +// Examples: +// +// Prev("1.2") = "1.1" +// Prev("1.3rc4") = "1.2" +func Prev(x string) string { + v := gover.Parse(x) + if gover.CmpInt(v.Minor, "1") <= 0 { + return v.Major + } + return v.Major + "." + gover.DecInt(v.Minor) +} + +// IsValid reports whether the version x is valid. +func IsValid(x string) bool { + return gover.IsValid(x) +} diff --git a/pkg/3rdparty/gover/internal/gover.go b/pkg/3rdparty/gover/internal/gover.go new file mode 100644 index 00000000..2ad06846 --- /dev/null +++ b/pkg/3rdparty/gover/internal/gover.go @@ -0,0 +1,223 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gover implements support for Go toolchain versions like 1.21.0 and 1.21rc1. +// (For historical reasons, Go does not use semver for its toolchains.) +// This package provides the same basic analysis that golang.org/x/mod/semver does for semver. +// +// The go/version package should be imported instead of this one when possible. +// Note that this package works on "1.21" while go/version works on "go1.21". +package gover + +import ( + "cmp" +) + +// A Version is a parsed Go version: major[.Minor[.Patch]][kind[pre]] +// The numbers are the original decimal strings to avoid integer overflows +// and since there is very little actual math. (Probably overflow doesn't matter in practice, +// but at the time this code was written, there was an existing test that used +// go1.99999999999, which does not fit in an int on 32-bit platforms. +// The "big decimal" representation avoids the problem entirely.) +type Version struct { + Major string // decimal + Minor string // decimal or "" + Patch string // decimal or "" + Kind string // "", "alpha", "beta", "rc" + Pre string // decimal or "" +} + +// Compare returns -1, 0, or +1 depending on whether +// x < y, x == y, or x > y, interpreted as toolchain versions. +// The versions x and y must not begin with a "go" prefix: just "1.21" not "go1.21". +// Malformed versions compare less than well-formed versions and equal to each other. +// The language version "1.21" compares less than the release candidate and eventual releases "1.21rc1" and "1.21.0". +func Compare(x, y string) int { + vx := Parse(x) + vy := Parse(y) + + if c := CmpInt(vx.Major, vy.Major); c != 0 { + return c + } + if c := CmpInt(vx.Minor, vy.Minor); c != 0 { + return c + } + if c := CmpInt(vx.Patch, vy.Patch); c != 0 { + return c + } + if c := cmp.Compare(vx.Kind, vy.Kind); c != 0 { // "" < alpha < beta < rc + return c + } + if c := CmpInt(vx.Pre, vy.Pre); c != 0 { + return c + } + return 0 +} + +// Max returns the maximum of x and y interpreted as toolchain versions, +// compared using Compare. +// If x and y compare equal, Max returns x. +func Max(x, y string) string { + if Compare(x, y) < 0 { + return y + } + return x +} + +// IsLang reports whether v denotes the overall Go language version +// and not a specific release. Starting with the Go 1.21 release, "1.x" denotes +// the overall language version; the first release is "1.x.0". +// The distinction is important because the relative ordering is +// +// 1.21 < 1.21rc1 < 1.21.0 +// +// meaning that Go 1.21rc1 and Go 1.21.0 will both handle go.mod files that +// say "go 1.21", but Go 1.21rc1 will not handle files that say "go 1.21.0". +func IsLang(x string) bool { + v := Parse(x) + return v != Version{} && v.Patch == "" && v.Kind == "" && v.Pre == "" +} + +// Lang returns the Go language version. For example, Lang("1.2.3") == "1.2". +func Lang(x string) string { + v := Parse(x) + if v.Minor == "" || v.Major == "1" && v.Minor == "0" { + return v.Major + } + return v.Major + "." + v.Minor +} + +// IsValid reports whether the version x is valid. +func IsValid(x string) bool { + return Parse(x) != Version{} +} + +// Parse parses the Go version string x into a version. +// It returns the zero version if x is malformed. +func Parse(x string) Version { + var v Version + + // Parse major version. + var ok bool + v.Major, x, ok = cutInt(x) + if !ok { + return Version{} + } + if x == "" { + // Interpret "1" as "1.0.0". + v.Minor = "0" + v.Patch = "0" + return v + } + + // Parse . before minor version. + if x[0] != '.' { + return Version{} + } + + // Parse minor version. + v.Minor, x, ok = cutInt(x[1:]) + if !ok { + return Version{} + } + if x == "" { + // Patch missing is same as "0" for older versions. + // Starting in Go 1.21, patch missing is different from explicit .0. + if CmpInt(v.Minor, "21") < 0 { + v.Patch = "0" + } + return v + } + + // Parse patch if present. + if x[0] == '.' { + v.Patch, x, ok = cutInt(x[1:]) + if !ok || x != "" { + // Note that we are disallowing prereleases (alpha, beta, rc) for patch releases here (x != ""). + // Allowing them would be a bit confusing because we already have: + // 1.21 < 1.21rc1 + // But a prerelease of a patch would have the opposite effect: + // 1.21.3rc1 < 1.21.3 + // We've never needed them before, so let's not start now. + return Version{} + } + return v + } + + // Parse prerelease. + i := 0 + for i < len(x) && (x[i] < '0' || '9' < x[i]) { + if x[i] < 'a' || 'z' < x[i] { + return Version{} + } + i++ + } + if i == 0 { + return Version{} + } + v.Kind, x = x[:i], x[i:] + if x == "" { + return v + } + v.Pre, x, ok = cutInt(x) + if !ok || x != "" { + return Version{} + } + + return v +} + +// cutInt scans the leading decimal number at the start of x to an integer +// and returns that value and the rest of the string. +func cutInt(x string) (n, rest string, ok bool) { + i := 0 + for i < len(x) && '0' <= x[i] && x[i] <= '9' { + i++ + } + if i == 0 || x[0] == '0' && i != 1 { // no digits or unnecessary leading zero + return "", "", false + } + return x[:i], x[i:], true +} + +// CmpInt returns cmp.Compare(x, y) interpreting x and y as decimal numbers. +// (Copied from golang.org/x/mod/semver's compareInt.) +func CmpInt(x, y string) int { + if x == y { + return 0 + } + if len(x) < len(y) { + return -1 + } + if len(x) > len(y) { + return +1 + } + if x < y { + return -1 + } else { + return +1 + } +} + +// DecInt returns the decimal string decremented by 1, or the empty string +// if the decimal is all zeroes. +// (Copied from golang.org/x/mod/module's decDecimal.) +func DecInt(decimal string) string { + // Scan right to left turning 0s to 9s until you find a digit to decrement. + digits := []byte(decimal) + i := len(digits) - 1 + for ; i >= 0 && digits[i] == '0'; i-- { + digits[i] = '9' + } + if i < 0 { + // decimal is all zeros + return "" + } + if i == 0 && digits[i] == '1' && len(digits) > 1 { + digits = digits[1:] + } else { + digits[i]-- + } + return string(digits) +} diff --git a/pkg/3rdparty/gover/mod.go b/pkg/3rdparty/gover/mod.go new file mode 100644 index 00000000..d3cc1706 --- /dev/null +++ b/pkg/3rdparty/gover/mod.go @@ -0,0 +1,127 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gover + +import ( + "sort" + "strings" + + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +// IsToolchain reports whether the module path corresponds to the +// virtual, non-downloadable module tracking go or toolchain directives in the go.mod file. +// +// Note that IsToolchain only matches "go" and "toolchain", not the +// real, downloadable module "golang.org/toolchain" containing toolchain files. +// +// IsToolchain("go") = true +// IsToolchain("toolchain") = true +// IsToolchain("golang.org/x/tools") = false +// IsToolchain("golang.org/toolchain") = false +func IsToolchain(path string) bool { + return path == "go" || path == "toolchain" +} + +// ModCompare returns the result of comparing the versions x and y +// for the module with the given path. +// The path is necessary because the "go" and "toolchain" modules +// use a different version syntax and semantics (gover, this package) +// than most modules (semver). +func ModCompare(path string, x, y string) int { + if path == "go" { + return Compare(x, y) + } + if path == "toolchain" { + return Compare(maybeToolchainVersion(x), maybeToolchainVersion(y)) + } + return semver.Compare(x, y) +} + +// ModSort is like module.Sort but understands the "go" and "toolchain" +// modules and their version ordering. +func ModSort(list []module.Version) { + sort.Slice(list, func(i, j int) bool { + mi := list[i] + mj := list[j] + if mi.Path != mj.Path { + return mi.Path < mj.Path + } + // To help go.sum formatting, allow version/file. + // Compare semver prefix by semver rules, + // file by string order. + vi := mi.Version + vj := mj.Version + var fi, fj string + if k := strings.Index(vi, "/"); k >= 0 { + vi, fi = vi[:k], vi[k:] + } + if k := strings.Index(vj, "/"); k >= 0 { + vj, fj = vj[:k], vj[k:] + } + if vi != vj { + return ModCompare(mi.Path, vi, vj) < 0 + } + return fi < fj + }) +} + +// ModIsValid reports whether vers is a valid version syntax for the module with the given path. +func ModIsValid(path, vers string) bool { + if IsToolchain(path) { + if path == "toolchain" { + return IsValid(FromToolchain(vers)) + } + return IsValid(vers) + } + return semver.IsValid(vers) +} + +// ModIsPrefix reports whether v is a valid version syntax prefix for the module with the given path. +// The caller is assumed to have checked that ModIsValid(path, vers) is true. +func ModIsPrefix(path, vers string) bool { + if IsToolchain(path) { + if path == "toolchain" { + return IsLang(FromToolchain(vers)) + } + return IsLang(vers) + } + // Semver + dots := 0 + for i := 0; i < len(vers); i++ { + switch vers[i] { + case '-', '+': + return false + case '.': + dots++ + if dots >= 2 { + return false + } + } + } + return true +} + +// ModIsPrerelease reports whether v is a prerelease version for the module with the given path. +// The caller is assumed to have checked that ModIsValid(path, vers) is true. +func ModIsPrerelease(path, vers string) bool { + if IsToolchain(path) { + return IsPrerelease(vers) + } + return semver.Prerelease(vers) != "" +} + +// ModMajorMinor returns the "major.minor" truncation of the version v, +// for use as a prefix in "@patch" queries. +func ModMajorMinor(path, vers string) string { + if IsToolchain(path) { + if path == "toolchain" { + return "go" + Lang(FromToolchain(vers)) + } + return Lang(vers) + } + return semver.MajorMinor(vers) +} diff --git a/pkg/3rdparty/gover/toolchain.go b/pkg/3rdparty/gover/toolchain.go new file mode 100644 index 00000000..fda95adc --- /dev/null +++ b/pkg/3rdparty/gover/toolchain.go @@ -0,0 +1,90 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gover + +import ( + "context" + "errors" + "strings" +) + +// FromToolchain returns the Go version for the named toolchain, +// derived from the name itself (not by running the toolchain). +// A toolchain is named "goVERSION". +// A suffix after the VERSION introduced by a -, space, or tab is removed. +// Examples: +// +// FromToolchain("go1.2.3") == "1.2.3" +// FromToolchain("go1.2.3-bigcorp") == "1.2.3" +// FromToolchain("invalid") == "" +func FromToolchain(name string) string { + if strings.ContainsAny(name, "\\/") { + // The suffix must not include a path separator, since that would cause + // exec.LookPath to resolve it from a relative directory instead of from + // $PATH. + return "" + } + + var v string + if strings.HasPrefix(name, "go") { + v = name[2:] + } else { + return "" + } + // Some builds use custom suffixes; strip them. + if i := strings.IndexAny(v, " \t-"); i >= 0 { + v = v[:i] + } + if !IsValid(v) { + return "" + } + return v +} + +func maybeToolchainVersion(name string) string { + if IsValid(name) { + return name + } + return FromToolchain(name) +} + +// ToolchainMax returns the maximum of x and y interpreted as toolchain names, +// compared using Compare(FromToolchain(x), FromToolchain(y)). +// If x and y compare equal, Max returns x. +func ToolchainMax(x, y string) string { + if Compare(FromToolchain(x), FromToolchain(y)) < 0 { + return y + } + return x +} + +// Startup records the information that went into the startup-time version switch. +// It is initialized by switchGoToolchain. +var Startup struct { + GOTOOLCHAIN string // $GOTOOLCHAIN setting + AutoFile string // go.mod or go.work file consulted + AutoGoVersion string // go line found in file + AutoToolchain string // toolchain line found in file +} + +// A TooNewError explains that a module is too new for this version of Go. +type TooNewError struct { + What string + GoVersion string + Toolchain string // for callers if they want to use it, but not printed +} + +var ErrTooNew = errors.New("module too new") + +func (e *TooNewError) Is(err error) bool { + return err == ErrTooNew +} + +// A Switcher provides the ability to switch to a new toolchain in response to TooNewErrors. +// See [cmd/go/internal/toolchain.Switcher] for documentation. +type Switcher interface { + Error(err error) + Switch(ctx context.Context) +} diff --git a/pkg/3rdparty/mvs/errors.go b/pkg/3rdparty/mvs/errors.go new file mode 100644 index 00000000..8db65d65 --- /dev/null +++ b/pkg/3rdparty/mvs/errors.go @@ -0,0 +1,105 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mvs + +import ( + "fmt" + "strings" + + "golang.org/x/mod/module" +) + +// BuildListError decorates an error that occurred gathering requirements +// while constructing a build list. BuildListError prints the chain +// of requirements to the module where the error occurred. +type BuildListError struct { + Err error + stack []buildListErrorElem +} + +type buildListErrorElem struct { + m module.Version + + // nextReason is the reason this module depends on the next module in the + // stack. Typically either "requires", or "updating to". + nextReason string +} + +// NewBuildListError returns a new BuildListError wrapping an error that +// occurred at a module found along the given path of requirements and/or +// upgrades, which must be non-empty. +// +// The isVersionChange function reports whether a path step is due to an +// explicit upgrade or downgrade (as opposed to an existing requirement in a +// go.mod file). A nil isVersionChange function indicates that none of the path +// steps are due to explicit version changes. +func NewBuildListError(err error, path []module.Version, isVersionChange func(from, to module.Version) bool) *BuildListError { + stack := make([]buildListErrorElem, 0, len(path)) + for len(path) > 1 { + reason := "requires" + if isVersionChange != nil && isVersionChange(path[0], path[1]) { + reason = "updating to" + } + stack = append(stack, buildListErrorElem{ + m: path[0], + nextReason: reason, + }) + path = path[1:] + } + stack = append(stack, buildListErrorElem{m: path[0]}) + + return &BuildListError{ + Err: err, + stack: stack, + } +} + +// Module returns the module where the error occurred. If the module stack +// is empty, this returns a zero value. +func (e *BuildListError) Module() module.Version { + if len(e.stack) == 0 { + return module.Version{} + } + return e.stack[len(e.stack)-1].m +} + +func (e *BuildListError) Error() string { + b := &strings.Builder{} + stack := e.stack + + // Don't print modules at the beginning of the chain without a + // version. These always seem to be the main module or a + // synthetic module ("target@"). + for len(stack) > 0 && stack[0].m.Version == "" { + stack = stack[1:] + } + + if len(stack) == 0 { + b.WriteString(e.Err.Error()) + } else { + for _, elem := range stack[:len(stack)-1] { + fmt.Fprintf(b, "%s %s\n\t", elem.m, elem.nextReason) + } + // Ensure that the final module path and version are included as part of the + // error message. + m := stack[len(stack)-1].m + if mErr, ok := e.Err.(*module.ModuleError); ok { + actual := module.Version{Path: mErr.Path, Version: mErr.Version} + if v, ok := mErr.Err.(*module.InvalidVersionError); ok { + actual.Version = v.Version + } + if actual == m { + fmt.Fprintf(b, "%v", e.Err) + } else { + fmt.Fprintf(b, "%s (replaced by %s): %v", m, actual, mErr.Err) + } + } else { + fmt.Fprintf(b, "%v", module.VersionError(m, e.Err)) + } + } + return b.String() +} + +func (e *BuildListError) Unwrap() error { return e.Err } diff --git a/pkg/3rdparty/mvs/graph.go b/pkg/3rdparty/mvs/graph.go new file mode 100644 index 00000000..85489e86 --- /dev/null +++ b/pkg/3rdparty/mvs/graph.go @@ -0,0 +1,226 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mvs + +import ( + "fmt" + "slices" + + "kcl-lang.io/kpm/pkg/3rdparty/gover" + + "golang.org/x/mod/module" +) + +// Graph implements an incremental version of the MVS algorithm, with the +// requirements pushed by the caller instead of pulled by the MVS traversal. +type Graph struct { + cmp func(p, v1, v2 string) int + roots []module.Version + + required map[module.Version][]module.Version + + isRoot map[module.Version]bool // contains true for roots and false for reachable non-roots + selected map[string]string // path → version +} + +// NewGraph returns an incremental MVS graph containing only a set of root +// dependencies and using the given max function for version strings. +// +// The caller must ensure that the root slice is not modified while the Graph +// may be in use. +func NewGraph(cmp func(p, v1, v2 string) int, roots []module.Version) *Graph { + g := &Graph{ + cmp: cmp, + roots: slices.Clip(roots), + required: make(map[module.Version][]module.Version), + isRoot: make(map[module.Version]bool), + selected: make(map[string]string), + } + + for _, m := range roots { + g.isRoot[m] = true + if g.cmp(m.Path, g.Selected(m.Path), m.Version) < 0 { + g.selected[m.Path] = m.Version + } + } + + return g +} + +// Require adds the information that module m requires all modules in reqs. +// The reqs slice must not be modified after it is passed to Require. +// +// m must be reachable by some existing chain of requirements from g's target, +// and Require must not have been called for it already. +// +// If any of the modules in reqs has the same path as g's target, +// the target must have higher precedence than the version in req. +func (g *Graph) Require(m module.Version, reqs []module.Version) { + // To help catch disconnected-graph bugs, enforce that all required versions + // are actually reachable from the roots (and therefore should affect the + // selected versions of the modules they name). + if _, reachable := g.isRoot[m]; !reachable { + panic(fmt.Sprintf("%v is not reachable from any root", m)) + } + + // Truncate reqs to its capacity to avoid aliasing bugs if it is later + // returned from RequiredBy and appended to. + reqs = slices.Clip(reqs) + + if _, dup := g.required[m]; dup { + panic(fmt.Sprintf("requirements of %v have already been set", m)) + } + g.required[m] = reqs + + for _, dep := range reqs { + // Mark dep reachable, regardless of whether it is selected. + if _, ok := g.isRoot[dep]; !ok { + g.isRoot[dep] = false + } + + if g.cmp(dep.Path, g.Selected(dep.Path), dep.Version) < 0 { + g.selected[dep.Path] = dep.Version + } + } +} + +// RequiredBy returns the slice of requirements passed to Require for m, if any, +// with its capacity reduced to its length. +// If Require has not been called for m, RequiredBy(m) returns ok=false. +// +// The caller must not modify the returned slice, but may safely append to it +// and may rely on it not to be modified. +func (g *Graph) RequiredBy(m module.Version) (reqs []module.Version, ok bool) { + reqs, ok = g.required[m] + return reqs, ok +} + +// Selected returns the selected version of the given module path. +// +// If no version is selected, Selected returns version "none". +func (g *Graph) Selected(path string) (version string) { + v, ok := g.selected[path] + if !ok { + return "none" + } + return v +} + +// BuildList returns the selected versions of all modules present in the Graph, +// beginning with the selected versions of each module path in the roots of g. +// +// The order of the remaining elements in the list is deterministic +// but arbitrary. +func (g *Graph) BuildList() []module.Version { + seenRoot := make(map[string]bool, len(g.roots)) + + var list []module.Version + for _, r := range g.roots { + if seenRoot[r.Path] { + // Multiple copies of the same root, with the same or different versions, + // are a bit of a degenerate case: we will take the transitive + // requirements of both roots into account, but only the higher one can + // possibly be selected. However — especially given that we need the + // seenRoot map for later anyway — it is simpler to support this + // degenerate case than to forbid it. + continue + } + + if v := g.Selected(r.Path); v != "none" { + list = append(list, module.Version{Path: r.Path, Version: v}) + } + seenRoot[r.Path] = true + } + uniqueRoots := list + + for path, version := range g.selected { + if !seenRoot[path] { + list = append(list, module.Version{Path: path, Version: version}) + } + } + gover.ModSort(list[len(uniqueRoots):]) + + return list +} + +// WalkBreadthFirst invokes f once, in breadth-first order, for each module +// version other than "none" that appears in the graph, regardless of whether +// that version is selected. +func (g *Graph) WalkBreadthFirst(f func(m module.Version)) { + var queue []module.Version + enqueued := make(map[module.Version]bool) + for _, m := range g.roots { + if m.Version != "none" { + queue = append(queue, m) + enqueued[m] = true + } + } + + for len(queue) > 0 { + m := queue[0] + queue = queue[1:] + + f(m) + + reqs, _ := g.RequiredBy(m) + for _, r := range reqs { + if !enqueued[r] && r.Version != "none" { + queue = append(queue, r) + enqueued[r] = true + } + } + } +} + +// FindPath reports a shortest requirement path starting at one of the roots of +// the graph and ending at a module version m for which f(m) returns true, or +// nil if no such path exists. +func (g *Graph) FindPath(f func(module.Version) bool) []module.Version { + // firstRequires[a] = b means that in a breadth-first traversal of the + // requirement graph, the module version a was first required by b. + firstRequires := make(map[module.Version]module.Version) + + queue := g.roots + for _, m := range g.roots { + firstRequires[m] = module.Version{} + } + + for len(queue) > 0 { + m := queue[0] + queue = queue[1:] + + if f(m) { + // Construct the path reversed (because we're starting from the far + // endpoint), then reverse it. + path := []module.Version{m} + for { + m = firstRequires[m] + if m.Path == "" { + break + } + path = append(path, m) + } + + i, j := 0, len(path)-1 + for i < j { + path[i], path[j] = path[j], path[i] + i++ + j-- + } + + return path + } + + reqs, _ := g.RequiredBy(m) + for _, r := range reqs { + if _, seen := firstRequires[r]; !seen { + queue = append(queue, r) + firstRequires[r] = m + } + } + } + + return nil +} diff --git a/pkg/3rdparty/mvs/mvs.go b/pkg/3rdparty/mvs/mvs.go new file mode 100644 index 00000000..4376c9d1 --- /dev/null +++ b/pkg/3rdparty/mvs/mvs.go @@ -0,0 +1,487 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package mvs implements Minimal Version Selection. +// See https://research.swtch.com/vgo-mvs. +package mvs + +import ( + "fmt" + "reflect" + "sort" + "sync" + + "golang.org/x/mod/module" + "kcl-lang.io/kpm/pkg/3rdparty/par" +) + +// A Reqs is the requirement graph on which Minimal Version Selection (MVS) operates. +// +// The version strings are opaque except for the special version "none" +// (see the documentation for module.Version). In particular, MVS does not +// assume that the version strings are semantic versions; instead, the Max method +// gives access to the comparison operation. +// +// It must be safe to call methods on a Reqs from multiple goroutines simultaneously. +// Because a Reqs may read the underlying graph from the network on demand, +// the MVS algorithms parallelize the traversal to overlap network delays. +type Reqs interface { + // Required returns the module versions explicitly required by m itself. + // The caller must not modify the returned list. + Required(m module.Version) ([]module.Version, error) + + // Max returns the maximum of v1 and v2 (it returns either v1 or v2) + // in the module with path p. + // + // For all versions v, Max(v, "none") must be v, + // and for the target passed as the first argument to MVS functions, + // Max(target, v) must be target. + // + // Note that v1 < v2 can be written Max(v1, v2) != v1 + // and similarly v1 <= v2 can be written Max(v1, v2) == v2. + Max(p, v1, v2 string) string +} + +// An UpgradeReqs is a Reqs that can also identify available upgrades. +type UpgradeReqs interface { + Reqs + + // Upgrade returns the upgraded version of m, + // for use during an UpgradeAll operation. + // If m should be kept as is, Upgrade returns m. + // If m is not yet used in the build, then m.Version will be "none". + // More typically, m.Version will be the version required + // by some other module in the build. + // + // If no module version is available for the given path, + // Upgrade returns a non-nil error. + // TODO(rsc): Upgrade must be able to return errors, + // but should "no latest version" just return m instead? + Upgrade(m module.Version) (module.Version, error) +} + +// A DowngradeReqs is a Reqs that can also identify available downgrades. +type DowngradeReqs interface { + Reqs + + // Previous returns the version of m.Path immediately prior to m.Version, + // or "none" if no such version is known. + Previous(m module.Version) (module.Version, error) +} + +// BuildList returns the build list for the target module. +// +// target is the root vertex of a module requirement graph. For cmd/go, this is +// typically the main module, but note that this algorithm is not intended to +// be Go-specific: module paths and versions are treated as opaque values. +// +// reqs describes the module requirement graph and provides an opaque method +// for comparing versions. +// +// BuildList traverses the graph and returns a list containing the highest +// version for each visited module. The first element of the returned list is +// target itself; reqs.Max requires target.Version to compare higher than all +// other versions, so no other version can be selected. The remaining elements +// of the list are sorted by path. +// +// See https://research.swtch.com/vgo-mvs for details. +func BuildList(targets []module.Version, reqs Reqs) ([]module.Version, error) { + return buildList(targets, reqs, nil) +} + +func buildList(targets []module.Version, reqs Reqs, upgrade func(module.Version) (module.Version, error)) ([]module.Version, error) { + cmp := func(p, v1, v2 string) int { + if reqs.Max(p, v1, v2) != v1 { + return -1 + } + if reqs.Max(p, v2, v1) != v2 { + return 1 + } + return 0 + } + + var ( + mu sync.Mutex + g = NewGraph(cmp, targets) + upgrades = map[module.Version]module.Version{} + errs = map[module.Version]error{} // (non-nil errors only) + ) + + // Explore work graph in parallel in case reqs.Required + // does high-latency network operations. + var work par.Work[module.Version] + for _, target := range targets { + work.Add(target) + } + work.Do(10, func(m module.Version) { + + var required []module.Version + var err error + if m.Version != "none" { + required, err = reqs.Required(m) + } + + u := m + if upgrade != nil { + upgradeTo, upErr := upgrade(m) + if upErr == nil { + u = upgradeTo + } else if err == nil { + err = upErr + } + } + + mu.Lock() + if err != nil { + errs[m] = err + } + if u != m { + upgrades[m] = u + required = append([]module.Version{u}, required...) + } + g.Require(m, required) + mu.Unlock() + + for _, r := range required { + work.Add(r) + } + }) + + // If there was an error, find the shortest path from the target to the + // node where the error occurred so we can report a useful error message. + if len(errs) > 0 { + errPath := g.FindPath(func(m module.Version) bool { + return errs[m] != nil + }) + if len(errPath) == 0 { + panic("internal error: could not reconstruct path to module with error") + } + + err := errs[errPath[len(errPath)-1]] + isUpgrade := func(from, to module.Version) bool { + if u, ok := upgrades[from]; ok { + return u == to + } + return false + } + return nil, NewBuildListError(err, errPath, isUpgrade) + } + + // The final list is the minimum version of each module found in the graph. + list := g.BuildList() + if vs := list[:len(targets)]; !reflect.DeepEqual(vs, targets) { + // target.Version will be "" for modload, the main client of MVS. + // "" denotes the main module, which has no version. However, MVS treats + // version strings as opaque, so "" is not a special value here. + // See golang.org/issue/31491, golang.org/issue/29773. + panic(fmt.Sprintf("mistake: chose versions %+v instead of targets %+v", vs, targets)) + } + return list, nil +} + +// Req returns the minimal requirement list for the target module, +// with the constraint that all module paths listed in base must +// appear in the returned list. +func Req(mainModule module.Version, base []string, reqs Reqs) ([]module.Version, error) { + list, err := BuildList([]module.Version{mainModule}, reqs) + if err != nil { + return nil, err + } + + // Note: Not running in parallel because we assume + // that list came from a previous operation that paged + // in all the requirements, so there's no I/O to overlap now. + + max := map[string]string{} + for _, m := range list { + max[m.Path] = m.Version + } + + // Compute postorder, cache requirements. + var postorder []module.Version + reqCache := map[module.Version][]module.Version{} + reqCache[mainModule] = nil + + var walk func(module.Version) error + walk = func(m module.Version) error { + _, ok := reqCache[m] + if ok { + return nil + } + required, err := reqs.Required(m) + if err != nil { + return err + } + reqCache[m] = required + for _, m1 := range required { + if err := walk(m1); err != nil { + return err + } + } + postorder = append(postorder, m) + return nil + } + for _, m := range list { + if err := walk(m); err != nil { + return nil, err + } + } + + // Walk modules in reverse post-order, only adding those not implied already. + have := map[module.Version]bool{} + walk = func(m module.Version) error { + if have[m] { + return nil + } + have[m] = true + for _, m1 := range reqCache[m] { + walk(m1) + } + return nil + } + // First walk the base modules that must be listed. + var min []module.Version + haveBase := map[string]bool{} + for _, path := range base { + if haveBase[path] { + continue + } + m := module.Version{Path: path, Version: max[path]} + min = append(min, m) + walk(m) + haveBase[path] = true + } + // Now the reverse postorder to bring in anything else. + for i := len(postorder) - 1; i >= 0; i-- { + m := postorder[i] + if max[m.Path] != m.Version { + // Older version. + continue + } + if !have[m] { + min = append(min, m) + walk(m) + } + } + sort.Slice(min, func(i, j int) bool { + return min[i].Path < min[j].Path + }) + return min, nil +} + +// UpgradeAll returns a build list for the target module +// in which every module is upgraded to its latest version. +func UpgradeAll(target module.Version, reqs UpgradeReqs) ([]module.Version, error) { + return buildList([]module.Version{target}, reqs, func(m module.Version) (module.Version, error) { + if m.Path == target.Path { + return target, nil + } + + return reqs.Upgrade(m) + }) +} + +// Upgrade returns a build list for the target module +// in which the given additional modules are upgraded. +func Upgrade(target module.Version, reqs UpgradeReqs, upgrade ...module.Version) ([]module.Version, error) { + list, err := reqs.Required(target) + if err != nil { + return nil, err + } + + pathInList := make(map[string]bool, len(list)) + for _, m := range list { + pathInList[m.Path] = true + } + list = append([]module.Version(nil), list...) + + upgradeTo := make(map[string]string, len(upgrade)) + for _, u := range upgrade { + if !pathInList[u.Path] { + list = append(list, module.Version{Path: u.Path, Version: "none"}) + } + if prev, dup := upgradeTo[u.Path]; dup { + upgradeTo[u.Path] = reqs.Max(u.Path, prev, u.Version) + } else { + upgradeTo[u.Path] = u.Version + } + } + + return buildList([]module.Version{target}, &override{target, list, reqs}, func(m module.Version) (module.Version, error) { + if v, ok := upgradeTo[m.Path]; ok { + return module.Version{Path: m.Path, Version: v}, nil + } + return m, nil + }) +} + +// Downgrade returns a build list for the target module +// in which the given additional modules are downgraded, +// potentially overriding the requirements of the target. +// +// The versions to be downgraded may be unreachable from reqs.Latest and +// reqs.Previous, but the methods of reqs must otherwise handle such versions +// correctly. +func Downgrade(target module.Version, reqs DowngradeReqs, downgrade ...module.Version) ([]module.Version, error) { + // Per https://research.swtch.com/vgo-mvs#algorithm_4: + // “To avoid an unnecessary downgrade to E 1.1, we must also add a new + // requirement on E 1.2. We can apply Algorithm R to find the minimal set of + // new requirements to write to go.mod.” + // + // In order to generate those new requirements, we need to identify versions + // for every module in the build list — not just reqs.Required(target). + list, err := BuildList([]module.Version{target}, reqs) + if err != nil { + return nil, err + } + list = list[1:] // remove target + + max := make(map[string]string) + for _, r := range list { + max[r.Path] = r.Version + } + for _, d := range downgrade { + if v, ok := max[d.Path]; !ok || reqs.Max(d.Path, v, d.Version) != d.Version { + max[d.Path] = d.Version + } + } + + var ( + added = make(map[module.Version]bool) + rdeps = make(map[module.Version][]module.Version) + excluded = make(map[module.Version]bool) + ) + var exclude func(module.Version) + exclude = func(m module.Version) { + if excluded[m] { + return + } + excluded[m] = true + for _, p := range rdeps[m] { + exclude(p) + } + } + var add func(module.Version) + add = func(m module.Version) { + if added[m] { + return + } + added[m] = true + if v, ok := max[m.Path]; ok && reqs.Max(m.Path, m.Version, v) != v { + // m would upgrade an existing dependency — it is not a strict downgrade, + // and because it was already present as a dependency, it could affect the + // behavior of other relevant packages. + exclude(m) + return + } + list, err := reqs.Required(m) + if err != nil { + // If we can't load the requirements, we couldn't load the go.mod file. + // There are a number of reasons this can happen, but this usually + // means an older version of the module had a missing or invalid + // go.mod file. For example, if example.com/mod released v2.0.0 before + // migrating to modules (v2.0.0+incompatible), then added a valid go.mod + // in v2.0.1, downgrading from v2.0.1 would cause this error. + // + // TODO(golang.org/issue/31730, golang.org/issue/30134): if the error + // is transient (we couldn't download go.mod), return the error from + // Downgrade. Currently, we can't tell what kind of error it is. + exclude(m) + return + } + for _, r := range list { + add(r) + if excluded[r] { + exclude(m) + return + } + rdeps[r] = append(rdeps[r], m) + } + } + + downgraded := make([]module.Version, 0, len(list)+1) + downgraded = append(downgraded, target) +List: + for _, r := range list { + add(r) + for excluded[r] { + p, err := reqs.Previous(r) + if err != nil { + // This is likely a transient error reaching the repository, + // rather than a permanent error with the retrieved version. + // + // TODO(golang.org/issue/31730, golang.org/issue/30134): + // decode what to do based on the actual error. + return nil, err + } + // If the target version is a pseudo-version, it may not be + // included when iterating over prior versions using reqs.Previous. + // Insert it into the right place in the iteration. + // If v is excluded, p should be returned again by reqs.Previous on the next iteration. + if v := max[r.Path]; reqs.Max(r.Path, v, r.Version) != v && reqs.Max(r.Path, p.Version, v) != p.Version { + p.Version = v + } + if p.Version == "none" { + continue List + } + add(p) + r = p + } + downgraded = append(downgraded, r) + } + + // The downgrades we computed above only downgrade to versions enumerated by + // reqs.Previous. However, reqs.Previous omits some versions — such as + // pseudo-versions and retracted versions — that may be selected as transitive + // requirements of other modules. + // + // If one of those requirements pulls the version back up above the version + // identified by reqs.Previous, then the transitive dependencies of that that + // initially-downgraded version should no longer matter — in particular, we + // should not add new dependencies on module paths that nothing else in the + // updated module graph even requires. + // + // In order to eliminate those spurious dependencies, we recompute the build + // list with the actual versions of the downgraded modules as selected by MVS, + // instead of our initial downgrades. + // (See the downhiddenartifact and downhiddencross test cases). + actual, err := BuildList([]module.Version{target}, &override{ + target: target, + list: downgraded, + Reqs: reqs, + }) + if err != nil { + return nil, err + } + actualVersion := make(map[string]string, len(actual)) + for _, m := range actual { + actualVersion[m.Path] = m.Version + } + + downgraded = downgraded[:0] + for _, m := range list { + if v, ok := actualVersion[m.Path]; ok { + downgraded = append(downgraded, module.Version{Path: m.Path, Version: v}) + } + } + + return BuildList([]module.Version{target}, &override{ + target: target, + list: downgraded, + Reqs: reqs, + }) +} + +type override struct { + target module.Version + list []module.Version + Reqs +} + +func (r *override) Required(m module.Version) ([]module.Version, error) { + if m == r.target { + return r.list, nil + } + return r.Reqs.Required(m) +} diff --git a/pkg/3rdparty/mvs/mvs_test.go b/pkg/3rdparty/mvs/mvs_test.go new file mode 100644 index 00000000..6e1e71cd --- /dev/null +++ b/pkg/3rdparty/mvs/mvs_test.go @@ -0,0 +1,635 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mvs + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "golang.org/x/mod/module" +) + +var tests = ` +# Scenario from blog. +name: blog +A: B1 C2 +B1: D3 +C1: D2 +C2: D4 +C3: D5 +C4: G1 +D2: E1 +D3: E2 +D4: E2 F1 +D5: E2 +G1: C4 +A2: B1 C4 D4 +build A: A B1 C2 D4 E2 F1 +upgrade* A: A B1 C4 D5 E2 F1 G1 +upgrade A C4: A B1 C4 D4 E2 F1 G1 +build A2: A2 B1 C4 D4 E2 F1 G1 +downgrade A2 D2: A2 C4 D2 E2 F1 G1 + +name: trim +A: B1 C2 +B1: D3 +C2: B2 +B2: +build A: A B2 C2 D3 + +# Cross-dependency between D and E. +# No matter how it arises, should get result of merging all build lists via max, +# which leads to including both D2 and E2. + +name: cross1 +A: B C +B: D1 +C: D2 +D1: E2 +D2: E1 +build A: A B C D2 E2 + +name: cross1V +A: B2 C D2 E1 +B1: +B2: D1 +C: D2 +D1: E2 +D2: E1 +build A: A B2 C D2 E2 + +name: cross1U +A: B1 C +B1: +B2: D1 +C: D2 +D1: E2 +D2: E1 +build A: A B1 C D2 E1 +upgrade A B2: A B2 C D2 E2 + +name: cross1R +A: B C +B: D2 +C: D1 +D1: E2 +D2: E1 +build A: A B C D2 E2 + +name: cross1X +A: B C +B: D1 E2 +C: D2 +D1: E2 +D2: E1 +build A: A B C D2 E2 + +name: cross2 +A: B D2 +B: D1 +D1: E2 +D2: E1 +build A: A B D2 E2 + +name: cross2X +A: B D2 +B: D1 E2 +C: D2 +D1: E2 +D2: E1 +build A: A B D2 E2 + +name: cross3 +A: B D2 E1 +B: D1 +D1: E2 +D2: E1 +build A: A B D2 E2 + +name: cross3X +A: B D2 E1 +B: D1 E2 +D1: E2 +D2: E1 +build A: A B D2 E2 + +# Should not get E2 here, because B has been updated +# not to depend on D1 anymore. +name: cross4 +A1: B1 D2 +A2: B2 D2 +B1: D1 +B2: D2 +D1: E2 +D2: E1 +build A1: A1 B1 D2 E2 +build A2: A2 B2 D2 E1 + +# But the upgrade from A1 preserves the E2 dep explicitly. +upgrade A1 B2: A1 B2 D2 E2 +upgradereq A1 B2: B2 E2 + +name: cross5 +A: D1 +D1: E2 +D2: E1 +build A: A D1 E2 +upgrade* A: A D2 E2 +upgrade A D2: A D2 E2 +upgradereq A D2: D2 E2 + +name: cross6 +A: D2 +D1: E2 +D2: E1 +build A: A D2 E1 +upgrade* A: A D2 E2 +upgrade A E2: A D2 E2 + +name: cross7 +A: B C +B: D1 +C: E1 +D1: E2 +E1: D2 +build A: A B C D2 E2 + +# golang.org/issue/31248: +# Even though we select X2, the requirement on I1 +# via X1 should be preserved. +name: cross8 +M: A1 B1 +A1: X1 +B1: X2 +X1: I1 +X2: +build M: M A1 B1 I1 X2 + +# Upgrade from B1 to B2 should not drop the transitive dep on D. +name: drop +A: B1 C1 +B1: D1 +B2: +C2: +D2: +build A: A B1 C1 D1 +upgrade* A: A B2 C2 D2 + +name: simplify +A: B1 C1 +B1: C2 +C1: D1 +C2: +build A: A B1 C2 D1 + +name: up1 +A: B1 C1 +B1: +B2: +B3: +B4: +B5.hidden: +C2: +C3: +build A: A B1 C1 +upgrade* A: A B4 C3 + +name: up2 +A: B5.hidden C1 +B1: +B2: +B3: +B4: +B5.hidden: +C2: +C3: +build A: A B5.hidden C1 +upgrade* A: A B5.hidden C3 + +name: down1 +A: B2 +B1: C1 +B2: C2 +build A: A B2 C2 +downgrade A C1: A B1 C1 + +name: down2 +A: B2 E2 +B1: +B2: C2 F2 +C1: +D1: +C2: D2 E2 +D2: B2 +E2: D2 +E1: +F1: +build A: A B2 C2 D2 E2 F2 +downgrade A F1: A B1 C1 D1 E1 F1 + +# https://research.swtch.com/vgo-mvs#algorithm_4: +# “[D]owngrades are constrained to only downgrade packages, not also upgrade +# them; if an upgrade before downgrade is needed, the user must ask for it +# explicitly.” +# +# Here, downgrading B2 to B1 upgrades C1 to C2, and C2 does not depend on D2. +# However, C2 would be an upgrade — not a downgrade — so B1 must also be +# rejected. +name: downcross1 +A: B2 C1 +B1: C2 +B2: C1 +C1: D2 +C2: +D1: +D2: +build A: A B2 C1 D2 +downgrade A D1: A D1 + +# https://research.swtch.com/vgo-mvs#algorithm_4: +# “Unlike upgrades, downgrades must work by removing requirements, not adding +# them.” +# +# However, downgrading a requirement may introduce a new requirement on a +# previously-unrequired module. If each dependency's requirements are complete +# (“tidy”), that can't change the behavior of any other package whose version is +# not also being downgraded, so we should allow it. +name: downcross2 +A: B2 +B1: C1 +B2: D2 +C1: +D1: +D2: +build A: A B2 D2 +downgrade A D1: A B1 C1 D1 + +name: downcycle +A: A B2 +B2: A +B1: +build A: A B2 +downgrade A B1: A B1 + +# Both B3 and C2 require D2. +# If we downgrade D to D1, then in isolation B3 would downgrade to B1, +# because B2 is hidden — B1 is the next-highest version that is not hidden. +# However, if we downgrade D, we will also downgrade C to C1. +# And C1 requires B2.hidden, and B2.hidden also meets our requirements: +# it is compatible with D1 and a strict downgrade from B3. +# +# Since neither the initial nor the final build list includes B1, +# and the nothing in the final downgraded build list requires E at all, +# no dependency on E1 (required by only B1) should be introduced. +# +name: downhiddenartifact +A: B3 C2 +A1: B3 +B1: E1 +B2.hidden: +B3: D2 +C1: B2.hidden +C2: D2 +D1: +D2: +build A1: A1 B3 D2 +downgrade A1 D1: A1 B1 D1 E1 +build A: A B3 C2 D2 +downgrade A D1: A B2.hidden C1 D1 + +# Both B3 and C3 require D2. +# If we downgrade D to D1, then in isolation B3 would downgrade to B1, +# and C3 would downgrade to C1. +# But C1 requires B2.hidden, and B1 requires C2.hidden, so we can't +# downgrade to either of those without pulling the other back up a little. +# +# B2.hidden and C2.hidden are both compatible with D1, so that still +# meets our requirements — but then we're in an odd state in which +# B and C have both been downgraded to hidden versions, without any +# remaining requirements to explain how those hidden versions got there. +# +# TODO(bcmills): Would it be better to force downgrades to land on non-hidden +# versions? +# In this case, that would remove the dependencies on B and C entirely. +# +name: downhiddencross +A: B3 C3 +B1: C2.hidden +B2.hidden: +B3: D2 +C1: B2.hidden +C2.hidden: +C3: D2 +D1: +D2: +build A: A B3 C3 D2 +downgrade A D1: A B2.hidden C2.hidden D1 + +# golang.org/issue/25542. +name: noprev1 +A: B4 C2 +B2.hidden: +C2: +build A: A B4 C2 +downgrade A B2.hidden: A B2.hidden C2 + +name: noprev2 +A: B4 C2 +B2.hidden: +B1: +C2: +build A: A B4 C2 +downgrade A B2.hidden: A B2.hidden C2 + +name: noprev3 +A: B4 C2 +B3: +B2.hidden: +C2: +build A: A B4 C2 +downgrade A B2.hidden: A B2.hidden C2 + +# Cycles involving the target. + +# The target must be the newest version of itself. +name: cycle1 +A: B1 +B1: A1 +B2: A2 +B3: A3 +build A: A B1 +upgrade A B2: A B2 +upgrade* A: A B3 + +# golang.org/issue/29773: +# Requirements of older versions of the target +# must be carried over. +name: cycle2 +A: B1 +A1: C1 +A2: D1 +B1: A1 +B2: A2 +C1: A2 +C2: +D2: +build A: A B1 C1 D1 +upgrade* A: A B2 C2 D2 + +# Cycles with multiple possible solutions. +# (golang.org/issue/34086) +name: cycle3 +M: A1 C2 +A1: B1 +B1: C1 +B2: C2 +C1: +C2: B2 +build M: M A1 B2 C2 +req M: A1 B2 +req M A: A1 B2 +req M C: A1 C2 + +# Requirement minimization. + +name: req1 +A: B1 C1 D1 E1 F1 +B1: C1 E1 F1 +req A: B1 D1 +req A C: B1 C1 D1 + +name: req2 +A: G1 H1 +G1: H1 +H1: G1 +req A: G1 +req A G: G1 +req A H: H1 + +name: req3 +M: A1 B1 +A1: X1 +B1: X2 +X1: I1 +X2: +req M: A1 B1 + +name: reqnone +M: Anone B1 D1 E1 +B1: Cnone D1 +E1: Fnone +build M: M B1 D1 E1 +req M: B1 E1 + +name: reqdup +M: A1 B1 +A1: B1 +B1: +req M A A: A1 + +name: reqcross +M: A1 B1 C1 +A1: B1 C1 +B1: C1 +C1: +req M A B: A1 B1 +` + +func Test(t *testing.T) { + var ( + name string + reqs reqsMap + fns []func(*testing.T) + ) + flush := func() { + if name != "" { + t.Run(name, func(t *testing.T) { + for _, fn := range fns { + fn(t) + } + if len(fns) == 0 { + t.Errorf("no functions tested") + } + }) + } + } + m := func(s string) module.Version { + return module.Version{Path: s[:1], Version: s[1:]} + } + ms := func(list []string) []module.Version { + var mlist []module.Version + for _, s := range list { + mlist = append(mlist, m(s)) + } + return mlist + } + checkList := func(t *testing.T, desc string, list []module.Version, err error, val string) { + if err != nil { + t.Fatalf("%s: %v", desc, err) + } + vs := ms(strings.Fields(val)) + if !reflect.DeepEqual(list, vs) { + t.Errorf("%s = %v, want %v", desc, list, vs) + } + } + + for _, line := range strings.Split(tests, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "#") || line == "" { + continue + } + i := strings.Index(line, ":") + if i < 0 { + t.Fatalf("missing colon: %q", line) + } + key := strings.TrimSpace(line[:i]) + val := strings.TrimSpace(line[i+1:]) + if key == "" { + t.Fatalf("missing key: %q", line) + } + kf := strings.Fields(key) + switch kf[0] { + case "name": + if len(kf) != 1 { + t.Fatalf("name takes no arguments: %q", line) + } + flush() + reqs = make(reqsMap) + fns = nil + name = val + continue + case "build": + if len(kf) != 2 { + t.Fatalf("build takes one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := BuildList([]module.Version{m(kf[1])}, reqs) + checkList(t, key, list, err, val) + }) + continue + case "upgrade*": + if len(kf) != 2 { + t.Fatalf("upgrade* takes one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := UpgradeAll(m(kf[1]), reqs) + checkList(t, key, list, err, val) + }) + continue + case "upgradereq": + if len(kf) < 2 { + t.Fatalf("upgrade takes at least one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := Upgrade(m(kf[1]), reqs, ms(kf[2:])...) + if err == nil { + // Copy the reqs map, but substitute the upgraded requirements in + // place of the target's original requirements. + upReqs := make(reqsMap, len(reqs)) + for m, r := range reqs { + upReqs[m] = r + } + upReqs[m(kf[1])] = list + + list, err = Req(m(kf[1]), nil, upReqs) + } + checkList(t, key, list, err, val) + }) + continue + case "upgrade": + if len(kf) < 2 { + t.Fatalf("upgrade takes at least one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := Upgrade(m(kf[1]), reqs, ms(kf[2:])...) + checkList(t, key, list, err, val) + }) + continue + case "downgrade": + if len(kf) < 2 { + t.Fatalf("downgrade takes at least one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := Downgrade(m(kf[1]), reqs, ms(kf[1:])...) + checkList(t, key, list, err, val) + }) + continue + case "req": + if len(kf) < 2 { + t.Fatalf("req takes at least one argument: %q", line) + } + fns = append(fns, func(t *testing.T) { + list, err := Req(m(kf[1]), kf[2:], reqs) + checkList(t, key, list, err, val) + }) + continue + } + if len(kf) == 1 && 'A' <= key[0] && key[0] <= 'Z' { + var rs []module.Version + for _, f := range strings.Fields(val) { + r := m(f) + if reqs[r] == nil { + reqs[r] = []module.Version{} + } + rs = append(rs, r) + } + reqs[m(key)] = rs + continue + } + t.Fatalf("bad line: %q", line) + } + flush() +} + +type reqsMap map[module.Version][]module.Version + +func (r reqsMap) Max(_, v1, v2 string) string { + if v1 == "none" || v2 == "" { + return v2 + } + if v2 == "none" || v1 == "" { + return v1 + } + if v1 < v2 { + return v2 + } + return v1 +} + +func (r reqsMap) Upgrade(m module.Version) (module.Version, error) { + u := module.Version{Version: "none"} + for k := range r { + if k.Path == m.Path && r.Max(k.Path, u.Version, k.Version) == k.Version && !strings.HasSuffix(k.Version, ".hidden") { + u = k + } + } + if u.Path == "" { + return module.Version{}, fmt.Errorf("missing module: %v", module.Version{Path: m.Path}) + } + return u, nil +} + +func (r reqsMap) Previous(m module.Version) (module.Version, error) { + var p module.Version + for k := range r { + if k.Path == m.Path && p.Version < k.Version && k.Version < m.Version && !strings.HasSuffix(k.Version, ".hidden") { + p = k + } + } + if p.Path == "" { + return module.Version{Path: m.Path, Version: "none"}, nil + } + return p, nil +} + +func (r reqsMap) Required(m module.Version) ([]module.Version, error) { + rr, ok := r[m] + if !ok { + return nil, fmt.Errorf("missing module: %v", m) + } + return rr, nil +} diff --git a/pkg/3rdparty/par/queue.go b/pkg/3rdparty/par/queue.go new file mode 100644 index 00000000..180bc75e --- /dev/null +++ b/pkg/3rdparty/par/queue.go @@ -0,0 +1,88 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package par + +import "fmt" + +// Queue manages a set of work items to be executed in parallel. The number of +// active work items is limited, and excess items are queued sequentially. +type Queue struct { + maxActive int + st chan queueState +} + +type queueState struct { + active int // number of goroutines processing work; always nonzero when len(backlog) > 0 + backlog []func() + idle chan struct{} // if non-nil, closed when active becomes 0 +} + +// NewQueue returns a Queue that executes up to maxActive items in parallel. +// +// maxActive must be positive. +func NewQueue(maxActive int) *Queue { + if maxActive < 1 { + panic(fmt.Sprintf("par.NewQueue called with nonpositive limit (%d)", maxActive)) + } + + q := &Queue{ + maxActive: maxActive, + st: make(chan queueState, 1), + } + q.st <- queueState{} + return q +} + +// Add adds f as a work item in the queue. +// +// Add returns immediately, but the queue will be marked as non-idle until after +// f (and any subsequently-added work) has completed. +func (q *Queue) Add(f func()) { + st := <-q.st + if st.active == q.maxActive { + st.backlog = append(st.backlog, f) + q.st <- st + return + } + if st.active == 0 { + // Mark q as non-idle. + st.idle = nil + } + st.active++ + q.st <- st + + go func() { + for { + f() + + st := <-q.st + if len(st.backlog) == 0 { + if st.active--; st.active == 0 && st.idle != nil { + close(st.idle) + } + q.st <- st + return + } + f, st.backlog = st.backlog[0], st.backlog[1:] + q.st <- st + } + }() +} + +// Idle returns a channel that will be closed when q has no (active or enqueued) +// work outstanding. +func (q *Queue) Idle() <-chan struct{} { + st := <-q.st + defer func() { q.st <- st }() + + if st.idle == nil { + st.idle = make(chan struct{}) + if st.active == 0 { + close(st.idle) + } + } + + return st.idle +} diff --git a/pkg/3rdparty/par/work.go b/pkg/3rdparty/par/work.go new file mode 100644 index 00000000..5b6de942 --- /dev/null +++ b/pkg/3rdparty/par/work.go @@ -0,0 +1,223 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package par implements parallel execution helpers. +package par + +import ( + "errors" + "math/rand" + "sync" + "sync/atomic" +) + +// Work manages a set of work items to be executed in parallel, at most once each. +// The items in the set must all be valid map keys. +type Work[T comparable] struct { + f func(T) // function to run for each item + running int // total number of runners + + mu sync.Mutex + added map[T]bool // items added to set + todo []T // items yet to be run + wait sync.Cond // wait when todo is empty + waiting int // number of runners waiting for todo +} + +func (w *Work[T]) init() { + if w.added == nil { + w.added = make(map[T]bool) + } +} + +// Add adds item to the work set, if it hasn't already been added. +func (w *Work[T]) Add(item T) { + w.mu.Lock() + w.init() + if !w.added[item] { + w.added[item] = true + w.todo = append(w.todo, item) + if w.waiting > 0 { + w.wait.Signal() + } + } + w.mu.Unlock() +} + +// Do runs f in parallel on items from the work set, +// with at most n invocations of f running at a time. +// It returns when everything added to the work set has been processed. +// At least one item should have been added to the work set +// before calling Do (or else Do returns immediately), +// but it is allowed for f(item) to add new items to the set. +// Do should only be used once on a given Work. +func (w *Work[T]) Do(n int, f func(item T)) { + if n < 1 { + panic("par.Work.Do: n < 1") + } + if w.running >= 1 { + panic("par.Work.Do: already called Do") + } + + w.running = n + w.f = f + w.wait.L = &w.mu + + for i := 0; i < n-1; i++ { + go w.runner() + } + w.runner() +} + +// runner executes work in w until both nothing is left to do +// and all the runners are waiting for work. +// (Then all the runners return.) +func (w *Work[T]) runner() { + for { + // Wait for something to do. + w.mu.Lock() + for len(w.todo) == 0 { + w.waiting++ + if w.waiting == w.running { + // All done. + w.wait.Broadcast() + w.mu.Unlock() + return + } + w.wait.Wait() + w.waiting-- + } + + // Pick something to do at random, + // to eliminate pathological contention + // in case items added at about the same time + // are most likely to contend. + i := rand.Intn(len(w.todo)) + item := w.todo[i] + w.todo[i] = w.todo[len(w.todo)-1] + w.todo = w.todo[:len(w.todo)-1] + w.mu.Unlock() + + w.f(item) + } +} + +// ErrCache is like Cache except that it also stores +// an error value alongside the cached value V. +type ErrCache[K comparable, V any] struct { + Cache[K, errValue[V]] +} + +type errValue[V any] struct { + v V + err error +} + +func (c *ErrCache[K, V]) Do(key K, f func() (V, error)) (V, error) { + v := c.Cache.Do(key, func() errValue[V] { + v, err := f() + return errValue[V]{v, err} + }) + return v.v, v.err +} + +var ErrCacheEntryNotFound = errors.New("cache entry not found") + +// Get returns the cached result associated with key. +// It returns ErrCacheEntryNotFound if there is no such result. +func (c *ErrCache[K, V]) Get(key K) (V, error) { + v, ok := c.Cache.Get(key) + if !ok { + v.err = ErrCacheEntryNotFound + } + return v.v, v.err +} + +// Cache runs an action once per key and caches the result. +type Cache[K comparable, V any] struct { + m sync.Map +} + +type cacheEntry[V any] struct { + done atomic.Bool + mu sync.Mutex + result V +} + +// Do calls the function f if and only if Do is being called for the first time with this key. +// No call to Do with a given key returns until the one call to f returns. +// Do returns the value returned by the one call to f. +func (c *Cache[K, V]) Do(key K, f func() V) V { + entryIface, ok := c.m.Load(key) + if !ok { + entryIface, _ = c.m.LoadOrStore(key, new(cacheEntry[V])) + } + e := entryIface.(*cacheEntry[V]) + if !e.done.Load() { + e.mu.Lock() + if !e.done.Load() { + e.result = f() + e.done.Store(true) + } + e.mu.Unlock() + } + return e.result +} + +// Get returns the cached result associated with key +// and reports whether there is such a result. +// +// If the result for key is being computed, Get does not wait for the computation to finish. +func (c *Cache[K, V]) Get(key K) (V, bool) { + entryIface, ok := c.m.Load(key) + if !ok { + return *new(V), false + } + e := entryIface.(*cacheEntry[V]) + if !e.done.Load() { + return *new(V), false + } + return e.result, true +} + +// Clear removes all entries in the cache. +// +// Concurrent calls to Get may return old values. Concurrent calls to Do +// may return old values or store results in entries that have been deleted. +// +// TODO(jayconrod): Delete this after the package cache clearing functions +// in internal/load have been removed. +func (c *Cache[K, V]) Clear() { + c.m.Range(func(key, value any) bool { + c.m.Delete(key) + return true + }) +} + +// Delete removes an entry from the map. It is safe to call Delete for an +// entry that does not exist. Delete will return quickly, even if the result +// for a key is still being computed; the computation will finish, but the +// result won't be accessible through the cache. +// +// TODO(jayconrod): Delete this after the package cache clearing functions +// in internal/load have been removed. +func (c *Cache[K, V]) Delete(key K) { + c.m.Delete(key) +} + +// DeleteIf calls pred for each key in the map. If pred returns true for a key, +// DeleteIf removes the corresponding entry. If the result for a key is +// still being computed, DeleteIf will remove the entry without waiting for +// the computation to finish. The result won't be accessible through the cache. +// +// TODO(jayconrod): Delete this after the package cache clearing functions +// in internal/load have been removed. +func (c *Cache[K, V]) DeleteIf(pred func(key K) bool) { + c.m.Range(func(key, _ any) bool { + if key := key.(K); pred(key) { + c.Delete(key) + } + return true + }) +} diff --git a/pkg/client/client.go b/pkg/client/client.go index a2a17a56..0732784e 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -13,6 +13,7 @@ import ( "github.com/dominikbraun/graph" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/otiai10/copy" + "golang.org/x/mod/module" "kcl-lang.io/kcl-go/pkg/kcl" "oras.land/oras-go/v2" @@ -840,12 +841,12 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (* // return nil, err // } dep.FullName = dep.GenDepFullName() - // If the dependency is from git commit, the version is the commit id. - // If the dependency is from git tag, the version is the tag. - dep.Version, err = dep.Source.Git.GetValidGitReference() + + modFile, err := c.LoadModFile(localPath) if err != nil { return nil, err } + dep.Version = modFile.Pkg.Version } if dep.Source.Oci != nil { @@ -1277,18 +1278,21 @@ func (c *KpmClient) ParseOciOptionFromString(oci string, tag string) (*opt.OciOp } // InitGraphAndDownloadDeps initializes a dependency graph and call downloadDeps function. -func (c *KpmClient) InitGraphAndDownloadDeps(kclPkg *pkg.KclPkg) (*pkg.Dependencies, graph.Graph[string, string], error) { +func (c *KpmClient) InitGraphAndDownloadDeps(kclPkg *pkg.KclPkg) (*pkg.Dependencies, graph.Graph[module.Version, module.Version], error) { - depGraph := graph.New(graph.StringHash, graph.Directed(), graph.PreventCycles()) + moduleHash := func(m module.Version) module.Version { + return m + } + depGraph := graph.New(moduleHash, graph.Directed(), graph.PreventCycles()) // add the root vertex(package name) to the dependency graph. - root := fmt.Sprintf("%s@%s", kclPkg.GetPkgName(), kclPkg.GetPkgVersion()) + root := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} err := depGraph.AddVertex(root) if err != nil { return nil, nil, err } - changedDeps, err := c.downloadDeps(&kclPkg.ModFile.Dependencies, &kclPkg.Dependencies, depGraph, kclPkg.HomePath, root) + changedDeps, err := c.DownloadDeps(&kclPkg.ModFile.Dependencies, &kclPkg.Dependencies, depGraph, kclPkg.HomePath, root) if err != nil { return nil, nil, err } @@ -1320,7 +1324,7 @@ func (c *KpmClient) dependencyExists(dep *pkg.Dependency, lockDeps *pkg.Dependen } // downloadDeps will download all the dependencies of the current kcl package. -func (c *KpmClient) downloadDeps(deps *pkg.Dependencies, lockDeps *pkg.Dependencies, depGraph graph.Graph[string, string], pkghome, parent string) (*pkg.Dependencies, error) { +func (c *KpmClient) DownloadDeps(deps *pkg.Dependencies, lockDeps *pkg.Dependencies, depGraph graph.Graph[module.Version, module.Version], pkghome string, parent module.Version) (*pkg.Dependencies, error) { newDeps := pkg.Dependencies{ Deps: make(map[string]pkg.Dependency), @@ -1397,27 +1401,29 @@ func (c *KpmClient) downloadDeps(deps *pkg.Dependencies, lockDeps *pkg.Dependenc return nil, err } - source := fmt.Sprintf("%s@%s", d.Name, d.Version) - source = strings.TrimRight(source, "@") - err = depGraph.AddVertex(source) + source := module.Version{Path: d.Name, Version: d.Version} + + err = depGraph.AddVertex(source, graph.VertexAttribute(d.GetSourceType(), d.GetDownloadPath())) if err != nil && err != graph.ErrVertexAlreadyExists { return nil, err } - err = depGraph.AddEdge(parent, source) - if err != nil { - if err == graph.ErrEdgeCreatesCycle { - return nil, reporter.NewErrorEvent( - reporter.CircularDependencyExist, - nil, - fmt.Sprintf("adding %s as a dependency results in a cycle", source), - ) + if parent != (module.Version{}) { + err = depGraph.AddEdge(parent, source) + if err != nil { + if err == graph.ErrEdgeCreatesCycle { + return nil, reporter.NewErrorEvent( + reporter.CircularDependencyExist, + nil, + fmt.Sprintf("adding %s as a dependency results in a cycle", source), + ) + } + return nil, err } - return nil, err } // Download the indirect dependencies. - nested, err := c.downloadDeps(&deppkg.ModFile.Dependencies, lockDeps, depGraph, deppkg.HomePath, source) + nested, err := c.DownloadDeps(&deppkg.ModFile.Dependencies, lockDeps, depGraph, deppkg.HomePath, source) if err != nil { return nil, err } diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 17adb1f1..79ee1665 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -16,6 +16,7 @@ import ( "github.com/dominikbraun/graph" "github.com/otiai10/copy" "github.com/stretchr/testify/assert" + "golang.org/x/mod/module" "gopkg.in/yaml.v3" "kcl-lang.io/kcl-go/pkg/kcl" "kcl-lang.io/kpm/pkg/downloader" @@ -149,28 +150,32 @@ func TestDependencyGraph(t *testing.T) { adjMap, err := depGraph.AdjacencyMap() assert.Equal(t, err, nil) + m := func(Path, Version string) module.Version { + return module.Version{Path: Path, Version: Version} + } + edgeProp := graph.EdgeProperties{ Attributes: map[string]string{}, Weight: 0, Data: nil, } assert.Equal(t, adjMap, - map[string]map[string]graph.Edge[string]{ - "dependency_graph@0.0.1": { - "teleport@0.1.0": {Source: "dependency_graph@0.0.1", Target: "teleport@0.1.0", Properties: edgeProp}, - "rabbitmq@0.0.1": {Source: "dependency_graph@0.0.1", Target: "rabbitmq@0.0.1", Properties: edgeProp}, - "agent@0.1.0": {Source: "dependency_graph@0.0.1", Target: "agent@0.1.0", Properties: edgeProp}, + map[module.Version]map[module.Version]graph.Edge[module.Version]{ + m("dependency_graph", "0.0.1"): { + m("teleport", "0.1.0"): {Source: m("dependency_graph", "0.0.1"), Target: m("teleport", "0.1.0"), Properties: edgeProp}, + m("rabbitmq", "0.0.1"): {Source: m("dependency_graph", "0.0.1"), Target: m("rabbitmq", "0.0.1"), Properties: edgeProp}, + m("agent", "0.1.0"): {Source: m("dependency_graph", "0.0.1"), Target: m("agent", "0.1.0"), Properties: edgeProp}, }, - "teleport@0.1.0": { - "k8s@1.28": {Source: "teleport@0.1.0", Target: "k8s@1.28", Properties: edgeProp}, + m("teleport", "0.1.0"): { + m("k8s", "1.28"): {Source: m("teleport", "0.1.0"), Target: m("k8s", "1.28"), Properties: edgeProp}, }, - "rabbitmq@0.0.1": { - "k8s@1.28": {Source: "rabbitmq@0.0.1", Target: "k8s@1.28", Properties: edgeProp}, + m("rabbitmq", "0.0.1"): { + m("k8s", "1.28"): {Source: m("rabbitmq", "0.0.1"), Target: m("k8s", "1.28"), Properties: edgeProp}, }, - "agent@0.1.0": { - "k8s@1.28": {Source: "agent@0.1.0", Target: "k8s@1.28", Properties: edgeProp}, + m("agent", "0.1.0"): { + m("k8s", "1.28"): {Source: m("agent", "0.1.0"), Target: m("k8s", "1.28"), Properties: edgeProp}, }, - "k8s@1.28": {}, + m("k8s", "1.28"): {}, }, ) } @@ -1464,3 +1469,4 @@ func testRunWithOciDownloader(t *testing.T) { assert.Equal(t, buf.String(), "downloading 'zong-zhe/helloworld:0.0.3' from 'ghcr.io/zong-zhe/helloworld:0.0.3'\n") assert.Equal(t, res.GetRawYamlResult(), "The_first_kcl_program: Hello World!") } + diff --git a/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect b/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect index ccc62bba..dbb61a69 100644 --- a/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect +++ b/pkg/client/test_data/add_with_git_commit/test_pkg/kcl.mod.lock.expect @@ -2,7 +2,7 @@ [dependencies.catalog] name = "catalog" full_name = "catalog_a29e3db" - version = "a29e3db" + version = "0.1.0" sum = "kFmlrYJbJUFFTEXjC9cquc80WB+UpZ/6oMPKrfgyeks=" url = "https://github.com/KusionStack/catalog.git" - commit = "a29e3db" + commit = "a29e3db" \ No newline at end of file diff --git a/pkg/client/test_data/add_with_git_commit/test_pkg_win/kcl.mod.lock.expect b/pkg/client/test_data/add_with_git_commit/test_pkg_win/kcl.mod.lock.expect index 67893893..cda1fc63 100644 --- a/pkg/client/test_data/add_with_git_commit/test_pkg_win/kcl.mod.lock.expect +++ b/pkg/client/test_data/add_with_git_commit/test_pkg_win/kcl.mod.lock.expect @@ -2,7 +2,7 @@ [dependencies.catalog] name = "catalog" full_name = "catalog_a29e3db" - version = "a29e3db" + version = "0.1.0" sum = "zhh1yHk5TrNi9apHUQF3hPOlwi5Kc75cNHjcVmGv+Qo=" url = "https://github.com/KusionStack/catalog.git" - commit = "a29e3db" + commit = "a29e3db" \ No newline at end of file diff --git a/pkg/cmd/cmd_graph.go b/pkg/cmd/cmd_graph.go index 7fbc0b5c..8bb4409a 100644 --- a/pkg/cmd/cmd_graph.go +++ b/pkg/cmd/cmd_graph.go @@ -8,6 +8,7 @@ import ( "github.com/dominikbraun/graph" "github.com/urfave/cli/v2" + "golang.org/x/mod/module" "kcl-lang.io/kpm/pkg/client" "kcl-lang.io/kpm/pkg/env" pkg "kcl-lang.io/kpm/pkg/package" @@ -72,12 +73,20 @@ func KpmGraph(c *cli.Context, kpmcli *client.KpmClient) error { return err } + format := func(m module.Version) string { + formattedMsg := m.Path + if m.Version != "" { + formattedMsg += "@" + m.Version + } + return formattedMsg + } + // print the dependency graph to stdout. - root := fmt.Sprintf("%s@%s", kclPkg.GetPkgName(), kclPkg.GetPkgVersion()) - err = graph.BFS(depGraph, root, func(source string) bool { + root := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} + err = graph.BFS(depGraph, root, func(source module.Version) bool { for target := range adjMap[source] { reporter.ReportMsgTo( - fmt.Sprint(source, " ", target), + fmt.Sprint(format(source), " ", format(target)), kpmcli.GetLogWriter(), ) } diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 5ae56535..5ec109c4 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -24,6 +24,9 @@ var InvalidAddOptionsInvalidOciRef = errors.New("invalid 'kpm add' argument, you var InvalidAddOptionsInvalidOciReg = errors.New("invalid 'kpm add' argument, you must provide a Reg for the package.") var InvalidAddOptionsInvalidOciRepo = errors.New("invalid 'kpm add' argument, you must provide a Repo for the package.") +// Invalid 'kpm update' +var MultipleSources = errors.New("multiple sources found, there must be a single source.") + // Invalid 'kpm run' var InvalidRunOptionsWithoutEntryFiles = errors.New("invalid 'kpm run' argument, you must provide an entry file.") var EntryFileNotFound = errors.New("entry file cannot be found, please make sure the '--input' entry file can be found") diff --git a/pkg/git/git.go b/pkg/git/git.go index 0af8db18..9e8466d0 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -1,12 +1,18 @@ package git import ( + "encoding/json" "errors" + "fmt" "io" + "net/http" + "regexp" + "time" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/hashicorp/go-getter" + giturl "github.com/kubescape/go-git-url" ) // CloneOptions is a struct for specifying options for cloning a git repository @@ -140,6 +146,7 @@ func CloneWithOpts(opts ...CloneOption) (*git.Repository, error) { return nil, err } + return cloneOpts.Clone() } @@ -153,3 +160,86 @@ func Clone(repoURL string, tagName string, localPath string, writer io.Writer) ( }) return repo, err } + +type GitHubRelease struct { + TagName string `json:"tag_name"` +} + +// parseNextPageURL extracts the 'next' page URL from the 'Link' header +func parseNextPageURL(linkHeader string) (string, error) { + // Regex to extract 'next' page URL from the link header + r := regexp.MustCompile(`<([^>]+)>;\s*rel="next"`) + matches := r.FindStringSubmatch(linkHeader) + + if len(matches) < 2 { + return "", errors.New("next page URL not found") + } + return matches[1], nil +} + +// GetAllGithubReleases fetches all releases from a GitHub repository +func GetAllGithubReleases(url string) ([]string, error) { + // Initialize and parse the URL to extract owner and repo names + gitURL, err := giturl.NewGitURL(url) + if err != nil { + return nil, err + } + + if gitURL.GetHostName() != "github.com" { + return nil, errors.New("only GitHub repositories are currently supported") + } + + // Construct initial API URL for the first page + apiBase := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases", gitURL.GetOwnerName(), gitURL.GetRepoName()) + apiURL := fmt.Sprintf("%s?per_page=100&page=1", apiBase) + + client := http.Client{ + Timeout: 10 * time.Second, + } + + var releaseTags []string + + for apiURL != "" { + req, err := http.NewRequest("GET", apiURL, nil) + if err != nil { + return nil, err + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to fetch tags, status code: %d", resp.StatusCode) + } + + // Decode the JSON response into a slice of releases + var releases []GitHubRelease + if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil { + return nil, err + } + + // Extract tag names from the releases + for _, release := range releases { + releaseTags = append(releaseTags, release.TagName) + } + + // Read the `Link` header to get the next page URL, if available + linkHeader := resp.Header.Get("Link") + if linkHeader != "" { + nextURL, err := parseNextPageURL(linkHeader) + if err != nil { + apiURL = "" + } else { + apiURL = nextURL + } + } else { + apiURL = "" + } + fmt.Println(apiURL) + } + + return releaseTags, nil +} diff --git a/pkg/mvs/mvs.go b/pkg/mvs/mvs.go new file mode 100644 index 00000000..fbd817c7 --- /dev/null +++ b/pkg/mvs/mvs.go @@ -0,0 +1,185 @@ +package mvs + +import ( + "fmt" + + "github.com/dominikbraun/graph" + "github.com/hashicorp/go-version" + "golang.org/x/mod/module" + "kcl-lang.io/kpm/pkg/client" + errInt "kcl-lang.io/kpm/pkg/errors" + "kcl-lang.io/kpm/pkg/git" + "kcl-lang.io/kpm/pkg/oci" + pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/reporter" + "kcl-lang.io/kpm/pkg/semver" +) + +type ReqsGraph struct { + graph.Graph[module.Version, module.Version] + kpmClient *client.KpmClient + kpmPkg *pkg.KclPkg +} + +func (r ReqsGraph) Max(path, v1, v2 string) string { + if v1 == "none" || v2 == "" { + return v2 + } + if v2 == "none" || v1 == "" { + return v1 + } + version1, err := version.NewVersion(v1) + if err != nil { + reporter.Fatal(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s for module %s", v1, path)) + return "" + } + version2, err := version.NewVersion(v2) + if err != nil { + reporter.Fatal(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s for module %s", v2, path)) + return "" + } + if version1.GreaterThan(version2) { + return v1 + } + return v2 +} + +func (r ReqsGraph) Upgrade(m module.Version) (module.Version, error) { + _, properties, err := r.VertexWithProperties(m) + if err != nil { + return module.Version{}, err + } + + releases, err := getReleasesFromSource(properties) + if err != nil { + return module.Version{}, err + } + + if releases == nil { + return m, nil + } + + m.Version, err = semver.LatestCompatibleVersion(releases, m.Version) + if err != nil { + return module.Version{}, err + } + _, err = r.Vertex(m) + if err == graph.ErrVertexNotFound { + d := pkg.Dependency{ + Name: m.Path, + Version: m.Version, + } + d.FullName = d.GenDepFullName() + for sourceType, uri := range properties.Attributes { + d.Source, err = pkg.GenSource(sourceType, uri, m.Version) + if err != nil { + return module.Version{}, err + } + } + deps := pkg.Dependencies{ + Deps: map[string]pkg.Dependency{ + m.Path: d, + }, + } + lockDeps := pkg.Dependencies{ + Deps: make(map[string]pkg.Dependency), + } + _, err = r.kpmClient.DownloadDeps(&deps, &lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) + if err != nil { + return module.Version{}, err + } + } + return m, nil +} + +func (r ReqsGraph) Previous(m module.Version) (module.Version, error) { + _, properties, err := r.VertexWithProperties(m) + if err != nil { + return module.Version{}, err + } + + releases, err := getReleasesFromSource(properties) + if err != nil { + return module.Version{}, err + } + + if releases == nil { + return m, nil + } + + // copy the version to compare it later + v := m.Version + + m.Version, err = semver.LeastOldCompatibleVersion(releases, m.Version) + if err != nil && err != errInt.InvalidVersionFormat { + return module.Version{}, err + } + + if v == m.Version { + return module.Version{Path: m.Path, Version: "none"}, nil + } + + _, err = r.Vertex(m) + if err == graph.ErrVertexNotFound { + d := pkg.Dependency{ + Name: m.Path, + Version: m.Version, + } + d.FullName = d.GenDepFullName() + for sourceType, uri := range properties.Attributes { + d.Source, err = pkg.GenSource(sourceType, uri, m.Version) + if err != nil { + return module.Version{}, err + } + } + deps := pkg.Dependencies{ + Deps: map[string]pkg.Dependency{ + m.Path: d, + }, + } + lockDeps := pkg.Dependencies{ + Deps: make(map[string]pkg.Dependency), + } + _, err = r.kpmClient.DownloadDeps(&deps, &lockDeps, r.Graph, r.kpmPkg.HomePath, module.Version{}) + if err != nil { + return module.Version{}, err + } + } + return m, nil +} + +func (r ReqsGraph) Required(m module.Version) ([]module.Version, error) { + adjMap, err := r.AdjacencyMap() + if err != nil { + return nil, err + } + var reqs []module.Version + for v := range adjMap[m] { + reqs = append(reqs, v) + } + return reqs, nil +} + +func getReleasesFromSource(properties graph.VertexProperties) ([]string, error) { + var releases []string + var err error + + // there must be only one property depending on the download source type + if len(properties.Attributes) != 1 { + return nil, errInt.MultipleSources + } + + for k, v := range properties.Attributes { + switch k { + case pkg.GIT: + releases, err = git.GetAllGithubReleases(v) + case pkg.OCI: + releases, err = oci.GetAllImageTags(v) + } + if err != nil { + return nil, err + } + } + + return releases, nil +} diff --git a/pkg/mvs/mvs_test.go b/pkg/mvs/mvs_test.go new file mode 100644 index 00000000..b90da6d5 --- /dev/null +++ b/pkg/mvs/mvs_test.go @@ -0,0 +1,236 @@ +package mvs + +import ( + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/mod/module" + "kcl-lang.io/kpm/pkg/3rdparty/mvs" + "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/utils" +) + +const testDataDir = "test_data" + +func getTestDir(subDir string) string { + pwd, _ := os.Getwd() + testDir := filepath.Join(pwd, testDataDir) + testDir = filepath.Join(testDir, subDir) + + return testDir +} + +func TestMax(t *testing.T) { + reqs := ReqsGraph{} + assert.Equal(t, reqs.Max("", "1.0.0", "2.0.0"), "2.0.0") + assert.Equal(t, reqs.Max("", "1.2", "2.0"), "2.0") + assert.Equal(t, reqs.Max("", "2.5.0", "2.6"), "2.6") + assert.Equal(t, reqs.Max("", "2.0.0", "v3.0"), "v3.0") +} + +func TestRequired(t *testing.T) { + pkg_path := filepath.Join(getTestDir("test_with_internal_deps"), "aaa") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + kpmcli, + kclPkg, + } + + req, err := reqs.Required(module.Version{Path: "aaa", Version: "0.0.1"}) + assert.Equal(t, err, nil) + assert.Equal(t, len(req), 2) + + expectedReqs := []module.Version{ + {Path:"bbb", Version:"0.0.1"}, + {Path:"ccc", Version:"0.0.1"}, + } + sort.Slice(req, func(i, j int) bool { + return req[i].Path < req[j].Path + }) + assert.Equal(t, req, expectedReqs) +} + +func TestUpgrade(t *testing.T) { + pkg_path := getTestDir("test_with_external_deps") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + kpmcli, + kclPkg, + } + + target := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} + upgradeList := []module.Version{ + {Path: "argo-cd-order", Version: "0.2.0"}, + {Path: "helloworld", Version: "0.1.1"}, + } + upgrade, err := mvs.Upgrade(target, reqs, upgradeList...) + assert.Equal(t, err, nil) + + expectedReqs := []module.Version{ + {Path: "test_with_external_deps", Version: "0.0.1"}, + {Path: "argo-cd-order", Version: "0.2.0"}, + {Path: "helloworld", Version: "0.1.1"}, + {Path: "json_merge_patch", Version: "0.1.0"}, + {Path: "k8s", Version: "1.29"}, + {Path: "podinfo", Version: "0.1.1"}, + } + assert.Equal(t, upgrade, expectedReqs) +} + +func TestUpgradeToLatest(t *testing.T) { + pkg_path := getTestDir("test_with_external_deps") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + kpmcli, + kclPkg, + } + + upgrade, err := reqs.Upgrade(module.Version{Path: "k8s", Version: "1.27"}) + assert.Equal(t, err, nil) + assert.Equal(t, upgrade, module.Version{Path: "k8s", Version: "1.29"}) +} + +func TestUpgradeAllToLatest(t *testing.T) { + pkg_path := getTestDir("test_with_external_deps") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + kpmcli, + kclPkg, + } + + target := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} + + upgrade, err := mvs.UpgradeAll(target, reqs) + assert.Equal(t, err, nil) + + expectedReqs := []module.Version{ + {Path: "test_with_external_deps", Version: "0.0.1"}, + {Path: "argo-cd-order", Version: "0.2.0"}, + {Path: "helloworld", Version: "0.1.2"}, + {Path: "json_merge_patch", Version: "0.1.1"}, + {Path: "k8s", Version: "1.29"}, + {Path: "podinfo", Version: "0.1.1"}, + } + assert.Equal(t, upgrade, expectedReqs) +} + +func TestPrevious(t *testing.T) { + pkg_path := getTestDir("test_with_external_deps") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + kpmcli, + kclPkg, + } + + downgrade, err := reqs.Previous(module.Version{Path: "k8s", Version: "1.27"}) + assert.Equal(t, err, nil) + assert.Equal(t, downgrade, module.Version{Path: "k8s", Version: "1.14"}) +} + +func TestUpgradePreviousOfLocalDependency(t *testing.T) { + pkg_path := filepath.Join(getTestDir("test_with_internal_deps"), "aaa") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + kpmcli, + kclPkg, + } + + upgrade, err := reqs.Upgrade(module.Version{Path: "bbb", Version: "0.0.1"}) + assert.Equal(t, err, nil) + assert.Equal(t, upgrade, module.Version{Path: "bbb", Version: "0.0.1"}) + + downgrade, err := reqs.Previous(module.Version{Path: "bbb", Version: "0.0.1"}) + assert.Equal(t, err, nil) + assert.Equal(t, downgrade, module.Version{Path: "bbb", Version: "0.0.1"}) +} + +func TestDowngrade(t *testing.T) { + pkg_path := getTestDir("test_with_external_deps") + assert.Equal(t, utils.DirExists(filepath.Join(pkg_path, "kcl.mod")), true) + kpmcli, err := client.NewKpmClient() + assert.Equal(t, err, nil) + kclPkg, err := kpmcli.LoadPkgFromPath(pkg_path) + assert.Equal(t, err, nil) + + _, depGraph, err := kpmcli.InitGraphAndDownloadDeps(kclPkg) + assert.Equal(t, err, nil) + + reqs := ReqsGraph{ + depGraph, + kpmcli, + kclPkg, + } + + target := module.Version{Path: kclPkg.GetPkgName(), Version: kclPkg.GetPkgVersion()} + downgradeList := []module.Version{ + {Path: "k8s", Version: "1.17"}, + } + downgrade, err := mvs.Downgrade(target, reqs, downgradeList...) + assert.Equal(t, err, nil) + + expectedReqs := []module.Version{ + {Path:"test_with_external_deps", Version:"0.0.1"}, + {Path:"argo-cd-order", Version:"0.1.2"}, + {Path:"helloworld", Version:"0.1.0"}, + {Path:"json_merge_patch", Version:"0.1.0"}, + {Path:"k8s", Version:"1.17"}, + } + assert.Equal(t, downgrade, expectedReqs) +} diff --git a/pkg/mvs/test_data/test_with_external_deps/kcl.mod b/pkg/mvs/test_data/test_with_external_deps/kcl.mod new file mode 100644 index 00000000..67278fa8 --- /dev/null +++ b/pkg/mvs/test_data/test_with_external_deps/kcl.mod @@ -0,0 +1,10 @@ +[package] +name = "test_with_external_deps" +edition = "0.0.1" +version = "0.0.1" + +[dependencies] +k8s = "1.27" +helloworld = "0.1.0" +argo-cd-order = "0.1.2" +podinfo = "0.1.1" \ No newline at end of file diff --git a/pkg/mvs/test_data/test_with_internal_deps/aaa/kcl.mod b/pkg/mvs/test_data/test_with_internal_deps/aaa/kcl.mod new file mode 100644 index 00000000..fdd2927f --- /dev/null +++ b/pkg/mvs/test_data/test_with_internal_deps/aaa/kcl.mod @@ -0,0 +1,8 @@ +[package] +name = "aaa" +edition = "0.0.1" +version = "0.0.1" + +[dependencies] +bbb = { path = "../bbb" } +ccc = { path = "../ccc" } diff --git a/pkg/mvs/test_data/test_with_internal_deps/bbb/kcl.mod b/pkg/mvs/test_data/test_with_internal_deps/bbb/kcl.mod new file mode 100644 index 00000000..e9ea10a5 --- /dev/null +++ b/pkg/mvs/test_data/test_with_internal_deps/bbb/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "bbb" +edition = "0.0.1" +version = "0.0.1" + diff --git a/pkg/mvs/test_data/test_with_internal_deps/ccc/kcl.mod b/pkg/mvs/test_data/test_with_internal_deps/ccc/kcl.mod new file mode 100644 index 00000000..9a762a4f --- /dev/null +++ b/pkg/mvs/test_data/test_with_internal_deps/ccc/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "ccc" +edition = "0.0.1" +version = "0.0.1" + diff --git a/pkg/oci/oci.go b/pkg/oci/oci.go index 6a9c73fa..bdb45427 100644 --- a/pkg/oci/oci.go +++ b/pkg/oci/oci.go @@ -5,12 +5,15 @@ import ( "encoding/json" "fmt" "io" + "log" "os" "path/filepath" "reflect" "runtime" "strings" + "github.com/containers/image/docker" + "github.com/containers/image/types" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/thoas/go-funk" "oras.land/oras-go/pkg/auth" @@ -399,6 +402,20 @@ func GenOciManifestFromPkg(kclPkg *pkg.KclPkg) (map[string]string, error) { return res, nil } +func GetAllImageTags(imageName string) ([]string, error) { + sysCtx := &types.SystemContext{} + ref, err := docker.ParseReference("//" + strings.TrimPrefix(imageName, "oci://")) + if err != nil { + log.Fatalf("Error parsing reference: %v", err) + } + + tags, err := docker.GetRepositoryTags(context.Background(), sysCtx, ref) + if err != nil { + log.Fatalf("Error getting tags: %v", err) + } + return tags, nil +} + const ( MediaTypeConfig = "application/vnd.docker.container.image.v1+json" MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" @@ -488,4 +505,4 @@ func ParsePlatform(platform string) (*v1.Platform, error) { } return &p, nil -} +} \ No newline at end of file diff --git a/pkg/package/modfile.go b/pkg/package/modfile.go index 1f906bac..38946a04 100644 --- a/pkg/package/modfile.go +++ b/pkg/package/modfile.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/BurntSushi/toml" + "kcl-lang.io/kcl-go/pkg/kcl" "oras.land/oras-go/v2/registry" @@ -24,6 +25,9 @@ import ( const ( MOD_FILE = "kcl.mod" MOD_LOCK_FILE = "kcl.mod.lock" + GIT = "git" + OCI = "oci" + LOCAL = "local" ) // 'Package' is the kcl package section of 'kcl.mod'. @@ -251,6 +255,51 @@ func (dep *Dependency) GenDepFullName() string { return dep.FullName } +// GetDownloadPath will get the download path of a dependency. +func (dep *Dependency) GetDownloadPath() string { + if dep.Source.Git != nil { + return dep.Source.Git.Url + } + if dep.Source.Oci != nil { + return dep.Source.Oci.IntoOciUrl() + } + return "" +} + +func GenSource(sourceType string, uri string, tagName string) (Source, error) { + source := Source{} + if sourceType == GIT { + source.Git = &Git{ + Url: uri, + Tag: tagName, + } + return source, nil + } + if sourceType == OCI { + oci := Oci{} + _, err := oci.FromString(uri + ":" + tagName) + if err != nil { + return Source{}, err + } + source.Oci = &oci + } + return source, nil +} + +// GetSourceType will get the source type of a dependency. +func (dep *Dependency) GetSourceType() string { + if dep.Source.Git != nil { + return GIT + } + if dep.Source.Oci != nil { + return OCI + } + if dep.Source.Local != nil { + return LOCAL + } + return "" +} + // Source is the package source from registry. type Source struct { *Git @@ -305,10 +354,11 @@ func (oci *Oci) FromString(ociUrl string) (*Oci, error) { // Git is the package source from git registry. type Git struct { - Url string `toml:"url,omitempty"` - Branch string `toml:"branch,omitempty"` - Commit string `toml:"commit,omitempty"` - Tag string `toml:"git_tag,omitempty"` + Url string `toml:"url,omitempty"` + Branch string `toml:"branch,omitempty"` + Commit string `toml:"commit,omitempty"` + Tag string `toml:"git_tag,omitempty"` + Version string `toml:"version,omitempty"` } // GetValidGitReference will get the valid git reference from git source. diff --git a/pkg/package/modfile_test.go b/pkg/package/modfile_test.go index f3f05e2e..ebc3bd0d 100644 --- a/pkg/package/modfile_test.go +++ b/pkg/package/modfile_test.go @@ -235,3 +235,16 @@ func TestGetFilePath(t *testing.T) { assert.Equal(t, mfile.GetModFilePath(), filepath.Join(testPath, MOD_FILE)) assert.Equal(t, mfile.GetModLockFilePath(), filepath.Join(testPath, MOD_LOCK_FILE)) } + +func TestGenSource(t *testing.T) { + src, err := GenSource("git", "https://github.com/kcl-lang/kcl", "0.8.7") + assert.Equal(t, err, nil) + assert.Equal(t, src.Git.Url, "https://github.com/kcl-lang/kcl") + assert.Equal(t, src.Git.Tag, "0.8.7") + + src, err = GenSource("oci", "oci://ghcr.io/kcl-lang/k8s", "1.24") + assert.Equal(t, err, nil) + assert.Equal(t, src.Oci.Reg, "ghcr.io") + assert.Equal(t, src.Oci.Repo, "kcl-lang/k8s") + assert.Equal(t, src.Oci.Tag, "1.24") +} \ No newline at end of file diff --git a/pkg/package/toml.go b/pkg/package/toml.go index 02d582cc..c21ef54b 100644 --- a/pkg/package/toml.go +++ b/pkg/package/toml.go @@ -118,6 +118,7 @@ func (source *Source) MarshalTOML() string { const GIT_URL_PATTERN = "git = \"%s\"" const TAG_PATTERN = "tag = \"%s\"" const GIT_COMMIT_PATTERN = "commit = \"%s\"" +const VERSION_PATTERN = "version = \"%s\"" const SEPARATOR = ", " func (git *Git) MarshalTOML() string { @@ -133,6 +134,10 @@ func (git *Git) MarshalTOML() string { sb.WriteString(SEPARATOR) sb.WriteString(fmt.Sprintf(GIT_COMMIT_PATTERN, git.Commit)) } + if len(git.Version) != 0 { + sb.WriteString(SEPARATOR) + sb.WriteString(fmt.Sprintf(VERSION_PATTERN, git.Version)) + } return sb.String() } diff --git a/pkg/semver/semver.go b/pkg/semver/semver.go index 7245a19b..6aec2e49 100644 --- a/pkg/semver/semver.go +++ b/pkg/semver/semver.go @@ -31,3 +31,56 @@ func LatestVersion(versions []string) (string, error) { return latest.Original(), nil } + +func OldestVersion(versions []string) (string, error) { + var oldest *version.Version + for _, v := range versions { + ver, err := version.NewVersion(v) + if err != nil { + return "", reporter.NewErrorEvent(reporter.FailedParseVersion, err, fmt.Sprintf("failed to parse version %s", v)) + } + if oldest == nil || ver.LessThan(oldest) { + oldest = ver + } + } + + if oldest == nil { + return "", errors.InvalidVersionFormat + } + + return oldest.Original(), nil +} + +func filterCompatibleVersions(versions []string, baseVersion string) ([]string, error) { + base, err := version.NewVersion(baseVersion) + if err != nil { + return nil, fmt.Errorf("invalid base version: %v", err) + } + var compatibleVersions []string + for _, v := range versions { + ver, err := version.NewVersion(v) + if err != nil { + continue // skip versions that fail to parse + } + if ver.Segments()[0] == base.Segments()[0] && ver.Prerelease() == "" { + compatibleVersions = append(compatibleVersions, ver.Original()) + } + } + return compatibleVersions, nil +} + +func LatestCompatibleVersion(versions []string, baseVersion string) (string, error) { + compatibleVersions, err := filterCompatibleVersions(versions, baseVersion) + if err != nil { + return "", err + } + return LatestVersion(compatibleVersions) +} + +func LeastOldCompatibleVersion(versions []string, baseVersion string) (string, error) { + compatibleVersions, err := filterCompatibleVersions(versions, baseVersion) + if err != nil { + return "", err + } + return OldestVersion(compatibleVersions) +} diff --git a/pkg/semver/semver_test.go b/pkg/semver/semver_test.go index 1a2ee484..90b873c1 100644 --- a/pkg/semver/semver_test.go +++ b/pkg/semver/semver_test.go @@ -42,3 +42,29 @@ func TestTheLatestTagWithMissingVersion(t *testing.T) { assert.Equal(t, err, nil) assert.Equal(t, latest, "5.5") } + +func TestOldestVersion(t *testing.T) { + oldest, err := OldestVersion([]string{"1.2.3", "1.4.0", "2.0.0", "1.3.5", "1.0.0"}) + assert.Equal(t, err, nil) + assert.Equal(t, oldest, "1.0.0") + + oldest, err = OldestVersion([]string{"2.2.0", "2.4.0", "3.0.0", "2.3.5"}) + assert.Equal(t, err, nil) + assert.Equal(t, oldest, "2.2.0") +} + +func TestFilterCompatibleVersions(t *testing.T) { + compatible, err := filterCompatibleVersions([]string{"1.2.3", "1.4.0", "2.0.0", "1.3.5", "1.0.0"}, "1.2.0") + assert.Equal(t, err, nil) + expCompatible := []string{"1.2.3", "1.4.0", "1.3.5", "1.0.0"} + for i, v := range compatible { + assert.Equal(t, v, expCompatible[i]) + } + + compatible, err = filterCompatibleVersions([]string{"2.2.0", "2.4.0", "3.0.0", "2.3.5"}, "2.0.0") + assert.Equal(t, err, nil) + expCompatible = []string{"2.2.0", "2.4.0", "2.3.5"} + for i, v := range compatible { + assert.Equal(t, v, expCompatible[i]) + } +}