From fd166ff053ddc7dbf6365b19e78c11587fba076b Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Thu, 18 Jan 2024 16:40:39 -0800 Subject: [PATCH] feat: adds updater CLI tool (#165) --- .config/dictionaries/project.dic | 1 + tools/updater/Earthfile | 55 +++++++++++++++++++++ tools/updater/README.md | 43 +++++++++++++++++ tools/updater/cmd/main.go | 46 ++++++++++++++++++ tools/updater/go.mod | 25 ++++++++++ tools/updater/go.sum | 65 +++++++++++++++++++++++++ tools/updater/pkg/cue.go | 74 +++++++++++++++++++++++++++++ tools/updater/pkg/cue_test.go | 69 +++++++++++++++++++++++++++ tools/updater/pkg/pkg_suite_test.go | 13 +++++ 9 files changed, 391 insertions(+) create mode 100644 tools/updater/Earthfile create mode 100644 tools/updater/README.md create mode 100644 tools/updater/cmd/main.go create mode 100644 tools/updater/go.mod create mode 100644 tools/updater/go.sum create mode 100644 tools/updater/pkg/cue.go create mode 100644 tools/updater/pkg/cue_test.go create mode 100644 tools/updater/pkg/pkg_suite_test.go diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index 906f221e2..32cdfbf17 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -53,6 +53,7 @@ subproject subprojects superfences testpackage +Timoni transpiling UDCs uniseg diff --git a/tools/updater/Earthfile b/tools/updater/Earthfile new file mode 100644 index 000000000..d3d3b0cfc --- /dev/null +++ b/tools/updater/Earthfile @@ -0,0 +1,55 @@ +VERSION 0.7 +FROM golang:1.20-alpine3.18 + +# cspell: words onsi ldflags extldflags + +fmt: + DO ../../earthly/go+FMT --src="go.mod go.sum cmd pkg" + +lint: + DO ../../earthly/go+LINT --src="go.mod go.sum cmd pkg" + +deps: + WORKDIR /work + + RUN apk add --no-cache gcc musl-dev + DO ../../earthly/go+DEPS + +src: + FROM +deps + + COPY --dir cmd pkg . + +check: + FROM +src + + BUILD +fmt + BUILD +lint + +build: + FROM +src + + ENV CGO_ENABLED=0 + RUN go build -ldflags="-extldflags=-static" -o bin/updater cmd/main.go + + SAVE ARTIFACT bin/updater updater + +test: + FROM +build + + RUN ginkgo ./... + +release: + FROM +build + + SAVE ARTIFACT bin/updater updater + +publish: + FROM debian:bookworm-slim + WORKDIR /workspace + ARG tag=latest + + COPY +build/updater /usr/local/bin/updater + + ENTRYPOINT ["/usr/local/bin/updater"] + SAVE IMAGE --push ci-updater:${tag} \ No newline at end of file diff --git a/tools/updater/README.md b/tools/updater/README.md new file mode 100644 index 000000000..ea2344a0f --- /dev/null +++ b/tools/updater/README.md @@ -0,0 +1,43 @@ +# updater + +> A helper tool for modifying CUE files to override arbitrary values. +> Useful for updating Timoni bundles. + +The `updater` CLI provides an interface for overriding existing concrete values in a given CUE file. +Normally, concrete values in CUE files are immutable and thus not possible to override using the CUE CLI. +However, in some cases, it may be desirable to override an existing concrete value. +This is especially true in GitOps scenarios where a source of truth needs to be updated. + +## Usage + +The `updater` CLI is most commonly used to update Timoni bundle image tags. +Assuming you have a `bundle.cue` file like this: + +```cue +bundle: { + apiVersion: "v1alpha1" + name: "bundle" + instances: { + instance: { + module: { + url: "oci://332405224602.dkr.ecr.eu-central-1.amazonaws.com/instance-deployment" + version: "0.0.1" + } + namespace: "default" + values: { + server: image: tag: "ed2951cf049e779bba8d97413653bb06d4c28144" + } + } + } +} +``` + +You can update the value of `server.image.tag` like so: + +```shell +updater -b bundle.cue "bundle.instances.instance.values.server.image.tag" "0fe74bf77739a3ef78de5fcc81c5c7a8dcae6199" +``` + +The `updater` CLI will overwrite the image tag with the provided one and update the `bundle.cue` file in place. +Note that the CLI uses the CUE API underneath the hood which may format the existing CUE syntax slightly differently. +In some cases, the resulting syntax might be a bit unsightly, so it's recommended to run `cue fmt` on the file after processing. diff --git a/tools/updater/cmd/main.go b/tools/updater/cmd/main.go new file mode 100644 index 000000000..0480df5d7 --- /dev/null +++ b/tools/updater/cmd/main.go @@ -0,0 +1,46 @@ +package main + +// cspell: words alecthomas cuelang cuecontext cuectx existingfile Timoni nolint + +import ( + "os" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/cuecontext" + "cuelang.org/go/cue/format" + "github.com/alecthomas/kong" + "github.com/input-output-hk/catalyst-ci/tools/updater/pkg" +) + +var cli struct { + BundleFile string `type:"existingfile" short:"b" help:"Path to the Timoni bundle file to modify." required:"true"` + Path string `arg:"" help:"A dot separated path to the value to override (must already exist)."` + Value string `arg:"" help:"The value to override the value at the path with."` +} + +func main() { + ctx := kong.Parse(&cli, + kong.Name("updater"), + kong.Description("A helper tool for modifying CUE files to override arbitrary values. Useful for updating Timoni bundles.")) + + cuectx := cuecontext.New() + v, err := pkg.ReadFile(cuectx, cli.BundleFile) + ctx.FatalIfErrorf(err) + + if !v.LookupPath(cue.ParsePath(cli.Path)).Exists() { + ctx.Fatalf("path %q does not exist", cli.Path) + } + + v, err = pkg.FillPathOverride(cuectx, v, cli.Path, cli.Value) + ctx.FatalIfErrorf(err) + + node := v.Syntax(cue.Final(), cue.Concrete(true)) + src, err := format.Node(node) + ctx.FatalIfErrorf(err) + + if err := os.WriteFile(cli.BundleFile, src, 0644); err != nil { //nolint:gosec + ctx.Fatalf("failed to write file %q: %v", cli.BundleFile, err) + } + + os.Exit(0) +} diff --git a/tools/updater/go.mod b/tools/updater/go.mod new file mode 100644 index 000000000..fb3922d2a --- /dev/null +++ b/tools/updater/go.mod @@ -0,0 +1,25 @@ +module github.com/input-output-hk/catalyst-ci/tools/updater + +go 1.20 + +require ( + cuelang.org/go v0.7.0 + github.com/alecthomas/kong v0.8.1 + github.com/onsi/ginkgo/v2 v2.15.0 + github.com/onsi/gomega v1.31.0 +) + +require ( + github.com/cockroachdb/apd/v3 v3.2.1 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect + github.com/google/uuid v1.2.0 // indirect + github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.16.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/tools/updater/go.sum b/tools/updater/go.sum new file mode 100644 index 000000000..2a68e4d58 --- /dev/null +++ b/tools/updater/go.sum @@ -0,0 +1,65 @@ +cuelabs.dev/go/oci/ociregistry v0.0.0-20231103182354-93e78c079a13 h1:zkiIe8AxZ/kDjqQN+mDKc5BxoVJOqioSdqApjc+eB1I= +cuelang.org/go v0.7.0 h1:gMztinxuKfJwMIxtboFsNc6s8AxwJGgsJV+3CuLffHI= +cuelang.org/go v0.7.0/go.mod h1:ix+3dM/bSpdG9xg6qpCgnJnpeLtciZu+O/rDbywoMII= +github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0= +github.com/alecthomas/kong v0.8.1 h1:acZdn3m4lLRobeh3Zi2S2EpnXTd1mOL6U7xVml+vfkY= +github.com/alecthomas/kong v0.8.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= +github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= +github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/proto v1.10.0 h1:pDGyFRVV5RvV+nkBK9iy3q67FBy9Xa7vwrOTE+g5aGw= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9ZPiLVHXz3UFw2+psEX+gYcto= +github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuHZgRjZUWWuH1DTxCtxbHDOIJsudS8jzY= +github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= +github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= +github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0 h1:sadMIsgmHpEOGbUs6VtHBXRR1OHevnj7hLx9ZcdNGW4= +github.com/rogpeppe/go-internal v1.11.1-0.20231026093722-fa6a31e0812c h1:fPpdjePK1atuOg28PXfNSqgwf9I/qD1Hlo39JFwKBXk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/updater/pkg/cue.go b/tools/updater/pkg/cue.go new file mode 100644 index 000000000..ca6d6b798 --- /dev/null +++ b/tools/updater/pkg/cue.go @@ -0,0 +1,74 @@ +package pkg + +// cspell: words cuelang + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "cuelang.org/go/cue" +) + +// FillPathOverride is like cue.Value.FillPath, but it allows you to override concrete values. +// The default behavior of CUE is to support immutable values, so you can't normally override a concrete value. +// This function first converts the cue.Value to JSON, then to a map, and then modifies the map. +// Finally, it converts the map back to JSON and then to a cue.Value. +func FillPathOverride(ctx *cue.Context, v cue.Value, path string, value interface{}) (cue.Value, error) { + j, err := v.MarshalJSON() + if err != nil { + return cue.Value{}, fmt.Errorf("failed to marshal cue.Value to JSON: %w", err) + } + + var data map[string]interface{} + if err := json.Unmarshal(j, &data); err != nil { + return cue.Value{}, fmt.Errorf("failed to unmarshal JSON to map: %w", err) + } + + if err := setField(data, path, value); err != nil { + return cue.Value{}, fmt.Errorf("failed to set field %q to value %v: %w", path, value, err) + } + + modifiedJSON, err := json.Marshal(data) + if err != nil { + return cue.Value{}, fmt.Errorf("failed to marshal map to JSON: %w", err) + } + + return ctx.CompileBytes(modifiedJSON), nil +} + +// ReadFile reads a CUE file and returns a cue.Value. +func ReadFile(ctx *cue.Context, path string) (cue.Value, error) { + contents, err := os.ReadFile(path) + if err != nil { + return cue.Value{}, fmt.Errorf("failed to read file %q: %w", path, err) + } + + v := ctx.CompileBytes(contents) + if err := v.Err(); err != nil { + return cue.Value{}, fmt.Errorf("failed to compile CUE file %q: %w", path, err) + } + + return v, nil +} + +// setField sets the value at the given path in a map, creating nested maps as necessary. +func setField(m map[string]interface{}, path string, value interface{}) error { + parts := strings.Split(path, ".") + for i, part := range parts { + if i == len(parts)-1 { + m[part] = value + } else { + if _, ok := m[part]; !ok { + m[part] = make(map[string]interface{}) + } + var ok bool + m, ok = m[part].(map[string]interface{}) + if !ok { + return fmt.Errorf("invalid path: %s", path) + } + } + } + return nil +} diff --git a/tools/updater/pkg/cue_test.go b/tools/updater/pkg/cue_test.go new file mode 100644 index 000000000..a610ccf79 --- /dev/null +++ b/tools/updater/pkg/cue_test.go @@ -0,0 +1,69 @@ +package pkg_test + +// cspell: words cuelang cuecontext + +import ( + "cuelang.org/go/cue" + "cuelang.org/go/cue/cuecontext" + "github.com/input-output-hk/catalyst-ci/tools/updater/pkg" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Cue", func() { + Describe("FillPathOverride", func() { + var path, str string + + When("given a nested CUE source", func() { + When("the path exists", func() { + BeforeEach(func() { + path = "bundles.instances.module.values.image.tag" + str = ` + bundle: { + instances: { + module: { + values: { + image: tag: "test" + } + } + } + } + ` + }) + + It("should override the value at the given path", func() { + ctx := cuecontext.New() + v := ctx.CompileString(str) + + v, err := pkg.FillPathOverride(ctx, v, path, "test1") + Expect(err).ToNot(HaveOccurred()) + + result, err := v.LookupPath(cue.ParsePath(path)).String() + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(Equal("test1")) + }) + }) + + When("the path does not include structs", func() { + BeforeEach(func() { + path = "bundles.instances.module.values.image.tag" + str = ` + bundles: { + instances: { + module: "test" + } + } + ` + }) + + It("should return an error", func() { + ctx := cuecontext.New() + v := ctx.CompileString(str) + + _, err := pkg.FillPathOverride(ctx, v, path, "test1") + Expect(err).To(HaveOccurred()) + }) + }) + }) + }) +}) diff --git a/tools/updater/pkg/pkg_suite_test.go b/tools/updater/pkg/pkg_suite_test.go new file mode 100644 index 000000000..b9657da96 --- /dev/null +++ b/tools/updater/pkg/pkg_suite_test.go @@ -0,0 +1,13 @@ +package pkg_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestPkg(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Pkg Suite") +}