diff --git a/.github/workflows/jsonschema.yml b/.github/workflows/jsonschema.yml new file mode 100644 index 000000000..8c16bc3ec --- /dev/null +++ b/.github/workflows/jsonschema.yml @@ -0,0 +1,35 @@ +# name of the action +name: jsonschema + +# trigger on release events +on: + release: + types: [created] + +# pipeline to execute +jobs: + schema: + runs-on: ubuntu-latest + + steps: + - name: clone + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - name: install go + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + # use version from go.mod file + go-version-file: "go.mod" + cache: true + check-latest: true + + - name: build + run: | + make jsonschema + + - name: upload + uses: skx/github-action-publish-binaries@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + args: "schema.json" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 272bbf582..4987db435 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,23 +12,29 @@ jobs: runs-on: ubuntu-latest steps: - - name: clone - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - - - name: install go - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 - with: - # use version from go.mod file - go-version-file: 'go.mod' - cache: true - check-latest: true - - - name: test - run: | - make test - - - name: coverage - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - file: coverage.out \ No newline at end of file + - name: clone + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - name: install go + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + # use version from go.mod file + go-version-file: "go.mod" + cache: true + check-latest: true + + - name: test + run: | + make test + + - name: test jsonschema + run: | + go install github.com/santhosh-tekuri/jsonschema/cmd/jv@v0.7.0 + make test-jsonschema + + - name: coverage + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.out + diff --git a/.gitignore b/.gitignore index c557bc18b..a0b951aa0 100644 --- a/.gitignore +++ b/.gitignore @@ -67,4 +67,6 @@ __debug_bin .history .ionide -# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode \ No newline at end of file +# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode + +schema.json diff --git a/Makefile b/Makefile index 7c7a3ca31..227c4480a 100644 --- a/Makefile +++ b/Makefile @@ -326,6 +326,50 @@ spec-version-update: .PHONY: spec spec: spec-gen spec-version-update spec-validate +# The `jsonschema` target is intended to create +# a jsonschema for a Vela pipeline. +# +# Usage: `make jsonschema` +.PHONY: jsonschema +jsonschema: + @echo + @echo "### Generating JSON schema" + @go run cmd/jsonschema-gen/main.go > schema.json + +# The `test-jsonschema` target is intended to test +# the created jsonschema against a set of failing +# and passing example vela templates located in +# schema/testdata/pipeline. +# +# The test relies on the `jv` command line tool, +# which can be installed via: +# +# go install github.com/santhosh-tekuri/jsonschema/cmd/jv@latest +# +# Usage: `make test-jsonschema` +.PHONY: test-jsonschema +test-jsonschema: jsonschema + @echo + @echo "### Testing Pipelines against JSON Schema" + @echo + @echo "=== Expected Failing Tests" + @for file in schema/testdata/pipeline/fail/*.yml; do \ + echo "› Test: $$file"; \ + if jv schema.json $$file >/dev/null 2>&1; then \ + echo "Unexpected success for $$file"; \ + exit 1; \ + fi; \ + done + @echo + @echo "=== Expected Passing Tests" + @for file in schema/testdata/pipeline/pass/*.yml; do \ + echo "› Test: $$file"; \ + if ! jv schema.json $$file >/dev/null 2>&1; then \ + echo "Unexpected failure for $$file"; \ + exit 1; \ + fi; \ + done + # The `lint` target is intended to lint the # Go source code with golangci-lint. # diff --git a/cmd/jsonschema-gen/main.go b/cmd/jsonschema-gen/main.go new file mode 100644 index 000000000..ce6b49b37 --- /dev/null +++ b/cmd/jsonschema-gen/main.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 + +//go:build ignore + +package main + +import ( + "encoding/json" + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/schema" +) + +func main() { + js, err := schema.NewPipelineSchema() + if err != nil { + logrus.Fatal("schema generation failed:", err) + } + + // output json + j, err := json.MarshalIndent(js, "", " ") + if err != nil { + logrus.Fatal(err) + } + + fmt.Printf("%s\n", j) +} diff --git a/compiler/types/raw/map.go b/compiler/types/raw/map.go index ed8fff427..ed569949a 100644 --- a/compiler/types/raw/map.go +++ b/compiler/types/raw/map.go @@ -7,6 +7,8 @@ import ( "encoding/json" "errors" "strings" + + "github.com/invopop/jsonschema" ) // StringSliceMap represents an array of strings or a map of strings. @@ -138,3 +140,27 @@ func (s *StringSliceMap) UnmarshalYAML(unmarshal func(interface{}) error) error return errors.New("unable to unmarshal into StringSliceMap") } + +// JSONSchema handles some overrides that need to be in place +// for this type for the jsonschema generation. +// +// Without these changes it would only allow a map of string, +// but we do some special handling to support array of strings. +func (StringSliceMap) JSONSchema() *jsonschema.Schema { + return &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + { + Type: "array", + Items: &jsonschema.Schema{ + Type: "string", + }, + }, + { + Type: "object", + AdditionalProperties: &jsonschema.Schema{ + Type: "string", + }, + }, + }, + } +} diff --git a/compiler/types/raw/slice.go b/compiler/types/raw/slice.go index 11746363e..fb6ac59b2 100644 --- a/compiler/types/raw/slice.go +++ b/compiler/types/raw/slice.go @@ -5,6 +5,8 @@ package raw import ( "encoding/json" "errors" + + "github.com/invopop/jsonschema" ) // StringSlice represents a string or an array of strings. @@ -71,3 +73,24 @@ func (s *StringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { return errors.New("unable to unmarshal into StringSlice") } + +// JSONSchema handles some overrides that need to be in place +// for this type for the jsonschema generation. +// +// Without these changes it would only allow an array of strings, +// but we do some special handling to support plain string also. +func (StringSlice) JSONSchema() *jsonschema.Schema { + return &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + { + Type: "string", + }, + { + Type: "array", + Items: &jsonschema.Schema{ + Type: "string", + }, + }, + }, + } +} diff --git a/compiler/types/yaml/ruleset.go b/compiler/types/yaml/ruleset.go index c30e3b109..a75433311 100644 --- a/compiler/types/yaml/ruleset.go +++ b/compiler/types/yaml/ruleset.go @@ -3,6 +3,8 @@ package yaml import ( + "github.com/invopop/jsonschema" + "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" "github.com/go-vela/server/constants" @@ -22,13 +24,15 @@ type ( // Rules is the yaml representation of the ruletypes // from a ruleset block for a step in a pipeline. Rules struct { - Branch []string `yaml:"branch,omitempty,flow" json:"branch,omitempty" jsonschema:"description=Limits the execution of a step to matching build branches.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` - Comment []string `yaml:"comment,omitempty,flow" json:"comment,omitempty" jsonschema:"description=Limits the execution of a step to matching a pull request comment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` - Event []string `yaml:"event,omitempty,flow" json:"event,omitempty" jsonschema:"description=Limits the execution of a step to matching build events.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` - Path []string `yaml:"path,omitempty,flow" json:"path,omitempty" jsonschema:"description=Limits the execution of a step to matching files changed in a repository.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` - Repo []string `yaml:"repo,omitempty,flow" json:"repo,omitempty" jsonschema:"description=Limits the execution of a step to matching repos.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` - Sender []string `yaml:"sender,omitempty,flow" json:"sender,omitempty" jsonschema:"description=Limits the execution of a step to matching build senders.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` - Status []string `yaml:"status,omitempty,flow" json:"status,omitempty" jsonschema:"enum=[failure],enum=[success],description=Limits the execution of a step to matching build statuses.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Branch []string `yaml:"branch,omitempty,flow" json:"branch,omitempty" jsonschema:"description=Limits the execution of a step to matching build branches.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Comment []string `yaml:"comment,omitempty,flow" json:"comment,omitempty" jsonschema:"description=Limits the execution of a step to matching a pull request comment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + // enums for 'event' jsonschema are set in JSONSchemaExtend() method below + Event []string `yaml:"event,omitempty,flow" json:"event,omitempty" jsonschema:"description=Limits the execution of a step to matching build events.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Path []string `yaml:"path,omitempty,flow" json:"path,omitempty" jsonschema:"description=Limits the execution of a step to matching files changed in a repository.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Repo []string `yaml:"repo,omitempty,flow" json:"repo,omitempty" jsonschema:"description=Limits the execution of a step to matching repos.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Sender []string `yaml:"sender,omitempty,flow" json:"sender,omitempty" jsonschema:"description=Limits the execution of a step to matching build senders.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + // enums for 'status' jsonschema are set in JSONSchemaExtend() method below + Status []string `yaml:"status,omitempty,flow" json:"status,omitempty" jsonschema:"description=Limits the execution of a step to matching build statuses.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` Tag []string `yaml:"tag,omitempty,flow" json:"tag,omitempty" jsonschema:"description=Limits the execution of a step to matching build tag references.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` Target []string `yaml:"target,omitempty,flow" json:"target,omitempty" jsonschema:"description=Limits the execution of a step to matching build deployment targets.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` Label []string `yaml:"label,omitempty,flow" json:"label,omitempty" jsonschema:"description=Limits step execution to match on pull requests labels.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` @@ -186,3 +190,89 @@ func (r *Rules) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } + +// JSONSchemaExtend handles some overrides that need to be in place +// for this type for the jsonschema generation. +// +// Mainly it handles the fact that all Rules fields are raw.StringSlice +// but also handles adding enums to select fields as they would be too +// cumbersome to maintain in the jsonschema struct tag. +func (Rules) JSONSchemaExtend(schema *jsonschema.Schema) { + for item := schema.Properties.Newest(); item != nil; item = item.Prev() { + currSchema := *item.Value + + // store the current description so we can lift it to top level + currDescription := currSchema.Description + currSchema.Description = "" + + // handle each field as needed + switch item.Key { + case "status": + // possible values for 'status' + enums := []string{ + "success", + "failure", + } + + for _, str := range enums { + currSchema.Items.Enum = append(currSchema.Items.Enum, str) + } + + schema.Properties.Set(item.Key, &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + &currSchema, + { + Type: "string", + Enum: currSchema.Items.Enum, + }, + }, + Description: currDescription, + }) + case "event": + // possible values for 'event' + enums := []string{ + "comment", + "comment:created", + "comment:edited", + "delete:branch", + "delete:tag", + "deployment", + "pull_request", + "pull_request*", + "pull_request:edited", + "pull_request:labeled", + "pull_request:opened", + "pull_request:reopened", + "pull_request:synchronize", + "pull_request:unlabeled", + "push", + "schedule", + "tag", + } + + for _, str := range enums { + currSchema.Items.Enum = append(currSchema.Items.Enum, str) + } + + schema.Properties.Set(item.Key, &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + &currSchema, + { + Type: "string", + Enum: currSchema.Items.Enum, + }, + }, + Description: currDescription, + }) + default: + // all other fields are raw.StringSlice + schema.Properties.Set(item.Key, &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + &currSchema, + {Type: "string"}, + }, + Description: currDescription, + }) + } + } +} diff --git a/compiler/types/yaml/secret.go b/compiler/types/yaml/secret.go index d0779e5c6..97b836a09 100644 --- a/compiler/types/yaml/secret.go +++ b/compiler/types/yaml/secret.go @@ -7,6 +7,8 @@ import ( "fmt" "strings" + "github.com/invopop/jsonschema" + "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" "github.com/go-vela/server/constants" @@ -203,8 +205,8 @@ type ( // StepSecret is the yaml representation of a secret // from a secrets block for a step in a pipeline. StepSecret struct { - Source string `yaml:"source,omitempty"` - Target string `yaml:"target,omitempty"` + Source string `yaml:"source,omitempty" json:"source"` + Target string `yaml:"target,omitempty" json:"target"` } ) @@ -269,3 +271,22 @@ func (s *StepSecretSlice) UnmarshalYAML(unmarshal func(interface{}) error) error return errors.New("failed to unmarshal StepSecretSlice") } + +// JSONSchemaExtend handles some overrides that need to be in place +// for this type for the jsonschema generation. +// +// Allows using simple strings or objects. +func (StepSecret) JSONSchemaExtend(schema *jsonschema.Schema) { + old := *schema + schema.OneOf = []*jsonschema.Schema{ + { + Type: "string", + AdditionalProperties: jsonschema.FalseSchema, + }, + &old, + } + schema.AdditionalProperties = nil + schema.Properties = nil + schema.Required = nil + schema.Type = "" +} diff --git a/compiler/types/yaml/stage.go b/compiler/types/yaml/stage.go index 93bc8c8ca..79b635b05 100644 --- a/compiler/types/yaml/stage.go +++ b/compiler/types/yaml/stage.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/buildkite/yaml" + "github.com/invopop/jsonschema" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" @@ -125,6 +126,22 @@ func (s StageSlice) MarshalYAML() (interface{}, error) { return output, nil } +// JSONSchemaExtend handles some overrides that need to be in place +// for this type for the jsonschema generation. +// +// Stages are not really a slice of stages to the user. This change +// supports the map they really are. +func (StageSlice) JSONSchemaExtend(schema *jsonschema.Schema) { + schema.AdditionalProperties = jsonschema.FalseSchema + schema.Items = nil + schema.PatternProperties = map[string]*jsonschema.Schema{ + ".*": { + Ref: "#/$defs/Stage", + }, + } + schema.Type = "object" +} + // MergeEnv takes a list of environment variables and attempts // to set them in the stage environment. If the environment // variable already exists in the stage, than this will diff --git a/compiler/types/yaml/template.go b/compiler/types/yaml/template.go index ef2005540..4055fc9dc 100644 --- a/compiler/types/yaml/template.go +++ b/compiler/types/yaml/template.go @@ -17,7 +17,7 @@ type ( Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique identifier for the template.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-name-key"` Source string `yaml:"source,omitempty" json:"source,omitempty" jsonschema:"required,minLength=1,description=Path to template in remote system.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-source-key"` Format string `yaml:"format,omitempty" json:"format,omitempty" jsonschema:"enum=starlark,enum=golang,enum=go,default=go,minLength=1,description=language used within the template file \nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-format-key"` - Type string `yaml:"type,omitempty" json:"type,omitempty" jsonschema:"minLength=1,example=github,description=Type of template provided from the remote system.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-type-key"` + Type string `yaml:"type,omitempty" json:"type,omitempty" jsonschema:"minLength=1,enum=github,enum=file,example=github,description=Type of template provided from the remote system.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-type-key"` Variables map[string]interface{} `yaml:"vars,omitempty" json:"vars,omitempty" jsonschema:"description=Variables injected into the template.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-variables-key"` } diff --git a/compiler/types/yaml/ulimit.go b/compiler/types/yaml/ulimit.go index 14d96c287..af0fa544b 100644 --- a/compiler/types/yaml/ulimit.go +++ b/compiler/types/yaml/ulimit.go @@ -7,6 +7,8 @@ import ( "strconv" "strings" + "github.com/invopop/jsonschema" + "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" ) @@ -130,3 +132,32 @@ func (u *UlimitSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } + +// JSONSchemaExtend handles some overrides that need to be in place +// for this type for the jsonschema generation. +// +// Without these changes it would only allow an object per the struct, +// but we do some special handling to allow specially formatted strings. +func (Ulimit) JSONSchemaExtend(schema *jsonschema.Schema) { + oldAddProps := schema.AdditionalProperties + oldProps := schema.Properties + oldReq := schema.Required + + schema.AdditionalProperties = nil + schema.OneOf = []*jsonschema.Schema{ + { + Type: "string", + Pattern: "[a-z]+=[0-9]+:[0-9]+", + AdditionalProperties: oldAddProps, + }, + { + Type: "object", + Properties: oldProps, + Required: oldReq, + AdditionalProperties: oldAddProps, + }, + } + schema.Properties = nil + schema.Required = nil + schema.Type = "" +} diff --git a/compiler/types/yaml/volume.go b/compiler/types/yaml/volume.go index 7b7b87d32..58740f652 100644 --- a/compiler/types/yaml/volume.go +++ b/compiler/types/yaml/volume.go @@ -6,6 +6,8 @@ import ( "fmt" "strings" + "github.com/invopop/jsonschema" + "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" ) @@ -119,3 +121,32 @@ func (v *VolumeSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } + +// JSONSchemaExtend handles some overrides that need to be in place +// for this type for the jsonschema generation. +// +// Without these changes it would only allow an object per the struct, +// but we do some special handling to allow specially formatted strings. +func (Volume) JSONSchemaExtend(schema *jsonschema.Schema) { + oldAddProps := schema.AdditionalProperties + oldProps := schema.Properties + oldReq := schema.Required + + schema.AdditionalProperties = nil + schema.OneOf = []*jsonschema.Schema{ + { + Type: "string", + Pattern: "[a-z\\/]+:[a-z\\/]+:[row]+", + AdditionalProperties: oldAddProps, + }, + { + Type: "object", + Properties: oldProps, + Required: oldReq, + AdditionalProperties: oldAddProps, + }, + } + schema.Properties = nil + schema.Required = nil + schema.Type = "" +} diff --git a/go.mod b/go.mod index 290078d95..0c68f970b 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-retryablehttp v0.7.7 github.com/hashicorp/vault/api v1.15.0 + github.com/invopop/jsonschema v0.12.0 github.com/joho/godotenv v1.5.1 github.com/lestrrat-go/jwx/v2 v2.1.1 github.com/lib/pq v1.10.9 @@ -62,7 +63,9 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.12.2 // indirect github.com/bytedance/sonic/loader v0.2.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -112,6 +115,7 @@ require ( github.com/lestrrat-go/httprc v1.0.6 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -134,6 +138,7 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect diff --git a/go.sum b/go.sum index 1957fb598..fd4582b91 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -36,6 +38,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 h1:q+sMKdA6L8LyGVudTkpGoC73h6ak2iWSPFiFo/pFOU8= github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3/go.mod h1:5hCug3EZaHXU3FdCA3gJm0YTNi+V+ooA2qNTiVpky4A= github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= @@ -158,6 +162,8 @@ github.com/hashicorp/vault/api v1.15.0 h1:O24FYQCWwhwKnF7CuSqP30S51rTV7vz1iACXE/ github.com/hashicorp/vault/api v1.15.0/go.mod h1:+5YTO09JGn0u+b6ySD/LLVf8WkJCPLAL2Vkmrn2+CM8= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= +github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= @@ -176,6 +182,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -209,6 +216,8 @@ github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNB github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -297,6 +306,8 @@ github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 h1:ZjUj9BLYf9PEqBn8W/Oa github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2/go.mod h1:O8bHQfyinKwTXKkiKNGmLQS7vRsqRxIQTFZpYpHK3IQ= github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8= github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= diff --git a/schema/pipeline.go b/schema/pipeline.go new file mode 100644 index 000000000..0daf45115 --- /dev/null +++ b/schema/pipeline.go @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: Apache-2.0 + +// This program utilizes the json/jsonschema tags on structs in compiler/types/yaml +// to generate the majority of the final jsonschema for a Vela pipeline. +// +// Some manual intervention is needed for custom types and/or custom unmarshaling +// that is in place. For reference, we use the mechanisms provided by the schema lib: +// https://github.com/invopop/jsonschema?tab=readme-ov-file#custom-type-definitions +// for hooking into the schema generation process. Some types will have a JSONSchema +// or JSONSchemaExtend method attached to handle these overrides. + +package schema + +import ( + "fmt" + + "github.com/invopop/jsonschema" + + types "github.com/go-vela/server/compiler/types/yaml" +) + +// NewPipelineSchema generates the JSON schema object for a Vela pipeline configuration. +// +// The returned value can be marshaled into actual JSON. +func NewPipelineSchema() (*jsonschema.Schema, error) { + ref := jsonschema.Reflector{ + ExpandedStruct: true, + } + s := ref.Reflect(types.Build{}) + + // very unlikely scenario + if s == nil { + return nil, fmt.Errorf("schema generation failed") + } + + s.Title = "Vela Pipeline Configuration" + + // allows folks to have other top level arbitrary + // keys without validation errors + s.AdditionalProperties = nil + + // apply Ruleset modification + // + // note: we have to do the modification here, + // because the custom type hooks can't provide + // access to the top level definitions, even if + // they were already processed, so we have to + // do it at this top level. + modRulesetSchema(s) + + return s, nil +} + +// modRulesetSchema applies modifications to the Ruleset definition. +// +// rules can currently live at ruleset level or nested within +// 'if' (default) or 'unless'. without changes the struct would +// only allow the nested version. +func modRulesetSchema(schema *jsonschema.Schema) { + if schema.Definitions == nil { + return + } + + rules, hasRules := schema.Definitions["Rules"] + ruleSet, hasRuleset := schema.Definitions["Ruleset"] + + // exit early if we don't have what we need + if !hasRules || !hasRuleset { + return + } + + // create copies + _rulesWithRuleset := *rules + _ruleSet := *ruleSet + + // copy every property from Ruleset, other than `if` and `unless` + for item := _ruleSet.Properties.Newest(); item != nil; item = item.Prev() { + if item.Key != "if" && item.Key != "unless" { + _rulesWithRuleset.Properties.Set(item.Key, item.Value) + } + } + + // create a new definition for Ruleset + schema.Definitions["Ruleset"].AnyOf = []*jsonschema.Schema{ + &_ruleSet, + &_rulesWithRuleset, + } + schema.Definitions["Ruleset"].Properties = nil + schema.Definitions["Ruleset"].Type = "" + schema.Definitions["Ruleset"].AdditionalProperties = nil +} diff --git a/schema/pipeline_test.go b/schema/pipeline_test.go new file mode 100644 index 000000000..d1a1a2d95 --- /dev/null +++ b/schema/pipeline_test.go @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 + +package schema + +import ( + "testing" +) + +func TestSchema_NewPipelineSchema(t *testing.T) { + tests := []struct { + name string + wantErr bool + }{ + { + name: "basic schema generation", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewPipelineSchema() + if (err != nil) != tt.wantErr { + t.Errorf("NewPipelineSchema() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && got == nil { + t.Error("NewPipelineSchema() returned nil schema without error") + } + if !tt.wantErr { + if got.Title != "Vela Pipeline Configuration" { + t.Errorf("NewPipelineSchema() title = %v, want %v", got.Title, "Vela Pipeline Configuration") + } + if got.AdditionalProperties != nil { + t.Error("NewPipelineSchema() AdditionalProperties should be nil") + } + } + }) + } +} diff --git a/schema/testdata/pipeline/fail/image_and_template.yml b/schema/testdata/pipeline/fail/image_and_template.yml new file mode 100644 index 000000000..4770aacfb --- /dev/null +++ b/schema/testdata/pipeline/fail/image_and_template.yml @@ -0,0 +1,7 @@ +version: "1" + +steps: + - name: deploy + image: alpine + template: + name: deploy-template \ No newline at end of file diff --git a/schema/testdata/pipeline/fail/invalid_prop.yml b/schema/testdata/pipeline/fail/invalid_prop.yml new file mode 100644 index 000000000..41e7407a8 --- /dev/null +++ b/schema/testdata/pipeline/fail/invalid_prop.yml @@ -0,0 +1,11 @@ +version: "1" + +metadata: + auto_cancel: + invalid_key: true + +steps: + - name: test + image: alpine + commands: + - echo "Hello World" diff --git a/schema/testdata/pipeline/fail/invalid_pull.yml b/schema/testdata/pipeline/fail/invalid_pull.yml new file mode 100644 index 000000000..caa54cfd9 --- /dev/null +++ b/schema/testdata/pipeline/fail/invalid_pull.yml @@ -0,0 +1,8 @@ +version: "1" + +steps: + - name: build + pull: sometimes + image: alpine + commands: + - echo "Hello World" diff --git a/schema/testdata/pipeline/fail/missing_name.yml b/schema/testdata/pipeline/fail/missing_name.yml new file mode 100644 index 000000000..893237c0d --- /dev/null +++ b/schema/testdata/pipeline/fail/missing_name.yml @@ -0,0 +1,8 @@ +version: "1" + +stages: + test: + steps: + - image: alpine + commands: + - npm test \ No newline at end of file diff --git a/schema/testdata/pipeline/fail/ruleset_invalid_matcher.yml b/schema/testdata/pipeline/fail/ruleset_invalid_matcher.yml new file mode 100644 index 000000000..ebefd382a --- /dev/null +++ b/schema/testdata/pipeline/fail/ruleset_invalid_matcher.yml @@ -0,0 +1,8 @@ +version: "1" + +steps: + - name: test + image: python + ruleset: + matcher: invalid_matcher + branch: [main] \ No newline at end of file diff --git a/schema/testdata/pipeline/fail/ruleset_invalid_status.yml b/schema/testdata/pipeline/fail/ruleset_invalid_status.yml new file mode 100644 index 000000000..c82f15ef4 --- /dev/null +++ b/schema/testdata/pipeline/fail/ruleset_invalid_status.yml @@ -0,0 +1,10 @@ +version: "1" + +steps: + - name: deploy + image: alpine + ruleset: + if: + status: [pending] + unless: + branch: main \ No newline at end of file diff --git a/schema/testdata/pipeline/fail/ruleset_invalid_values.yml b/schema/testdata/pipeline/fail/ruleset_invalid_values.yml new file mode 100644 index 000000000..ac14ac7e4 --- /dev/null +++ b/schema/testdata/pipeline/fail/ruleset_invalid_values.yml @@ -0,0 +1,9 @@ +version: "1" + +steps: + - name: build + image: node + ruleset: + if: + event: invalid_event + operator: invalid \ No newline at end of file diff --git a/schema/testdata/pipeline/fail/stages_and_steps.yml b/schema/testdata/pipeline/fail/stages_and_steps.yml new file mode 100644 index 000000000..1418f4daf --- /dev/null +++ b/schema/testdata/pipeline/fail/stages_and_steps.yml @@ -0,0 +1,15 @@ +version: "1" + +stages: + test: + steps: + - name: test + image: alpine:latest + commands: + - echo "hello world" + +steps: + - name: test + image: alpine:latest + commands: + - echo "hello world" diff --git a/schema/testdata/pipeline/pass/basic.yml b/schema/testdata/pipeline/pass/basic.yml new file mode 100644 index 000000000..6787adfe1 --- /dev/null +++ b/schema/testdata/pipeline/pass/basic.yml @@ -0,0 +1,7 @@ +version: "1" + +steps: + - name: test + image: alpine:latest + commands: + - echo "hello world" diff --git a/schema/testdata/pipeline/pass/complex.yml b/schema/testdata/pipeline/pass/complex.yml new file mode 100644 index 000000000..c36e71f48 --- /dev/null +++ b/schema/testdata/pipeline/pass/complex.yml @@ -0,0 +1,59 @@ +version: "1" + +step_image: &step_image + image: something + +templates: + - name: go + source: github.com/octocat/hello-world/.vela/build.yml + format: go + type: github + +metadata: + clone: false + +worker: + flavor: large + +stages: + greeting: + steps: + - name: Greeting + secrets: [ docker_username ] + image: alpine + commands: + - echo "Hello, World" + - name: Template + template: + name: go + vars: + image: golang:latest + + welcome: + steps: + - name: Welcome + <<: *step_image + ruleset: + unless: + event: push + branch: main + if: + event: pull_request + continue: true + commands: | + echo "Welcome to the Vela docs" + go build something + + goodbye: + # will wait for greeting and welcome to finish + needs: [greeting, welcome] + steps: + - name: Goodbye + image: alpine + commands: + - echo "Goodbye, World" +secrets: + - name: docker_username + key: go-vela/docs/username + engine: native + type: repo \ No newline at end of file diff --git a/schema/testdata/pipeline/pass/ruleset_if.yml b/schema/testdata/pipeline/pass/ruleset_if.yml new file mode 100644 index 000000000..506eb4a1a --- /dev/null +++ b/schema/testdata/pipeline/pass/ruleset_if.yml @@ -0,0 +1,10 @@ +version: "1" + +steps: + - name: deploy + image: alpine + ruleset: + if: + event: [deployment, push] + branch: main + operator: and \ No newline at end of file diff --git a/schema/testdata/pipeline/pass/ruleset_path.yml b/schema/testdata/pipeline/pass/ruleset_path.yml new file mode 100644 index 000000000..59276f0f9 --- /dev/null +++ b/schema/testdata/pipeline/pass/ruleset_path.yml @@ -0,0 +1,10 @@ +version: "1" + +steps: + - name: test + image: golang + ruleset: + branch: feature/* + event: pull_request + path: ["src/*/*.go"] + matcher: filepath \ No newline at end of file diff --git a/schema/testdata/pipeline/pass/ruleset_status.yml b/schema/testdata/pipeline/pass/ruleset_status.yml new file mode 100644 index 000000000..3c7d7ff42 --- /dev/null +++ b/schema/testdata/pipeline/pass/ruleset_status.yml @@ -0,0 +1,9 @@ +version: "1" + +steps: + - name: notify + image: slack + ruleset: + unless: + status: success + continue: true \ No newline at end of file diff --git a/schema/testdata/pipeline/pass/stages.yml b/schema/testdata/pipeline/pass/stages.yml new file mode 100644 index 000000000..8fe7820c6 --- /dev/null +++ b/schema/testdata/pipeline/pass/stages.yml @@ -0,0 +1,20 @@ +version: "1" + +stages: + test: + steps: + - name: test + image: alpine:latest + commands: + - echo "hello world" + - name: test2 + image: alpine:latest + commands: + - echo "hello world" + + test2: + steps: + - name: test + image: alpine:latest + commands: + - echo "hello world" \ No newline at end of file