From 1b95cf6ffa26197e174c348a493cdef9c1feb689 Mon Sep 17 00:00:00 2001 From: Colin Dean Date: Mon, 28 Aug 2023 16:20:11 -0400 Subject: [PATCH 1/8] Increase Starlark execution limit, abstract limit resolver 5,000 was too few to enable the example added to the testdata to work, so was 6,000. I chose 7,500 arbitrarily after a test at 10,000 and both worked. In the long term, this should probably be configurable so as not to require recompilation. For now, this kicks the can down the road while allowing this build matrix use case to exist. --- compiler/template/starlark/render.go | 17 +- compiler/template/starlark/render_test.go | 6 +- .../starlark/testdata/build/large/build.star | 202 ++++++++++++++++++ 3 files changed, 219 insertions(+), 6 deletions(-) create mode 100644 compiler/template/starlark/testdata/build/large/build.star diff --git a/compiler/template/starlark/render.go b/compiler/template/starlark/render.go index 0be1063f5..96b2f1b3d 100644 --- a/compiler/template/starlark/render.go +++ b/compiler/template/starlark/render.go @@ -38,7 +38,7 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM // arbitrarily limiting the steps of the thread to 5000 to help prevent infinite loops // may need to further investigate spawning a separate POSIX process if user input is problematic // see https://github.com/google/starlark-go/issues/160#issuecomment-466794230 for further details - thread.SetMaxExecutionSteps(5000) + thread.SetMaxExecutionSteps(GetStarlarkExecutionStepLimit()) predeclared := starlark.StringDict{"struct": starlark.NewBuiltin("struct", starlarkstruct.Make)} @@ -136,6 +136,15 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM return &types.Build{Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment}, nil } +// GetStarlarkExecutionStepLimit may eventually look up config or calculate it +func GetStarlarkExecutionStepLimit() uint64 { + // arbitrarily limiting the steps of the thread to help prevent infinite loops + // may need to further investigate spawning a separate POSIX process if user input is problematic + // see https://github.com/google/starlark-go/issues/160#issuecomment-466794230 for further details + // This value was previously 5000 and that inhibited a four-dimensional build matrix from working. + return 7500 +} + // RenderBuild renders the templated build. // //nolint:lll // ignore function length due to input args @@ -143,10 +152,8 @@ func RenderBuild(tmpl string, b string, envs map[string]string, variables map[st config := new(types.Build) thread := &starlark.Thread{Name: "templated-base"} - // arbitrarily limiting the steps of the thread to 5000 to help prevent infinite loops - // may need to further investigate spawning a separate POSIX process if user input is problematic - // see https://github.com/google/starlark-go/issues/160#issuecomment-466794230 for further details - thread.SetMaxExecutionSteps(5000) + + thread.SetMaxExecutionSteps(GetStarlarkExecutionStepLimit()) predeclared := starlark.StringDict{"struct": starlark.NewBuiltin("struct", starlarkstruct.Make)} diff --git a/compiler/template/starlark/render_test.go b/compiler/template/starlark/render_test.go index 8cc089e47..c602bac38 100644 --- a/compiler/template/starlark/render_test.go +++ b/compiler/template/starlark/render_test.go @@ -92,6 +92,8 @@ func TestStarlark_Render(t *testing.T) { } func TestNative_RenderBuild(t *testing.T) { + noWantFile := "none" + type args struct { velaFile string } @@ -106,6 +108,7 @@ func TestNative_RenderBuild(t *testing.T) { {"stages", args{velaFile: "testdata/build/basic_stages/build.star"}, "testdata/build/basic_stages/want.yml", false}, {"conditional match", args{velaFile: "testdata/build/conditional/build.star"}, "testdata/build/conditional/want.yml", false}, {"steps, with structs", args{velaFile: "testdata/build/with_struct/build.star"}, "testdata/build/with_struct/want.yml", false}, + {"large build to stress execution steps", args{velaFile: "testdata/build/large/build.star"}, noWantFile, false}, } for _, tt := range tests { @@ -118,13 +121,14 @@ func TestNative_RenderBuild(t *testing.T) { got, err := RenderBuild("build", string(sFile), map[string]string{ "VELA_REPO_FULL_NAME": "octocat/hello-world", "VELA_BUILD_BRANCH": "master", + "VELA_REPO_ORG": "octocat", }, map[string]interface{}{}) if (err != nil) != tt.wantErr { t.Errorf("RenderBuild() error = %v, wantErr %v", err, tt.wantErr) return } - if tt.wantErr != true { + if tt.wantErr != true && tt.wantFile != noWantFile { wFile, err := os.ReadFile(tt.wantFile) if err != nil { t.Error(err) diff --git a/compiler/template/starlark/testdata/build/large/build.star b/compiler/template/starlark/testdata/build/large/build.star new file mode 100644 index 000000000..5ffbc482a --- /dev/null +++ b/compiler/template/starlark/testdata/build/large/build.star @@ -0,0 +1,202 @@ +###### +## Setup the build matrix with the base versions a human will maintain. +###### + +DISTRO_WITH_VERSIONS = { + # n.b. these reduce to DockerHub tags + # https://hub.docker.com/_/python/tags?name=alpine + # https://endoflife.date/alpine + 'alpine': [ + '3.17', # EOL 22 Nov 2024 + '3.18' # EOL 09 May 2025 + ], + # https://hub.docker.com/_/python/tags?name=slim + # https://endoflife.date/debian + 'debian': [ + 'slim-bullseye', # EOL 30 Jun 2026 + 'slim-bookworm' # EOL 10 Jun 2028 + ] +} +PYTHON_VERSIONS = [ + '3.8', + '3.9', + '3.10', + '3.11' +] +POETRY_VERSIONS = [ + '1.6.1' +] + +KANIKO_IMAGE = 'target/vela-kaniko:latest' +HADOLINT_IMAGE = 'hadolint/hadolint:v2.12.0-alpine' + + +## The base Docker container build step's config for push builds +def base(): + return { + 'image': KANIKO_IMAGE, + 'ruleset': { + 'event': 'push', + 'branch': 'main' + }, + 'pull': 'not_present', + 'secrets': [ + { + 'source': 'artifactory_password', + 'target': 'docker_password' + } + ] + } + + +## The base Docker container plugin params for push builds +## +## These are parameters passed to Kaniko. +def base_params(): + return { + 'username': 'ibuildallthings', + 'registry': 'docker.example.com', + 'repo': 'docker.example.com/app/multibuild' + } + + +## The step config for pull request builds +def pull_request(): + pr = base() + pr['ruleset']['event'] = 'pull_request' + pr['ruleset'].pop('branch') + return pr + + +## The Kaniko params for pull request builds +def pull_request_params(): + prp = base_params() + prp['dry_run'] = True + return prp + + +## Define a linting stage that uses Hadolint inside of a Make task +## +## This keeps our Dockerfiles tidy and compliant with conventions +def stage_linting(): + return { + 'linting': { + 'steps': [{ + 'name': 'check-docker', + 'image': HADOLINT_IMAGE, + 'pull': 'not_present', + 'commands': [ + 'time apk add --no-cache make', + 'time make check-docker' + ] + }] + } + } + + +## Build stages comprised of a step for push and pull_request builds +def stage_build_tuple(distro, distro_version, python_version, poetry_version): + pr = build_template("build", distro, distro_version, python_version, poetry_version, pull_request(), pull_request_params()) + base_step = build_template("publish", distro, distro_version, python_version, poetry_version, base(), base_params()) + combined = base_step | pr + return combined + + +## Build a single stage for a build tuple, with its base step config and plugin parameters +def build_template(step_name, distro, distro_version, python_version, poetry_version, step_def_base, step_def_params): + return { + ('python_%s_%s_%s %s' % (python_version, distro, distro_version, step_def_base['ruleset']['event'])): { + 'steps': [step_def_base | { + 'name': ('%s python-%s %s %s' % (step_name, python_version, distro, distro_version)), + 'parameters': step_def_params | { + 'dockerfile': ('python-%s.Dockerfile' % distro), + 'build_args': [ + 'PYTHON_VERSION=%s' % python_version, + '%s_VERSION=%s' % (distro.upper(), distro_version), + 'POETRY_VERSION=%s' % poetry_version + ], + 'tags': [ + '%s-%s-%s-%s' % (python_version, distro, distro_version, poetry_version), + '%s-%s-%s' % (python_version, distro, distro_version), + '%s-%s' % (python_version, distro) + ] + } + }] + } + } + + +## Define a stage that uses the Slack template +def stage_slack_notify(needs): + return { + 'slack': { + 'needs': needs, + 'steps': [{ + 'name': 'slack', + 'template': { + 'name': 'slack' + } + }] + } + } + + +## Builds the build matrix in the form of list of tuples from the constants defined at the top of the file +def build_matrix(): + BUILD_MATRIX = [] + for poetry_version in POETRY_VERSIONS: + for python_version in PYTHON_VERSIONS: + for distro in DISTRO_WITH_VERSIONS: + for distro_version in DISTRO_WITH_VERSIONS[distro]: + BUILD_MATRIX.append((distro, + distro_version, + python_version, + poetry_version)) + return BUILD_MATRIX + + +## Construct a secret +def secret(name, key, secret_type, engine='native'): + return {'name': name, 'key': key, 'engine': engine, 'type': secret_type} + + +## Construct a template +def template(name, source, version=None, template_type='github'): + real_source = '%s@%s' % (source, version) if version else source + return { + 'name': name, + 'source': real_source, + 'type': template_type + } + +## The main method, the real deal. +## +## Vela actually calls this function, its return is what Vela uses. +def main(ctx): + # Retrieve the org dynamically since we're using some org secrets + vela_repo_org = ctx['vela']['repo']['org'] if 'vela' in ctx else "UNKNOWN-ORG" + + # Build the stages from the build matrix + build_stages = {} + for (distro, distro_version, python_version, poetry_version) in build_matrix(): + build_stages = build_stages | (stage_build_tuple(distro, distro_version, python_version, poetry_version)) + + # assemble the stage list with the bookends of linting and notifications in place + stages = stage_linting() | build_stages | stage_slack_notify(build_stages.keys()) + + # Build the final output + final = { + 'version': '1', + 'templates': [ + template(name='slack', + source='git.example.com/vela/vela-templates/slack/slack.yml') + ], + 'stages': stages, + 'secrets': [ + secret('artifactory_password','platform/vela-secrets/artifactory_password_for_ibuildallthings', 'shared'), + secret('slack_webhook', vela_repo_org + '/slack_webhook', 'org') + ] + } + + return final + From 3547e2ab210cd58e049d343b050fd33e035d48e9 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Tue, 29 Aug 2023 10:04:53 -0400 Subject: [PATCH 2/8] refactor(db): return user on created and updated (#935) --- api/admin/user.go | 4 +-- api/auth/get_token.go | 4 +-- api/auth/logout.go | 2 +- api/user/create.go | 5 +-- api/user/create_token.go | 2 +- api/user/delete_token.go | 2 +- api/user/update.go | 5 +-- api/user/update_current.go | 12 +------- database/integration_test.go | 9 ++---- database/user/count_test.go | 4 +-- database/user/create.go | 19 +++++++----- database/user/create_test.go | 7 ++++- database/user/delete_test.go | 2 +- database/user/get_name_test.go | 2 +- database/user/get_test.go | 2 +- database/user/interface.go | 4 +-- database/user/list_lite_test.go | 4 +-- database/user/list_test.go | 4 +-- database/user/update.go | 19 +++++++----- database/user/update_test.go | 9 ++++-- internal/token/refresh_test.go | 4 +-- router/middleware/claims/claims_test.go | 2 +- router/middleware/perm/perm_test.go | 34 ++++++++++----------- router/middleware/pipeline/pipeline_test.go | 2 +- router/middleware/user/user_test.go | 2 +- 25 files changed, 82 insertions(+), 83 deletions(-) diff --git a/api/admin/user.go b/api/admin/user.go index f7ef70ba3..c0c2aacfd 100644 --- a/api/admin/user.go +++ b/api/admin/user.go @@ -66,7 +66,7 @@ func UpdateUser(c *gin.Context) { } // send API call to update the user - err = database.FromContext(c).UpdateUser(input) + u, err := database.FromContext(c).UpdateUser(input) if err != nil { retErr := fmt.Errorf("unable to update user %d: %w", input.GetID(), err) @@ -75,5 +75,5 @@ func UpdateUser(c *gin.Context) { return } - c.JSON(http.StatusOK, input) + c.JSON(http.StatusOK, u) } diff --git a/api/auth/get_token.go b/api/auth/get_token.go index 29829f072..2b6d7da66 100644 --- a/api/auth/get_token.go +++ b/api/auth/get_token.go @@ -121,7 +121,7 @@ func GetAuthToken(c *gin.Context) { u.SetRefreshToken(rt) // send API call to create the user in the database - err = database.FromContext(c).CreateUser(u) + _, err = database.FromContext(c).CreateUser(u) if err != nil { retErr := fmt.Errorf("unable to create user %s: %w", u.GetName(), err) @@ -154,7 +154,7 @@ func GetAuthToken(c *gin.Context) { u.SetRefreshToken(rt) // send API call to update the user in the database - err = database.FromContext(c).UpdateUser(u) + _, err = database.FromContext(c).UpdateUser(u) if err != nil { retErr := fmt.Errorf("unable to update user %s: %w", u.GetName(), err) diff --git a/api/auth/logout.go b/api/auth/logout.go index 8accdf42e..1bc8f8d58 100644 --- a/api/auth/logout.go +++ b/api/auth/logout.go @@ -75,7 +75,7 @@ func Logout(c *gin.Context) { u.SetRefreshToken("") // send API call to update the user in the database - err = database.FromContext(c).UpdateUser(u) + _, err = database.FromContext(c).UpdateUser(u) if err != nil { retErr := fmt.Errorf("unable to update user %s: %w", u.GetName(), err) diff --git a/api/user/create.go b/api/user/create.go index d6348810e..bb7823546 100644 --- a/api/user/create.go +++ b/api/user/create.go @@ -72,7 +72,7 @@ func CreateUser(c *gin.Context) { }).Infof("creating new user %s", input.GetName()) // send API call to create the user - err = database.FromContext(c).CreateUser(input) + user, err := database.FromContext(c).CreateUser(input) if err != nil { retErr := fmt.Errorf("unable to create user: %w", err) @@ -81,8 +81,5 @@ func CreateUser(c *gin.Context) { return } - // send API call to capture the created user - user, _ := database.FromContext(c).GetUserForName(input.GetName()) - c.JSON(http.StatusCreated, user) } diff --git a/api/user/create_token.go b/api/user/create_token.go index dee7044cb..427fab34d 100644 --- a/api/user/create_token.go +++ b/api/user/create_token.go @@ -65,7 +65,7 @@ func CreateToken(c *gin.Context) { u.SetRefreshToken(rt) // send API call to update the user - err = database.FromContext(c).UpdateUser(u) + _, err = database.FromContext(c).UpdateUser(u) if err != nil { retErr := fmt.Errorf("unable to update user %s: %w", u.GetName(), err) diff --git a/api/user/delete_token.go b/api/user/delete_token.go index 22fa9724d..05a7a3699 100644 --- a/api/user/delete_token.go +++ b/api/user/delete_token.go @@ -65,7 +65,7 @@ func DeleteToken(c *gin.Context) { u.SetRefreshToken(rt) // send API call to update the user - err = database.FromContext(c).UpdateUser(u) + _, err = database.FromContext(c).UpdateUser(u) if err != nil { retErr := fmt.Errorf("unable to update user %s: %w", u.GetName(), err) diff --git a/api/user/update.go b/api/user/update.go index d2fb631ab..21a089e04 100644 --- a/api/user/update.go +++ b/api/user/update.go @@ -108,7 +108,7 @@ func UpdateUser(c *gin.Context) { } // send API call to update the user - err = database.FromContext(c).UpdateUser(u) + u, err = database.FromContext(c).UpdateUser(u) if err != nil { retErr := fmt.Errorf("unable to update user %s: %w", user, err) @@ -117,8 +117,5 @@ func UpdateUser(c *gin.Context) { return } - // send API call to capture the updated user - u, _ = database.FromContext(c).GetUserForName(user) - c.JSON(http.StatusOK, u) } diff --git a/api/user/update_current.go b/api/user/update_current.go index ed861ae61..0eeb4e1be 100644 --- a/api/user/update_current.go +++ b/api/user/update_current.go @@ -82,7 +82,7 @@ func UpdateCurrentUser(c *gin.Context) { } // send API call to update the user - err = database.FromContext(c).UpdateUser(u) + u, err = database.FromContext(c).UpdateUser(u) if err != nil { retErr := fmt.Errorf("unable to update user %s: %w", u.GetName(), err) @@ -91,15 +91,5 @@ func UpdateCurrentUser(c *gin.Context) { return } - // send API call to capture the updated user - u, err = database.FromContext(c).GetUserForName(u.GetName()) - if err != nil { - retErr := fmt.Errorf("unable to get updated user %s: %w", u.GetName(), err) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - c.JSON(http.StatusOK, u) } diff --git a/database/integration_test.go b/database/integration_test.go index 30d6ff696..f982b25cb 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -1656,7 +1656,7 @@ func testUsers(t *testing.T, db Interface, resources *Resources) { // create the users for _, user := range resources.Users { - err := db.CreateUser(user) + _, err := db.CreateUser(user) if err != nil { t.Errorf("unable to create user %d: %v", user.GetID(), err) } @@ -1711,16 +1711,11 @@ func testUsers(t *testing.T, db Interface, resources *Resources) { // update the users for _, user := range resources.Users { user.SetActive(false) - err = db.UpdateUser(user) + got, err := db.UpdateUser(user) if err != nil { t.Errorf("unable to update user %d: %v", user.GetID(), err) } - // lookup the user by ID - got, err := db.GetUser(user.GetID()) - if err != nil { - t.Errorf("unable to get user %d by ID: %v", user.GetID(), err) - } if !reflect.DeepEqual(got, user) { t.Errorf("GetUser() is %v, want %v", got, user) } diff --git a/database/user/count_test.go b/database/user/count_test.go index be7cb6437..be4c77f0c 100644 --- a/database/user/count_test.go +++ b/database/user/count_test.go @@ -37,12 +37,12 @@ func TestUser_Engine_CountUsers(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateUser(_userOne) + _, err := _sqlite.CreateUser(_userOne) if err != nil { t.Errorf("unable to create test user for sqlite: %v", err) } - err = _sqlite.CreateUser(_userTwo) + _, err = _sqlite.CreateUser(_userTwo) if err != nil { t.Errorf("unable to create test user for sqlite: %v", err) } diff --git a/database/user/create.go b/database/user/create.go index f527979b9..4a45f0d89 100644 --- a/database/user/create.go +++ b/database/user/create.go @@ -15,7 +15,7 @@ import ( ) // CreateUser creates a new user in the database. -func (e *engine) CreateUser(u *library.User) error { +func (e *engine) CreateUser(u *library.User) (*library.User, error) { e.logger.WithFields(logrus.Fields{ "user": u.GetName(), }).Tracef("creating user %s in the database", u.GetName()) @@ -30,7 +30,7 @@ func (e *engine) CreateUser(u *library.User) error { // https://pkg.go.dev/github.com/go-vela/types/database#User.Validate err := user.Validate() if err != nil { - return err + return nil, err } // encrypt the fields for the user @@ -38,12 +38,17 @@ func (e *engine) CreateUser(u *library.User) error { // https://pkg.go.dev/github.com/go-vela/types/database#User.Encrypt err = user.Encrypt(e.config.EncryptionKey) if err != nil { - return fmt.Errorf("unable to encrypt user %s: %w", u.GetName(), err) + return nil, fmt.Errorf("unable to encrypt user %s: %w", u.GetName(), err) } // send query to the database - return e.client. - Table(constants.TableUser). - Create(user). - Error + result := e.client.Table(constants.TableUser).Create(user) + + // decrypt fields to return user + err = user.Decrypt(e.config.EncryptionKey) + if err != nil { + return nil, fmt.Errorf("unable to decrypt user %s: %w", u.GetName(), err) + } + + return user.ToLibrary(), result.Error } diff --git a/database/user/create_test.go b/database/user/create_test.go index 12de80f96..815b3ef88 100644 --- a/database/user/create_test.go +++ b/database/user/create_test.go @@ -5,6 +5,7 @@ package user import ( + "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -55,7 +56,7 @@ VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING "id"`). // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err := test.database.CreateUser(_user) + got, err := test.database.CreateUser(_user) if test.failure { if err == nil { @@ -68,6 +69,10 @@ VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING "id"`). if err != nil { t.Errorf("CreateUser for %s returned err: %v", test.name, err) } + + if !reflect.DeepEqual(got, _user) { + t.Errorf("CreateUser for %s returned %s, want %s", test.name, got, _user) + } }) } } diff --git a/database/user/delete_test.go b/database/user/delete_test.go index 0dec0a6b2..937df9bc3 100644 --- a/database/user/delete_test.go +++ b/database/user/delete_test.go @@ -29,7 +29,7 @@ func TestUser_Engine_DeleteUser(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateUser(_user) + _, err := _sqlite.CreateUser(_user) if err != nil { t.Errorf("unable to create test user for sqlite: %v", err) } diff --git a/database/user/get_name_test.go b/database/user/get_name_test.go index 4b0a6446b..cb93ffd0b 100644 --- a/database/user/get_name_test.go +++ b/database/user/get_name_test.go @@ -35,7 +35,7 @@ func TestUser_Engine_GetUserForName(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateUser(_user) + _, err := _sqlite.CreateUser(_user) if err != nil { t.Errorf("unable to create test user for sqlite: %v", err) } diff --git a/database/user/get_test.go b/database/user/get_test.go index 4593ce48f..2ecac65f2 100644 --- a/database/user/get_test.go +++ b/database/user/get_test.go @@ -35,7 +35,7 @@ func TestUser_Engine_GetUser(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateUser(_user) + _, err := _sqlite.CreateUser(_user) if err != nil { t.Errorf("unable to create test user for sqlite: %v", err) } diff --git a/database/user/interface.go b/database/user/interface.go index ad5986fad..dc67c933c 100644 --- a/database/user/interface.go +++ b/database/user/interface.go @@ -29,7 +29,7 @@ type UserInterface interface { // CountUsers defines a function that gets the count of all users. CountUsers() (int64, error) // CreateUser defines a function that creates a new user. - CreateUser(*library.User) error + CreateUser(*library.User) (*library.User, error) // DeleteUser defines a function that deletes an existing user. DeleteUser(*library.User) error // GetUser defines a function that gets a user by ID. @@ -41,5 +41,5 @@ type UserInterface interface { // ListLiteUsers defines a function that gets a lite list of users. ListLiteUsers(int, int) ([]*library.User, int64, error) // UpdateUser defines a function that updates an existing user. - UpdateUser(*library.User) error + UpdateUser(*library.User) (*library.User, error) } diff --git a/database/user/list_lite_test.go b/database/user/list_lite_test.go index 4c44bc20b..6dd88aa75 100644 --- a/database/user/list_lite_test.go +++ b/database/user/list_lite_test.go @@ -49,12 +49,12 @@ func TestUser_Engine_ListLiteUsers(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateUser(_userOne) + _, err := _sqlite.CreateUser(_userOne) if err != nil { t.Errorf("unable to create test user for sqlite: %v", err) } - err = _sqlite.CreateUser(_userTwo) + _, err = _sqlite.CreateUser(_userTwo) if err != nil { t.Errorf("unable to create test user for sqlite: %v", err) } diff --git a/database/user/list_test.go b/database/user/list_test.go index 61293d44c..9e088e72f 100644 --- a/database/user/list_test.go +++ b/database/user/list_test.go @@ -49,12 +49,12 @@ func TestUser_Engine_ListUsers(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateUser(_userOne) + _, err := _sqlite.CreateUser(_userOne) if err != nil { t.Errorf("unable to create test user for sqlite: %v", err) } - err = _sqlite.CreateUser(_userTwo) + _, err = _sqlite.CreateUser(_userTwo) if err != nil { t.Errorf("unable to create test user for sqlite: %v", err) } diff --git a/database/user/update.go b/database/user/update.go index c7efc5e7f..2a215a75f 100644 --- a/database/user/update.go +++ b/database/user/update.go @@ -15,7 +15,7 @@ import ( ) // UpdateUser updates an existing user in the database. -func (e *engine) UpdateUser(u *library.User) error { +func (e *engine) UpdateUser(u *library.User) (*library.User, error) { e.logger.WithFields(logrus.Fields{ "user": u.GetName(), }).Tracef("updating user %s in the database", u.GetName()) @@ -30,7 +30,7 @@ func (e *engine) UpdateUser(u *library.User) error { // https://pkg.go.dev/github.com/go-vela/types/database#User.Validate err := user.Validate() if err != nil { - return err + return nil, err } // encrypt the fields for the user @@ -38,12 +38,17 @@ func (e *engine) UpdateUser(u *library.User) error { // https://pkg.go.dev/github.com/go-vela/types/database#User.Encrypt err = user.Encrypt(e.config.EncryptionKey) if err != nil { - return fmt.Errorf("unable to encrypt user %s: %w", u.GetName(), err) + return nil, fmt.Errorf("unable to encrypt user %s: %w", u.GetName(), err) } // send query to the database - return e.client. - Table(constants.TableUser). - Save(user). - Error + result := e.client.Table(constants.TableUser).Save(user) + + // decrypt fields to return user + err = user.Decrypt(e.config.EncryptionKey) + if err != nil { + return nil, fmt.Errorf("unable to decrypt user %s: %w", u.GetName(), err) + } + + return user.ToLibrary(), result.Error } diff --git a/database/user/update_test.go b/database/user/update_test.go index 4253ac435..44f2cb0fe 100644 --- a/database/user/update_test.go +++ b/database/user/update_test.go @@ -5,6 +5,7 @@ package user import ( + "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -31,7 +32,7 @@ WHERE "id" = $8`). _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateUser(_user) + _, err := _sqlite.CreateUser(_user) if err != nil { t.Errorf("unable to create test user for sqlite: %v", err) } @@ -57,7 +58,7 @@ WHERE "id" = $8`). // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err = test.database.UpdateUser(_user) + got, err := test.database.UpdateUser(_user) if test.failure { if err == nil { @@ -70,6 +71,10 @@ WHERE "id" = $8`). if err != nil { t.Errorf("UpdateUser for %s returned err: %v", test.name, err) } + + if !reflect.DeepEqual(got, _user) { + t.Errorf("UpdateUser for %s returned %s, want %s", test.name, got, _user) + } }) } } diff --git a/internal/token/refresh_test.go b/internal/token/refresh_test.go index e306f9ed6..ee9a30c4d 100644 --- a/internal/token/refresh_test.go +++ b/internal/token/refresh_test.go @@ -55,7 +55,7 @@ func TestTokenManager_Refresh(t *testing.T) { db.Close() }() - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) // set up context gin.SetMode(gin.TestMode) @@ -114,7 +114,7 @@ func TestTokenManager_Refresh_Expired(t *testing.T) { db.Close() }() - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) // set up context gin.SetMode(gin.TestMode) diff --git a/router/middleware/claims/claims_test.go b/router/middleware/claims/claims_test.go index da28eb319..fd1e0ccfd 100644 --- a/router/middleware/claims/claims_test.go +++ b/router/middleware/claims/claims_test.go @@ -278,7 +278,7 @@ func TestClaims_Establish_BadToken(t *testing.T) { db.Close() }() - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) mto := &token.MintTokenOpts{ User: u, diff --git a/router/middleware/perm/perm_test.go b/router/middleware/perm/perm_test.go index b1495ab19..2b384250e 100644 --- a/router/middleware/perm/perm_test.go +++ b/router/middleware/perm/perm_test.go @@ -64,7 +64,7 @@ func TestPerm_MustPlatformAdmin(t *testing.T) { db.Close() }() - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) // setup context gin.SetMode(gin.TestMode) @@ -152,7 +152,7 @@ func TestPerm_MustPlatformAdmin_NotAdmin(t *testing.T) { db.Close() }() - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/admin/users", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -279,7 +279,7 @@ func TestPerm_MustWorkerRegisterToken_PlatAdmin(t *testing.T) { db.Close() }() - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -544,7 +544,7 @@ func TestPerm_MustBuildAccess_PlatAdmin(t *testing.T) { _, _ = db.CreateRepo(_context.TODO(), r) _, _ = db.CreateBuild(ctx, b) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar/builds/1", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -963,7 +963,7 @@ func TestPerm_MustAdmin(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -1061,7 +1061,7 @@ func TestPerm_MustAdmin_PlatAdmin(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -1159,7 +1159,7 @@ func TestPerm_MustAdmin_NotAdmin(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -1257,7 +1257,7 @@ func TestPerm_MustWrite(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -1355,7 +1355,7 @@ func TestPerm_MustWrite_PlatAdmin(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -1453,7 +1453,7 @@ func TestPerm_MustWrite_RepoAdmin(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -1551,7 +1551,7 @@ func TestPerm_MustWrite_NotWrite(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -1649,7 +1649,7 @@ func TestPerm_MustRead(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -1747,7 +1747,7 @@ func TestPerm_MustRead_PlatAdmin(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -1931,7 +1931,7 @@ func TestPerm_MustRead_RepoAdmin(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -2029,7 +2029,7 @@ func TestPerm_MustRead_RepoWrite(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -2127,7 +2127,7 @@ func TestPerm_MustRead_RepoPublic(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -2225,7 +2225,7 @@ func TestPerm_MustRead_NotRead(t *testing.T) { }() _, _ = db.CreateRepo(_context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) diff --git a/router/middleware/pipeline/pipeline_test.go b/router/middleware/pipeline/pipeline_test.go index 0fc1e8b29..ead046065 100644 --- a/router/middleware/pipeline/pipeline_test.go +++ b/router/middleware/pipeline/pipeline_test.go @@ -295,7 +295,7 @@ func TestPipeline_Establish_NoPipeline(t *testing.T) { }() _, _ = db.CreateRepo(context.TODO(), r) - _ = db.CreateUser(u) + _, _ = db.CreateUser(u) // setup context gin.SetMode(gin.TestMode) diff --git a/router/middleware/user/user_test.go b/router/middleware/user/user_test.go index 0ea18dfcc..c90cb53fc 100644 --- a/router/middleware/user/user_test.go +++ b/router/middleware/user/user_test.go @@ -96,7 +96,7 @@ func TestUser_Establish(t *testing.T) { db.Close() }() - _ = db.CreateUser(want) + _, _ = db.CreateUser(want) // setup context gin.SetMode(gin.TestMode) From 85a477d5a10eac4fa6369afba888ad03f387397b Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 29 Aug 2023 12:12:18 -0500 Subject: [PATCH 3/8] enhance(starlark): make execution step limit configurable --- cmd/vela-server/main.go | 6 + compiler/native/native.go | 6 + compiler/native/parse.go | 2 +- compiler/template/starlark/render.go | 9 +- compiler/template/starlark/render_test.go | 115 ++- .../starlark/testdata/build/large/build.star | 1 - .../starlark/testdata/build/large/want.yml | 946 ++++++++++++++++++ docker-compose.yml | 1 + scm/github/webhook.go | 1 + 9 files changed, 1071 insertions(+), 16 deletions(-) create mode 100644 compiler/template/starlark/testdata/build/large/want.yml diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 788f6c26b..94fe7016b 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -175,6 +175,12 @@ func main() { Name: "github-token", Usage: "github token, used by compiler, for pulling registry templates", }, + &cli.Uint64Flag{ + EnvVars: []string{"VELA_COMPILER_STARLARK_EXEC_LIMIT", "COMPILER_STARLARK_EXEC_LIMIT"}, + Name: "compiler-starlark-exec-limit", + Usage: "set the starlark execution step limit for compiling starlark pipelines", + Value: 7500, + }, &cli.StringFlag{ EnvVars: []string{"VELA_MODIFICATION_ADDR", "MODIFICATION_ADDR"}, Name: "modification-addr", diff --git a/compiler/native/native.go b/compiler/native/native.go index c394af933..d2b5ebd19 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -33,6 +33,7 @@ type client struct { ModificationService ModificationConfig CloneImage string TemplateDepth int + StarlarkExecLimit uint64 build *library.Build comment string @@ -73,8 +74,12 @@ func New(ctx *cli.Context) (*client, error) { // set the clone image to use for the injected clone step c.CloneImage = ctx.String("clone-image") + // set the template depth to use for nested templates c.TemplateDepth = ctx.Int("max-template-depth") + // set the starlark execution step limit for compiling starlark pipelines + c.StarlarkExecLimit = ctx.Uint64("compiler-starlark-exec-limit") + if ctx.Bool("github-driver") { logrus.Tracef("setting up Private GitHub Client for %s", ctx.String("github-url")) // setup private github service @@ -115,6 +120,7 @@ func (c *client) Duplicate() compiler.Engine { cc.ModificationService = c.ModificationService cc.CloneImage = c.CloneImage cc.TemplateDepth = c.TemplateDepth + cc.StarlarkExecLimit = c.StarlarkExecLimit return cc } diff --git a/compiler/native/parse.go b/compiler/native/parse.go index b3ff020f4..96e9ec25a 100644 --- a/compiler/native/parse.go +++ b/compiler/native/parse.go @@ -73,7 +73,7 @@ func (c *client) Parse(v interface{}, pipelineType string, template *types.Templ // capture the raw pipeline configuration raw = []byte(parsedRaw) - p, err = starlark.RenderBuild(template.Name, parsedRaw, c.EnvironmentBuild(), template.Variables) + p, err = starlark.RenderBuild(template.Name, parsedRaw, c.EnvironmentBuild(), template.Variables, c.StarlarkExecLimit) if err != nil { return nil, raw, err } diff --git a/compiler/template/starlark/render.go b/compiler/template/starlark/render.go index 96b2f1b3d..194ea2885 100644 --- a/compiler/template/starlark/render.go +++ b/compiler/template/starlark/render.go @@ -8,6 +8,7 @@ import ( "bytes" "errors" "fmt" + "github.com/go-vela/types/raw" "go.starlark.net/starlarkstruct" @@ -148,12 +149,14 @@ func GetStarlarkExecutionStepLimit() uint64 { // RenderBuild renders the templated build. // //nolint:lll // ignore function length due to input args -func RenderBuild(tmpl string, b string, envs map[string]string, variables map[string]interface{}) (*types.Build, error) { +func RenderBuild(tmpl string, b string, envs map[string]string, variables map[string]interface{}, limit uint64) (*types.Build, error) { config := new(types.Build) thread := &starlark.Thread{Name: "templated-base"} - - thread.SetMaxExecutionSteps(GetStarlarkExecutionStepLimit()) + // arbitrarily limiting the steps of the thread to 5000 to help prevent infinite loops + // may need to further investigate spawning a separate POSIX process if user input is problematic + // see https://github.com/google/starlark-go/issues/160#issuecomment-466794230 for further details + thread.SetMaxExecutionSteps(limit) predeclared := starlark.StringDict{"struct": starlark.NewBuiltin("struct", starlarkstruct.Make)} diff --git a/compiler/template/starlark/render_test.go b/compiler/template/starlark/render_test.go index c602bac38..8d23b1c6a 100644 --- a/compiler/template/starlark/render_test.go +++ b/compiler/template/starlark/render_test.go @@ -6,6 +6,7 @@ package starlark import ( "os" + "strings" "testing" goyaml "github.com/buildkite/yaml" @@ -26,11 +27,51 @@ func TestStarlark_Render(t *testing.T) { wantFile string wantErr bool }{ - {"basic", args{velaFile: "testdata/step/basic/step.yml", starlarkFile: "testdata/step/basic/template.py"}, "testdata/step/basic/want.yml", false}, - {"with method", args{velaFile: "testdata/step/with_method/step.yml", starlarkFile: "testdata/step/with_method/template.star"}, "testdata/step/with_method/want.yml", false}, - {"user vars", args{velaFile: "testdata/step/with_vars/step.yml", starlarkFile: "testdata/step/with_vars/template.star"}, "testdata/step/with_vars/want.yml", false}, - {"platform vars", args{velaFile: "testdata/step/with_vars_plat/step.yml", starlarkFile: "testdata/step/with_vars_plat/template.star"}, "testdata/step/with_vars_plat/want.yml", false}, - {"cancel due to complexity", args{velaFile: "testdata/step/cancel/step.yml", starlarkFile: "testdata/step/cancel/template.star"}, "", true}, + { + name: "basic", + args: args{ + velaFile: "testdata/step/basic/step.yml", + starlarkFile: "testdata/step/basic/template.py", + }, + wantFile: "testdata/step/basic/want.yml", + wantErr: false, + }, + { + name: "with method", + args: args{ + velaFile: "testdata/step/with_method/step.yml", + starlarkFile: "testdata/step/with_method/template.star", + }, + wantFile: "testdata/step/with_method/want.yml", + wantErr: false, + }, + { + name: "user vars", + args: args{ + velaFile: "testdata/step/with_vars/step.yml", + starlarkFile: "testdata/step/with_vars/template.star", + }, + wantFile: "testdata/step/with_vars/want.yml", + wantErr: false, + }, + { + name: "platform vars", + args: args{ + velaFile: "testdata/step/with_vars_plat/step.yml", + starlarkFile: "testdata/step/with_vars_plat/template.star", + }, + wantFile: "testdata/step/with_vars_plat/want.yml", + wantErr: false, + }, + { + name: "cancel due to complexity", + args: args{ + velaFile: "testdata/step/cancel/step.yml", + starlarkFile: "testdata/step/cancel/template.star", + }, + wantFile: "", + wantErr: true, + }, } for _, tt := range tests { @@ -104,11 +145,54 @@ func TestNative_RenderBuild(t *testing.T) { wantFile string wantErr bool }{ - {"steps", args{velaFile: "testdata/build/basic/build.star"}, "testdata/build/basic/want.yml", false}, - {"stages", args{velaFile: "testdata/build/basic_stages/build.star"}, "testdata/build/basic_stages/want.yml", false}, - {"conditional match", args{velaFile: "testdata/build/conditional/build.star"}, "testdata/build/conditional/want.yml", false}, - {"steps, with structs", args{velaFile: "testdata/build/with_struct/build.star"}, "testdata/build/with_struct/want.yml", false}, - {"large build to stress execution steps", args{velaFile: "testdata/build/large/build.star"}, noWantFile, false}, + { + name: "steps", + args: args{ + velaFile: "testdata/build/basic/build.star", + }, + wantFile: "testdata/build/basic/want.yml", + wantErr: false, + }, + { + name: "stages", + args: args{ + velaFile: "testdata/build/basic_stages/build.star", + }, + wantFile: "testdata/build/basic_stages/want.yml", + wantErr: false, + }, + { + name: "conditional match", + args: args{ + velaFile: "testdata/build/conditional/build.star", + }, + wantFile: "testdata/build/conditional/want.yml", + wantErr: false, + }, + { + name: "steps, with structs", + args: args{ + velaFile: "testdata/build/with_struct/build.star", + }, + wantFile: "testdata/build/with_struct/want.yml", + wantErr: false, + }, + { + name: "large build - exec step limit good", + args: args{ + velaFile: "testdata/build/large/build.star", + }, + wantFile: "testdata/build/large/want.yml", + wantErr: false, + }, + { + name: "large build - exec step limit too low", + args: args{ + velaFile: "testdata/build/large/build.star", + }, + wantFile: "", + wantErr: true, + }, } for _, tt := range tests { @@ -118,11 +202,20 @@ func TestNative_RenderBuild(t *testing.T) { t.Error(err) } + // set execLimit based on test name + var execLimit uint64 + + if strings.EqualFold(tt.name, "large build - exec step limit too low") { + execLimit = 5000 + } else { + execLimit = 7500 + } + got, err := RenderBuild("build", string(sFile), map[string]string{ "VELA_REPO_FULL_NAME": "octocat/hello-world", "VELA_BUILD_BRANCH": "master", "VELA_REPO_ORG": "octocat", - }, map[string]interface{}{}) + }, map[string]interface{}{}, execLimit) if (err != nil) != tt.wantErr { t.Errorf("RenderBuild() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/compiler/template/starlark/testdata/build/large/build.star b/compiler/template/starlark/testdata/build/large/build.star index 5ffbc482a..b53a188bc 100644 --- a/compiler/template/starlark/testdata/build/large/build.star +++ b/compiler/template/starlark/testdata/build/large/build.star @@ -199,4 +199,3 @@ def main(ctx): } return final - diff --git a/compiler/template/starlark/testdata/build/large/want.yml b/compiler/template/starlark/testdata/build/large/want.yml new file mode 100644 index 000000000..a9aaca543 --- /dev/null +++ b/compiler/template/starlark/testdata/build/large/want.yml @@ -0,0 +1,946 @@ +version: "1" +metadata: + environment: + - steps + - services + - secrets +secrets: +- name: artifactory_password + key: platform/vela-secrets/artifactory_password_for_ibuildallthings + engine: native + type: shared +- name: slack_webhook + key: octocat/slack_webhook + engine: native + type: org +stages: + linting: + needs: [clone] + steps: + - commands: + - time apk add --no-cache make + - time make check-docker + image: hadolint/hadolint:v2.12.0-alpine + name: check-docker + pull: not_present + python_3.8_alpine_3.17 push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.8 alpine 3.17 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.8 + - ALPINE_VERSION=3.17 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.8-alpine-3.17-1.6.1 + - 3.8-alpine-3.17 + - 3.8-alpine + username: ibuildallthings + python_3.8_alpine_3.17 pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.8 alpine 3.17 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.8 + - ALPINE_VERSION=3.17 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.8-alpine-3.17-1.6.1 + - 3.8-alpine-3.17 + - 3.8-alpine + username: ibuildallthings + python_3.8_alpine_3.18 push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.8 alpine 3.18 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.8 + - ALPINE_VERSION=3.18 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.8-alpine-3.18-1.6.1 + - 3.8-alpine-3.18 + - 3.8-alpine + username: ibuildallthings + python_3.8_alpine_3.18 pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.8 alpine 3.18 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.8 + - ALPINE_VERSION=3.18 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.8-alpine-3.18-1.6.1 + - 3.8-alpine-3.18 + - 3.8-alpine + username: ibuildallthings + python_3.8_debian_slim-bullseye push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.8 debian slim-bullseye + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.8 + - DEBIAN_VERSION=slim-bullseye + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.8-debian-slim-bullseye-1.6.1 + - 3.8-debian-slim-bullseye + - 3.8-debian + username: ibuildallthings + python_3.8_debian_slim-bullseye pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.8 debian slim-bullseye + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.8 + - DEBIAN_VERSION=slim-bullseye + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.8-debian-slim-bullseye-1.6.1 + - 3.8-debian-slim-bullseye + - 3.8-debian + username: ibuildallthings + python_3.8_debian_slim-bookworm push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.8 debian slim-bookworm + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.8 + - DEBIAN_VERSION=slim-bookworm + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.8-debian-slim-bookworm-1.6.1 + - 3.8-debian-slim-bookworm + - 3.8-debian + username: ibuildallthings + python_3.8_debian_slim-bookworm pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.8 debian slim-bookworm + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.8 + - DEBIAN_VERSION=slim-bookworm + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.8-debian-slim-bookworm-1.6.1 + - 3.8-debian-slim-bookworm + - 3.8-debian + username: ibuildallthings + python_3.9_alpine_3.17 push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.9 alpine 3.17 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.9 + - ALPINE_VERSION=3.17 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.9-alpine-3.17-1.6.1 + - 3.9-alpine-3.17 + - 3.9-alpine + username: ibuildallthings + python_3.9_alpine_3.17 pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.9 alpine 3.17 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.9 + - ALPINE_VERSION=3.17 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.9-alpine-3.17-1.6.1 + - 3.9-alpine-3.17 + - 3.9-alpine + username: ibuildallthings + python_3.9_alpine_3.18 push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.9 alpine 3.18 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.9 + - ALPINE_VERSION=3.18 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.9-alpine-3.18-1.6.1 + - 3.9-alpine-3.18 + - 3.9-alpine + username: ibuildallthings + python_3.9_alpine_3.18 pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.9 alpine 3.18 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.9 + - ALPINE_VERSION=3.18 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.9-alpine-3.18-1.6.1 + - 3.9-alpine-3.18 + - 3.9-alpine + username: ibuildallthings + python_3.9_debian_slim-bullseye push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.9 debian slim-bullseye + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.9 + - DEBIAN_VERSION=slim-bullseye + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.9-debian-slim-bullseye-1.6.1 + - 3.9-debian-slim-bullseye + - 3.9-debian + username: ibuildallthings + python_3.9_debian_slim-bullseye pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.9 debian slim-bullseye + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.9 + - DEBIAN_VERSION=slim-bullseye + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.9-debian-slim-bullseye-1.6.1 + - 3.9-debian-slim-bullseye + - 3.9-debian + username: ibuildallthings + python_3.9_debian_slim-bookworm push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.9 debian slim-bookworm + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.9 + - DEBIAN_VERSION=slim-bookworm + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.9-debian-slim-bookworm-1.6.1 + - 3.9-debian-slim-bookworm + - 3.9-debian + username: ibuildallthings + python_3.9_debian_slim-bookworm pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.9 debian slim-bookworm + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.9 + - DEBIAN_VERSION=slim-bookworm + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.9-debian-slim-bookworm-1.6.1 + - 3.9-debian-slim-bookworm + - 3.9-debian + username: ibuildallthings + python_3.10_alpine_3.17 push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.10 alpine 3.17 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.10 + - ALPINE_VERSION=3.17 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.10-alpine-3.17-1.6.1 + - 3.10-alpine-3.17 + - 3.10-alpine + username: ibuildallthings + python_3.10_alpine_3.17 pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.10 alpine 3.17 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.10 + - ALPINE_VERSION=3.17 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.10-alpine-3.17-1.6.1 + - 3.10-alpine-3.17 + - 3.10-alpine + username: ibuildallthings + python_3.10_alpine_3.18 push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.10 alpine 3.18 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.10 + - ALPINE_VERSION=3.18 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.10-alpine-3.18-1.6.1 + - 3.10-alpine-3.18 + - 3.10-alpine + username: ibuildallthings + python_3.10_alpine_3.18 pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.10 alpine 3.18 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.10 + - ALPINE_VERSION=3.18 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.10-alpine-3.18-1.6.1 + - 3.10-alpine-3.18 + - 3.10-alpine + username: ibuildallthings + python_3.10_debian_slim-bullseye push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.10 debian slim-bullseye + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.10 + - DEBIAN_VERSION=slim-bullseye + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.10-debian-slim-bullseye-1.6.1 + - 3.10-debian-slim-bullseye + - 3.10-debian + username: ibuildallthings + python_3.10_debian_slim-bullseye pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.10 debian slim-bullseye + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.10 + - DEBIAN_VERSION=slim-bullseye + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.10-debian-slim-bullseye-1.6.1 + - 3.10-debian-slim-bullseye + - 3.10-debian + username: ibuildallthings + python_3.10_debian_slim-bookworm push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.10 debian slim-bookworm + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.10 + - DEBIAN_VERSION=slim-bookworm + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.10-debian-slim-bookworm-1.6.1 + - 3.10-debian-slim-bookworm + - 3.10-debian + username: ibuildallthings + python_3.10_debian_slim-bookworm pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.10 debian slim-bookworm + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.10 + - DEBIAN_VERSION=slim-bookworm + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.10-debian-slim-bookworm-1.6.1 + - 3.10-debian-slim-bookworm + - 3.10-debian + username: ibuildallthings + python_3.11_alpine_3.17 push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.11 alpine 3.17 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.11 + - ALPINE_VERSION=3.17 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.11-alpine-3.17-1.6.1 + - 3.11-alpine-3.17 + - 3.11-alpine + username: ibuildallthings + python_3.11_alpine_3.17 pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.11 alpine 3.17 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.11 + - ALPINE_VERSION=3.17 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.11-alpine-3.17-1.6.1 + - 3.11-alpine-3.17 + - 3.11-alpine + username: ibuildallthings + python_3.11_alpine_3.18 push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.11 alpine 3.18 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.11 + - ALPINE_VERSION=3.18 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.11-alpine-3.18-1.6.1 + - 3.11-alpine-3.18 + - 3.11-alpine + username: ibuildallthings + python_3.11_alpine_3.18 pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.11 alpine 3.18 + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.11 + - ALPINE_VERSION=3.18 + - POETRY_VERSION=1.6.1 + dockerfile: python-alpine.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.11-alpine-3.18-1.6.1 + - 3.11-alpine-3.18 + - 3.11-alpine + username: ibuildallthings + python_3.11_debian_slim-bullseye push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.11 debian slim-bullseye + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.11 + - DEBIAN_VERSION=slim-bullseye + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.11-debian-slim-bullseye-1.6.1 + - 3.11-debian-slim-bullseye + - 3.11-debian + username: ibuildallthings + python_3.11_debian_slim-bullseye pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.11 debian slim-bullseye + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.11 + - DEBIAN_VERSION=slim-bullseye + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.11-debian-slim-bullseye-1.6.1 + - 3.11-debian-slim-bullseye + - 3.11-debian + username: ibuildallthings + python_3.11_debian_slim-bookworm push: + needs: [clone] + steps: + - ruleset: + if: + branch: [main] + event: [push] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: publish python-3.11 debian slim-bookworm + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.11 + - DEBIAN_VERSION=slim-bookworm + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.11-debian-slim-bookworm-1.6.1 + - 3.11-debian-slim-bookworm + - 3.11-debian + username: ibuildallthings + python_3.11_debian_slim-bookworm pull_request: + needs: [clone] + steps: + - ruleset: + if: + event: ['pull_request:opened', 'pull_request:synchronize'] + matcher: filepath + operator: and + secrets: + - source: artifactory_password + target: docker_password + image: target/vela-kaniko:latest + name: build python-3.11 debian slim-bookworm + pull: not_present + parameters: + build_args: + - PYTHON_VERSION=3.11 + - DEBIAN_VERSION=slim-bookworm + - POETRY_VERSION=1.6.1 + dockerfile: python-debian.Dockerfile + dry_run: true + registry: docker.example.com + repo: docker.example.com/app/multibuild + tags: + - 3.11-debian-slim-bookworm-1.6.1 + - 3.11-debian-slim-bookworm + - 3.11-debian + username: ibuildallthings + slack: + needs: [python_3.8_alpine_3.17 push, python_3.8_alpine_3.17 pull_request, python_3.8_alpine_3.18 + push, python_3.8_alpine_3.18 pull_request, python_3.8_debian_slim-bullseye + push, python_3.8_debian_slim-bullseye pull_request, python_3.8_debian_slim-bookworm + push, python_3.8_debian_slim-bookworm pull_request, python_3.9_alpine_3.17 + push, python_3.9_alpine_3.17 pull_request, python_3.9_alpine_3.18 push, python_3.9_alpine_3.18 + pull_request, python_3.9_debian_slim-bullseye push, python_3.9_debian_slim-bullseye + pull_request, python_3.9_debian_slim-bookworm push, python_3.9_debian_slim-bookworm + pull_request, python_3.10_alpine_3.17 push, python_3.10_alpine_3.17 pull_request, + python_3.10_alpine_3.18 push, python_3.10_alpine_3.18 pull_request, python_3.10_debian_slim-bullseye + push, python_3.10_debian_slim-bullseye pull_request, python_3.10_debian_slim-bookworm + push, python_3.10_debian_slim-bookworm pull_request, python_3.11_alpine_3.17 + push, python_3.11_alpine_3.17 pull_request, python_3.11_alpine_3.18 push, + python_3.11_alpine_3.18 pull_request, python_3.11_debian_slim-bullseye push, + python_3.11_debian_slim-bullseye pull_request, python_3.11_debian_slim-bookworm + push, python_3.11_debian_slim-bookworm pull_request, clone] + steps: + - template: + name: slack + name: slack + pull: not_present +templates: +- name: slack + source: git.example.com/vela/vela-templates/slack/slack.yml + type: github \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 6a596e45e..33afcbaba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,7 @@ services: networks: - vela environment: + COMPILER_STARLARK_EXEC_LIMIT: 7500 DATABASE_DRIVER: postgres DATABASE_ADDR: 'postgres://vela:zB7mrKDTZqNeNTD8z47yG4DHywspAh@postgres:5432/vela?sslmode=disable' DATABASE_COMPRESSION_LEVEL: 3 diff --git a/scm/github/webhook.go b/scm/github/webhook.go index 43615e8cb..fb719a4d7 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -29,6 +29,7 @@ import ( func (c *client) ProcessWebhook(request *http.Request) (*types.Webhook, error) { c.Logger.Tracef("processing GitHub webhook") + // create our own record of the hook and populate its fields h := new(library.Hook) h.SetNumber(1) h.SetSourceID(request.Header.Get("X-GitHub-Delivery")) From 4d983280fc7999847992f16dcdf8391926eb97f1 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 29 Aug 2023 12:17:09 -0500 Subject: [PATCH 4/8] remove compose configuration --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 33afcbaba..6a596e45e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,6 @@ services: networks: - vela environment: - COMPILER_STARLARK_EXEC_LIMIT: 7500 DATABASE_DRIVER: postgres DATABASE_ADDR: 'postgres://vela:zB7mrKDTZqNeNTD8z47yG4DHywspAh@postgres:5432/vela?sslmode=disable' DATABASE_COMPRESSION_LEVEL: 3 From 20a9da6d4816afa027f45b72642c8dc1190bc29e Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 29 Aug 2023 12:22:38 -0500 Subject: [PATCH 5/8] make execlimit part of test struct --- compiler/template/starlark/render_test.go | 51 +++++++++++------------ 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/compiler/template/starlark/render_test.go b/compiler/template/starlark/render_test.go index 8d23b1c6a..736563b84 100644 --- a/compiler/template/starlark/render_test.go +++ b/compiler/template/starlark/render_test.go @@ -6,7 +6,6 @@ package starlark import ( "os" - "strings" "testing" goyaml "github.com/buildkite/yaml" @@ -140,58 +139,65 @@ func TestNative_RenderBuild(t *testing.T) { } tests := []struct { - name string - args args - wantFile string - wantErr bool + name string + args args + wantFile string + wantErr bool + execLimit uint64 }{ { name: "steps", args: args{ velaFile: "testdata/build/basic/build.star", }, - wantFile: "testdata/build/basic/want.yml", - wantErr: false, + wantFile: "testdata/build/basic/want.yml", + wantErr: false, + execLimit: 7500, }, { name: "stages", args: args{ velaFile: "testdata/build/basic_stages/build.star", }, - wantFile: "testdata/build/basic_stages/want.yml", - wantErr: false, + wantFile: "testdata/build/basic_stages/want.yml", + wantErr: false, + execLimit: 7500, }, { name: "conditional match", args: args{ velaFile: "testdata/build/conditional/build.star", }, - wantFile: "testdata/build/conditional/want.yml", - wantErr: false, + wantFile: "testdata/build/conditional/want.yml", + wantErr: false, + execLimit: 7500, }, { name: "steps, with structs", args: args{ velaFile: "testdata/build/with_struct/build.star", }, - wantFile: "testdata/build/with_struct/want.yml", - wantErr: false, + wantFile: "testdata/build/with_struct/want.yml", + wantErr: false, + execLimit: 7500, }, { name: "large build - exec step limit good", args: args{ velaFile: "testdata/build/large/build.star", }, - wantFile: "testdata/build/large/want.yml", - wantErr: false, + wantFile: "testdata/build/large/want.yml", + wantErr: false, + execLimit: 7500, }, { name: "large build - exec step limit too low", args: args{ velaFile: "testdata/build/large/build.star", }, - wantFile: "", - wantErr: true, + wantFile: "", + wantErr: true, + execLimit: 5000, }, } @@ -202,20 +208,11 @@ func TestNative_RenderBuild(t *testing.T) { t.Error(err) } - // set execLimit based on test name - var execLimit uint64 - - if strings.EqualFold(tt.name, "large build - exec step limit too low") { - execLimit = 5000 - } else { - execLimit = 7500 - } - got, err := RenderBuild("build", string(sFile), map[string]string{ "VELA_REPO_FULL_NAME": "octocat/hello-world", "VELA_BUILD_BRANCH": "master", "VELA_REPO_ORG": "octocat", - }, map[string]interface{}{}, execLimit) + }, map[string]interface{}{}, tt.execLimit) if (err != nil) != tt.wantErr { t.Errorf("RenderBuild() error = %v, wantErr %v", err, tt.wantErr) return From df7a19671bcecf6144ed9c650d587f9d3c3705d8 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Tue, 29 Aug 2023 13:36:19 -0500 Subject: [PATCH 6/8] update limit for Render --- compiler/native/expand.go | 2 +- compiler/template/starlark/render.go | 4 ++-- compiler/template/starlark/render_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/native/expand.go b/compiler/native/expand.go index d593bc26e..02faface2 100644 --- a/compiler/native/expand.go +++ b/compiler/native/expand.go @@ -313,7 +313,7 @@ func (c *client) mergeTemplate(bytes []byte, tmpl *yaml.Template, step *yaml.Ste return native.Render(string(bytes), step.Name, step.Template.Name, step.Environment, step.Template.Variables) case constants.PipelineTypeStarlark: //nolint:lll // ignore long line length due to return - return starlark.Render(string(bytes), step.Name, step.Template.Name, step.Environment, step.Template.Variables) + return starlark.Render(string(bytes), step.Name, step.Template.Name, step.Environment, step.Template.Variables, c.StarlarkExecLimit) default: //nolint:lll // ignore long line length due to return return &yaml.Build{}, fmt.Errorf("format of %s is unsupported", tmpl.Format) diff --git a/compiler/template/starlark/render.go b/compiler/template/starlark/render.go index 194ea2885..8f6dbed27 100644 --- a/compiler/template/starlark/render.go +++ b/compiler/template/starlark/render.go @@ -32,14 +32,14 @@ var ( ) // Render combines the template with the step in the yaml pipeline. -func Render(tmpl string, name string, tName string, environment raw.StringSliceMap, variables map[string]interface{}) (*types.Build, error) { +func Render(tmpl string, name string, tName string, environment raw.StringSliceMap, variables map[string]interface{}, limit uint64) (*types.Build, error) { config := new(types.Build) thread := &starlark.Thread{Name: name} // arbitrarily limiting the steps of the thread to 5000 to help prevent infinite loops // may need to further investigate spawning a separate POSIX process if user input is problematic // see https://github.com/google/starlark-go/issues/160#issuecomment-466794230 for further details - thread.SetMaxExecutionSteps(GetStarlarkExecutionStepLimit()) + thread.SetMaxExecutionSteps(limit) predeclared := starlark.StringDict{"struct": starlark.NewBuiltin("struct", starlarkstruct.Make)} diff --git a/compiler/template/starlark/render_test.go b/compiler/template/starlark/render_test.go index 736563b84..7bd25b06b 100644 --- a/compiler/template/starlark/render_test.go +++ b/compiler/template/starlark/render_test.go @@ -93,7 +93,7 @@ func TestStarlark_Render(t *testing.T) { t.Error(err) } - tmplBuild, err := Render(string(tmpl), b.Steps[0].Name, b.Steps[0].Template.Name, b.Steps[0].Environment, b.Steps[0].Template.Variables) + tmplBuild, err := Render(string(tmpl), b.Steps[0].Name, b.Steps[0].Template.Name, b.Steps[0].Environment, b.Steps[0].Template.Variables, 7500) if (err != nil) != tt.wantErr { t.Errorf("Render() error = %v, wantErr %v", err, tt.wantErr) return From 294a8935ac5e6375ddc459d855339e06cdf1534a Mon Sep 17 00:00:00 2001 From: dave vader <48764154+plyr4@users.noreply.github.com> Date: Tue, 29 Aug 2023 12:32:23 -0500 Subject: [PATCH 7/8] enhance: use consistent variable aliases (#946) --- queue/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/queue/flags.go b/queue/flags.go index b5ede4733..2e78b11da 100644 --- a/queue/flags.go +++ b/queue/flags.go @@ -51,13 +51,13 @@ var Flags = []cli.Flag{ Value: 60 * time.Second, }, &cli.StringFlag{ - EnvVars: []string{"QUEUE_PRIVATE_KEY"}, + EnvVars: []string{"VELA_QUEUE_PRIVATE_KEY", "QUEUE_PRIVATE_KEY"}, FilePath: "/vela/signing.key", Name: "queue.private-key", Usage: "set value of base64 encoded queue signing private key", }, &cli.StringFlag{ - EnvVars: []string{"QUEUE_PUBLIC_KEY"}, + EnvVars: []string{"VELA_QUEUE_PUBLIC_KEY", "QUEUE_PUBLIC_KEY"}, FilePath: "/vela/signing.pub", Name: "queue.public-key", Usage: "set value of base64 encoded queue signing public key", From b7ee3eb7d492b29f10d0cb73068fef3fc026d4fc Mon Sep 17 00:00:00 2001 From: ecrupper Date: Wed, 30 Aug 2023 11:29:24 -0500 Subject: [PATCH 8/8] remove helper function to get limit --- compiler/template/starlark/render.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/compiler/template/starlark/render.go b/compiler/template/starlark/render.go index 8f6dbed27..3ed9cbb3c 100644 --- a/compiler/template/starlark/render.go +++ b/compiler/template/starlark/render.go @@ -137,15 +137,6 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM return &types.Build{Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment}, nil } -// GetStarlarkExecutionStepLimit may eventually look up config or calculate it -func GetStarlarkExecutionStepLimit() uint64 { - // arbitrarily limiting the steps of the thread to help prevent infinite loops - // may need to further investigate spawning a separate POSIX process if user input is problematic - // see https://github.com/google/starlark-go/issues/160#issuecomment-466794230 for further details - // This value was previously 5000 and that inhibited a four-dimensional build matrix from working. - return 7500 -} - // RenderBuild renders the templated build. // //nolint:lll // ignore function length due to input args