From cba432fcd1feaf89db7a7fc69bb460bfba2cbcba Mon Sep 17 00:00:00 2001 From: David May <1301201+wass3r@users.noreply.github.com> Date: Thu, 11 May 2023 14:58:31 -0500 Subject: [PATCH 01/53] fix(docker-compose): fix env var names for tokens (#841) --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 91c830d0c..cb4ef0684 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,8 +38,8 @@ services: VELA_LOG_LEVEL: trace VELA_SECRET: 'zB7mrKDTZqNeNTD8z47yG4DHywspAh' VELA_SERVER_PRIVATE_KEY: 'F534FF2A080E45F38E05DC70752E6787' - VELA_REFRESH_TOKEN_DURATION: 90m - VELA_ACCESS_TOKEN_DURATION: 60m + VELA_USER_REFRESH_TOKEN_DURATION: 90m + VELA_USER_ACCESS_TOKEN_DURATION: 60m VELA_DISABLE_WEBHOOK_VALIDATION: 'true' VELA_ENABLE_SECURE_COOKIE: 'false' VELA_REPO_ALLOWLIST: '*' From 5a7b9c53fd675d99298cbcafb8acbbb65a32a0e5 Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Mon, 15 May 2023 09:03:10 -0500 Subject: [PATCH 02/53] refactor(database): move service logic into separate package (#816) --- api/build.go | 6 +- api/metrics.go | 4 +- api/service.go | 22 +- api/step.go | 2 +- api/webhook.go | 2 +- cmd/vela-server/database.go | 2 +- cmd/vela-server/secret.go | 2 +- database/context.go | 10 +- database/database.go | 2 +- database/hook/hook.go | 4 +- database/hook/{service.go => interface.go} | 4 +- database/{service.go => interface.go} | 73 ++--- database/log/{service.go => interface.go} | 4 +- database/log/log.go | 4 +- .../pipeline/{service.go => interface.go} | 4 +- database/pipeline/pipeline.go | 4 +- database/postgres/ddl/service.go | 32 --- database/postgres/dml/service.go | 67 ----- database/postgres/postgres.go | 89 +++--- database/postgres/postgres_test.go | 7 +- database/postgres/service.go | 94 ------ database/postgres/service_count.go | 90 ------ database/postgres/service_count_test.go | 194 ------------- database/postgres/service_list.go | 71 ----- database/postgres/service_list_test.go | 168 ----------- database/postgres/service_test.go | 267 ------------------ database/repo/{service.go => interface.go} | 4 +- database/repo/repo.go | 4 +- database/secret/{service.go => interface.go} | 4 +- database/secret/secret.go | 4 +- database/service/count.go | 25 ++ database/service/count_build.go | 31 ++ database/service/count_build_test.go | 104 +++++++ database/service/count_test.go | 97 +++++++ database/service/create.go | 38 +++ database/service/create_test.go | 75 +++++ database/service/delete.go | 30 ++ database/service/delete_test.go | 75 +++++ database/service/get.go | 34 +++ database/service/get_build.go | 39 +++ database/service/get_build_test.go | 92 ++++++ database/service/get_test.go | 87 ++++++ database/service/interface.go | 49 ++++ database/service/list.go | 54 ++++ database/service/list_build.go | 65 +++++ database/service/list_build_test.go | 114 ++++++++ database/service/list_image.go | 44 +++ database/service/list_image_test.go | 97 +++++++ database/service/list_status.go | 50 ++++ database/service/list_status_test.go | 114 ++++++++ database/service/list_test.go | 107 +++++++ database/service/opts.go | 44 +++ database/service/opts_test.go | 161 +++++++++++ database/service/service.go | 74 +++++ database/service/service_test.go | 224 +++++++++++++++ database/service/table.go | 76 +++++ database/service/table_test.go | 59 ++++ database/service/update.go | 38 +++ database/service/update_test.go | 75 +++++ database/setup.go | 4 +- database/sqlite/ddl/service.go | 32 --- database/sqlite/dml/service.go | 67 ----- database/sqlite/service.go | 94 ------ database/sqlite/service_count.go | 90 ------ database/sqlite/service_count_test.go | 198 ------------- database/sqlite/service_list.go | 71 ----- database/sqlite/service_list_test.go | 174 ------------ database/sqlite/service_test.go | 266 ----------------- database/sqlite/sqlite.go | 85 +++--- database/step/{service.go => interface.go} | 4 +- database/step/step.go | 4 +- database/user/{service.go => interface.go} | 4 +- database/user/user.go | 4 +- database/worker/{service.go => interface.go} | 4 +- database/worker/worker.go | 4 +- router/middleware/database.go | 2 +- router/middleware/database_test.go | 2 +- router/middleware/service/service.go | 2 +- secret/native/native.go | 4 +- secret/native/native_test.go | 4 +- secret/native/opts.go | 2 +- secret/native/opts_test.go | 4 +- secret/setup.go | 3 +- 83 files changed, 2363 insertions(+), 2181 deletions(-) rename database/hook/{service.go => interface.go} (95%) rename database/{service.go => interface.go} (63%) rename database/log/{service.go => interface.go} (95%) rename database/pipeline/{service.go => interface.go} (95%) delete mode 100644 database/postgres/ddl/service.go delete mode 100644 database/postgres/dml/service.go delete mode 100644 database/postgres/service.go delete mode 100644 database/postgres/service_count.go delete mode 100644 database/postgres/service_count_test.go delete mode 100644 database/postgres/service_list.go delete mode 100644 database/postgres/service_list_test.go delete mode 100644 database/postgres/service_test.go rename database/repo/{service.go => interface.go} (96%) rename database/secret/{service.go => interface.go} (97%) create mode 100644 database/service/count.go create mode 100644 database/service/count_build.go create mode 100644 database/service/count_build_test.go create mode 100644 database/service/count_test.go create mode 100644 database/service/create.go create mode 100644 database/service/create_test.go create mode 100644 database/service/delete.go create mode 100644 database/service/delete_test.go create mode 100644 database/service/get.go create mode 100644 database/service/get_build.go create mode 100644 database/service/get_build_test.go create mode 100644 database/service/get_test.go create mode 100644 database/service/interface.go create mode 100644 database/service/list.go create mode 100644 database/service/list_build.go create mode 100644 database/service/list_build_test.go create mode 100644 database/service/list_image.go create mode 100644 database/service/list_image_test.go create mode 100644 database/service/list_status.go create mode 100644 database/service/list_status_test.go create mode 100644 database/service/list_test.go create mode 100644 database/service/opts.go create mode 100644 database/service/opts_test.go create mode 100644 database/service/service.go create mode 100644 database/service/service_test.go create mode 100644 database/service/table.go create mode 100644 database/service/table_test.go create mode 100644 database/service/update.go create mode 100644 database/service/update_test.go delete mode 100644 database/sqlite/ddl/service.go delete mode 100644 database/sqlite/dml/service.go delete mode 100644 database/sqlite/service.go delete mode 100644 database/sqlite/service_count.go delete mode 100644 database/sqlite/service_count_test.go delete mode 100644 database/sqlite/service_list.go delete mode 100644 database/sqlite/service_list_test.go delete mode 100644 database/sqlite/service_test.go rename database/step/{service.go => interface.go} (95%) rename database/user/{service.go => interface.go} (94%) rename database/worker/{service.go => interface.go} (94%) diff --git a/api/build.go b/api/build.go index bf5394eef..60f7f8eb1 100644 --- a/api/build.go +++ b/api/build.go @@ -1571,7 +1571,7 @@ func getPRNumberFromBuild(b *library.Build) (int, error) { // and services, for the build in the configured backend. // TODO: // - return build and error. -func planBuild(database database.Service, p *pipeline.Build, b *library.Build, r *library.Repo) error { +func planBuild(database database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo) error { // update fields in build object b.SetCreated(time.Now().UTC().Unix()) @@ -1625,7 +1625,7 @@ func planBuild(database database.Service, p *pipeline.Build, b *library.Build, r // without execution. This will kill all resources, // like steps and services, for the build in the // configured backend. -func cleanBuild(database database.Service, b *library.Build, services []*library.Service, steps []*library.Step, e error) { +func cleanBuild(database database.Interface, b *library.Build, services []*library.Service, steps []*library.Step, e error) { // update fields in build object b.SetError(fmt.Sprintf("unable to publish to queue: %s", e.Error())) b.SetStatus(constants.StatusError) @@ -1881,7 +1881,7 @@ func CancelBuild(c *gin.Context) { for page > 0 { // retrieve build services (per page) from the database - servicesPart, err := database.FromContext(c).GetBuildServiceList(b, page, perPage) + servicesPart, _, err := database.FromContext(c).ListServicesForBuild(b, map[string]interface{}{}, page, perPage) if err != nil { retErr := fmt.Errorf("unable to retrieve services for build %s: %w", entry, err) util.HandleError(c, http.StatusNotFound, retErr) diff --git a/api/metrics.go b/api/metrics.go index 4bdeb044e..d68ab35e7 100644 --- a/api/metrics.go +++ b/api/metrics.go @@ -350,7 +350,7 @@ func recordGauges(c *gin.Context) { // service_image_count if q.ServiceImageCount { // send API call to capture the total number of service images - serviceImageMap, err := database.FromContext(c).GetServiceImageCount() + serviceImageMap, err := database.FromContext(c).ListServiceImageCount() if err != nil { logrus.Errorf("unable to get count of all service images: %v", err) } @@ -363,7 +363,7 @@ func recordGauges(c *gin.Context) { // service_status_count if q.ServiceStatusCount { // send API call to capture the total number of service statuses - serviceStatusMap, err := database.FromContext(c).GetServiceStatusCount() + serviceStatusMap, err := database.FromContext(c).ListServiceStatusCount() if err != nil { logrus.Errorf("unable to get count of all service statuses: %v", err) } diff --git a/api/service.go b/api/service.go index a2136a96e..66b81b080 100644 --- a/api/service.go +++ b/api/service.go @@ -130,7 +130,7 @@ func CreateService(c *gin.Context) { } // send API call to capture the created service - s, _ := database.FromContext(c).GetService(input.GetNumber(), b) + s, _ := database.FromContext(c).GetServiceForBuild(b, input.GetNumber()) c.JSON(http.StatusCreated, s) } @@ -238,18 +238,8 @@ func GetServices(c *gin.Context) { // ensure per_page isn't above or below allowed values perPage = util.MaxInt(1, util.MinInt(100, perPage)) - // send API call to capture the total number of services for the build - t, err := database.FromContext(c).GetBuildServiceCount(b) - if err != nil { - retErr := fmt.Errorf("unable to get services count for build %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - // send API call to capture the list of services for the build - s, err := database.FromContext(c).GetBuildServiceList(b, page, perPage) + s, t, err := database.FromContext(c).ListServicesForBuild(b, map[string]interface{}{}, page, perPage) if err != nil { retErr := fmt.Errorf("unable to get services for build %s: %w", entry, err) @@ -462,7 +452,7 @@ func UpdateService(c *gin.Context) { } // send API call to capture the updated service - s, _ = database.FromContext(c).GetService(s.GetNumber(), b) + s, _ = database.FromContext(c).GetServiceForBuild(b, s.GetNumber()) c.JSON(http.StatusOK, s) } @@ -534,7 +524,7 @@ func DeleteService(c *gin.Context) { }).Infof("deleting service %s", entry) // send API call to remove the service - err := database.FromContext(c).DeleteService(s.GetID()) + err := database.FromContext(c).DeleteService(s) if err != nil { retErr := fmt.Errorf("unable to delete service %s: %w", entry, err) @@ -549,7 +539,7 @@ func DeleteService(c *gin.Context) { // planServices is a helper function to plan all services // in the build for execution. This creates the services // for the build in the configured backend. -func planServices(database database.Service, p *pipeline.Build, b *library.Build) ([]*library.Service, error) { +func planServices(database database.Interface, p *pipeline.Build, b *library.Build) ([]*library.Service, error) { // variable to store planned services services := []*library.Service{} @@ -572,7 +562,7 @@ func planServices(database database.Service, p *pipeline.Build, b *library.Build } // send API call to capture the created service - s, err = database.GetService(s.GetNumber(), b) + s, err = database.GetServiceForBuild(b, s.GetNumber()) if err != nil { return services, fmt.Errorf("unable to get service %s: %w", s.GetName(), err) } diff --git a/api/step.go b/api/step.go index 465ce89f0..23d72fac4 100644 --- a/api/step.go +++ b/api/step.go @@ -540,7 +540,7 @@ func DeleteStep(c *gin.Context) { // planSteps is a helper function to plan all steps // in the build for execution. This creates the steps // for the build in the configured backend. -func planSteps(database database.Service, p *pipeline.Build, b *library.Build) ([]*library.Step, error) { +func planSteps(database database.Interface, p *pipeline.Build, b *library.Build) ([]*library.Step, error) { // variable to store planned steps steps := []*library.Step{} diff --git a/api/webhook.go b/api/webhook.go index 6bbf7cab3..501caffa6 100644 --- a/api/webhook.go +++ b/api/webhook.go @@ -707,7 +707,7 @@ func PostWebhook(c *gin.Context) { // publishToQueue is a helper function that creates // a build item and publishes it to the queue. -func publishToQueue(queue queue.Service, db database.Service, p *pipeline.Build, b *library.Build, r *library.Repo, u *library.User) { +func publishToQueue(queue queue.Service, db database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo, u *library.User) { item := types.ToItem(p, b, r, u) logrus.Infof("Converting queue item to json for build %d for %s", b.GetNumber(), r.GetFullName()) diff --git a/cmd/vela-server/database.go b/cmd/vela-server/database.go index cee4f9a83..e4db992ec 100644 --- a/cmd/vela-server/database.go +++ b/cmd/vela-server/database.go @@ -13,7 +13,7 @@ import ( ) // helper function to setup the database from the CLI arguments. -func setupDatabase(c *cli.Context) (database.Service, error) { +func setupDatabase(c *cli.Context) (database.Interface, error) { logrus.Debug("Creating database client from CLI configuration") // database configuration diff --git a/cmd/vela-server/secret.go b/cmd/vela-server/secret.go index ee780dce0..56e6c0572 100644 --- a/cmd/vela-server/secret.go +++ b/cmd/vela-server/secret.go @@ -15,7 +15,7 @@ import ( ) // helper function to setup the secrets engines from the CLI arguments. -func setupSecrets(c *cli.Context, d database.Service) (map[string]secret.Service, error) { +func setupSecrets(c *cli.Context, d database.Interface) (map[string]secret.Service, error) { logrus.Debug("Creating secret clients from CLI configuration") secrets := make(map[string]secret.Service) diff --git a/database/context.go b/database/context.go index 7e315511c..f3a872602 100644 --- a/database/context.go +++ b/database/context.go @@ -15,14 +15,14 @@ type Setter interface { Set(string, interface{}) } -// FromContext returns the database Service associated with this context. -func FromContext(c context.Context) Service { +// FromContext returns the database Interface associated with this context. +func FromContext(c context.Context) Interface { v := c.Value(key) if v == nil { return nil } - d, ok := v.(Service) + d, ok := v.(Interface) if !ok { return nil } @@ -30,8 +30,8 @@ func FromContext(c context.Context) Service { return d } -// ToContext adds the database Service to this context if it supports +// ToContext adds the database Interface to this context if it supports // the Setter interface. -func ToContext(c Setter, d Service) { +func ToContext(c Setter, d Interface) { c.Set(key, d) } diff --git a/database/database.go b/database/database.go index a6e8c8958..ad26dd82a 100644 --- a/database/database.go +++ b/database/database.go @@ -20,7 +20,7 @@ import ( // * Postgres // * Sqlite // . -func New(s *Setup) (Service, error) { +func New(s *Setup) (Interface, error) { // validate the setup being provided // // https://pkg.go.dev/github.com/go-vela/server/database?tab=doc#Setup.Validate diff --git a/database/hook/hook.go b/database/hook/hook.go index 279bc3d2a..5225f901a 100644 --- a/database/hook/hook.go +++ b/database/hook/hook.go @@ -14,13 +14,13 @@ import ( ) type ( - // config represents the settings required to create the engine that implements the HookService interface. + // config represents the settings required to create the engine that implements the HookInterface interface. config struct { // specifies to skip creating tables and indexes for the Hook engine SkipCreation bool } - // engine represents the hook functionality that implements the HookService interface. + // engine represents the hook functionality that implements the HookInterface interface. engine struct { // engine configuration settings used in hook functions config *config diff --git a/database/hook/service.go b/database/hook/interface.go similarity index 95% rename from database/hook/service.go rename to database/hook/interface.go index 99a389fef..a1e5c5fe0 100644 --- a/database/hook/service.go +++ b/database/hook/interface.go @@ -8,11 +8,11 @@ import ( "github.com/go-vela/types/library" ) -// HookService represents the Vela interface for hook +// HookInterface represents the Vela interface for hook // functions with the supported Database backends. // //nolint:revive // ignore name stutter -type HookService interface { +type HookInterface interface { // Hook Data Definition Language Functions // // https://en.wikipedia.org/wiki/Data_definition_language diff --git a/database/service.go b/database/interface.go similarity index 63% rename from database/service.go rename to database/interface.go index 306c77720..b7d1b6fee 100644 --- a/database/service.go +++ b/database/interface.go @@ -10,15 +10,16 @@ import ( "github.com/go-vela/server/database/pipeline" "github.com/go-vela/server/database/repo" "github.com/go-vela/server/database/secret" + "github.com/go-vela/server/database/service" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" "github.com/go-vela/types/library" ) -// Service represents the interface for Vela integrating +// Interface represents the interface for Vela integrating // with the different supported Database backends. -type Service interface { +type Interface interface { // Database Interface Functions // Driver defines a function that outputs @@ -76,67 +77,39 @@ type Service interface { // deletes a build by unique ID. DeleteBuild(int64) error - // HookService provides the interface for functionality + // HookInterface provides the interface for functionality // related to hooks stored in the database. - hook.HookService + hook.HookInterface - // LogService provides the interface for functionality + // LogInterface provides the interface for functionality // related to logs stored in the database. - log.LogService + log.LogInterface - // PipelineService provides the interface for functionality + // PipelineInterface provides the interface for functionality // related to pipelines stored in the database. - pipeline.PipelineService + pipeline.PipelineInterface - // RepoService provides the interface for functionality + // RepoInterface provides the interface for functionality // related to repos stored in the database. - repo.RepoService + repo.RepoInterface - // SecretService provides the interface for functionality + // SecretInterface provides the interface for functionality // related to secrets stored in the database. - secret.SecretService + secret.SecretInterface - // StepService provides the interface for functionality - // related to steps stored in the database. - step.StepService - - // Service Database Interface Functions + // ServiceInterface provides the interface for functionality + // related to services stored in the database. + service.ServiceInterface - // GetService defines a function that - // gets a step by number and build ID. - GetService(int, *library.Build) (*library.Service, error) - // GetServiceList defines a function that - // gets a list of all steps. - GetServiceList() ([]*library.Service, error) - // GetBuildServiceList defines a function - // that gets a list of steps by build ID. - GetBuildServiceList(*library.Build, int, int) ([]*library.Service, error) - // GetBuildServiceCount defines a function - // that gets the count of steps by build ID. - GetBuildServiceCount(*library.Build) (int64, error) - // GetServiceImageCount defines a function that - // gets a list of all service images and the - // count of their occurrence. - GetServiceImageCount() (map[string]float64, error) - // GetServiceStatusCount defines a function that - // gets a list of all service statuses and the - // count of their occurrence. - GetServiceStatusCount() (map[string]float64, error) - // CreateService defines a function that - // creates a new step. - CreateService(*library.Service) error - // UpdateService defines a function that - // updates a step. - UpdateService(*library.Service) error - // DeleteService defines a function that - // deletes a step by unique ID. - DeleteService(int64) error + // StepInterface provides the interface for functionality + // related to steps stored in the database. + step.StepInterface - // UserService provides the interface for functionality + // UserInterface provides the interface for functionality // related to users stored in the database. - user.UserService + user.UserInterface - // WorkerService provides the interface for functionality + // WorkerInterface provides the interface for functionality // related to workers stored in the database. - worker.WorkerService + worker.WorkerInterface } diff --git a/database/log/service.go b/database/log/interface.go similarity index 95% rename from database/log/service.go rename to database/log/interface.go index 00e686ee9..8c72a5098 100644 --- a/database/log/service.go +++ b/database/log/interface.go @@ -8,11 +8,11 @@ import ( "github.com/go-vela/types/library" ) -// LogService represents the Vela interface for log +// LogInterface represents the Vela interface for log // functions with the supported Database backends. // //nolint:revive // ignore name stutter -type LogService interface { +type LogInterface interface { // Log Data Definition Language Functions // // https://en.wikipedia.org/wiki/Data_definition_language diff --git a/database/log/log.go b/database/log/log.go index 22604e411..35d25400f 100644 --- a/database/log/log.go +++ b/database/log/log.go @@ -14,7 +14,7 @@ import ( ) type ( - // config represents the settings required to create the engine that implements the LogService interface. + // config represents the settings required to create the engine that implements the LogInterface interface. config struct { // specifies the level of compression to use for the Log engine CompressionLevel int @@ -22,7 +22,7 @@ type ( SkipCreation bool } - // engine represents the log functionality that implements the LogService interface. + // engine represents the log functionality that implements the LogInterface interface. engine struct { // engine configuration settings used in log functions config *config diff --git a/database/pipeline/service.go b/database/pipeline/interface.go similarity index 95% rename from database/pipeline/service.go rename to database/pipeline/interface.go index c619fda46..eaafa59fe 100644 --- a/database/pipeline/service.go +++ b/database/pipeline/interface.go @@ -8,11 +8,11 @@ import ( "github.com/go-vela/types/library" ) -// PipelineService represents the Vela interface for pipeline +// PipelineInterface represents the Vela interface for pipeline // functions with the supported Database backends. // //nolint:revive // ignore name stutter -type PipelineService interface { +type PipelineInterface interface { // Pipeline Data Definition Language Functions // // https://en.wikipedia.org/wiki/Data_definition_language diff --git a/database/pipeline/pipeline.go b/database/pipeline/pipeline.go index b99bdbfef..a48cc6e07 100644 --- a/database/pipeline/pipeline.go +++ b/database/pipeline/pipeline.go @@ -14,7 +14,7 @@ import ( ) type ( - // config represents the settings required to create the engine that implements the PipelineService interface. + // config represents the settings required to create the engine that implements the PipelineInterface interface. config struct { // specifies the level of compression to use for the Pipeline engine CompressionLevel int @@ -22,7 +22,7 @@ type ( SkipCreation bool } - // engine represents the pipeline functionality that implements the PipelineService interface. + // engine represents the pipeline functionality that implements the PipelineInterface interface. engine struct { // engine configuration settings used in pipeline functions config *config diff --git a/database/postgres/ddl/service.go b/database/postgres/ddl/service.go deleted file mode 100644 index b714586b5..000000000 --- a/database/postgres/ddl/service.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package ddl - -const ( - // CreateServiceTable represents a query to - // create the services table for Vela. - CreateServiceTable = ` -CREATE TABLE -IF NOT EXISTS -services ( - id SERIAL PRIMARY KEY, - repo_id INTEGER, - build_id INTEGER, - number INTEGER, - name VARCHAR(250), - image VARCHAR(500), - status VARCHAR(250), - error VARCHAR(500), - exit_code INTEGER, - created INTEGER, - started INTEGER, - finished INTEGER, - host VARCHAR(250), - runtime VARCHAR(250), - distribution VARCHAR(250), - UNIQUE(build_id, number) -); -` -) diff --git a/database/postgres/dml/service.go b/database/postgres/dml/service.go deleted file mode 100644 index 66e009406..000000000 --- a/database/postgres/dml/service.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package dml - -const ( - // ListServices represents a query to - // list all services in the database. - ListServices = ` -SELECT * -FROM services; -` - - // ListBuildServices represents a query to list - // all services for a build_id in the database. - ListBuildServices = ` -SELECT * -FROM services -WHERE build_id = ? -ORDER BY id DESC -LIMIT ? -OFFSET ?; -` - - // SelectBuildServicesCount represents a query to select - // the count of services for a build_id in the database. - SelectBuildServicesCount = ` -SELECT count(*) as count -FROM services -WHERE build_id = ? -` - - // SelectServiceImagesCount represents a query to select - // the count of an images appearances in the database. - SelectServiceImagesCount = ` -SELECT image, count(image) as count -FROM services -GROUP BY image -` - - // SelectServiceStatusesCount represents a query to select - // the count of service status appearances in the database. - SelectServiceStatusesCount = ` -SELECT status, count(status) as count -FROM services -GROUP BY status; -` - - // SelectBuildService represents a query to select a - // service for a build_id and number in the database. - SelectBuildService = ` -SELECT * -FROM services -WHERE build_id = ? -AND number = ? -LIMIT 1; -` - - // DeleteService represents a query to - // remove a service from the database. - DeleteService = ` -DELETE -FROM services -WHERE id = ?; -` -) diff --git a/database/postgres/postgres.go b/database/postgres/postgres.go index c96883b59..2adbfdd31 100644 --- a/database/postgres/postgres.go +++ b/database/postgres/postgres.go @@ -15,6 +15,7 @@ import ( "github.com/go-vela/server/database/postgres/ddl" "github.com/go-vela/server/database/repo" "github.com/go-vela/server/database/secret" + "github.com/go-vela/server/database/service" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" @@ -49,22 +50,24 @@ type ( Postgres *gorm.DB // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry - // https://pkg.go.dev/github.com/go-vela/server/database/hook#HookService - hook.HookService - // https://pkg.go.dev/github.com/go-vela/server/database/log#LogService - log.LogService - // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#PipelineService - pipeline.PipelineService - // https://pkg.go.dev/github.com/go-vela/server/database/repo#RepoService - repo.RepoService - // https://pkg.go.dev/github.com/go-vela/server/database/secret#SecretService - secret.SecretService - // https://pkg.go.dev/github.com/go-vela/server/database/step#StepService - step.StepService - // https://pkg.go.dev/github.com/go-vela/server/database/user#UserService - user.UserService - // https://pkg.go.dev/github.com/go-vela/server/database/worker#WorkerService - worker.WorkerService + // https://pkg.go.dev/github.com/go-vela/server/database/hook#HookInterface + hook.HookInterface + // https://pkg.go.dev/github.com/go-vela/server/database/log#LogInterface + log.LogInterface + // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#PipelineInterface + pipeline.PipelineInterface + // https://pkg.go.dev/github.com/go-vela/server/database/repo#RepoInterface + repo.RepoInterface + // https://pkg.go.dev/github.com/go-vela/server/database/secret#SecretInterface + secret.SecretInterface + // https://pkg.go.dev/github.com/go-vela/server/database/service#ServiceInterface + service.ServiceInterface + // https://pkg.go.dev/github.com/go-vela/server/database/step#StepInterface + step.StepInterface + // https://pkg.go.dev/github.com/go-vela/server/database/user#UserInterface + user.UserInterface + // https://pkg.go.dev/github.com/go-vela/server/database/worker#WorkerInterface + worker.WorkerInterface } ) @@ -178,6 +181,8 @@ func NewTest() (*client, sqlmock.Sqlmock, error) { _mock.ExpectExec(secret.CreateTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(secret.CreateTypeOrgTeam).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(secret.CreateTypeOrg).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the service queries + _mock.ExpectExec(service.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the step queries _mock.ExpectExec(step.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the user queries @@ -272,12 +277,6 @@ func createTables(c *client) error { return fmt.Errorf("unable to create %s table: %w", constants.TableBuild, err) } - // create the services table - err = c.Postgres.Exec(ddl.CreateServiceTable).Error - if err != nil { - return fmt.Errorf("unable to create %s table: %w", constants.TableService, err) - } - return nil } @@ -317,10 +316,10 @@ func createIndexes(c *client) error { func createServices(c *client) error { var err error - // create the database agnostic hook service + // create the database agnostic engine for hooks // // https://pkg.go.dev/github.com/go-vela/server/database/hook#New - c.HookService, err = hook.New( + c.HookInterface, err = hook.New( hook.WithClient(c.Postgres), hook.WithLogger(c.Logger), hook.WithSkipCreation(c.config.SkipCreation), @@ -329,10 +328,10 @@ func createServices(c *client) error { return err } - // create the database agnostic log service + // create the database agnostic engine for logs // // https://pkg.go.dev/github.com/go-vela/server/database/log#New - c.LogService, err = log.New( + c.LogInterface, err = log.New( log.WithClient(c.Postgres), log.WithCompressionLevel(c.config.CompressionLevel), log.WithLogger(c.Logger), @@ -342,10 +341,10 @@ func createServices(c *client) error { return err } - // create the database agnostic pipeline service + // create the database agnostic engine for pipelines // // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#New - c.PipelineService, err = pipeline.New( + c.PipelineInterface, err = pipeline.New( pipeline.WithClient(c.Postgres), pipeline.WithCompressionLevel(c.config.CompressionLevel), pipeline.WithLogger(c.Logger), @@ -355,10 +354,10 @@ func createServices(c *client) error { return err } - // create the database agnostic repo service + // create the database agnostic engine for repos // // https://pkg.go.dev/github.com/go-vela/server/database/repo#New - c.RepoService, err = repo.New( + c.RepoInterface, err = repo.New( repo.WithClient(c.Postgres), repo.WithEncryptionKey(c.config.EncryptionKey), repo.WithLogger(c.Logger), @@ -368,10 +367,10 @@ func createServices(c *client) error { return err } - // create the database agnostic secret service + // create the database agnostic engine for secrets // // https://pkg.go.dev/github.com/go-vela/server/database/secret#New - c.SecretService, err = secret.New( + c.SecretInterface, err = secret.New( secret.WithClient(c.Postgres), secret.WithEncryptionKey(c.config.EncryptionKey), secret.WithLogger(c.Logger), @@ -381,10 +380,22 @@ func createServices(c *client) error { return err } - // create the database agnostic step service + // create the database agnostic engine for services // - // https://pkg.go.dev/github.com/go-vela/server/database/repo#New - c.StepService, err = step.New( + // https://pkg.go.dev/github.com/go-vela/server/database/service#New + c.ServiceInterface, err = service.New( + service.WithClient(c.Postgres), + service.WithLogger(c.Logger), + service.WithSkipCreation(c.config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for steps + // + // https://pkg.go.dev/github.com/go-vela/server/database/step#New + c.StepInterface, err = step.New( step.WithClient(c.Postgres), step.WithLogger(c.Logger), step.WithSkipCreation(c.config.SkipCreation), @@ -393,10 +404,10 @@ func createServices(c *client) error { return err } - // create the database agnostic user service + // create the database agnostic engine for users // // https://pkg.go.dev/github.com/go-vela/server/database/user#New - c.UserService, err = user.New( + c.UserInterface, err = user.New( user.WithClient(c.Postgres), user.WithEncryptionKey(c.config.EncryptionKey), user.WithLogger(c.Logger), @@ -406,10 +417,10 @@ func createServices(c *client) error { return err } - // create the database agnostic worker service + // create the database agnostic engine for workers // // https://pkg.go.dev/github.com/go-vela/server/database/worker#New - c.WorkerService, err = worker.New( + c.WorkerInterface, err = worker.New( worker.WithClient(c.Postgres), worker.WithLogger(c.Logger), worker.WithSkipCreation(c.config.SkipCreation), diff --git a/database/postgres/postgres_test.go b/database/postgres/postgres_test.go index 03c49a7d6..6074c65ca 100644 --- a/database/postgres/postgres_test.go +++ b/database/postgres/postgres_test.go @@ -16,6 +16,7 @@ import ( "github.com/go-vela/server/database/postgres/ddl" "github.com/go-vela/server/database/repo" "github.com/go-vela/server/database/secret" + "github.com/go-vela/server/database/service" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" @@ -82,7 +83,6 @@ func TestPostgres_setupDatabase(t *testing.T) { // ensure the mock expects the table queries _mock.ExpectExec(ddl.CreateBuildTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateServiceTable).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the index queries _mock.ExpectExec(ddl.CreateBuildRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -107,6 +107,8 @@ func TestPostgres_setupDatabase(t *testing.T) { _mock.ExpectExec(secret.CreateTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(secret.CreateTypeOrgTeam).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(secret.CreateTypeOrg).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the service queries + _mock.ExpectExec(service.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the step queries _mock.ExpectExec(step.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the user queries @@ -176,7 +178,6 @@ func TestPostgres_createTables(t *testing.T) { // ensure the mock expects the table queries _mock.ExpectExec(ddl.CreateBuildTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateServiceTable).WillReturnResult(sqlmock.NewResult(1, 1)) tests := []struct { failure bool @@ -273,6 +274,8 @@ func TestPostgres_createServices(t *testing.T) { _mock.ExpectExec(secret.CreateTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(secret.CreateTypeOrgTeam).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(secret.CreateTypeOrg).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the service queries + _mock.ExpectExec(service.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the step queries _mock.ExpectExec(step.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the user queries diff --git a/database/postgres/service.go b/database/postgres/service.go deleted file mode 100644 index 011a09770..000000000 --- a/database/postgres/service.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "errors" - - "github.com/sirupsen/logrus" - - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -// GetService gets a service by number and build ID from the database. -func (c *client) GetService(number int, b *library.Build) (*library.Service, error) { - c.Logger.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "service": number, - }).Tracef("getting service %d for build %d from the database", number, b.GetNumber()) - - // variable to store query results - s := new(database.Service) - - // send query to the database and store result in variable - result := c.Postgres. - Table(constants.TableService). - Raw(dml.SelectBuildService, b.GetID(), number). - Scan(s) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - return s.ToLibrary(), result.Error -} - -// CreateService creates a new service in the database. -func (c *client) CreateService(s *library.Service) error { - c.Logger.WithFields(logrus.Fields{ - "service": s.GetNumber(), - }).Tracef("creating service %s in the database", s.GetName()) - - // cast to database type - service := database.ServiceFromLibrary(s) - - // validate the necessary fields are populated - err := service.Validate() - if err != nil { - return err - } - - // send query to the database - return c.Postgres. - Table(constants.TableService). - Create(service).Error -} - -// UpdateService updates a service in the database. -func (c *client) UpdateService(s *library.Service) error { - c.Logger.WithFields(logrus.Fields{ - "service": s.GetNumber(), - }).Tracef("updating service %s in the database", s.GetName()) - - // cast to database type - service := database.ServiceFromLibrary(s) - - // validate the necessary fields are populated - err := service.Validate() - if err != nil { - return err - } - - // send query to the database - return c.Postgres. - Table(constants.TableService). - Save(service).Error -} - -// DeleteService deletes a service by unique ID from the database. -func (c *client) DeleteService(id int64) error { - c.Logger.Tracef("deleting service %d from the database", id) - - // send query to the database - return c.Postgres. - Table(constants.TableService). - Exec(dml.DeleteService, id).Error -} diff --git a/database/postgres/service_count.go b/database/postgres/service_count.go deleted file mode 100644 index 426239bc2..000000000 --- a/database/postgres/service_count.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" -) - -// GetBuildServiceCount gets a count of all services by build ID from the database. -func (c *client) GetBuildServiceCount(b *library.Build) (int64, error) { - c.Logger.WithFields(logrus.Fields{ - "build": b.GetNumber(), - }).Tracef("getting count of services for build %d from the database", b.GetNumber()) - - // variable to store query results - var s int64 - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableService). - Raw(dml.SelectBuildServicesCount, b.GetID()). - Pluck("count", &s).Error - - return s, err -} - -// GetServiceImageCount gets a count of all service images -// and the count of their occurrence in the database. -func (c *client) GetServiceImageCount() (map[string]float64, error) { - c.Logger.Tracef("getting count of all images for services from the database") - - type imageCount struct { - Image string - Count int - } - - // variable to store query results - images := new([]imageCount) - counts := make(map[string]float64) - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableService). - Raw(dml.SelectServiceImagesCount). - Scan(images).Error - - for _, image := range *images { - counts[image.Image] = float64(image.Count) - } - - return counts, err -} - -// GetServiceStatusCount gets a list of all service statuses -// and the count of their occurrence in the database. -func (c *client) GetServiceStatusCount() (map[string]float64, error) { - c.Logger.Trace("getting count of all statuses for services from the database") - - type statusCount struct { - Status string - Count int - } - - // variable to store query results - s := new([]statusCount) - counts := map[string]float64{ - "pending": 0, - "failure": 0, - "killed": 0, - "running": 0, - "success": 0, - } - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableService). - Raw(dml.SelectServiceStatusesCount). - Scan(s).Error - - for _, status := range *s { - counts[status.Status] = float64(status.Count) - } - - return counts, err -} diff --git a/database/postgres/service_count_test.go b/database/postgres/service_count_test.go deleted file mode 100644 index 52226e296..000000000 --- a/database/postgres/service_count_test.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "reflect" - "testing" - - sqlmock "github.com/DATA-DOG/go-sqlmock" - - "github.com/go-vela/server/database/postgres/dml" - - "gorm.io/gorm" -) - -func TestPostgres_Client_GetBuildServiceCount(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectBuildServicesCount, 1).Statement - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) - - // ensure the mock expects the query - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetBuildServiceCount(_build) - - if test.failure { - if err == nil { - t.Errorf("GetBuildServiceCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildServiceCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildServiceCount is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetServiceImageCount(t *testing.T) { - // setup types - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectServiceImagesCount).Statement - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"image", "count"}).AddRow("foo", 0) - - // ensure the mock expects the query - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want map[string]float64 - }{ - { - failure: false, - want: map[string]float64{"foo": 0}, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetServiceImageCount() - - if test.failure { - if err == nil { - t.Errorf("GetServiceImageCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetServiceImageCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetServiceImageCount is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetServiceStatusCount(t *testing.T) { - // setup types - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectServiceStatusesCount).Statement - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"status", "count"}). - AddRow("failure", 0). - AddRow("killed", 0). - AddRow("pending", 0). - AddRow("running", 0). - AddRow("success", 0) - - // ensure the mock expects the query - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want map[string]float64 - }{ - { - failure: false, - want: map[string]float64{ - "pending": 0, - "failure": 0, - "killed": 0, - "running": 0, - "success": 0, - }, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetServiceStatusCount() - - if test.failure { - if err == nil { - t.Errorf("GetServiceStatusCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetServiceStatusCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetServiceStatusCount is %v, want %v", got, test.want) - } - } -} diff --git a/database/postgres/service_list.go b/database/postgres/service_list.go deleted file mode 100644 index b26536d56..000000000 --- a/database/postgres/service_list.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" -) - -// GetServiceList gets a list of all services from the database. -func (c *client) GetServiceList() ([]*library.Service, error) { - c.Logger.Trace("listing services from the database") - - // variable to store query results - s := new([]database.Service) - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableService). - Raw(dml.ListServices). - Scan(s).Error - - // variable we want to return - services := []*library.Service{} - // iterate through all query results - for _, service := range *s { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := service - - // convert query result to library type - services = append(services, tmp.ToLibrary()) - } - - return services, err -} - -// GetBuildServiceList gets a list of services by build ID from the database. -func (c *client) GetBuildServiceList(b *library.Build, page, perPage int) ([]*library.Service, error) { - c.Logger.WithFields(logrus.Fields{ - "build": b.GetNumber(), - }).Tracef("listing services for build %d from the database", b.GetNumber()) - - // variable to store query results - s := new([]database.Service) - // calculate offset for pagination through results - offset := perPage * (page - 1) - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableService). - Raw(dml.ListBuildServices, b.GetID(), perPage, offset). - Scan(s).Error - - // variable we want to return - services := []*library.Service{} - // iterate through all query results - for _, service := range *s { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := service - - // convert query result to library type - services = append(services, tmp.ToLibrary()) - } - - return services, err -} diff --git a/database/postgres/service_list_test.go b/database/postgres/service_list_test.go deleted file mode 100644 index 40c267d27..000000000 --- a/database/postgres/service_list_test.go +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "reflect" - "testing" - - sqlmock "github.com/DATA-DOG/go-sqlmock" - - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -func TestPostgres_Client_GetServiceList(t *testing.T) { - // setup types - _serviceOne := testService() - _serviceOne.SetID(1) - _serviceOne.SetRepoID(1) - _serviceOne.SetBuildID(1) - _serviceOne.SetNumber(1) - _serviceOne.SetName("foo") - _serviceOne.SetImage("bar") - - _serviceTwo := testService() - _serviceTwo.SetID(2) - _serviceTwo.SetRepoID(1) - _serviceTwo.SetBuildID(1) - _serviceTwo.SetNumber(1) - _serviceTwo.SetName("bar") - _serviceTwo.SetImage("foo") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.ListServices).Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "repo_id", "build_id", "number", "name", "image", "status", "error", "exit_code", "created", "started", "finished", "host", "runtime", "distribution"}, - ).AddRow(1, 1, 1, 1, "foo", "bar", "", "", 0, 0, 0, 0, "", "", ""). - AddRow(2, 1, 1, 1, "bar", "foo", "", "", 0, 0, 0, 0, "", "", "") - - // ensure the mock expects the query - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want []*library.Service - }{ - { - failure: false, - want: []*library.Service{_serviceOne, _serviceTwo}, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetServiceList() - - if test.failure { - if err == nil { - t.Errorf("GetServiceList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetServiceList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetServiceList is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetBuildServiceList(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - - _serviceOne := testService() - _serviceOne.SetID(1) - _serviceOne.SetRepoID(1) - _serviceOne.SetBuildID(1) - _serviceOne.SetNumber(1) - _serviceOne.SetName("foo") - _serviceOne.SetImage("bar") - - _serviceTwo := testService() - _serviceTwo.SetID(2) - _serviceTwo.SetRepoID(1) - _serviceTwo.SetBuildID(1) - _serviceTwo.SetNumber(1) - _serviceTwo.SetName("bar") - _serviceTwo.SetImage("foo") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.ListBuildServices, 1, 1, 10).Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "repo_id", "build_id", "number", "name", "image", "status", "error", "exit_code", "created", "started", "finished", "host", "runtime", "distribution"}, - ).AddRow(1, 1, 1, 1, "foo", "bar", "", "", 0, 0, 0, 0, "", "", ""). - AddRow(2, 1, 1, 1, "bar", "foo", "", "", 0, 0, 0, 0, "", "", "") - - // ensure the mock expects the query - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want []*library.Service - }{ - { - failure: false, - want: []*library.Service{_serviceOne, _serviceTwo}, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetBuildServiceList(_build, 1, 10) - - if test.failure { - if err == nil { - t.Errorf("GetBuildServiceList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildServiceList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildServiceList is %v, want %v", got, test.want) - } - } -} diff --git a/database/postgres/service_test.go b/database/postgres/service_test.go deleted file mode 100644 index 5c88561e5..000000000 --- a/database/postgres/service_test.go +++ /dev/null @@ -1,267 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "reflect" - "testing" - - sqlmock "github.com/DATA-DOG/go-sqlmock" - - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -func TestPostgres_Client_GetService(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - - _service := testService() - _service.SetID(1) - _service.SetRepoID(1) - _service.SetBuildID(1) - _service.SetNumber(1) - _service.SetName("foo") - _service.SetImage("bar") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectBuildService, 1, 1).Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "repo_id", "build_id", "number", "name", "image", "status", "error", "exit_code", "created", "started", "finished", "host", "runtime", "distribution"}, - ).AddRow(1, 1, 1, 1, "foo", "bar", "", "", 0, 0, 0, 0, "", "", "") - - // ensure the mock expects the query for test case 1 - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - // ensure the mock expects the error for test case 2 - _mock.ExpectQuery(_query.SQL.String()).WillReturnError(gorm.ErrRecordNotFound) - - // setup tests - tests := []struct { - failure bool - want *library.Service - }{ - { - failure: false, - want: _service, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetService(1, _build) - - if test.failure { - if err == nil { - t.Errorf("GetService should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetService returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetService is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_CreateService(t *testing.T) { - // setup types - _service := testService() - _service.SetID(1) - _service.SetRepoID(1) - _service.SetBuildID(1) - _service.SetNumber(1) - _service.SetName("foo") - _service.SetImage("bar") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) - - // ensure the mock expects the query - _mock.ExpectQuery(`INSERT INTO "services" ("build_id","repo_id","number","name","image","status","error","exit_code","created","started","finished","host","runtime","distribution","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). - WithArgs(1, 1, 1, "foo", "bar", nil, nil, nil, nil, nil, nil, nil, nil, nil, 1). - WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := _database.CreateService(_service) - - if test.failure { - if err == nil { - t.Errorf("CreateService should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("CreateService returned err: %v", err) - } - } -} - -func TestPostgres_Client_UpdateService(t *testing.T) { - // setup types - _service := testService() - _service.SetID(1) - _service.SetRepoID(1) - _service.SetBuildID(1) - _service.SetNumber(1) - _service.SetName("foo") - _service.SetImage("bar") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // ensure the mock expects the query - _mock.ExpectExec(`UPDATE "services" SET "build_id"=$1,"repo_id"=$2,"number"=$3,"name"=$4,"image"=$5,"status"=$6,"error"=$7,"exit_code"=$8,"created"=$9,"started"=$10,"finished"=$11,"host"=$12,"runtime"=$13,"distribution"=$14 WHERE "id" = $15`). - WithArgs(1, 1, 1, "foo", "bar", nil, nil, nil, nil, nil, nil, nil, nil, nil, 1). - WillReturnResult(sqlmock.NewResult(1, 1)) - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := _database.UpdateService(_service) - - if test.failure { - if err == nil { - t.Errorf("UpdateService should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("UpdateService returned err: %v", err) - } - } -} - -func TestPostgres_Client_DeleteService(t *testing.T) { - // setup types - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Exec(dml.DeleteService, 1).Statement - - // ensure the mock expects the query - _mock.ExpectExec(_query.SQL.String()).WillReturnResult(sqlmock.NewResult(1, 1)) - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := _database.DeleteService(1) - - if test.failure { - if err == nil { - t.Errorf("DeleteService should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("DeleteService returned err: %v", err) - } - } -} - -// testService is a test helper function to create a -// library Service type with all fields set to their -// zero values. -func testService() *library.Service { - i64 := int64(0) - i := 0 - str := "" - - return &library.Service{ - ID: &i64, - BuildID: &i64, - RepoID: &i64, - Number: &i, - Name: &str, - Image: &str, - Status: &str, - Error: &str, - ExitCode: &i, - Created: &i64, - Started: &i64, - Finished: &i64, - Host: &str, - Runtime: &str, - Distribution: &str, - } -} diff --git a/database/repo/service.go b/database/repo/interface.go similarity index 96% rename from database/repo/service.go rename to database/repo/interface.go index 7695333ff..a53ad3a0a 100644 --- a/database/repo/service.go +++ b/database/repo/interface.go @@ -8,11 +8,11 @@ import ( "github.com/go-vela/types/library" ) -// RepoService represents the Vela interface for repo +// RepoInterface represents the Vela interface for repo // functions with the supported Database backends. // //nolint:revive // ignore name stutter -type RepoService interface { +type RepoInterface interface { // Repo Data Definition Language Functions // // https://en.wikipedia.org/wiki/Data_definition_language diff --git a/database/repo/repo.go b/database/repo/repo.go index e0f768741..1da6a8ab8 100644 --- a/database/repo/repo.go +++ b/database/repo/repo.go @@ -14,7 +14,7 @@ import ( ) type ( - // config represents the settings required to create the engine that implements the RepoService interface. + // config represents the settings required to create the engine that implements the RepoInterface interface. config struct { // specifies the encryption key to use for the Repo engine EncryptionKey string @@ -22,7 +22,7 @@ type ( SkipCreation bool } - // engine represents the repo functionality that implements the RepoService interface. + // engine represents the repo functionality that implements the RepoInterface interface. engine struct { // engine configuration settings used in repo functions config *config diff --git a/database/secret/service.go b/database/secret/interface.go similarity index 97% rename from database/secret/service.go rename to database/secret/interface.go index 7637a91b8..c6bee369a 100644 --- a/database/secret/service.go +++ b/database/secret/interface.go @@ -8,11 +8,11 @@ import ( "github.com/go-vela/types/library" ) -// SecretService represents the Vela interface for secret +// SecretInterface represents the Vela interface for secret // functions with the supported Database backends. // //nolint:revive // ignore name stutter -type SecretService interface { +type SecretInterface interface { // Secret Data Definition Language Functions // // https://en.wikipedia.org/wiki/Data_definition_language diff --git a/database/secret/secret.go b/database/secret/secret.go index 9f19279f4..fa342f5d5 100644 --- a/database/secret/secret.go +++ b/database/secret/secret.go @@ -14,7 +14,7 @@ import ( ) type ( - // config represents the settings required to create the engine that implements the SecretService interface. + // config represents the settings required to create the engine that implements the SecretInterface interface. config struct { // specifies the encryption key to use for the Secret engine EncryptionKey string @@ -22,7 +22,7 @@ type ( SkipCreation bool } - // engine represents the secret functionality that implements the SecretService interface. + // engine represents the secret functionality that implements the SecretInterface interface. engine struct { // engine configuration settings used in secret functions config *config diff --git a/database/service/count.go b/database/service/count.go new file mode 100644 index 000000000..08b488933 --- /dev/null +++ b/database/service/count.go @@ -0,0 +1,25 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "github.com/go-vela/types/constants" +) + +// CountServices gets the count of all services from the database. +func (e *engine) CountServices() (int64, error) { + e.logger.Tracef("getting count of all services from the database") + + // variable to store query results + var s int64 + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableService). + Count(&s). + Error + + return s, err +} diff --git a/database/service/count_build.go b/database/service/count_build.go new file mode 100644 index 000000000..0cd773570 --- /dev/null +++ b/database/service/count_build.go @@ -0,0 +1,31 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// CountServicesForBuild gets the count of services by build ID from the database. +func (e *engine) CountServicesForBuild(b *library.Build, filters map[string]interface{}) (int64, error) { + e.logger.WithFields(logrus.Fields{ + "build": b.GetNumber(), + }).Tracef("getting count of services for build %d from the database", b.GetNumber()) + + // variable to store query results + var s int64 + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableService). + Where("build_id = ?", b.GetID()). + Where(filters). + Count(&s). + Error + + return s, err +} diff --git a/database/service/count_build_test.go b/database/service/count_build_test.go new file mode 100644 index 000000000..08c964da7 --- /dev/null +++ b/database/service/count_build_test.go @@ -0,0 +1,104 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestService_Engine_CountServicesForBuild(t *testing.T) { + // setup types + _build := testBuild() + _build.SetID(1) + _build.SetRepoID(1) + _build.SetNumber(1) + + _serviceOne := testService() + _serviceOne.SetID(1) + _serviceOne.SetRepoID(1) + _serviceOne.SetBuildID(1) + _serviceOne.SetNumber(1) + _serviceOne.SetName("foo") + _serviceOne.SetImage("bar") + + _serviceTwo := testService() + _serviceTwo.SetID(2) + _serviceTwo.SetRepoID(1) + _serviceTwo.SetBuildID(2) + _serviceTwo.SetNumber(1) + _serviceTwo.SetName("foo") + _serviceTwo.SetImage("bar") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(1) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "services" WHERE build_id = $1`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateService(_serviceOne) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + err = _sqlite.CreateService(_serviceTwo) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 1, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 1, + }, + } + + filters := map[string]interface{}{} + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountServicesForBuild(_build, filters) + + if test.failure { + if err == nil { + t.Errorf("CountServicesForBuild for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountServicesForBuild for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountServicesForBuild for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/service/count_test.go b/database/service/count_test.go new file mode 100644 index 000000000..226d4b879 --- /dev/null +++ b/database/service/count_test.go @@ -0,0 +1,97 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestService_Engine_CountServices(t *testing.T) { + // setup types + _serviceOne := testService() + _serviceOne.SetID(1) + _serviceOne.SetRepoID(1) + _serviceOne.SetBuildID(1) + _serviceOne.SetNumber(1) + _serviceOne.SetName("foo") + _serviceOne.SetImage("bar") + + _serviceTwo := testService() + _serviceTwo.SetID(2) + _serviceTwo.SetRepoID(1) + _serviceTwo.SetBuildID(2) + _serviceTwo.SetNumber(1) + _serviceTwo.SetName("foo") + _serviceTwo.SetImage("bar") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "services"`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateService(_serviceOne) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + err = _sqlite.CreateService(_serviceTwo) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 2, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 2, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountServices() + + if test.failure { + if err == nil { + t.Errorf("CountServices for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountServices for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountServices for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/service/create.go b/database/service/create.go new file mode 100644 index 000000000..56574e23b --- /dev/null +++ b/database/service/create.go @@ -0,0 +1,38 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// CreateService creates a new service in the database. +func (e *engine) CreateService(s *library.Service) error { + e.logger.WithFields(logrus.Fields{ + "service": s.GetNumber(), + }).Tracef("creating service %s in the database", s.GetName()) + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#ServiceFromLibrary + service := database.ServiceFromLibrary(s) + + // validate the necessary fields are populated + // + // https://pkg.go.dev/github.com/go-vela/types/database#Service.Validate + err := service.Validate() + if err != nil { + return err + } + + // send query to the database + return e.client. + Table(constants.TableService). + Create(service). + Error +} diff --git a/database/service/create_test.go b/database/service/create_test.go new file mode 100644 index 000000000..4e84ed090 --- /dev/null +++ b/database/service/create_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestService_Engine_CreateService(t *testing.T) { + // setup types + _service := testService() + _service.SetID(1) + _service.SetRepoID(1) + _service.SetBuildID(1) + _service.SetNumber(1) + _service.SetName("foo") + _service.SetImage("bar") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) + + // ensure the mock expects the query + _mock.ExpectQuery(`INSERT INTO "services" +("build_id","repo_id","number","name","image","status","error","exit_code","created","started","finished","host","runtime","distribution","id") +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). + WithArgs(1, 1, 1, "foo", "bar", nil, nil, nil, nil, nil, nil, nil, nil, nil, 1). + WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateService(_service) + + if test.failure { + if err == nil { + t.Errorf("CreateService for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateService for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/service/delete.go b/database/service/delete.go new file mode 100644 index 000000000..86c0c21b2 --- /dev/null +++ b/database/service/delete.go @@ -0,0 +1,30 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// DeleteService deletes an existing service from the database. +func (e *engine) DeleteService(s *library.Service) error { + e.logger.WithFields(logrus.Fields{ + "service": s.GetNumber(), + }).Tracef("deleting service %s from the database", s.GetName()) + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#ServiceFromLibrary + service := database.ServiceFromLibrary(s) + + // send query to the database + return e.client. + Table(constants.TableService). + Delete(service). + Error +} diff --git a/database/service/delete_test.go b/database/service/delete_test.go new file mode 100644 index 000000000..d63c55c05 --- /dev/null +++ b/database/service/delete_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestService_Engine_DeleteService(t *testing.T) { + // setup types + _service := testService() + _service.SetID(1) + _service.SetRepoID(1) + _service.SetBuildID(1) + _service.SetNumber(1) + _service.SetName("foo") + _service.SetImage("bar") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`DELETE FROM "services" WHERE "services"."id" = $1`). + WithArgs(1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateService(_service) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err = test.database.DeleteService(_service) + + if test.failure { + if err == nil { + t.Errorf("DeleteService for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("DeleteService for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/service/get.go b/database/service/get.go new file mode 100644 index 000000000..7678168b2 --- /dev/null +++ b/database/service/get.go @@ -0,0 +1,34 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// GetService gets a service by ID from the database. +func (e *engine) GetService(id int64) (*library.Service, error) { + e.logger.Tracef("getting service %d from the database", id) + + // variable to store query results + s := new(database.Service) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableService). + Where("id = ?", id). + Take(s). + Error + if err != nil { + return nil, err + } + + // return the service + // + // https://pkg.go.dev/github.com/go-vela/types/database#Service.ToLibrary + return s.ToLibrary(), nil +} diff --git a/database/service/get_build.go b/database/service/get_build.go new file mode 100644 index 000000000..5321cc728 --- /dev/null +++ b/database/service/get_build.go @@ -0,0 +1,39 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// GetServiceForBuild gets a service by number and build ID from the database. +func (e *engine) GetServiceForBuild(b *library.Build, number int) (*library.Service, error) { + e.logger.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "service": number, + }).Tracef("getting service %d from the database", number) + + // variable to store query results + s := new(database.Service) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableService). + Where("build_id = ?", b.GetID()). + Where("number = ?", number). + Take(s). + Error + if err != nil { + return nil, err + } + + // return the service + // + // https://pkg.go.dev/github.com/go-vela/types/database#Service.ToLibrary + return s.ToLibrary(), nil +} diff --git a/database/service/get_build_test.go b/database/service/get_build_test.go new file mode 100644 index 000000000..36747ad65 --- /dev/null +++ b/database/service/get_build_test.go @@ -0,0 +1,92 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestService_Engine_GetServiceForBuild(t *testing.T) { + // setup types + _build := testBuild() + _build.SetID(1) + _build.SetRepoID(1) + _build.SetNumber(1) + + _service := testService() + _service.SetID(1) + _service.SetRepoID(1) + _service.SetBuildID(1) + _service.SetNumber(1) + _service.SetName("foo") + _service.SetImage("bar") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "repo_id", "build_id", "number", "name", "image", "stage", "status", "error", "exit_code", "created", "started", "finished", "host", "runtime", "distribution"}). + AddRow(1, 1, 1, 1, "foo", "bar", "", "", "", 0, 0, 0, 0, "", "", "") + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "services" WHERE build_id = $1 AND number = $2 LIMIT 1`).WithArgs(1, 1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateService(_service) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.Service + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _service, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _service, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetServiceForBuild(_build, 1) + + if test.failure { + if err == nil { + t.Errorf("GetServiceForBuild for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetServiceForBuild for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetServiceForBuild for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/service/get_test.go b/database/service/get_test.go new file mode 100644 index 000000000..13bd6691a --- /dev/null +++ b/database/service/get_test.go @@ -0,0 +1,87 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestService_Engine_GetService(t *testing.T) { + // setup types + _service := testService() + _service.SetID(1) + _service.SetRepoID(1) + _service.SetBuildID(1) + _service.SetNumber(1) + _service.SetName("foo") + _service.SetImage("bar") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "repo_id", "build_id", "number", "name", "image", "stage", "status", "error", "exit_code", "created", "started", "finished", "host", "runtime", "distribution"}). + AddRow(1, 1, 1, 1, "foo", "bar", "", "", "", 0, 0, 0, 0, "", "", "") + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "services" WHERE id = $1 LIMIT 1`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateService(_service) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.Service + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _service, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _service, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetService(1) + + if test.failure { + if err == nil { + t.Errorf("GetService for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetService for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetService for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/service/interface.go b/database/service/interface.go new file mode 100644 index 000000000..29cb0f457 --- /dev/null +++ b/database/service/interface.go @@ -0,0 +1,49 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "github.com/go-vela/types/library" +) + +// ServiceInterface represents the Vela interface for service +// functions with the supported Database backends. +// +//nolint:revive // ignore name stutter +type ServiceInterface interface { + // Service Data Definition Language Functions + // + // https://en.wikipedia.org/wiki/Data_definition_language + + // CreateServiceTable defines a function that creates the services table. + CreateServiceTable(string) error + + // Service Data Manipulation Language Functions + // + // https://en.wikipedia.org/wiki/Data_manipulation_language + + // CountServices defines a function that gets the count of all services. + CountServices() (int64, error) + // CountServicesForBuild defines a function that gets the count of services by build ID. + CountServicesForBuild(*library.Build, map[string]interface{}) (int64, error) + // CreateService defines a function that creates a new service. + CreateService(*library.Service) error + // DeleteService defines a function that deletes an existing service. + DeleteService(*library.Service) error + // GetService defines a function that gets a service by ID. + GetService(int64) (*library.Service, error) + // GetServiceForBuild defines a function that gets a service by number and build ID. + GetServiceForBuild(*library.Build, int) (*library.Service, error) + // ListServices defines a function that gets a list of all services. + ListServices() ([]*library.Service, error) + // ListServicesForBuild defines a function that gets a list of services by build ID. + ListServicesForBuild(*library.Build, map[string]interface{}, int, int) ([]*library.Service, int64, error) + // ListServiceImageCount defines a function that gets a list of all service images and the count of their occurrence. + ListServiceImageCount() (map[string]float64, error) + // ListServiceStatusCount defines a function that gets a list of all service statuses and the count of their occurrence. + ListServiceStatusCount() (map[string]float64, error) + // UpdateService defines a function that updates an existing service. + UpdateService(*library.Service) error +} diff --git a/database/service/list.go b/database/service/list.go new file mode 100644 index 000000000..c7b1b6b13 --- /dev/null +++ b/database/service/list.go @@ -0,0 +1,54 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// ListServices gets a list of all services from the database. +func (e *engine) ListServices() ([]*library.Service, error) { + e.logger.Trace("listing all services from the database") + + // variables to store query results and return value + count := int64(0) + w := new([]database.Service) + services := []*library.Service{} + + // count the results + count, err := e.CountServices() + if err != nil { + return nil, err + } + + // short-circuit if there are no results + if count == 0 { + return services, nil + } + + // send query to the database and store result in variable + err = e.client. + Table(constants.TableService). + Find(&w). + Error + if err != nil { + return nil, err + } + + // iterate through all query results + for _, service := range *w { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := service + + // convert query result to library type + // + // https://pkg.go.dev/github.com/go-vela/types/database#Service.ToLibrary + services = append(services, tmp.ToLibrary()) + } + + return services, nil +} diff --git a/database/service/list_build.go b/database/service/list_build.go new file mode 100644 index 000000000..caffb2d5c --- /dev/null +++ b/database/service/list_build.go @@ -0,0 +1,65 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// ListServicesForBuild gets a list of all services from the database. +func (e *engine) ListServicesForBuild(b *library.Build, filters map[string]interface{}, page int, perPage int) ([]*library.Service, int64, error) { + e.logger.WithFields(logrus.Fields{ + "build": b.GetNumber(), + }).Tracef("listing services for build %d from the database", b.GetNumber()) + + // variables to store query results and return value + count := int64(0) + s := new([]database.Service) + services := []*library.Service{} + + // count the results + count, err := e.CountServicesForBuild(b, filters) + if err != nil { + return services, 0, err + } + + // short-circuit if there are no results + if count == 0 { + return services, 0, nil + } + + // calculate offset for pagination through results + offset := perPage * (page - 1) + + // send query to the database and store result in variable + err = e.client. + Table(constants.TableService). + Where("build_id = ?", b.GetID()). + Where(filters). + Order("id DESC"). + Limit(perPage). + Offset(offset). + Find(&s). + Error + if err != nil { + return nil, count, err + } + + // iterate through all query results + for _, service := range *s { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := service + + // convert query result to library type + // + // https://pkg.go.dev/github.com/go-vela/types/database#Service.ToLibrary + services = append(services, tmp.ToLibrary()) + } + + return services, count, nil +} diff --git a/database/service/list_build_test.go b/database/service/list_build_test.go new file mode 100644 index 000000000..3b6d97161 --- /dev/null +++ b/database/service/list_build_test.go @@ -0,0 +1,114 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestService_Engine_ListServicesForBuild(t *testing.T) { + // setup types + _build := testBuild() + _build.SetID(1) + _build.SetRepoID(1) + _build.SetNumber(1) + + _serviceOne := testService() + _serviceOne.SetID(1) + _serviceOne.SetRepoID(1) + _serviceOne.SetBuildID(1) + _serviceOne.SetNumber(1) + _serviceOne.SetName("foo") + _serviceOne.SetImage("bar") + + _serviceTwo := testService() + _serviceTwo.SetID(2) + _serviceTwo.SetRepoID(1) + _serviceTwo.SetBuildID(1) + _serviceTwo.SetNumber(2) + _serviceTwo.SetName("foo") + _serviceTwo.SetImage("bar") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "services" WHERE build_id = $1`).WithArgs(1).WillReturnRows(_rows) + + // create expected result in mock + _rows = sqlmock.NewRows( + []string{"id", "repo_id", "build_id", "number", "name", "image", "stage", "status", "error", "exit_code", "created", "started", "finished", "host", "runtime", "distribution"}). + AddRow(2, 1, 1, 2, "foo", "bar", "", "", "", 0, 0, 0, 0, "", "", ""). + AddRow(1, 1, 1, 1, "foo", "bar", "", "", "", 0, 0, 0, 0, "", "", "") + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "services" WHERE build_id = $1 ORDER BY id DESC LIMIT 10`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateService(_serviceOne) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + err = _sqlite.CreateService(_serviceTwo) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.Service + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.Service{_serviceTwo, _serviceOne}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.Service{_serviceTwo, _serviceOne}, + }, + } + + filters := map[string]interface{}{} + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, _, err := test.database.ListServicesForBuild(_build, filters, 1, 10) + + if test.failure { + if err == nil { + t.Errorf("ListServicesForBuild for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListServicesForBuild for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListServicesForBuild for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/service/list_image.go b/database/service/list_image.go new file mode 100644 index 000000000..6625420dc --- /dev/null +++ b/database/service/list_image.go @@ -0,0 +1,44 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "database/sql" + + "github.com/go-vela/types/constants" +) + +// ListServiceImageCount gets a list of all service images and the count of their occurrence from the database. +func (e *engine) ListServiceImageCount() (map[string]float64, error) { + e.logger.Tracef("getting count of all images for services from the database") + + // variables to store query results and return value + s := []struct { + Image sql.NullString + Count sql.NullInt32 + }{} + images := make(map[string]float64) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableService). + Select("image", " count(image) as count"). + Group("image"). + Find(&s). + Error + if err != nil { + return nil, err + } + + // iterate through all query results + for _, value := range s { + // check if the image returned is not empty + if value.Image.Valid { + images[value.Image.String] = float64(value.Count.Int32) + } + } + + return images, nil +} diff --git a/database/service/list_image_test.go b/database/service/list_image_test.go new file mode 100644 index 000000000..f4e2f228e --- /dev/null +++ b/database/service/list_image_test.go @@ -0,0 +1,97 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestService_Engine_ListServiceImageCount(t *testing.T) { + // setup types + _serviceOne := testService() + _serviceOne.SetID(1) + _serviceOne.SetRepoID(1) + _serviceOne.SetBuildID(1) + _serviceOne.SetNumber(1) + _serviceOne.SetName("foo") + _serviceOne.SetImage("bar") + + _serviceTwo := testService() + _serviceTwo.SetID(2) + _serviceTwo.SetRepoID(1) + _serviceTwo.SetBuildID(1) + _serviceTwo.SetNumber(2) + _serviceTwo.SetName("foo") + _serviceTwo.SetImage("bar") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"image", "count"}).AddRow("bar", 2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT "image", count(image) as count FROM "services" GROUP BY "image"`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateService(_serviceOne) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + err = _sqlite.CreateService(_serviceTwo) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want map[string]float64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: map[string]float64{"bar": 2}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: map[string]float64{"bar": 2}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.ListServiceImageCount() + + if test.failure { + if err == nil { + t.Errorf("ListServiceImageCount for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListServiceImageCount for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListServiceImageCount for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/service/list_status.go b/database/service/list_status.go new file mode 100644 index 000000000..01aedeac1 --- /dev/null +++ b/database/service/list_status.go @@ -0,0 +1,50 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "database/sql" + + "github.com/go-vela/types/constants" +) + +// ListServiceStatusCount gets a list of all service statuses and the count of their occurrence from the database. +func (e *engine) ListServiceStatusCount() (map[string]float64, error) { + e.logger.Tracef("getting count of all statuses for services from the database") + + // variables to store query results and return value + s := []struct { + Status sql.NullString + Count sql.NullInt32 + }{} + statuses := map[string]float64{ + "pending": 0, + "failure": 0, + "killed": 0, + "running": 0, + "success": 0, + } + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableService). + Select("status", " count(status) as count"). + Group("status"). + Find(&s). + Error + if err != nil { + return nil, err + } + + // iterate through all query results + for _, value := range s { + // check if the status returned is not empty + if value.Status.Valid { + statuses[value.Status.String] = float64(value.Count.Int32) + } + } + + return statuses, nil +} diff --git a/database/service/list_status_test.go b/database/service/list_status_test.go new file mode 100644 index 000000000..85b7808af --- /dev/null +++ b/database/service/list_status_test.go @@ -0,0 +1,114 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestService_Engine_ListServiceStatusCount(t *testing.T) { + // setup types + _serviceOne := testService() + _serviceOne.SetID(1) + _serviceOne.SetRepoID(1) + _serviceOne.SetBuildID(1) + _serviceOne.SetNumber(1) + _serviceOne.SetName("foo") + _serviceOne.SetImage("bar") + + _serviceTwo := testService() + _serviceTwo.SetID(2) + _serviceTwo.SetRepoID(1) + _serviceTwo.SetBuildID(1) + _serviceTwo.SetNumber(2) + _serviceTwo.SetName("foo") + _serviceTwo.SetImage("bar") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"status", "count"}). + AddRow("pending", 0). + AddRow("failure", 0). + AddRow("killed", 0). + AddRow("running", 0). + AddRow("success", 0) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT "status", count(status) as count FROM "services" GROUP BY "status"`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateService(_serviceOne) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + err = _sqlite.CreateService(_serviceTwo) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want map[string]float64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: map[string]float64{ + "pending": 0, + "failure": 0, + "killed": 0, + "running": 0, + "success": 0, + }, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: map[string]float64{ + "pending": 0, + "failure": 0, + "killed": 0, + "running": 0, + "success": 0, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.ListServiceStatusCount() + + if test.failure { + if err == nil { + t.Errorf("ListServiceStatusCount for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListServiceStatusCount for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListServiceStatusCount for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/service/list_test.go b/database/service/list_test.go new file mode 100644 index 000000000..72b60d5cf --- /dev/null +++ b/database/service/list_test.go @@ -0,0 +1,107 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestService_Engine_ListServices(t *testing.T) { + // setup types + _serviceOne := testService() + _serviceOne.SetID(1) + _serviceOne.SetRepoID(1) + _serviceOne.SetBuildID(1) + _serviceOne.SetNumber(1) + _serviceOne.SetName("foo") + _serviceOne.SetImage("bar") + + _serviceTwo := testService() + _serviceTwo.SetID(2) + _serviceTwo.SetRepoID(1) + _serviceTwo.SetBuildID(2) + _serviceTwo.SetNumber(1) + _serviceTwo.SetName("bar") + _serviceTwo.SetImage("foo") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "services"`).WillReturnRows(_rows) + + // create expected result in mock + _rows = sqlmock.NewRows( + []string{"id", "repo_id", "build_id", "number", "name", "image", "stage", "status", "error", "exit_code", "created", "started", "finished", "host", "runtime", "distribution"}). + AddRow(1, 1, 1, 1, "foo", "bar", "", "", "", 0, 0, 0, 0, "", "", ""). + AddRow(2, 1, 2, 1, "bar", "foo", "", "", "", 0, 0, 0, 0, "", "", "") + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "services"`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateService(_serviceOne) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + err = _sqlite.CreateService(_serviceTwo) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.Service + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.Service{_serviceOne, _serviceTwo}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.Service{_serviceOne, _serviceTwo}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.ListServices() + + if test.failure { + if err == nil { + t.Errorf("ListServices for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListServices for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListServices for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/service/opts.go b/database/service/opts.go new file mode 100644 index 000000000..2201b1abd --- /dev/null +++ b/database/service/opts.go @@ -0,0 +1,44 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +// EngineOpt represents a configuration option to initialize the database engine for Services. +type EngineOpt func(*engine) error + +// WithClient sets the gorm.io/gorm client in the database engine for Services. +func WithClient(client *gorm.DB) EngineOpt { + return func(e *engine) error { + // set the gorm.io/gorm client in the service engine + e.client = client + + return nil + } +} + +// WithLogger sets the github.com/sirupsen/logrus logger in the database engine for Services. +func WithLogger(logger *logrus.Entry) EngineOpt { + return func(e *engine) error { + // set the github.com/sirupsen/logrus logger in the service engine + e.logger = logger + + return nil + } +} + +// WithSkipCreation sets the skip creation logic in the database engine for Services. +func WithSkipCreation(skipCreation bool) EngineOpt { + return func(e *engine) error { + // set to skip creating tables and indexes in the service engine + e.config.SkipCreation = skipCreation + + return nil + } +} diff --git a/database/service/opts_test.go b/database/service/opts_test.go new file mode 100644 index 000000000..62ce5adf8 --- /dev/null +++ b/database/service/opts_test.go @@ -0,0 +1,161 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "reflect" + "testing" + + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +func TestService_EngineOpt_WithClient(t *testing.T) { + // setup types + e := &engine{client: new(gorm.DB)} + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + want *gorm.DB + }{ + { + failure: false, + name: "client set to new database", + client: new(gorm.DB), + want: new(gorm.DB), + }, + { + failure: false, + name: "client set to nil", + client: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithClient(test.client)(e) + + if test.failure { + if err == nil { + t.Errorf("WithClient for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithClient returned err: %v", err) + } + + if !reflect.DeepEqual(e.client, test.want) { + t.Errorf("WithClient is %v, want %v", e.client, test.want) + } + }) + } +} + +func TestService_EngineOpt_WithLogger(t *testing.T) { + // setup types + e := &engine{logger: new(logrus.Entry)} + + // setup tests + tests := []struct { + failure bool + name string + logger *logrus.Entry + want *logrus.Entry + }{ + { + failure: false, + name: "logger set to new entry", + logger: new(logrus.Entry), + want: new(logrus.Entry), + }, + { + failure: false, + name: "logger set to nil", + logger: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithLogger(test.logger)(e) + + if test.failure { + if err == nil { + t.Errorf("WithLogger for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithLogger returned err: %v", err) + } + + if !reflect.DeepEqual(e.logger, test.want) { + t.Errorf("WithLogger is %v, want %v", e.logger, test.want) + } + }) + } +} + +func TestService_EngineOpt_WithSkipCreation(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + skipCreation bool + want bool + }{ + { + failure: false, + name: "skip creation set to true", + skipCreation: true, + want: true, + }, + { + failure: false, + name: "skip creation set to false", + skipCreation: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithSkipCreation(test.skipCreation)(e) + + if test.failure { + if err == nil { + t.Errorf("WithSkipCreation for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithSkipCreation returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.SkipCreation, test.want) { + t.Errorf("WithSkipCreation is %v, want %v", e.config.SkipCreation, test.want) + } + }) + } +} diff --git a/database/service/service.go b/database/service/service.go new file mode 100644 index 000000000..548164c0f --- /dev/null +++ b/database/service/service.go @@ -0,0 +1,74 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "fmt" + + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +type ( + // config represents the settings required to create the engine that implements the ServiceInterface interface. + config struct { + // specifies to skip creating tables and indexes for the Service engine + SkipCreation bool + } + + // engine represents the service functionality that implements the ServiceInterface interface. + engine struct { + // engine configuration settings used in service functions + config *config + + // gorm.io/gorm database client used in service functions + // + // https://pkg.go.dev/gorm.io/gorm#DB + client *gorm.DB + + // sirupsen/logrus logger used in service functions + // + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry + logger *logrus.Entry + } +) + +// New creates and returns a Vela service for integrating with services in the database. +// +//nolint:revive // ignore returning unexported engine +func New(opts ...EngineOpt) (*engine, error) { + // create new Service engine + e := new(engine) + + // create new fields + e.client = new(gorm.DB) + e.config = new(config) + e.logger = new(logrus.Entry) + + // apply all provided configuration options + for _, opt := range opts { + err := opt(e) + if err != nil { + return nil, err + } + } + + // check if we should skip creating service database objects + if e.config.SkipCreation { + e.logger.Warning("skipping creation of services table in the database") + + return e, nil + } + + // create the services table + err := e.CreateServiceTable(e.client.Config.Dialector.Name()) + if err != nil { + return nil, fmt.Errorf("unable to create %s table: %w", constants.TableService, err) + } + + return e, nil +} diff --git a/database/service/service_test.go b/database/service/service_test.go new file mode 100644 index 000000000..7749f43d3 --- /dev/null +++ b/database/service/service_test.go @@ -0,0 +1,224 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" + + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestService_New(t *testing.T) { + // setup types + logger := logrus.NewEntry(logrus.StandardLogger()) + + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + defer _sql.Close() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _config := &gorm.Config{SkipDefaultTransaction: true} + + _postgres, err := gorm.Open(postgres.New(postgres.Config{Conn: _sql}), _config) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _sqlite, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), _config) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + defer func() { _sql, _ := _sqlite.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + key string + logger *logrus.Entry + skipCreation bool + want *engine + }{ + { + failure: false, + name: "postgres", + client: _postgres, + logger: logger, + skipCreation: false, + want: &engine{ + client: _postgres, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + { + failure: false, + name: "sqlite3", + client: _sqlite, + logger: logger, + skipCreation: false, + want: &engine{ + client: _sqlite, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := New( + WithClient(test.client), + WithLogger(test.logger), + WithSkipCreation(test.skipCreation), + ) + + if test.failure { + if err == nil { + t.Errorf("New for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("New for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("New for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} + +// testPostgres is a helper function to create a Postgres engine for testing. +func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { + // create the new mock sql database + // + // https://pkg.go.dev/github.com/DATA-DOG/go-sqlmock#New + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + // create the new mock Postgres database client + // + // https://pkg.go.dev/gorm.io/gorm#Open + _postgres, err := gorm.Open( + postgres.New(postgres.Config{Conn: _sql}), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _engine, err := New( + WithClient(_postgres), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new postgres service engine: %v", err) + } + + return _engine, _mock +} + +// testSqlite is a helper function to create a Sqlite engine for testing. +func testSqlite(t *testing.T) *engine { + _sqlite, err := gorm.Open( + sqlite.Open("file::memory:?cache=shared"), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + _engine, err := New( + WithClient(_sqlite), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new sqlite service engine: %v", err) + } + + return _engine +} + +// testBuild is a test helper function to create a library +// Build type with all fields set to their zero values. +func testBuild() *library.Build { + return &library.Build{ + ID: new(int64), + RepoID: new(int64), + PipelineID: new(int64), + Number: new(int), + Parent: new(int), + Event: new(string), + EventAction: new(string), + Status: new(string), + Error: new(string), + Enqueued: new(int64), + Created: new(int64), + Started: new(int64), + Finished: new(int64), + Deploy: new(string), + Clone: new(string), + Source: new(string), + Title: new(string), + Message: new(string), + Commit: new(string), + Sender: new(string), + Author: new(string), + Email: new(string), + Link: new(string), + Branch: new(string), + Ref: new(string), + BaseRef: new(string), + HeadRef: new(string), + Host: new(string), + Runtime: new(string), + Distribution: new(string), + } +} + +// testService is a test helper function to create a library +// Service type with all fields set to their zero values. +func testService() *library.Service { + return &library.Service{ + ID: new(int64), + BuildID: new(int64), + RepoID: new(int64), + Number: new(int), + Name: new(string), + Image: new(string), + Status: new(string), + Error: new(string), + ExitCode: new(int), + Created: new(int64), + Started: new(int64), + Finished: new(int64), + Host: new(string), + Runtime: new(string), + Distribution: new(string), + } +} diff --git a/database/service/table.go b/database/service/table.go new file mode 100644 index 000000000..48f8dc712 --- /dev/null +++ b/database/service/table.go @@ -0,0 +1,76 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "github.com/go-vela/types/constants" +) + +const ( + // CreatePostgresTable represents a query to create the Postgres services table. + CreatePostgresTable = ` +CREATE TABLE +IF NOT EXISTS +services ( + id SERIAL PRIMARY KEY, + repo_id INTEGER, + build_id INTEGER, + number INTEGER, + name VARCHAR(250), + image VARCHAR(500), + status VARCHAR(250), + error VARCHAR(500), + exit_code INTEGER, + created INTEGER, + started INTEGER, + finished INTEGER, + host VARCHAR(250), + runtime VARCHAR(250), + distribution VARCHAR(250), + UNIQUE(build_id, number) +); +` + + // CreateSqliteTable represents a query to create the Sqlite services table. + CreateSqliteTable = ` +CREATE TABLE +IF NOT EXISTS +services ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_id INTEGER, + build_id INTEGER, + number INTEGER, + name TEXT, + image TEXT, + status TEXT, + error TEXT, + exit_code INTEGER, + created INTEGER, + started INTEGER, + finished INTEGER, + host TEXT, + runtime TEXT, + distribution TEXT, + UNIQUE(build_id, number) +); +` +) + +// CreateServiceTable creates the services table in the database. +func (e *engine) CreateServiceTable(driver string) error { + e.logger.Tracef("creating services table in the database") + + // handle the driver provided to create the table + switch driver { + case constants.DriverPostgres: + // create the services table for Postgres + return e.client.Exec(CreatePostgresTable).Error + case constants.DriverSqlite: + fallthrough + default: + // create the services table for Sqlite + return e.client.Exec(CreateSqliteTable).Error + } +} diff --git a/database/service/table_test.go b/database/service/table_test.go new file mode 100644 index 000000000..1b4d1b7de --- /dev/null +++ b/database/service/table_test.go @@ -0,0 +1,59 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestService_Engine_CreateServiceTable(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateServiceTable(test.name) + + if test.failure { + if err == nil { + t.Errorf("CreateServiceTable for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateServiceTable for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/service/update.go b/database/service/update.go new file mode 100644 index 000000000..ceaa7b6b8 --- /dev/null +++ b/database/service/update.go @@ -0,0 +1,38 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// UpdateService updates an existing service in the database. +func (e *engine) UpdateService(s *library.Service) error { + e.logger.WithFields(logrus.Fields{ + "service": s.GetNumber(), + }).Tracef("updating service %s in the database", s.GetName()) + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#ServiceFromLibrary + service := database.ServiceFromLibrary(s) + + // validate the necessary fields are populated + // + // https://pkg.go.dev/github.com/go-vela/types/database#Service.Validate + err := service.Validate() + if err != nil { + return err + } + + // send query to the database + return e.client. + Table(constants.TableService). + Save(service). + Error +} diff --git a/database/service/update_test.go b/database/service/update_test.go new file mode 100644 index 000000000..e9fe61955 --- /dev/null +++ b/database/service/update_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestService_Engine_UpdateService(t *testing.T) { + // setup types + _service := testService() + _service.SetID(1) + _service.SetRepoID(1) + _service.SetBuildID(1) + _service.SetNumber(1) + _service.SetName("foo") + _service.SetImage("bar") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`UPDATE "services" SET "build_id"=$1,"repo_id"=$2,"number"=$3,"name"=$4,"image"=$5,"status"=$6,"error"=$7,"exit_code"=$8,"created"=$9,"started"=$10,"finished"=$11,"host"=$12,"runtime"=$13,"distribution"=$14 WHERE "id" = $15`). + WithArgs(1, 1, 1, "foo", "bar", nil, nil, nil, nil, nil, nil, nil, nil, nil, 1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateService(_service) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err = test.database.UpdateService(_service) + + if test.failure { + if err == nil { + t.Errorf("UpdateService for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("UpdateService for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/setup.go b/database/setup.go index fa13ecdff..a4b553d4d 100644 --- a/database/setup.go +++ b/database/setup.go @@ -41,7 +41,7 @@ type Setup struct { // Postgres creates and returns a Vela service capable of // integrating with a Postgres database system. -func (s *Setup) Postgres() (Service, error) { +func (s *Setup) Postgres() (Interface, error) { logrus.Trace("creating postgres database client from setup") // create new Postgres database service @@ -60,7 +60,7 @@ func (s *Setup) Postgres() (Service, error) { // Sqlite creates and returns a Vela service capable of // integrating with a Sqlite database system. -func (s *Setup) Sqlite() (Service, error) { +func (s *Setup) Sqlite() (Interface, error) { logrus.Trace("creating sqlite database client from setup") // create new Sqlite database service diff --git a/database/sqlite/ddl/service.go b/database/sqlite/ddl/service.go deleted file mode 100644 index 3b8f9527e..000000000 --- a/database/sqlite/ddl/service.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package ddl - -const ( - // CreateServiceTable represents a query to - // create the services table for Vela. - CreateServiceTable = ` -CREATE TABLE -IF NOT EXISTS -services ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - repo_id INTEGER, - build_id INTEGER, - number INTEGER, - name TEXT, - image TEXT, - status TEXT, - error TEXT, - exit_code INTEGER, - created INTEGER, - started INTEGER, - finished INTEGER, - host TEXT, - runtime TEXT, - distribution TEXT, - UNIQUE(build_id, number) -); -` -) diff --git a/database/sqlite/dml/service.go b/database/sqlite/dml/service.go deleted file mode 100644 index 66e009406..000000000 --- a/database/sqlite/dml/service.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package dml - -const ( - // ListServices represents a query to - // list all services in the database. - ListServices = ` -SELECT * -FROM services; -` - - // ListBuildServices represents a query to list - // all services for a build_id in the database. - ListBuildServices = ` -SELECT * -FROM services -WHERE build_id = ? -ORDER BY id DESC -LIMIT ? -OFFSET ?; -` - - // SelectBuildServicesCount represents a query to select - // the count of services for a build_id in the database. - SelectBuildServicesCount = ` -SELECT count(*) as count -FROM services -WHERE build_id = ? -` - - // SelectServiceImagesCount represents a query to select - // the count of an images appearances in the database. - SelectServiceImagesCount = ` -SELECT image, count(image) as count -FROM services -GROUP BY image -` - - // SelectServiceStatusesCount represents a query to select - // the count of service status appearances in the database. - SelectServiceStatusesCount = ` -SELECT status, count(status) as count -FROM services -GROUP BY status; -` - - // SelectBuildService represents a query to select a - // service for a build_id and number in the database. - SelectBuildService = ` -SELECT * -FROM services -WHERE build_id = ? -AND number = ? -LIMIT 1; -` - - // DeleteService represents a query to - // remove a service from the database. - DeleteService = ` -DELETE -FROM services -WHERE id = ?; -` -) diff --git a/database/sqlite/service.go b/database/sqlite/service.go deleted file mode 100644 index ecc9dad6b..000000000 --- a/database/sqlite/service.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "errors" - - "github.com/sirupsen/logrus" - - "github.com/go-vela/server/database/sqlite/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -// GetService gets a service by number and build ID from the database. -func (c *client) GetService(number int, b *library.Build) (*library.Service, error) { - c.Logger.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "service": number, - }).Tracef("getting service %d for build %d from the database", number, b.GetNumber()) - - // variable to store query results - s := new(database.Service) - - // send query to the database and store result in variable - result := c.Sqlite. - Table(constants.TableService). - Raw(dml.SelectBuildService, b.GetID(), number). - Scan(s) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - return s.ToLibrary(), result.Error -} - -// CreateService creates a new service in the database. -func (c *client) CreateService(s *library.Service) error { - c.Logger.WithFields(logrus.Fields{ - "service": s.GetNumber(), - }).Tracef("creating service %s in the database", s.GetName()) - - // cast to database type - service := database.ServiceFromLibrary(s) - - // validate the necessary fields are populated - err := service.Validate() - if err != nil { - return err - } - - // send query to the database - return c.Sqlite. - Table(constants.TableService). - Create(service).Error -} - -// UpdateService updates a service in the database. -func (c *client) UpdateService(s *library.Service) error { - c.Logger.WithFields(logrus.Fields{ - "service": s.GetNumber(), - }).Tracef("updating service %s in the database", s.GetName()) - - // cast to database type - service := database.ServiceFromLibrary(s) - - // validate the necessary fields are populated - err := service.Validate() - if err != nil { - return err - } - - // send query to the database - return c.Sqlite. - Table(constants.TableService). - Save(service).Error -} - -// DeleteService deletes a service by unique ID from the database. -func (c *client) DeleteService(id int64) error { - c.Logger.Tracef("deleting service %d from the database", id) - - // send query to the database - return c.Sqlite. - Table(constants.TableService). - Exec(dml.DeleteService, id).Error -} diff --git a/database/sqlite/service_count.go b/database/sqlite/service_count.go deleted file mode 100644 index 4979e9527..000000000 --- a/database/sqlite/service_count.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "github.com/go-vela/server/database/sqlite/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" -) - -// GetBuildServiceCount gets a count of all services by build ID from the database. -func (c *client) GetBuildServiceCount(b *library.Build) (int64, error) { - c.Logger.WithFields(logrus.Fields{ - "build": b.GetNumber(), - }).Tracef("getting count of services for build %d from the database", b.GetNumber()) - - // variable to store query results - var s int64 - - // send query to the database and store result in variable - err := c.Sqlite. - Table(constants.TableService). - Raw(dml.SelectBuildServicesCount, b.GetID()). - Pluck("count", &s).Error - - return s, err -} - -// GetServiceImageCount gets a count of all service images -// and the count of their occurrence in the database. -func (c *client) GetServiceImageCount() (map[string]float64, error) { - c.Logger.Tracef("getting count of all images for services from the database") - - type imageCount struct { - Image string - Count int - } - - // variable to store query results - images := new([]imageCount) - counts := make(map[string]float64) - - // send query to the database and store result in variable - err := c.Sqlite. - Table(constants.TableService). - Raw(dml.SelectServiceImagesCount). - Scan(images).Error - - for _, image := range *images { - counts[image.Image] = float64(image.Count) - } - - return counts, err -} - -// GetServiceStatusCount gets a list of all service statuses -// and the count of their occurrence in the database. -func (c *client) GetServiceStatusCount() (map[string]float64, error) { - c.Logger.Trace("getting count of all statuses for services from the database") - - type statusCount struct { - Status string - Count int - } - - // variable to store query results - s := new([]statusCount) - counts := map[string]float64{ - "pending": 0, - "failure": 0, - "killed": 0, - "running": 0, - "success": 0, - } - - // send query to the database and store result in variable - err := c.Sqlite. - Table(constants.TableService). - Raw(dml.SelectServiceStatusesCount). - Scan(s).Error - - for _, status := range *s { - counts[status.Status] = float64(status.Count) - } - - return counts, err -} diff --git a/database/sqlite/service_count_test.go b/database/sqlite/service_count_test.go deleted file mode 100644 index 13d533a82..000000000 --- a/database/sqlite/service_count_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "log" - "reflect" - "testing" - - "github.com/go-vela/server/database/sqlite/ddl" - "github.com/go-vela/types/constants" -) - -func init() { - // setup the test database client - _database, err := NewTest() - if err != nil { - log.Fatalf("unable to create new sqlite test database: %v", err) - } - - // create the service table - err = _database.Sqlite.Exec(ddl.CreateServiceTable).Error - if err != nil { - log.Fatalf("unable to create %s table: %v", constants.TableService, err) - } -} - -func TestSqlite_Client_GetBuildServiceCount(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - - _serviceOne := testService() - _serviceOne.SetID(1) - _serviceOne.SetRepoID(1) - _serviceOne.SetBuildID(1) - _serviceOne.SetNumber(1) - _serviceOne.SetName("foo") - _serviceOne.SetImage("bar") - - _serviceTwo := testService() - _serviceTwo.SetID(2) - _serviceTwo.SetRepoID(1) - _serviceTwo.SetBuildID(1) - _serviceTwo.SetNumber(2) - _serviceTwo.SetName("bar") - _serviceTwo.SetImage("foo") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the services table - defer _database.Sqlite.Exec("delete from services;") - - // create the services in the database - err := _database.CreateService(_serviceOne) - if err != nil { - t.Errorf("unable to create test service: %v", err) - } - - err = _database.CreateService(_serviceTwo) - if err != nil { - t.Errorf("unable to create test service: %v", err) - } - - got, err := _database.GetBuildServiceCount(_build) - - if test.failure { - if err == nil { - t.Errorf("GetBuildServiceCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildServiceCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildServiceCount is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetServiceImageCount(t *testing.T) { - // setup types - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want map[string]float64 - }{ - { - failure: false, - want: map[string]float64{}, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetServiceImageCount() - - if test.failure { - if err == nil { - t.Errorf("GetServiceImageCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetServiceImageCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetServiceImageCount is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetServiceStatusCount(t *testing.T) { - // setup types - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want map[string]float64 - }{ - { - failure: false, - want: map[string]float64{ - "pending": 0, - "failure": 0, - "killed": 0, - "running": 0, - "success": 0, - }, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetServiceStatusCount() - - if test.failure { - if err == nil { - t.Errorf("GetServiceStatusCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetServiceStatusCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetServiceStatusCount is %v, want %v", got, test.want) - } - } -} diff --git a/database/sqlite/service_list.go b/database/sqlite/service_list.go deleted file mode 100644 index eb16406af..000000000 --- a/database/sqlite/service_list.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "github.com/go-vela/server/database/sqlite/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" -) - -// GetServiceList gets a list of all services from the database. -func (c *client) GetServiceList() ([]*library.Service, error) { - c.Logger.Trace("listing services from the database") - - // variable to store query results - s := new([]database.Service) - - // send query to the database and store result in variable - err := c.Sqlite. - Table(constants.TableService). - Raw(dml.ListServices). - Scan(s).Error - - // variable we want to return - services := []*library.Service{} - // iterate through all query results - for _, service := range *s { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := service - - // convert query result to library type - services = append(services, tmp.ToLibrary()) - } - - return services, err -} - -// GetBuildServiceList gets a list of services by build ID from the database. -func (c *client) GetBuildServiceList(b *library.Build, page, perPage int) ([]*library.Service, error) { - c.Logger.WithFields(logrus.Fields{ - "build": b.GetNumber(), - }).Tracef("listing services for build %d from the database", b.GetNumber()) - - // variable to store query results - s := new([]database.Service) - // calculate offset for pagination through results - offset := perPage * (page - 1) - - // send query to the database and store result in variable - err := c.Sqlite. - Table(constants.TableService). - Raw(dml.ListBuildServices, b.GetID(), perPage, offset). - Scan(s).Error - - // variable we want to return - services := []*library.Service{} - // iterate through all query results - for _, service := range *s { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := service - - // convert query result to library type - services = append(services, tmp.ToLibrary()) - } - - return services, err -} diff --git a/database/sqlite/service_list_test.go b/database/sqlite/service_list_test.go deleted file mode 100644 index a9c556d97..000000000 --- a/database/sqlite/service_list_test.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "log" - "reflect" - "testing" - - "github.com/go-vela/server/database/sqlite/ddl" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" -) - -func init() { - // setup the test database client - _database, err := NewTest() - if err != nil { - log.Fatalf("unable to create new sqlite test database: %v", err) - } - - // create the service table - err = _database.Sqlite.Exec(ddl.CreateServiceTable).Error - if err != nil { - log.Fatalf("unable to create %s table: %v", constants.TableService, err) - } -} - -func TestSqlite_Client_GetServiceList(t *testing.T) { - // setup types - _serviceOne := testService() - _serviceOne.SetID(1) - _serviceOne.SetRepoID(1) - _serviceOne.SetBuildID(1) - _serviceOne.SetNumber(1) - _serviceOne.SetName("foo") - _serviceOne.SetImage("bar") - - _serviceTwo := testService() - _serviceTwo.SetID(2) - _serviceTwo.SetRepoID(1) - _serviceTwo.SetBuildID(1) - _serviceTwo.SetNumber(2) - _serviceTwo.SetName("bar") - _serviceTwo.SetImage("foo") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want []*library.Service - }{ - { - failure: false, - want: []*library.Service{_serviceOne, _serviceTwo}, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the services table - defer _database.Sqlite.Exec("delete from services;") - - for _, service := range test.want { - // create the service in the database - err := _database.CreateService(service) - if err != nil { - t.Errorf("unable to create test service: %v", err) - } - } - - got, err := _database.GetServiceList() - - if test.failure { - if err == nil { - t.Errorf("GetServiceList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetServiceList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetServiceList is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetBuildServiceList(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - - _serviceOne := testService() - _serviceOne.SetID(1) - _serviceOne.SetRepoID(1) - _serviceOne.SetBuildID(1) - _serviceOne.SetNumber(1) - _serviceOne.SetName("foo") - _serviceOne.SetImage("bar") - - _serviceTwo := testService() - _serviceTwo.SetID(2) - _serviceTwo.SetRepoID(1) - _serviceTwo.SetBuildID(1) - _serviceTwo.SetNumber(2) - _serviceTwo.SetName("bar") - _serviceTwo.SetImage("foo") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want []*library.Service - }{ - { - failure: false, - want: []*library.Service{_serviceTwo, _serviceOne}, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the services table - defer _database.Sqlite.Exec("delete from services;") - - for _, service := range test.want { - // create the service in the database - err := _database.CreateService(service) - if err != nil { - t.Errorf("unable to create test service: %v", err) - } - } - - got, err := _database.GetBuildServiceList(_build, 1, 10) - - if test.failure { - if err == nil { - t.Errorf("GetBuildServiceList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildServiceList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildServiceList is %v, want %v", got, test.want) - } - } -} diff --git a/database/sqlite/service_test.go b/database/sqlite/service_test.go deleted file mode 100644 index 3982ddf36..000000000 --- a/database/sqlite/service_test.go +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "reflect" - "testing" - - "github.com/go-vela/types/library" -) - -func TestSqlite_Client_GetService(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - - _service := testService() - _service.SetID(1) - _service.SetRepoID(1) - _service.SetBuildID(1) - _service.SetNumber(1) - _service.SetName("foo") - _service.SetImage("bar") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want *library.Service - }{ - { - failure: false, - want: _service, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - if test.want != nil { - // create the service in the database - err := _database.CreateService(test.want) - if err != nil { - t.Errorf("unable to create test service: %v", err) - } - } - - got, err := _database.GetService(1, _build) - - // cleanup the services table - _ = _database.Sqlite.Exec("DELETE FROM services;") - - if test.failure { - if err == nil { - t.Errorf("GetService should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetService returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetService is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_CreateService(t *testing.T) { - // setup types - _service := testService() - _service.SetID(1) - _service.SetRepoID(1) - _service.SetBuildID(1) - _service.SetNumber(1) - _service.SetName("foo") - _service.SetImage("bar") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the services table - defer _database.Sqlite.Exec("delete from services;") - - err := _database.CreateService(_service) - - if test.failure { - if err == nil { - t.Errorf("CreateService should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("CreateService returned err: %v", err) - } - } -} - -func TestSqlite_Client_UpdateService(t *testing.T) { - // setup types - _service := testService() - _service.SetID(1) - _service.SetRepoID(1) - _service.SetBuildID(1) - _service.SetNumber(1) - _service.SetName("foo") - _service.SetImage("bar") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the services table - defer _database.Sqlite.Exec("delete from services;") - - // create the service in the database - err := _database.CreateService(_service) - if err != nil { - t.Errorf("unable to create test service: %v", err) - } - - err = _database.UpdateService(_service) - - if test.failure { - if err == nil { - t.Errorf("UpdateService should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("UpdateService returned err: %v", err) - } - } -} - -func TestSqlite_Client_DeleteService(t *testing.T) { - // setup types - _service := testService() - _service.SetID(1) - _service.SetRepoID(1) - _service.SetBuildID(1) - _service.SetNumber(1) - _service.SetName("foo") - _service.SetImage("bar") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the services table - defer _database.Sqlite.Exec("delete from services;") - - // create the service in the database - err := _database.CreateService(_service) - if err != nil { - t.Errorf("unable to create test service: %v", err) - } - - err = _database.DeleteService(1) - - if test.failure { - if err == nil { - t.Errorf("DeleteService should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("DeleteService returned err: %v", err) - } - } -} - -// testService is a test helper function to create a -// library Service type with all fields set to their -// zero values. -func testService() *library.Service { - i64 := int64(0) - i := 0 - str := "" - - return &library.Service{ - ID: &i64, - BuildID: &i64, - RepoID: &i64, - Number: &i, - Name: &str, - Image: &str, - Status: &str, - Error: &str, - ExitCode: &i, - Created: &i64, - Started: &i64, - Finished: &i64, - Host: &str, - Runtime: &str, - Distribution: &str, - } -} diff --git a/database/sqlite/sqlite.go b/database/sqlite/sqlite.go index abc382a37..d16f7cf09 100644 --- a/database/sqlite/sqlite.go +++ b/database/sqlite/sqlite.go @@ -13,6 +13,7 @@ import ( "github.com/go-vela/server/database/pipeline" "github.com/go-vela/server/database/repo" "github.com/go-vela/server/database/secret" + "github.com/go-vela/server/database/service" "github.com/go-vela/server/database/sqlite/ddl" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" @@ -48,22 +49,24 @@ type ( Sqlite *gorm.DB // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry - // https://pkg.go.dev/github.com/go-vela/server/database/hook#HookService - hook.HookService - // https://pkg.go.dev/github.com/go-vela/server/database/log#LogService - log.LogService - // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#PipelineService - pipeline.PipelineService - // https://pkg.go.dev/github.com/go-vela/server/database/repo#RepoService - repo.RepoService - // https://pkg.go.dev/github.com/go-vela/server/database/secret#SecretService - secret.SecretService - // https://pkg.go.dev/github.com/go-vela/server/database/step#StepService - step.StepService - // https://pkg.go.dev/github.com/go-vela/server/database/user#UserService - user.UserService - // https://pkg.go.dev/github.com/go-vela/server/database/worker#WorkerService - worker.WorkerService + // https://pkg.go.dev/github.com/go-vela/server/database/hook#HookInterface + hook.HookInterface + // https://pkg.go.dev/github.com/go-vela/server/database/log#LogInterface + log.LogInterface + // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#PipelineInterface + pipeline.PipelineInterface + // https://pkg.go.dev/github.com/go-vela/server/database/repo#RepoInterface + repo.RepoInterface + // https://pkg.go.dev/github.com/go-vela/server/database/secret#SecretInterface + secret.SecretInterface + // https://pkg.go.dev/github.com/go-vela/server/database/service#ServiceInterface + service.ServiceInterface + // https://pkg.go.dev/github.com/go-vela/server/database/step#StepInterface + step.StepInterface + // https://pkg.go.dev/github.com/go-vela/server/database/user#UserInterface + user.UserInterface + // https://pkg.go.dev/github.com/go-vela/server/database/worker#WorkerInterface + worker.WorkerInterface } ) @@ -245,12 +248,6 @@ func createTables(c *client) error { return fmt.Errorf("unable to create %s table: %w", constants.TableBuild, err) } - // create the services table - err = c.Sqlite.Exec(ddl.CreateServiceTable).Error - if err != nil { - return fmt.Errorf("unable to create %s table: %w", constants.TableService, err) - } - return nil } @@ -290,10 +287,10 @@ func createIndexes(c *client) error { func createServices(c *client) error { var err error - // create the database agnostic hook service + // create the database agnostic engine for hooks // // https://pkg.go.dev/github.com/go-vela/server/database/hook#New - c.HookService, err = hook.New( + c.HookInterface, err = hook.New( hook.WithClient(c.Sqlite), hook.WithLogger(c.Logger), hook.WithSkipCreation(c.config.SkipCreation), @@ -302,10 +299,10 @@ func createServices(c *client) error { return err } - // create the database agnostic log service + // create the database agnostic engine for logs // // https://pkg.go.dev/github.com/go-vela/server/database/log#New - c.LogService, err = log.New( + c.LogInterface, err = log.New( log.WithClient(c.Sqlite), log.WithCompressionLevel(c.config.CompressionLevel), log.WithLogger(c.Logger), @@ -315,10 +312,10 @@ func createServices(c *client) error { return err } - // create the database agnostic pipeline service + // create the database agnostic engine for pipelines // // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#New - c.PipelineService, err = pipeline.New( + c.PipelineInterface, err = pipeline.New( pipeline.WithClient(c.Sqlite), pipeline.WithCompressionLevel(c.config.CompressionLevel), pipeline.WithLogger(c.Logger), @@ -328,10 +325,10 @@ func createServices(c *client) error { return err } - // create the database agnostic repo service + // create the database agnostic engine for repos // // https://pkg.go.dev/github.com/go-vela/server/database/repo#New - c.RepoService, err = repo.New( + c.RepoInterface, err = repo.New( repo.WithClient(c.Sqlite), repo.WithEncryptionKey(c.config.EncryptionKey), repo.WithLogger(c.Logger), @@ -341,10 +338,10 @@ func createServices(c *client) error { return err } - // create the database agnostic secret service + // create the database agnostic engine for secrets // // https://pkg.go.dev/github.com/go-vela/server/database/secret#New - c.SecretService, err = secret.New( + c.SecretInterface, err = secret.New( secret.WithClient(c.Sqlite), secret.WithEncryptionKey(c.config.EncryptionKey), secret.WithLogger(c.Logger), @@ -354,10 +351,22 @@ func createServices(c *client) error { return err } - // create the database agnostic step service + // create the database agnostic engine for services + // + // https://pkg.go.dev/github.com/go-vela/server/database/service#New + c.ServiceInterface, err = service.New( + service.WithClient(c.Sqlite), + service.WithLogger(c.Logger), + service.WithSkipCreation(c.config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for steps // // https://pkg.go.dev/github.com/go-vela/server/database/step#New - c.StepService, err = step.New( + c.StepInterface, err = step.New( step.WithClient(c.Sqlite), step.WithLogger(c.Logger), step.WithSkipCreation(c.config.SkipCreation), @@ -366,10 +375,10 @@ func createServices(c *client) error { return err } - // create the database agnostic user service + // create the database agnostic engine for users // // https://pkg.go.dev/github.com/go-vela/server/database/user#New - c.UserService, err = user.New( + c.UserInterface, err = user.New( user.WithClient(c.Sqlite), user.WithEncryptionKey(c.config.EncryptionKey), user.WithLogger(c.Logger), @@ -379,10 +388,10 @@ func createServices(c *client) error { return err } - // create the database agnostic worker service + // create the database agnostic engine for workers // // https://pkg.go.dev/github.com/go-vela/server/database/worker#New - c.WorkerService, err = worker.New( + c.WorkerInterface, err = worker.New( worker.WithClient(c.Sqlite), worker.WithLogger(c.Logger), worker.WithSkipCreation(c.config.SkipCreation), diff --git a/database/step/service.go b/database/step/interface.go similarity index 95% rename from database/step/service.go rename to database/step/interface.go index 4007e537a..a410f2600 100644 --- a/database/step/service.go +++ b/database/step/interface.go @@ -8,11 +8,11 @@ import ( "github.com/go-vela/types/library" ) -// StepService represents the Vela interface for step +// StepInterface represents the Vela interface for step // functions with the supported Database backends. // //nolint:revive // ignore name stutter -type StepService interface { +type StepInterface interface { // Step Data Definition Language Functions // // https://en.wikipedia.org/wiki/Data_definition_language diff --git a/database/step/step.go b/database/step/step.go index 94017b04c..43e905797 100644 --- a/database/step/step.go +++ b/database/step/step.go @@ -14,13 +14,13 @@ import ( ) type ( - // config represents the settings required to create the engine that implements the StepService interface. + // config represents the settings required to create the engine that implements the StepInterface interface. config struct { // specifies to skip creating tables and indexes for the Step engine SkipCreation bool } - // engine represents the step functionality that implements the StepService interface. + // engine represents the step functionality that implements the StepInterface interface. engine struct { // engine configuration settings used in step functions config *config diff --git a/database/user/service.go b/database/user/interface.go similarity index 94% rename from database/user/service.go rename to database/user/interface.go index b33c8cd63..ad5986fad 100644 --- a/database/user/service.go +++ b/database/user/interface.go @@ -8,11 +8,11 @@ import ( "github.com/go-vela/types/library" ) -// UserService represents the Vela interface for user +// UserInterface represents the Vela interface for user // functions with the supported Database backends. // //nolint:revive // ignore name stutter -type UserService interface { +type UserInterface interface { // User Data Definition Language Functions // // https://en.wikipedia.org/wiki/Data_definition_language diff --git a/database/user/user.go b/database/user/user.go index 9bef43843..99e1f9701 100644 --- a/database/user/user.go +++ b/database/user/user.go @@ -14,7 +14,7 @@ import ( ) type ( - // config represents the settings required to create the engine that implements the UserService interface. + // config represents the settings required to create the engine that implements the UserInterface interface. config struct { // specifies the encryption key to use for the User engine EncryptionKey string @@ -22,7 +22,7 @@ type ( SkipCreation bool } - // engine represents the user functionality that implements the UserService interface. + // engine represents the user functionality that implements the UserInterface interface. engine struct { // engine configuration settings used in user functions config *config diff --git a/database/worker/service.go b/database/worker/interface.go similarity index 94% rename from database/worker/service.go rename to database/worker/interface.go index b02c6f854..9e7fe1169 100644 --- a/database/worker/service.go +++ b/database/worker/interface.go @@ -8,11 +8,11 @@ import ( "github.com/go-vela/types/library" ) -// WorkerService represents the Vela interface for worker +// WorkerInterface represents the Vela interface for worker // functions with the supported Database backends. // //nolint:revive // ignore name stutter -type WorkerService interface { +type WorkerInterface interface { // Worker Data Definition Language Functions // // https://en.wikipedia.org/wiki/Data_definition_language diff --git a/database/worker/worker.go b/database/worker/worker.go index d870fb1ba..d18aa6408 100644 --- a/database/worker/worker.go +++ b/database/worker/worker.go @@ -14,13 +14,13 @@ import ( ) type ( - // config represents the settings required to create the engine that implements the WorkerService interface. + // config represents the settings required to create the engine that implements the WorkerInterface interface. config struct { // specifies to skip creating tables and indexes for the Worker engine SkipCreation bool } - // engine represents the worker functionality that implements the WorkerService interface. + // engine represents the worker functionality that implements the WorkerInterface interface. engine struct { // engine configuration settings used in worker functions config *config diff --git a/router/middleware/database.go b/router/middleware/database.go index f4037b7df..fbbf91eea 100644 --- a/router/middleware/database.go +++ b/router/middleware/database.go @@ -11,7 +11,7 @@ import ( // Database is a middleware function that initializes the database and // attaches to the context of every http.Request. -func Database(d database.Service) gin.HandlerFunc { +func Database(d database.Interface) gin.HandlerFunc { return func(c *gin.Context) { database.ToContext(c, d) c.Next() diff --git a/router/middleware/database_test.go b/router/middleware/database_test.go index dd8b95a37..a3480a9f5 100644 --- a/router/middleware/database_test.go +++ b/router/middleware/database_test.go @@ -17,7 +17,7 @@ import ( func TestMiddleware_Database(t *testing.T) { // setup types - var got database.Service + var got database.Interface want, _ := sqlite.NewTest() diff --git a/router/middleware/service/service.go b/router/middleware/service/service.go index be36fb7b6..30abf92d0 100644 --- a/router/middleware/service/service.go +++ b/router/middleware/service/service.go @@ -75,7 +75,7 @@ func Establish() gin.HandlerFunc { "user": u.GetName(), }).Debugf("reading service %s/%d/%d", r.GetFullName(), b.GetNumber(), number) - s, err := database.FromContext(c).GetService(number, b) + s, err := database.FromContext(c).GetServiceForBuild(b, number) if err != nil { retErr := fmt.Errorf("unable to read service %s/%d/%d: %w", r.GetFullName(), b.GetNumber(), number, err) util.HandleError(c, http.StatusNotFound, retErr) diff --git a/secret/native/native.go b/secret/native/native.go index 4109d8e4f..99b015715 100644 --- a/secret/native/native.go +++ b/secret/native/native.go @@ -12,7 +12,7 @@ import ( // client represents a struct to hold native secret setup. type client struct { // client to interact with database for secret operations - Database database.Service + Database database.Interface // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry } @@ -25,7 +25,7 @@ func New(opts ...ClientOpt) (*client, error) { c := new(client) // create new fields - c.Database = *new(database.Service) + c.Database = *new(database.Interface) // create new logger for the client // diff --git a/secret/native/native_test.go b/secret/native/native_test.go index 070dba97b..1b7daaa2f 100644 --- a/secret/native/native_test.go +++ b/secret/native/native_test.go @@ -23,8 +23,8 @@ func TestNative_New(t *testing.T) { // setup tests tests := []struct { failure bool - database database.Service - want database.Service + database database.Interface + want database.Interface }{ { failure: false, diff --git a/secret/native/opts.go b/secret/native/opts.go index 160b554c6..0c4396348 100644 --- a/secret/native/opts.go +++ b/secret/native/opts.go @@ -14,7 +14,7 @@ import ( type ClientOpt func(*client) error // WithDatabase sets the Vela database service in the secret client for Native. -func WithDatabase(d database.Service) ClientOpt { +func WithDatabase(d database.Interface) ClientOpt { return func(c *client) error { c.Logger.Trace("configuring database service in native secret client") diff --git a/secret/native/opts_test.go b/secret/native/opts_test.go index dea0ef22e..1cda40700 100644 --- a/secret/native/opts_test.go +++ b/secret/native/opts_test.go @@ -24,8 +24,8 @@ func TestNative_ClientOpt_WithDatabase(t *testing.T) { // setup tests tests := []struct { failure bool - database database.Service - want database.Service + database database.Interface + want database.Interface }{ { failure: false, diff --git a/secret/setup.go b/secret/setup.go index cdebe406b..e0a28a8a8 100644 --- a/secret/setup.go +++ b/secret/setup.go @@ -13,7 +13,6 @@ import ( "github.com/go-vela/server/secret/native" "github.com/go-vela/server/secret/vault" "github.com/go-vela/types/constants" - "github.com/sirupsen/logrus" ) @@ -27,7 +26,7 @@ type Setup struct { Driver string // specifies the database service to use for the secret client - Database database.Service + Database database.Interface // specifies the address to use for the secret client Address string From 3aa414e46ced00ce3019f0d58120801a7c4cd233 Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Mon, 15 May 2023 11:58:37 -0500 Subject: [PATCH 03/53] refactor(api): move worker logic to separate package (#828) * refactor(api): move worker handlers to new package * chore: address review feedback Co-authored-by: Jacob Floyd --------- Co-authored-by: Jacob Floyd Co-authored-by: Kelly Merrick Co-authored-by: David May <1301201+wass3r@users.noreply.github.com> Co-authored-by: Driscoll <43050003+chrispdriscoll@users.noreply.github.com> Co-authored-by: dave vader <48764154+plyr4@users.noreply.github.com> --- api/worker.go | 509 ------------------------------------------ api/worker/create.go | 141 ++++++++++++ api/worker/delete.go | 70 ++++++ api/worker/get.go | 69 ++++++ api/worker/list.go | 62 +++++ api/worker/refresh.go | 138 ++++++++++++ api/worker/update.go | 115 ++++++++++ router/worker.go | 20 +- 8 files changed, 605 insertions(+), 519 deletions(-) delete mode 100644 api/worker.go create mode 100644 api/worker/create.go create mode 100644 api/worker/delete.go create mode 100644 api/worker/get.go create mode 100644 api/worker/list.go create mode 100644 api/worker/refresh.go create mode 100644 api/worker/update.go diff --git a/api/worker.go b/api/worker.go deleted file mode 100644 index 5d949779e..000000000 --- a/api/worker.go +++ /dev/null @@ -1,509 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package api - -import ( - "fmt" - "net/http" - "strings" - "time" - - "github.com/go-vela/server/internal/token" - "github.com/go-vela/server/router/middleware/claims" - "github.com/go-vela/server/router/middleware/user" - "github.com/go-vela/types/constants" - - "github.com/go-vela/server/database" - "github.com/go-vela/server/router/middleware/worker" - "github.com/go-vela/server/util" - - "github.com/go-vela/types/library" - - "github.com/gin-gonic/gin" - "github.com/sirupsen/logrus" -) - -// swagger:operation POST /api/v1/workers workers CreateWorker -// -// Create a worker for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: body -// name: body -// description: Payload containing the worker to create -// required: true -// schema: -// "$ref": "#/definitions/Worker" -// security: -// - ApiKeyAuth: [] -// responses: -// '201': -// description: Successfully created the worker and retrieved auth token -// schema: -// "$ref": "#definitions/Token" -// '400': -// description: Unable to create the worker -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to create the worker -// schema: -// "$ref": "#/definitions/Error" - -// CreateWorker represents the API handler to -// create a worker in the configured backend. -func CreateWorker(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - cl := claims.Retrieve(c) - - // capture body from API request - input := new(library.Worker) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for new worker: %w", err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // verify input host name matches worker hostname - if !strings.EqualFold(cl.TokenType, constants.ServerWorkerTokenType) && !strings.EqualFold(cl.Subject, input.GetHostname()) { - retErr := fmt.Errorf("unable to add worker; claims subject %s does not match worker hostname %s", cl.Subject, input.GetHostname()) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - input.SetLastCheckedIn(time.Now().Unix()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "user": u.GetName(), - "worker": input.GetHostname(), - }).Infof("creating new worker %s", input.GetHostname()) - - err = database.FromContext(c).CreateWorker(input) - if err != nil { - retErr := fmt.Errorf("unable to create worker: %w", err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - switch cl.TokenType { - // if symmetric token configured, send back symmetric token - case constants.ServerWorkerTokenType: - if secret, ok := c.Value("secret").(string); ok { - tkn := new(library.Token) - tkn.SetToken(secret) - c.JSON(http.StatusCreated, tkn) - - return - } - - retErr := fmt.Errorf("symmetric token provided but not configured in server") - util.HandleError(c, http.StatusBadRequest, retErr) - - return - // if worker register token, send back auth token - default: - tm := c.MustGet("token-manager").(*token.Manager) - - wmto := &token.MintTokenOpts{ - TokenType: constants.WorkerAuthTokenType, - TokenDuration: tm.WorkerAuthTokenDuration, - Hostname: cl.Subject, - } - - tkn := new(library.Token) - - wt, err := tm.MintToken(wmto) - if err != nil { - retErr := fmt.Errorf("unable to generate auth token for worker %s: %w", input.GetHostname(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - tkn.SetToken(wt) - - c.JSON(http.StatusCreated, tkn) - } -} - -// swagger:operation GET /api/v1/workers workers GetWorkers -// -// Retrieve a list of workers for the configured backend -// -// --- -// produces: -// - application/json -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the list of workers -// schema: -// type: array -// items: -// "$ref": "#/definitions/Worker" -// '500': -// description: Unable to retrieve the list of workers -// schema: -// "$ref": "#/definitions/Error" - -// GetWorkers represents the API handler to capture a -// list of workers from the configured backend. -func GetWorkers(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "user": u.GetName(), - }).Info("reading workers") - - w, err := database.FromContext(c).ListWorkers() - if err != nil { - retErr := fmt.Errorf("unable to get workers: %w", err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, w) -} - -// swagger:operation GET /api/v1/workers/{worker} workers GetWorker -// -// Retrieve a worker for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: worker -// description: Hostname of the worker -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the worker -// schema: -// "$ref": "#/definitions/Worker" -// '404': -// description: Unable to retrieve the worker -// schema: -// "$ref": "#/definitions/Error" - -// GetWorker represents the API handler to capture a -// worker from the configured backend. -func GetWorker(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - w := worker.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "user": u.GetName(), - "worker": w.GetHostname(), - }).Infof("reading worker %s", w.GetHostname()) - - w, err := database.FromContext(c).GetWorkerForHostname(w.GetHostname()) - if err != nil { - retErr := fmt.Errorf("unable to get workers: %w", err) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - c.JSON(http.StatusOK, w) -} - -// swagger:operation PUT /api/v1/workers/{worker} workers UpdateWorker -// -// Update a worker for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: body -// name: body -// description: Payload containing the worker to update -// required: true -// schema: -// "$ref": "#/definitions/Worker" -// - in: path -// name: worker -// description: Name of the worker -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully updated the worker -// schema: -// "$ref": "#/definitions/Worker" -// '400': -// description: Unable to update the worker -// schema: -// "$ref": "#/definitions/Error" -// '404': -// description: Unable to update the worker -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to update the worker -// schema: -// "$ref": "#/definitions/Error" - -// UpdateWorker represents the API handler to -// update a worker in the configured backend. -func UpdateWorker(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - w := worker.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "user": u.GetName(), - "worker": w.GetHostname(), - }).Infof("updating worker %s", w.GetHostname()) - - // capture body from API request - input := new(library.Worker) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for worker %s: %w", w.GetHostname(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - if len(input.GetAddress()) > 0 { - // update admin if set - w.SetAddress(input.GetAddress()) - } - - if len(input.GetRoutes()) > 0 { - // update routes if set - w.SetRoutes(input.GetRoutes()) - } - - if input.GetActive() { - // update active if set - w.SetActive(input.GetActive()) - } - - // send API call to update the worker - err = database.FromContext(c).UpdateWorker(w) - if err != nil { - retErr := fmt.Errorf("unable to update worker %s: %w", w.GetHostname(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the updated worker - w, _ = database.FromContext(c).GetWorkerForHostname(w.GetHostname()) - - c.JSON(http.StatusOK, w) -} - -// swagger:operation POST /api/v1/workers/{worker}/refresh workers RefreshWorkerAuth -// -// Refresh authorization token for worker -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: worker -// description: Name of the worker -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully refreshed auth -// schema: -// "$ref": "#/definitions/Token" -// '400': -// description: Unable to refresh worker auth -// schema: -// "$ref": "#/definitions/Error" -// '404': -// description: Unable to refresh worker auth -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to refresh worker auth -// schema: -// "$ref": "#/definitions/Error" - -// RefreshWorkerAuth represents the API handler to -// refresh the auth token for a worker. -func RefreshWorkerAuth(c *gin.Context) { - // capture middleware values - w := worker.Retrieve(c) - cl := claims.Retrieve(c) - - // if we are not using a symmetric token, and the subject does not match the input, request should be denied - if !strings.EqualFold(cl.TokenType, constants.ServerWorkerTokenType) && !strings.EqualFold(cl.Subject, w.GetHostname()) { - retErr := fmt.Errorf("unable to refresh worker auth: claims subject %s does not match worker hostname %s", cl.Subject, w.GetHostname()) - - logrus.WithFields(logrus.Fields{ - "subject": cl.Subject, - "worker": w.GetHostname(), - }).Warnf("attempted refresh of worker %s using token from worker %s", w.GetHostname(), cl.Subject) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // set last checked in time - w.SetLastCheckedIn(time.Now().Unix()) - - // send API call to update the worker - err := database.FromContext(c).UpdateWorker(w) - if err != nil { - retErr := fmt.Errorf("unable to update worker %s: %w", w.GetHostname(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "worker": w.GetHostname(), - }).Infof("refreshing worker %s authentication", w.GetHostname()) - - switch cl.TokenType { - // if symmetric token configured, send back symmetric token - case constants.ServerWorkerTokenType: - if secret, ok := c.Value("secret").(string); ok { - tkn := new(library.Token) - tkn.SetToken(secret) - c.JSON(http.StatusOK, tkn) - - return - } - - retErr := fmt.Errorf("symmetric token provided but not configured in server") - util.HandleError(c, http.StatusBadRequest, retErr) - - return - // if worker auth / register token, send back auth token - case constants.WorkerAuthTokenType, constants.WorkerRegisterTokenType: - tm := c.MustGet("token-manager").(*token.Manager) - - wmto := &token.MintTokenOpts{ - TokenType: constants.WorkerAuthTokenType, - TokenDuration: tm.WorkerAuthTokenDuration, - Hostname: cl.Subject, - } - - tkn := new(library.Token) - - wt, err := tm.MintToken(wmto) - if err != nil { - retErr := fmt.Errorf("unable to generate auth token for worker %s: %w", w.GetHostname(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - tkn.SetToken(wt) - - c.JSON(http.StatusOK, tkn) - } -} - -// swagger:operation DELETE /api/v1/workers/{worker} workers DeleteWorker -// -// Delete a worker for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: worker -// description: Name of the worker -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully deleted of worker -// schema: -// type: string -// '500': -// description: Unable to delete worker -// schema: -// "$ref": "#/definitions/Error" - -// DeleteWorker represents the API handler to remove -// a worker from the configured backend. -func DeleteWorker(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - w := worker.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "user": u.GetName(), - "worker": w.GetHostname(), - }).Infof("deleting worker %s", w.GetHostname()) - - // send API call to remove the step - err := database.FromContext(c).DeleteWorker(w) - if err != nil { - retErr := fmt.Errorf("unable to delete worker %s: %w", w.GetHostname(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, fmt.Sprintf("worker %s deleted", w.GetHostname())) -} diff --git a/api/worker/create.go b/api/worker/create.go new file mode 100644 index 000000000..b7d1bf286 --- /dev/null +++ b/api/worker/create.go @@ -0,0 +1,141 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/internal/token" + "github.com/go-vela/server/router/middleware/claims" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation POST /api/v1/workers workers CreateWorker +// +// Create a worker for the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: body +// name: body +// description: Payload containing the worker to create +// required: true +// schema: +// "$ref": "#/definitions/Worker" +// security: +// - ApiKeyAuth: [] +// responses: +// '201': +// description: Successfully created the worker and retrieved auth token +// schema: +// "$ref": "#definitions/Token" +// '400': +// description: Unable to create the worker +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to create the worker +// schema: +// "$ref": "#/definitions/Error" + +// CreateWorker represents the API handler to +// create a worker in the configured backend. +func CreateWorker(c *gin.Context) { + // capture middleware values + u := user.Retrieve(c) + cl := claims.Retrieve(c) + + // capture body from API request + input := new(library.Worker) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for new worker: %w", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // verify input host name matches worker hostname + if !strings.EqualFold(cl.TokenType, constants.ServerWorkerTokenType) && !strings.EqualFold(cl.Subject, input.GetHostname()) { + retErr := fmt.Errorf("unable to add worker; claims subject %s does not match worker hostname %s", cl.Subject, input.GetHostname()) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + input.SetLastCheckedIn(time.Now().Unix()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "user": u.GetName(), + "worker": input.GetHostname(), + }).Infof("creating new worker %s", input.GetHostname()) + + err = database.FromContext(c).CreateWorker(input) + if err != nil { + retErr := fmt.Errorf("unable to create worker: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + switch cl.TokenType { + // if symmetric token configured, send back symmetric token + case constants.ServerWorkerTokenType: + if secret, ok := c.Value("secret").(string); ok { + tkn := new(library.Token) + tkn.SetToken(secret) + c.JSON(http.StatusCreated, tkn) + + return + } + + retErr := fmt.Errorf("symmetric token provided but not configured in server") + util.HandleError(c, http.StatusBadRequest, retErr) + + return + // if worker register token, send back auth token + default: + tm := c.MustGet("token-manager").(*token.Manager) + + wmto := &token.MintTokenOpts{ + TokenType: constants.WorkerAuthTokenType, + TokenDuration: tm.WorkerAuthTokenDuration, + Hostname: cl.Subject, + } + + tkn := new(library.Token) + + wt, err := tm.MintToken(wmto) + if err != nil { + retErr := fmt.Errorf("unable to generate auth token for worker %s: %w", input.GetHostname(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + tkn.SetToken(wt) + + c.JSON(http.StatusCreated, tkn) + } +} diff --git a/api/worker/delete.go b/api/worker/delete.go new file mode 100644 index 000000000..fa0ad5e65 --- /dev/null +++ b/api/worker/delete.go @@ -0,0 +1,70 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/router/middleware/worker" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation DELETE /api/v1/workers/{worker} workers DeleteWorker +// +// Delete a worker for the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: worker +// description: Name of the worker +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully deleted of worker +// schema: +// type: string +// '500': +// description: Unable to delete worker +// schema: +// "$ref": "#/definitions/Error" + +// DeleteWorker represents the API handler to remove +// a worker from the configured backend. +func DeleteWorker(c *gin.Context) { + // capture middleware values + u := user.Retrieve(c) + w := worker.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "user": u.GetName(), + "worker": w.GetHostname(), + }).Infof("deleting worker %s", w.GetHostname()) + + // send API call to remove the step + err := database.FromContext(c).DeleteWorker(w) + if err != nil { + retErr := fmt.Errorf("unable to delete worker %s: %w", w.GetHostname(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, fmt.Sprintf("worker %s deleted", w.GetHostname())) +} diff --git a/api/worker/get.go b/api/worker/get.go new file mode 100644 index 000000000..88b323532 --- /dev/null +++ b/api/worker/get.go @@ -0,0 +1,69 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/router/middleware/worker" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/workers/{worker} workers GetWorker +// +// Retrieve a worker for the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: worker +// description: Hostname of the worker +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the worker +// schema: +// "$ref": "#/definitions/Worker" +// '404': +// description: Unable to retrieve the worker +// schema: +// "$ref": "#/definitions/Error" + +// GetWorker represents the API handler to capture a +// worker from the configured backend. +func GetWorker(c *gin.Context) { + // capture middleware values + u := user.Retrieve(c) + w := worker.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "user": u.GetName(), + "worker": w.GetHostname(), + }).Infof("reading worker %s", w.GetHostname()) + + w, err := database.FromContext(c).GetWorkerForHostname(w.GetHostname()) + if err != nil { + retErr := fmt.Errorf("unable to get workers: %w", err) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + c.JSON(http.StatusOK, w) +} diff --git a/api/worker/list.go b/api/worker/list.go new file mode 100644 index 000000000..2587ba07f --- /dev/null +++ b/api/worker/list.go @@ -0,0 +1,62 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/workers workers ListWorkers +// +// Retrieve a list of workers for the configured backend +// +// --- +// produces: +// - application/json +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the list of workers +// schema: +// type: array +// items: +// "$ref": "#/definitions/Worker" +// '500': +// description: Unable to retrieve the list of workers +// schema: +// "$ref": "#/definitions/Error" + +// ListWorkers represents the API handler to capture a +// list of workers from the configured backend. +func ListWorkers(c *gin.Context) { + // capture middleware values + u := user.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "user": u.GetName(), + }).Info("reading workers") + + w, err := database.FromContext(c).ListWorkers() + if err != nil { + retErr := fmt.Errorf("unable to get workers: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, w) +} diff --git a/api/worker/refresh.go b/api/worker/refresh.go new file mode 100644 index 000000000..cd4aa7ef3 --- /dev/null +++ b/api/worker/refresh.go @@ -0,0 +1,138 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/internal/token" + "github.com/go-vela/server/router/middleware/claims" + "github.com/go-vela/server/router/middleware/worker" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation POST /api/v1/workers/{worker}/refresh workers RefreshWorkerAuth +// +// Refresh authorization token for worker +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: worker +// description: Name of the worker +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully refreshed auth +// schema: +// "$ref": "#/definitions/Token" +// '400': +// description: Unable to refresh worker auth +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: Unable to refresh worker auth +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to refresh worker auth +// schema: +// "$ref": "#/definitions/Error" + +// Refresh represents the API handler to +// refresh the auth token for a worker. +func Refresh(c *gin.Context) { + // capture middleware values + w := worker.Retrieve(c) + cl := claims.Retrieve(c) + + // if we are not using a symmetric token, and the subject does not match the input, request should be denied + if !strings.EqualFold(cl.TokenType, constants.ServerWorkerTokenType) && !strings.EqualFold(cl.Subject, w.GetHostname()) { + retErr := fmt.Errorf("unable to refresh worker auth: claims subject %s does not match worker hostname %s", cl.Subject, w.GetHostname()) + + logrus.WithFields(logrus.Fields{ + "subject": cl.Subject, + "worker": w.GetHostname(), + }).Warnf("attempted refresh of worker %s using token from worker %s", w.GetHostname(), cl.Subject) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // set last checked in time + w.SetLastCheckedIn(time.Now().Unix()) + + // send API call to update the worker + err := database.FromContext(c).UpdateWorker(w) + if err != nil { + retErr := fmt.Errorf("unable to update worker %s: %w", w.GetHostname(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "worker": w.GetHostname(), + }).Infof("refreshing worker %s authentication", w.GetHostname()) + + switch cl.TokenType { + // if symmetric token configured, send back symmetric token + case constants.ServerWorkerTokenType: + if secret, ok := c.Value("secret").(string); ok { + tkn := new(library.Token) + tkn.SetToken(secret) + c.JSON(http.StatusOK, tkn) + + return + } + + retErr := fmt.Errorf("symmetric token provided but not configured in server") + util.HandleError(c, http.StatusBadRequest, retErr) + + return + // if worker auth / register token, send back auth token + case constants.WorkerAuthTokenType, constants.WorkerRegisterTokenType: + tm := c.MustGet("token-manager").(*token.Manager) + + wmto := &token.MintTokenOpts{ + TokenType: constants.WorkerAuthTokenType, + TokenDuration: tm.WorkerAuthTokenDuration, + Hostname: cl.Subject, + } + + tkn := new(library.Token) + + wt, err := tm.MintToken(wmto) + if err != nil { + retErr := fmt.Errorf("unable to generate auth token for worker %s: %w", w.GetHostname(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + tkn.SetToken(wt) + + c.JSON(http.StatusOK, tkn) + } +} diff --git a/api/worker/update.go b/api/worker/update.go new file mode 100644 index 000000000..0c2f9c4fb --- /dev/null +++ b/api/worker/update.go @@ -0,0 +1,115 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/router/middleware/worker" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation PUT /api/v1/workers/{worker} workers UpdateWorker +// +// Update a worker for the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: body +// name: body +// description: Payload containing the worker to update +// required: true +// schema: +// "$ref": "#/definitions/Worker" +// - in: path +// name: worker +// description: Name of the worker +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully updated the worker +// schema: +// "$ref": "#/definitions/Worker" +// '400': +// description: Unable to update the worker +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: Unable to update the worker +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to update the worker +// schema: +// "$ref": "#/definitions/Error" + +// UpdateWorker represents the API handler to +// update a worker in the configured backend. +func UpdateWorker(c *gin.Context) { + // capture middleware values + u := user.Retrieve(c) + w := worker.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "user": u.GetName(), + "worker": w.GetHostname(), + }).Infof("updating worker %s", w.GetHostname()) + + // capture body from API request + input := new(library.Worker) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for worker %s: %w", w.GetHostname(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + if len(input.GetAddress()) > 0 { + // update address if set + w.SetAddress(input.GetAddress()) + } + + if len(input.GetRoutes()) > 0 { + // update routes if set + w.SetRoutes(input.GetRoutes()) + } + + if input.GetActive() { + // update active if set + w.SetActive(input.GetActive()) + } + + // send API call to update the worker + err = database.FromContext(c).UpdateWorker(w) + if err != nil { + retErr := fmt.Errorf("unable to update worker %s: %w", w.GetHostname(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the updated worker + w, _ = database.FromContext(c).GetWorkerForHostname(w.GetHostname()) + + c.JSON(http.StatusOK, w) +} diff --git a/router/worker.go b/router/worker.go index 100a24975..7bb114c69 100644 --- a/router/worker.go +++ b/router/worker.go @@ -6,10 +6,10 @@ package router import ( "github.com/gin-gonic/gin" - "github.com/go-vela/server/api" + "github.com/go-vela/server/api/worker" "github.com/go-vela/server/router/middleware" "github.com/go-vela/server/router/middleware/perm" - "github.com/go-vela/server/router/middleware/worker" + wmiddleware "github.com/go-vela/server/router/middleware/worker" ) // WorkerHandlers is a function that extends the provided base router group @@ -23,18 +23,18 @@ import ( // DELETE /api/v1/workers/:worker . func WorkerHandlers(base *gin.RouterGroup) { // Workers endpoints - workers := base.Group("/workers") + _workers := base.Group("/workers") { - workers.POST("", perm.MustWorkerRegisterToken(), middleware.Payload(), api.CreateWorker) - workers.GET("", api.GetWorkers) + _workers.POST("", perm.MustWorkerRegisterToken(), middleware.Payload(), worker.CreateWorker) + _workers.GET("", worker.ListWorkers) // Worker endpoints - w := workers.Group("/:worker") + _worker := _workers.Group("/:worker") { - w.GET("", worker.Establish(), api.GetWorker) - w.PUT("", perm.MustPlatformAdmin(), worker.Establish(), api.UpdateWorker) - w.POST("/refresh", perm.MustWorkerAuthToken(), worker.Establish(), api.RefreshWorkerAuth) - w.DELETE("", perm.MustPlatformAdmin(), worker.Establish(), api.DeleteWorker) + _worker.GET("", wmiddleware.Establish(), worker.GetWorker) + _worker.PUT("", perm.MustPlatformAdmin(), wmiddleware.Establish(), worker.UpdateWorker) + _worker.POST("/refresh", perm.MustWorkerAuthToken(), wmiddleware.Establish(), worker.Refresh) + _worker.DELETE("", perm.MustPlatformAdmin(), wmiddleware.Establish(), worker.DeleteWorker) } // end of worker endpoints } // end of workers endpoints } From fa4ffd73abd84f7d3f13160aa9cc1f3cbf49bf68 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Wed, 17 May 2023 15:15:53 -0600 Subject: [PATCH 04/53] refactor(api): move deployment logic to separate package (#847) --- api/deployment.go | 338 --------------------------------------- api/deployment/create.go | 107 +++++++++++++ api/deployment/doc.go | 10 ++ api/deployment/get.go | 100 ++++++++++++ api/deployment/list.go | 170 ++++++++++++++++++++ api/pipeline/doc.go | 10 ++ api/repo/doc.go | 10 ++ api/user/doc.go | 10 ++ api/worker/doc.go | 10 ++ router/deployment.go | 8 +- 10 files changed, 431 insertions(+), 342 deletions(-) delete mode 100644 api/deployment.go create mode 100644 api/deployment/create.go create mode 100644 api/deployment/doc.go create mode 100644 api/deployment/get.go create mode 100644 api/deployment/list.go create mode 100644 api/pipeline/doc.go create mode 100644 api/repo/doc.go create mode 100644 api/user/doc.go create mode 100644 api/worker/doc.go diff --git a/api/deployment.go b/api/deployment.go deleted file mode 100644 index 82433b613..000000000 --- a/api/deployment.go +++ /dev/null @@ -1,338 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package api - -import ( - "fmt" - "net/http" - "strconv" - - "github.com/gin-gonic/gin" - "github.com/go-vela/server/database" - "github.com/go-vela/server/router/middleware/org" - "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/user" - "github.com/go-vela/server/scm" - "github.com/go-vela/server/util" - "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" -) - -// swagger:operation POST /api/v1/deployments/{org}/{repo} deployment CreateDeployment -// -// Create a deployment for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '201': -// description: Successfully created the deployment -// schema: -// "$ref": "#/definitions/Deployment" -// '400': -// description: Unable to create the deployment -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to create the deployment -// schema: -// "$ref": "#/definitions/Error" - -// CreateDeployment represents the API handler to -// create a deployment in the configured backend. -func CreateDeployment(c *gin.Context) { - // capture middleware values - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("creating new deployment for repo %s", r.GetFullName()) - - // capture body from API request - input := new(library.Deployment) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for new deployment for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update fields in deployment object - input.SetRepoID(r.GetID()) - input.SetUser(u.GetName()) - - if len(input.GetDescription()) == 0 { - input.SetDescription("Deployment request from Vela") - } - - if len(input.GetTask()) == 0 { - input.SetTask("deploy:vela") - } - - // send API call to create the deployment - err = scm.FromContext(c).CreateDeployment(u, r, input) - if err != nil { - retErr := fmt.Errorf("unable to create new deployment for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusCreated, input) -} - -// swagger:operation GET /api/v1/deployments/{org}/{repo} deployment GetDeployments -// -// Get a list of deployments for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: query -// name: page -// description: The page of results to retrieve -// type: integer -// default: 1 -// - in: query -// name: per_page -// description: How many results per page to return -// type: integer -// maximum: 100 -// default: 10 -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the list of deployments -// schema: -// type: array -// items: -// "$ref": "#/definitions/Deployment" -// headers: -// X-Total-Count: -// description: Total number of results -// type: integer -// Link: -// description: see https://tools.ietf.org/html/rfc5988 -// type: string -// '400': -// description: Unable to retrieve the list of deployments -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to retrieve the list of deployments -// schema: -// "$ref": "#/definitions/Error" - -// GetDeployments represents the API handler to capture -// a list of deployments from the configured backend. -func GetDeployments(c *gin.Context) { - // capture middleware values - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("reading deployments for repo %s", r.GetFullName()) - - // capture page query parameter if present - page, err := strconv.Atoi(c.DefaultQuery("page", "1")) - if err != nil { - retErr := fmt.Errorf("unable to convert page query parameter for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // capture per_page query parameter if present - perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) - if err != nil { - retErr := fmt.Errorf("unable to convert per_page query parameter for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // ensure per_page isn't above or below allowed values - perPage = util.MaxInt(1, util.MinInt(100, perPage)) - - // send API call to capture the total number of deployments for the repo - t, err := scm.FromContext(c).GetDeploymentCount(u, r) - if err != nil { - retErr := fmt.Errorf("unable to get deployment count for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the list of deployments for the repo - d, err := scm.FromContext(c).GetDeploymentList(u, r, page, perPage) - if err != nil { - retErr := fmt.Errorf("unable to get deployments for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - dWithBs := []*library.Deployment{} - - for _, deployment := range d { - b, err := database.FromContext(c).GetDeploymentBuildList(*deployment.URL) - if err != nil { - retErr := fmt.Errorf("unable to get builds for deployment %d: %w", deployment.GetID(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - builds := []library.Build{} - for _, build := range b { - builds = append(builds, *build) - } - - deployment.SetBuilds(builds) - - dWithBs = append(dWithBs, deployment) - } - - // create pagination object - pagination := Pagination{ - Page: page, - PerPage: perPage, - Total: t, - } - // set pagination headers - pagination.SetHeaderLink(c) - - c.JSON(http.StatusOK, dWithBs) -} - -// swagger:operation GET /api/v1/deployments/{org}/{repo}/{deployment} deployment GetDeployment -// -// Get a deployment from the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: deployment -// description: Name of the org -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the deployment -// schema: -// "$ref": "#/definitions/Deployment" -// '400': -// description: Unable to retrieve the deployment -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to retrieve the deployment -// schema: -// "$ref": "#/definitions/Error" - -// GetDeployment represents the API handler to -// capture a deployment from the configured backend. -func GetDeployment(c *gin.Context) { - // capture middleware values - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - deployment := util.PathParameter(c, "deployment") - - entry := fmt.Sprintf("%s/%s", r.GetFullName(), deployment) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("reading deployment %s", entry) - - number, err := strconv.Atoi(deployment) - if err != nil { - retErr := fmt.Errorf("invalid deployment parameter provided: %s", deployment) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // send API call to capture the deployment - d, err := scm.FromContext(c).GetDeployment(u, r, int64(number)) - if err != nil { - retErr := fmt.Errorf("unable to get deployment %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, d) -} diff --git a/api/deployment/create.go b/api/deployment/create.go new file mode 100644 index 000000000..1ad549c07 --- /dev/null +++ b/api/deployment/create.go @@ -0,0 +1,107 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package deployment + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation POST /api/v1/deployments/{org}/{repo} deployments CreateDeployment +// +// Create a deployment for the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '201': +// description: Successfully created the deployment +// schema: +// "$ref": "#/definitions/Deployment" +// '400': +// description: Unable to create the deployment +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to create the deployment +// schema: +// "$ref": "#/definitions/Error" + +// CreateDeployment represents the API handler to +// create a deployment in the configured backend. +func CreateDeployment(c *gin.Context) { + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("creating new deployment for repo %s", r.GetFullName()) + + // capture body from API request + input := new(library.Deployment) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for new deployment for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update fields in deployment object + input.SetRepoID(r.GetID()) + input.SetUser(u.GetName()) + + if len(input.GetDescription()) == 0 { + input.SetDescription("Deployment request from Vela") + } + + if len(input.GetTask()) == 0 { + input.SetTask("deploy:vela") + } + + // send API call to create the deployment + err = scm.FromContext(c).CreateDeployment(u, r, input) + if err != nil { + retErr := fmt.Errorf("unable to create new deployment for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusCreated, input) +} diff --git a/api/deployment/doc.go b/api/deployment/doc.go new file mode 100644 index 000000000..c6118b8f3 --- /dev/null +++ b/api/deployment/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package deployment provides the deployment handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/deployment" +package deployment diff --git a/api/deployment/get.go b/api/deployment/get.go new file mode 100644 index 000000000..ff6827d3a --- /dev/null +++ b/api/deployment/get.go @@ -0,0 +1,100 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package deployment + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/deployments/{org}/{repo}/{deployment} deployments GetDeployment +// +// Get a deployment from the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: deployment +// description: Number of the deployment +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the deployment +// schema: +// "$ref": "#/definitions/Deployment" +// '400': +// description: Unable to retrieve the deployment +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to retrieve the deployment +// schema: +// "$ref": "#/definitions/Error" + +// GetDeployment represents the API handler to +// capture a deployment from the configured backend. +func GetDeployment(c *gin.Context) { + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + deployment := util.PathParameter(c, "deployment") + + entry := fmt.Sprintf("%s/%s", r.GetFullName(), deployment) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("reading deployment %s", entry) + + number, err := strconv.Atoi(deployment) + if err != nil { + retErr := fmt.Errorf("invalid deployment parameter provided: %s", deployment) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to capture the deployment + d, err := scm.FromContext(c).GetDeployment(u, r, int64(number)) + if err != nil { + retErr := fmt.Errorf("unable to get deployment %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, d) +} diff --git a/api/deployment/list.go b/api/deployment/list.go new file mode 100644 index 000000000..bc5f9e97a --- /dev/null +++ b/api/deployment/list.go @@ -0,0 +1,170 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package deployment + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/api" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/deployments/{org}/{repo} deployments ListDeployments +// +// Get a list of deployments for the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: query +// name: page +// description: The page of results to retrieve +// type: integer +// default: 1 +// - in: query +// name: per_page +// description: How many results per page to return +// type: integer +// maximum: 100 +// default: 10 +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the list of deployments +// schema: +// type: array +// items: +// "$ref": "#/definitions/Deployment" +// headers: +// X-Total-Count: +// description: Total number of results +// type: integer +// Link: +// description: see https://tools.ietf.org/html/rfc5988 +// type: string +// '400': +// description: Unable to retrieve the list of deployments +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to retrieve the list of deployments +// schema: +// "$ref": "#/definitions/Error" + +// ListDeployments represents the API handler to capture +// a list of deployments from the configured backend. +func ListDeployments(c *gin.Context) { + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("reading deployments for repo %s", r.GetFullName()) + + // capture page query parameter if present + page, err := strconv.Atoi(c.DefaultQuery("page", "1")) + if err != nil { + retErr := fmt.Errorf("unable to convert page query parameter for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // capture per_page query parameter if present + perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) + if err != nil { + retErr := fmt.Errorf("unable to convert per_page query parameter for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // ensure per_page isn't above or below allowed values + perPage = util.MaxInt(1, util.MinInt(100, perPage)) + + // send API call to capture the total number of deployments for the repo + t, err := scm.FromContext(c).GetDeploymentCount(u, r) + if err != nil { + retErr := fmt.Errorf("unable to get deployment count for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the list of deployments for the repo + d, err := scm.FromContext(c).GetDeploymentList(u, r, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to get deployments for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + dWithBs := []*library.Deployment{} + + for _, deployment := range d { + b, err := database.FromContext(c).GetDeploymentBuildList(*deployment.URL) + if err != nil { + retErr := fmt.Errorf("unable to get builds for deployment %d: %w", deployment.GetID(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + builds := []library.Build{} + for _, build := range b { + builds = append(builds, *build) + } + + deployment.SetBuilds(builds) + + dWithBs = append(dWithBs, deployment) + } + + // create pagination object + pagination := api.Pagination{ + Page: page, + PerPage: perPage, + Total: t, + } + // set pagination headers + pagination.SetHeaderLink(c) + + c.JSON(http.StatusOK, dWithBs) +} diff --git a/api/pipeline/doc.go b/api/pipeline/doc.go new file mode 100644 index 000000000..0b2c02901 --- /dev/null +++ b/api/pipeline/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package pipeline provides the pipeline handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/pipeline" +package pipeline diff --git a/api/repo/doc.go b/api/repo/doc.go new file mode 100644 index 000000000..82eb5304e --- /dev/null +++ b/api/repo/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package repo provides the repo handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/repo" +package repo diff --git a/api/user/doc.go b/api/user/doc.go new file mode 100644 index 000000000..7f1ce6bc5 --- /dev/null +++ b/api/user/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package user provides the user handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/user" +package user diff --git a/api/worker/doc.go b/api/worker/doc.go new file mode 100644 index 000000000..86da09ed7 --- /dev/null +++ b/api/worker/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package worker provides the worker handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/worker" +package worker diff --git a/router/deployment.go b/router/deployment.go index 9a5b15e27..eabbc48c6 100644 --- a/router/deployment.go +++ b/router/deployment.go @@ -8,7 +8,7 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/router/middleware/org" - "github.com/go-vela/server/api" + "github.com/go-vela/server/api/deployment" "github.com/go-vela/server/router/middleware/perm" "github.com/go-vela/server/router/middleware/repo" ) @@ -23,8 +23,8 @@ func DeploymentHandlers(base *gin.RouterGroup) { // Deployments endpoints deployments := base.Group("/deployments/:org/:repo", org.Establish(), repo.Establish()) { - deployments.POST("", perm.MustWrite(), api.CreateDeployment) - deployments.GET("", perm.MustRead(), api.GetDeployments) - deployments.GET("/:deployment", perm.MustRead(), api.GetDeployment) + deployments.POST("", perm.MustWrite(), deployment.CreateDeployment) + deployments.GET("", perm.MustRead(), deployment.ListDeployments) + deployments.GET("/:deployment", perm.MustRead(), deployment.GetDeployment) } // end of deployments endpoints } From cd6a651381fc8bea7dcc8318b69cddccd44a9a71 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Thu, 18 May 2023 09:00:57 -0600 Subject: [PATCH 05/53] refactor(api): move scm logic to separate package (#848) --- api/scm/doc.go | 10 +++ api/scm/sync.go | 137 ++++++++++++++++++++++++++++++++ api/{scm.go => scm/sync_org.go} | 127 +---------------------------- router/scm.go | 6 +- 4 files changed, 154 insertions(+), 126 deletions(-) create mode 100644 api/scm/doc.go create mode 100644 api/scm/sync.go rename api/{scm.go => scm/sync_org.go} (54%) diff --git a/api/scm/doc.go b/api/scm/doc.go new file mode 100644 index 000000000..5d66cafed --- /dev/null +++ b/api/scm/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package scm provides the scm handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/scm" +package scm diff --git a/api/scm/sync.go b/api/scm/sync.go new file mode 100644 index 000000000..b160f5fa0 --- /dev/null +++ b/api/scm/sync.go @@ -0,0 +1,137 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package scm + +import ( + "fmt" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/scm/repos/{org}/{repo}/sync scm SyncRepo +// +// Sync up scm service and database in the context of a specific repo +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully synchronized repo +// schema: +// type: string +// '500': +// description: Unable to synchronize repo +// schema: +// "$ref": "#/definitions/Error" + +// SyncRepo represents the API handler to +// synchronize a single repository between +// SCM service and the database should a discrepancy +// exist. Primarily used for deleted repos or to align +// subscribed events with allowed events. +func SyncRepo(c *gin.Context) { + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logger := logrus.WithFields(logrus.Fields{ + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }) + + logger.Infof("syncing repo %s", r.GetFullName()) + + // retrieve repo from source code manager service + _, err := scm.FromContext(c).GetRepo(u, r) + + // if there is an error retrieving repo, we know it is deleted: set to inactive + if err != nil { + // set repo to inactive - do not delete + r.SetActive(false) + + // update repo in database + err := database.FromContext(c).UpdateRepo(r) + if err != nil { + retErr := fmt.Errorf("unable to update repo for org %s: %w", o, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // exit with success as hook sync will be unnecessary + c.JSON(http.StatusOK, fmt.Sprintf("repo %s synced", r.GetFullName())) + + return + } + + // verify the user is an admin of the repo + // we cannot use our normal permissions check due to the possibility the repo was deleted + perm, err := scm.FromContext(c).RepoAccess(u, u.GetToken(), o, r.GetName()) + if err != nil { + logger.Errorf("unable to get user %s access level for org %s", u.GetName(), o) + } + + if !strings.EqualFold(perm, "admin") { + retErr := fmt.Errorf("user %s does not have 'admin' permissions for the repo %s", u.GetName(), r.GetFullName()) + + util.HandleError(c, http.StatusUnauthorized, retErr) + + return + } + + // if we have webhook validation, update the repo hook in the SCM + if c.Value("webhookvalidation").(bool) { + // grab last hook from repo to fetch the webhook ID + lastHook, err := database.FromContext(c).LastHookForRepo(r) + if err != nil { + retErr := fmt.Errorf("unable to retrieve last hook for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // update webhook + err = scm.FromContext(c).Update(u, r, lastHook.GetWebhookID()) + if err != nil { + retErr := fmt.Errorf("unable to update repo webhook for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + } + + c.JSON(http.StatusOK, fmt.Sprintf("repo %s synced", r.GetFullName())) +} diff --git a/api/scm.go b/api/scm/sync_org.go similarity index 54% rename from api/scm.go rename to api/scm/sync_org.go index ee0898ff8..ac18ec551 100644 --- a/api/scm.go +++ b/api/scm/sync_org.go @@ -2,17 +2,15 @@ // // Use of this source code is governed by the LICENSE file in this repository. -package api +package scm import ( "fmt" "net/http" - "strings" "github.com/gin-gonic/gin" "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/org" - "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" @@ -20,7 +18,7 @@ import ( "github.com/sirupsen/logrus" ) -// swagger:operation GET /api/v1/scm/orgs/{org}/sync scm SyncRepos +// swagger:operation GET /api/v1/scm/orgs/{org}/sync scm SyncReposForOrg // // Sync up repos from scm service and database in a specified org // @@ -45,12 +43,12 @@ import ( // schema: // "$ref": "#/definitions/Error" -// SyncRepos represents the API handler to +// SyncReposForOrg represents the API handler to // synchronize organization repositories between // SCM Service and the database should a discrepancy // exist. Primarily used for deleted repos or to align // subscribed events with allowed events. -func SyncRepos(c *gin.Context) { +func SyncReposForOrg(c *gin.Context) { // capture middleware values o := org.Retrieve(c) u := user.Retrieve(c) @@ -151,120 +149,3 @@ func SyncRepos(c *gin.Context) { c.JSON(http.StatusOK, fmt.Sprintf("org %s repos synced", o)) } - -// swagger:operation GET /api/v1/scm/repos/{org}/{repo}/sync scm SyncRepo -// -// Sync up scm service and database in the context of a specific repo -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully synchronized repo -// schema: -// type: string -// '500': -// description: Unable to synchronize repo -// schema: -// "$ref": "#/definitions/Error" - -// SyncRepo represents the API handler to -// synchronize a single repository between -// SCM service and the database should a discrepancy -// exist. Primarily used for deleted repos or to align -// subscribed events with allowed events. -func SyncRepo(c *gin.Context) { - // capture middleware values - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logger := logrus.WithFields(logrus.Fields{ - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }) - - logger.Infof("syncing repo %s", r.GetFullName()) - - // retrieve repo from source code manager service - _, err := scm.FromContext(c).GetRepo(u, r) - - // if there is an error retrieving repo, we know it is deleted: set to inactive - if err != nil { - // set repo to inactive - do not delete - r.SetActive(false) - - // update repo in database - err := database.FromContext(c).UpdateRepo(r) - if err != nil { - retErr := fmt.Errorf("unable to update repo for org %s: %w", o, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // exit with success as hook sync will be unnecessary - c.JSON(http.StatusOK, fmt.Sprintf("repo %s synced", r.GetFullName())) - - return - } - - // verify the user is an admin of the repo - // we cannot use our normal permissions check due to the possibility the repo was deleted - perm, err := scm.FromContext(c).RepoAccess(u, u.GetToken(), o, r.GetName()) - if err != nil { - logger.Errorf("unable to get user %s access level for org %s", u.GetName(), o) - } - - if !strings.EqualFold(perm, "admin") { - retErr := fmt.Errorf("user %s does not have 'admin' permissions for the repo %s", u.GetName(), r.GetFullName()) - - util.HandleError(c, http.StatusUnauthorized, retErr) - - return - } - - // if we have webhook validation, update the repo hook in the SCM - if c.Value("webhookvalidation").(bool) { - // grab last hook from repo to fetch the webhook ID - lastHook, err := database.FromContext(c).LastHookForRepo(r) - if err != nil { - retErr := fmt.Errorf("unable to retrieve last hook for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // update webhook - err = scm.FromContext(c).Update(u, r, lastHook.GetWebhookID()) - if err != nil { - retErr := fmt.Errorf("unable to update repo webhook for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - } - - c.JSON(http.StatusOK, fmt.Sprintf("repo %s synced", r.GetFullName())) -} diff --git a/router/scm.go b/router/scm.go index 85b2a0d9f..40d745087 100644 --- a/router/scm.go +++ b/router/scm.go @@ -6,7 +6,7 @@ package router import ( "github.com/gin-gonic/gin" - "github.com/go-vela/server/api" + "github.com/go-vela/server/api/scm" "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/repo" ) @@ -23,7 +23,7 @@ func ScmHandlers(base *gin.RouterGroup) { // SCM org endpoints org := orgs.Group("/:org", org.Establish()) { - org.GET("/sync", api.SyncRepos) + org.GET("/sync", scm.SyncReposForOrg) } // end of SCM org endpoints } // end of SCM orgs endpoints @@ -33,7 +33,7 @@ func ScmHandlers(base *gin.RouterGroup) { // SCM repo endpoints repo := repos.Group("/:org/:repo", org.Establish(), repo.Establish()) { - repo.GET("/sync", api.SyncRepo) + repo.GET("/sync", scm.SyncRepo) } // end of SCM repo endpoints } // end of SCM repos endpoints } From 61419f437d2809f2e6cffe8f999262d16b16de62 Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Thu, 18 May 2023 12:43:26 -0500 Subject: [PATCH 06/53] refactor(api): move hook handlers to new package (#829) Co-authored-by: Jacob Floyd --- api/hook.go | 674 ------------------------------------------ api/hook/create.go | 129 ++++++++ api/hook/delete.go | 115 +++++++ api/hook/get.go | 101 +++++++ api/hook/list.go | 136 +++++++++ api/hook/redeliver.go | 115 +++++++ api/hook/update.go | 173 +++++++++++ router/hook.go | 16 +- 8 files changed, 777 insertions(+), 682 deletions(-) delete mode 100644 api/hook.go create mode 100644 api/hook/create.go create mode 100644 api/hook/delete.go create mode 100644 api/hook/get.go create mode 100644 api/hook/list.go create mode 100644 api/hook/redeliver.go create mode 100644 api/hook/update.go diff --git a/api/hook.go b/api/hook.go deleted file mode 100644 index 1787f9975..000000000 --- a/api/hook.go +++ /dev/null @@ -1,674 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package api - -import ( - "fmt" - "net/http" - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/go-vela/server/database" - "github.com/go-vela/server/router/middleware/org" - "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/user" - "github.com/go-vela/server/scm" - "github.com/go-vela/server/util" - "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" -) - -// swagger:operation POST /api/v1/hooks/{org}/{repo} webhook CreateHook -// -// Create a webhook for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: body -// name: body -// description: Webhook payload that we expect from the user or VCS -// required: true -// schema: -// "$ref": "#/definitions/Webhook" -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '201': -// description: The webhook has been created -// schema: -// "$ref": "#/definitions/Webhook" -// '400': -// description: The webhook was unable to be created -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: The webhook was unable to be created -// schema: -// "$ref": "#/definitions/Error" - -// CreateHook represents the API handler to create -// a webhook in the configured backend. -func CreateHook(c *gin.Context) { - // capture middleware values - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("creating new hook for repo %s", r.GetFullName()) - - // capture body from API request - input := new(library.Hook) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for new hook for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // send API call to capture the last hook for the repo - lastHook, err := database.FromContext(c).LastHookForRepo(r) - if err != nil { - retErr := fmt.Errorf("unable to get last hook for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // update fields in webhook object - input.SetRepoID(r.GetID()) - input.SetNumber(1) - - if input.GetCreated() == 0 { - input.SetCreated(time.Now().UTC().Unix()) - } - - if lastHook != nil { - input.SetNumber( - lastHook.GetNumber() + 1, - ) - } - - // send API call to create the webhook - err = database.FromContext(c).CreateHook(input) - if err != nil { - retErr := fmt.Errorf("unable to create hook for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the created webhook - h, _ := database.FromContext(c).GetHookForRepo(r, input.GetNumber()) - - c.JSON(http.StatusCreated, h) -} - -// swagger:operation GET /api/v1/hooks/{org}/{repo} webhook GetHooks -// -// Retrieve the webhooks for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: query -// name: page -// description: The page of results to retrieve -// type: integer -// default: 1 -// - in: query -// name: per_page -// description: How many results per page to return -// type: integer -// maximum: 100 -// default: 10 -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved webhooks -// schema: -// type: array -// items: -// "$ref": "#/definitions/Webhook" -// headers: -// X-Total-Count: -// description: Total number of results -// type: integer -// Link: -// description: see https://tools.ietf.org/html/rfc5988 -// type: string -// '400': -// description: Unable to retrieve webhooks -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to retrieve webhooks -// schema: -// "$ref": "#/definitions/Error" - -// GetHooks represents the API handler to capture a list -// of webhooks from the configured backend. -func GetHooks(c *gin.Context) { - // capture middleware values - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("reading hooks for repo %s", r.GetFullName()) - - // capture page query parameter if present - page, err := strconv.Atoi(c.DefaultQuery("page", "1")) - if err != nil { - retErr := fmt.Errorf("unable to convert page query parameter for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // capture per_page query parameter if present - perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) - if err != nil { - retErr := fmt.Errorf("unable to convert per_page query parameter for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // ensure per_page isn't above or below allowed values - perPage = util.MaxInt(1, util.MinInt(100, perPage)) - - // send API call to capture the list of steps for the build - h, t, err := database.FromContext(c).ListHooksForRepo(r, page, perPage) - if err != nil { - retErr := fmt.Errorf("unable to get hooks for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // create pagination object - pagination := Pagination{ - Page: page, - PerPage: perPage, - Total: t, - } - // set pagination headers - pagination.SetHeaderLink(c) - - c.JSON(http.StatusOK, h) -} - -// swagger:operation GET /api/v1/hooks/{org}/{repo}/{hook} webhook GetHook -// -// Retrieve a webhook for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: hook -// description: Number of the hook -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the webhook -// schema: -// "$ref": "#/definitions/Webhook" -// '400': -// description: Unable to retrieve the webhook -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to retrieve the webhook -// schema: -// "$ref": "#/definitions/Error" - -// GetHook represents the API handler to capture a -// webhook from the configured backend. -func GetHook(c *gin.Context) { - // capture middleware values - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - hook := util.PathParameter(c, "hook") - - entry := fmt.Sprintf("%s/%s", r.GetFullName(), hook) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "org": o, - "hook": hook, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("reading hook %s", entry) - - number, err := strconv.Atoi(hook) - if err != nil { - retErr := fmt.Errorf("invalid hook parameter provided: %s", hook) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // send API call to capture the webhook - h, err := database.FromContext(c).GetHookForRepo(r, number) - if err != nil { - retErr := fmt.Errorf("unable to get hook %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, h) -} - -// swagger:operation PUT /api/v1/hooks/{org}/{repo}/{hook} webhook UpdateHook -// -// Update a webhook for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: hook -// description: Number of the hook -// required: true -// type: integer -// - in: body -// name: body -// description: Webhook payload that we expect from the user or VCS -// required: true -// schema: -// "$ref": "#/definitions/Webhook" -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully updated the webhook -// schema: -// "$ref": "#/definitions/Webhook" -// '400': -// description: The webhook was unable to be updated -// schema: -// "$ref": "#/definitions/Error" -// '404': -// description: The webhook was unable to be updated -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: The webhook was unable to be updated -// schema: -// "$ref": "#/definitions/Error" - -// UpdateHook represents the API handler to update -// a webhook in the configured backend. -func UpdateHook(c *gin.Context) { - // capture middleware values - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - hook := util.PathParameter(c, "hook") - - entry := fmt.Sprintf("%s/%s", r.GetFullName(), hook) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "org": o, - "hook": hook, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("updating hook %s", entry) - - // capture body from API request - input := new(library.Hook) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for hook %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - number, err := strconv.Atoi(hook) - if err != nil { - retErr := fmt.Errorf("invalid hook parameter provided: %s", hook) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // send API call to capture the webhook - h, err := database.FromContext(c).GetHookForRepo(r, number) - if err != nil { - retErr := fmt.Errorf("unable to get hook %s: %w", entry, err) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - // update webhook fields if provided - if input.GetCreated() > 0 { - // update created if set - h.SetCreated(input.GetCreated()) - } - - if len(input.GetHost()) > 0 { - // update host if set - h.SetHost(input.GetHost()) - } - - if len(input.GetEvent()) > 0 { - // update event if set - h.SetEvent(input.GetEvent()) - } - - if len(input.GetBranch()) > 0 { - // update branch if set - h.SetBranch(input.GetBranch()) - } - - if len(input.GetError()) > 0 { - // update error if set - h.SetError(input.GetError()) - } - - if len(input.GetStatus()) > 0 { - // update status if set - h.SetStatus(input.GetStatus()) - } - - if len(input.GetLink()) > 0 { - // update link if set - h.SetLink(input.GetLink()) - } - - // send API call to update the webhook - err = database.FromContext(c).UpdateHook(h) - if err != nil { - retErr := fmt.Errorf("unable to update hook %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the updated user - h, _ = database.FromContext(c).GetHookForRepo(r, h.GetNumber()) - - c.JSON(http.StatusOK, h) -} - -// swagger:operation DELETE /api/v1/hooks/{org}/{repo}/{hook} webhook DeleteHook -// -// Delete a webhook for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: hook -// description: Number of the hook -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully deleted the webhook -// schema: -// type: string -// '400': -// description: The webhook was unable to be deleted -// schema: -// "$ref": "#/definitions/Error" -// '404': -// description: The webhook was unable to be deleted -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: The webhook was unable to be deleted -// schema: -// "$ref": "#/definitions/Error" - -// DeleteHook represents the API handler to remove -// a webhook from the configured backend. -func DeleteHook(c *gin.Context) { - // capture middleware values - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - hook := util.PathParameter(c, "hook") - - entry := fmt.Sprintf("%s/%s", r.GetFullName(), hook) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "org": o, - "hook": hook, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("deleting hook %s", entry) - - number, err := strconv.Atoi(hook) - if err != nil { - retErr := fmt.Errorf("invalid hook parameter provided: %s", hook) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // send API call to capture the webhook - h, err := database.FromContext(c).GetHookForRepo(r, number) - if err != nil { - retErr := fmt.Errorf("unable to get hook %s: %w", hook, err) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - // send API call to remove the webhook - err = database.FromContext(c).DeleteHook(h) - if err != nil { - retErr := fmt.Errorf("unable to delete hook %s: %w", hook, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, fmt.Sprintf("hook %s deleted", entry)) -} - -// swagger:operation POST /api/v1/hooks/{org}/{repo}/{hook}/redeliver webhook RedeliverHook -// -// Redeliver a webhook from the SCM -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: hook -// description: Number of the hook -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully redelivered the webhook -// schema: -// "$ref": "#/definitions/Webhook" -// '400': -// description: The webhook was unable to be redelivered -// schema: -// "$ref": "#/definitions/Error" -// '404': -// description: The webhook was unable to be redelivered -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: The webhook was unable to be redelivered -// schema: -// "$ref": "#/definitions/Error" - -// RedeliverHook represents the API handler to redeliver -// a webhook from the SCM. - -func RedeliverHook(c *gin.Context) { - // capture middleware values - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - hook := util.PathParameter(c, "hook") - - entry := fmt.Sprintf("%s/%s", r.GetFullName(), hook) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "org": o, - "hook": hook, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("redelivering hook %s", entry) - - number, err := strconv.Atoi(hook) - if err != nil { - retErr := fmt.Errorf("invalid hook parameter provided: %s", hook) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // send API call to capture the webhook - h, err := database.FromContext(c).GetHookForRepo(r, number) - if err != nil { - retErr := fmt.Errorf("unable to get hook %s: %w", entry, err) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - err = scm.FromContext(c).RedeliverWebhook(c, u, r, h) - if err != nil { - retErr := fmt.Errorf("unable to redeliver hook %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, fmt.Sprintf("hook %s redelivered", entry)) -} diff --git a/api/hook/create.go b/api/hook/create.go new file mode 100644 index 000000000..70f524524 --- /dev/null +++ b/api/hook/create.go @@ -0,0 +1,129 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package hook + +import ( + "fmt" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation POST /api/v1/hooks/{org}/{repo} webhook CreateHook +// +// Create a webhook for the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: body +// name: body +// description: Webhook payload that we expect from the user or VCS +// required: true +// schema: +// "$ref": "#/definitions/Webhook" +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '201': +// description: The webhook has been created +// schema: +// "$ref": "#/definitions/Webhook" +// '400': +// description: The webhook was unable to be created +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: The webhook was unable to be created +// schema: +// "$ref": "#/definitions/Error" + +// CreateHook represents the API handler to create +// a webhook in the configured backend. +func CreateHook(c *gin.Context) { + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("creating new hook for repo %s", r.GetFullName()) + + // capture body from API request + input := new(library.Hook) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for new hook for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to capture the last hook for the repo + lastHook, err := database.FromContext(c).LastHookForRepo(r) + if err != nil { + retErr := fmt.Errorf("unable to get last hook for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // update fields in webhook object + input.SetRepoID(r.GetID()) + input.SetNumber(1) + + if input.GetCreated() == 0 { + input.SetCreated(time.Now().UTC().Unix()) + } + + if lastHook != nil { + input.SetNumber( + lastHook.GetNumber() + 1, + ) + } + + // send API call to create the webhook + err = database.FromContext(c).CreateHook(input) + if err != nil { + retErr := fmt.Errorf("unable to create hook for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the created webhook + h, _ := database.FromContext(c).GetHookForRepo(r, input.GetNumber()) + + c.JSON(http.StatusCreated, h) +} diff --git a/api/hook/delete.go b/api/hook/delete.go new file mode 100644 index 000000000..7ada1f0fb --- /dev/null +++ b/api/hook/delete.go @@ -0,0 +1,115 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package hook + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation DELETE /api/v1/hooks/{org}/{repo}/{hook} webhook DeleteHook +// +// Delete a webhook for the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: hook +// description: Number of the hook +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully deleted the webhook +// schema: +// type: string +// '400': +// description: The webhook was unable to be deleted +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: The webhook was unable to be deleted +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: The webhook was unable to be deleted +// schema: +// "$ref": "#/definitions/Error" + +// DeleteHook represents the API handler to remove +// a webhook from the configured backend. +func DeleteHook(c *gin.Context) { + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + hook := util.PathParameter(c, "hook") + + entry := fmt.Sprintf("%s/%s", r.GetFullName(), hook) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "hook": hook, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("deleting hook %s", entry) + + number, err := strconv.Atoi(hook) + if err != nil { + retErr := fmt.Errorf("invalid hook parameter provided: %s", hook) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to capture the webhook + h, err := database.FromContext(c).GetHookForRepo(r, number) + if err != nil { + retErr := fmt.Errorf("unable to get hook %s: %w", hook, err) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // send API call to remove the webhook + err = database.FromContext(c).DeleteHook(h) + if err != nil { + retErr := fmt.Errorf("unable to delete hook %s: %w", hook, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, fmt.Sprintf("hook %s deleted", entry)) +} diff --git a/api/hook/get.go b/api/hook/get.go new file mode 100644 index 000000000..458183dcb --- /dev/null +++ b/api/hook/get.go @@ -0,0 +1,101 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package hook + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/hooks/{org}/{repo}/{hook} webhook GetHook +// +// Retrieve a webhook for the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: hook +// description: Number of the hook +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the webhook +// schema: +// "$ref": "#/definitions/Webhook" +// '400': +// description: Unable to retrieve the webhook +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to retrieve the webhook +// schema: +// "$ref": "#/definitions/Error" + +// GetHook represents the API handler to capture a +// webhook from the configured backend. +func GetHook(c *gin.Context) { + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + hook := util.PathParameter(c, "hook") + + entry := fmt.Sprintf("%s/%s", r.GetFullName(), hook) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "hook": hook, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("reading hook %s", entry) + + number, err := strconv.Atoi(hook) + if err != nil { + retErr := fmt.Errorf("invalid hook parameter provided: %s", hook) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to capture the webhook + h, err := database.FromContext(c).GetHookForRepo(r, number) + if err != nil { + retErr := fmt.Errorf("unable to get hook %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, h) +} diff --git a/api/hook/list.go b/api/hook/list.go new file mode 100644 index 000000000..2cb6d05fe --- /dev/null +++ b/api/hook/list.go @@ -0,0 +1,136 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package hook + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/api" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/hooks/{org}/{repo} webhook ListHooks +// +// Retrieve the webhooks for the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: query +// name: page +// description: The page of results to retrieve +// type: integer +// default: 1 +// - in: query +// name: per_page +// description: How many results per page to return +// type: integer +// maximum: 100 +// default: 10 +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved webhooks +// schema: +// type: array +// items: +// "$ref": "#/definitions/Webhook" +// headers: +// X-Total-Count: +// description: Total number of results +// type: integer +// Link: +// description: see https://tools.ietf.org/html/rfc5988 +// type: string +// '400': +// description: Unable to retrieve webhooks +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to retrieve webhooks +// schema: +// "$ref": "#/definitions/Error" + +// ListHooks represents the API handler to capture a list +// of webhooks from the configured backend. +func ListHooks(c *gin.Context) { + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("reading hooks for repo %s", r.GetFullName()) + + // capture page query parameter if present + page, err := strconv.Atoi(c.DefaultQuery("page", "1")) + if err != nil { + retErr := fmt.Errorf("unable to convert page query parameter for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // capture per_page query parameter if present + perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) + if err != nil { + retErr := fmt.Errorf("unable to convert per_page query parameter for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // ensure per_page isn't above or below allowed values + perPage = util.MaxInt(1, util.MinInt(100, perPage)) + + // send API call to capture the list of steps for the build + h, t, err := database.FromContext(c).ListHooksForRepo(r, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to get hooks for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // create pagination object + pagination := api.Pagination{ + Page: page, + PerPage: perPage, + Total: t, + } + // set pagination headers + pagination.SetHeaderLink(c) + + c.JSON(http.StatusOK, h) +} diff --git a/api/hook/redeliver.go b/api/hook/redeliver.go new file mode 100644 index 000000000..4afcb8a79 --- /dev/null +++ b/api/hook/redeliver.go @@ -0,0 +1,115 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package hook + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation POST /api/v1/hooks/{org}/{repo}/{hook}/redeliver webhook RedeliverHook +// +// Redeliver a webhook from the SCM +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: hook +// description: Number of the hook +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully redelivered the webhook +// schema: +// "$ref": "#/definitions/Webhook" +// '400': +// description: The webhook was unable to be redelivered +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: The webhook was unable to be redelivered +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: The webhook was unable to be redelivered +// schema: +// "$ref": "#/definitions/Error" + +// RedeliverHook represents the API handler to redeliver +// a webhook from the SCM. +func RedeliverHook(c *gin.Context) { + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + hook := util.PathParameter(c, "hook") + + entry := fmt.Sprintf("%s/%s", r.GetFullName(), hook) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "hook": hook, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("redelivering hook %s", entry) + + number, err := strconv.Atoi(hook) + if err != nil { + retErr := fmt.Errorf("invalid hook parameter provided: %s", hook) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to capture the webhook + h, err := database.FromContext(c).GetHookForRepo(r, number) + if err != nil { + retErr := fmt.Errorf("unable to get hook %s: %w", entry, err) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + err = scm.FromContext(c).RedeliverWebhook(c, u, r, h) + if err != nil { + retErr := fmt.Errorf("unable to redeliver hook %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, fmt.Sprintf("hook %s redelivered", entry)) +} diff --git a/api/hook/update.go b/api/hook/update.go new file mode 100644 index 000000000..f8beeb1ec --- /dev/null +++ b/api/hook/update.go @@ -0,0 +1,173 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package hook + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation PUT /api/v1/hooks/{org}/{repo}/{hook} webhook UpdateHook +// +// Update a webhook for the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: hook +// description: Number of the hook +// required: true +// type: integer +// - in: body +// name: body +// description: Webhook payload that we expect from the user or VCS +// required: true +// schema: +// "$ref": "#/definitions/Webhook" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully updated the webhook +// schema: +// "$ref": "#/definitions/Webhook" +// '400': +// description: The webhook was unable to be updated +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: The webhook was unable to be updated +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: The webhook was unable to be updated +// schema: +// "$ref": "#/definitions/Error" + +// UpdateHook represents the API handler to update +// a webhook in the configured backend. +func UpdateHook(c *gin.Context) { + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + hook := util.PathParameter(c, "hook") + + entry := fmt.Sprintf("%s/%s", r.GetFullName(), hook) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "hook": hook, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("updating hook %s", entry) + + // capture body from API request + input := new(library.Hook) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for hook %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + number, err := strconv.Atoi(hook) + if err != nil { + retErr := fmt.Errorf("invalid hook parameter provided: %s", hook) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to capture the webhook + h, err := database.FromContext(c).GetHookForRepo(r, number) + if err != nil { + retErr := fmt.Errorf("unable to get hook %s: %w", entry, err) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // update webhook fields if provided + if input.GetCreated() > 0 { + // update created if set + h.SetCreated(input.GetCreated()) + } + + if len(input.GetHost()) > 0 { + // update host if set + h.SetHost(input.GetHost()) + } + + if len(input.GetEvent()) > 0 { + // update event if set + h.SetEvent(input.GetEvent()) + } + + if len(input.GetBranch()) > 0 { + // update branch if set + h.SetBranch(input.GetBranch()) + } + + if len(input.GetError()) > 0 { + // update error if set + h.SetError(input.GetError()) + } + + if len(input.GetStatus()) > 0 { + // update status if set + h.SetStatus(input.GetStatus()) + } + + if len(input.GetLink()) > 0 { + // update link if set + h.SetLink(input.GetLink()) + } + + // send API call to update the webhook + err = database.FromContext(c).UpdateHook(h) + if err != nil { + retErr := fmt.Errorf("unable to update hook %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the updated user + h, _ = database.FromContext(c).GetHookForRepo(r, h.GetNumber()) + + c.JSON(http.StatusOK, h) +} diff --git a/router/hook.go b/router/hook.go index 8237cc87a..f5e1aa1db 100644 --- a/router/hook.go +++ b/router/hook.go @@ -6,7 +6,7 @@ package router import ( "github.com/gin-gonic/gin" - "github.com/go-vela/server/api" + "github.com/go-vela/server/api/hook" "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/perm" "github.com/go-vela/server/router/middleware/repo" @@ -23,13 +23,13 @@ import ( // POST /api/v1/hooks/:org/:repo/:hook/redeliver . func HookHandlers(base *gin.RouterGroup) { // Hooks endpoints - hooks := base.Group("/hooks/:org/:repo", org.Establish(), repo.Establish()) + _hooks := base.Group("/hooks/:org/:repo", org.Establish(), repo.Establish()) { - hooks.POST("", perm.MustPlatformAdmin(), api.CreateHook) - hooks.GET("", perm.MustRead(), api.GetHooks) - hooks.GET("/:hook", perm.MustRead(), api.GetHook) - hooks.PUT("/:hook", perm.MustPlatformAdmin(), api.UpdateHook) - hooks.DELETE("/:hook", perm.MustPlatformAdmin(), api.DeleteHook) - hooks.POST("/:hook/redeliver", perm.MustWrite(), api.RedeliverHook) + _hooks.POST("", perm.MustPlatformAdmin(), hook.CreateHook) + _hooks.GET("", perm.MustRead(), hook.ListHooks) + _hooks.GET("/:hook", perm.MustRead(), hook.GetHook) + _hooks.PUT("/:hook", perm.MustPlatformAdmin(), hook.UpdateHook) + _hooks.DELETE("/:hook", perm.MustPlatformAdmin(), hook.DeleteHook) + _hooks.POST("/:hook/redeliver", perm.MustWrite(), hook.RedeliverHook) } // end of hooks endpoints } From d0f63b14f175096e69af41bbb678457b280d7a33 Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Thu, 18 May 2023 12:47:17 -0500 Subject: [PATCH 07/53] feat(database): add support for schedules (#834) * feat(api/types): add support for schedules * feat(database/types): add support for schedules * feat(database): add support for schedules * chore: update go dependencies * feat(database): add schedule engine * fix: parse entry for schedules * enhance: switch to adhocore/gronx * chore: update go deps * chore: address linter feedback * chore: remove new types * chore: updates for removed types * chore: update go dependencies * chore: address review feedback --------- Co-authored-by: dave vader <48764154+plyr4@users.noreply.github.com> Co-authored-by: Easton Crupper <65553218+ecrupper@users.noreply.github.com> --- database/interface.go | 5 + database/postgres/postgres.go | 18 +++ database/postgres/postgres_test.go | 8 + database/schedule/count.go | 25 +++ database/schedule/count_active.go | 26 +++ database/schedule/count_active_test.go | 102 ++++++++++++ database/schedule/count_repo.go | 31 ++++ database/schedule/count_repo_test.go | 106 +++++++++++++ database/schedule/count_test.go | 100 ++++++++++++ database/schedule/create.go | 35 ++++ database/schedule/create_test.go | 76 +++++++++ database/schedule/delete.go | 28 ++++ database/schedule/delete_test.go | 76 +++++++++ database/schedule/get.go | 31 ++++ database/schedule/get_repo.go | 37 +++++ database/schedule/get_repo_test.go | 94 +++++++++++ database/schedule/get_test.go | 88 ++++++++++ database/schedule/index.go | 24 +++ database/schedule/index_test.go | 59 +++++++ database/schedule/interface.go | 49 ++++++ database/schedule/list.go | 52 ++++++ database/schedule/list_active.go | 53 +++++++ database/schedule/list_active_test.go | 111 +++++++++++++ database/schedule/list_repo.go | 63 ++++++++ database/schedule/list_repo_test.go | 115 ++++++++++++++ database/schedule/list_test.go | 110 +++++++++++++ database/schedule/opts.go | 44 +++++ database/schedule/opts_test.go | 161 +++++++++++++++++++ database/schedule/schedule.go | 80 ++++++++++ database/schedule/schedule_test.go | 212 +++++++++++++++++++++++++ database/schedule/table.go | 66 ++++++++ database/schedule/table_test.go | 59 +++++++ database/schedule/update.go | 35 ++++ database/schedule/update_test.go | 85 ++++++++++ database/sqlite/sqlite.go | 15 ++ go.mod | 4 +- go.sum | 9 +- 37 files changed, 2288 insertions(+), 4 deletions(-) create mode 100644 database/schedule/count.go create mode 100644 database/schedule/count_active.go create mode 100644 database/schedule/count_active_test.go create mode 100644 database/schedule/count_repo.go create mode 100644 database/schedule/count_repo_test.go create mode 100644 database/schedule/count_test.go create mode 100644 database/schedule/create.go create mode 100644 database/schedule/create_test.go create mode 100644 database/schedule/delete.go create mode 100644 database/schedule/delete_test.go create mode 100644 database/schedule/get.go create mode 100644 database/schedule/get_repo.go create mode 100644 database/schedule/get_repo_test.go create mode 100644 database/schedule/get_test.go create mode 100644 database/schedule/index.go create mode 100644 database/schedule/index_test.go create mode 100644 database/schedule/interface.go create mode 100644 database/schedule/list.go create mode 100644 database/schedule/list_active.go create mode 100644 database/schedule/list_active_test.go create mode 100644 database/schedule/list_repo.go create mode 100644 database/schedule/list_repo_test.go create mode 100644 database/schedule/list_test.go create mode 100644 database/schedule/opts.go create mode 100644 database/schedule/opts_test.go create mode 100644 database/schedule/schedule.go create mode 100644 database/schedule/schedule_test.go create mode 100644 database/schedule/table.go create mode 100644 database/schedule/table_test.go create mode 100644 database/schedule/update.go create mode 100644 database/schedule/update_test.go diff --git a/database/interface.go b/database/interface.go index b7d1b6fee..9a2177b14 100644 --- a/database/interface.go +++ b/database/interface.go @@ -9,6 +9,7 @@ import ( "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" "github.com/go-vela/server/database/repo" + "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" "github.com/go-vela/server/database/step" @@ -93,6 +94,10 @@ type Interface interface { // related to repos stored in the database. repo.RepoInterface + // ScheduleInterface provides the interface for functionality + // related to schedules stored in the database. + schedule.ScheduleInterface + // SecretInterface provides the interface for functionality // related to secrets stored in the database. secret.SecretInterface diff --git a/database/postgres/postgres.go b/database/postgres/postgres.go index 2adbfdd31..0c316531b 100644 --- a/database/postgres/postgres.go +++ b/database/postgres/postgres.go @@ -14,6 +14,7 @@ import ( "github.com/go-vela/server/database/pipeline" "github.com/go-vela/server/database/postgres/ddl" "github.com/go-vela/server/database/repo" + "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" "github.com/go-vela/server/database/step" @@ -58,6 +59,8 @@ type ( pipeline.PipelineInterface // https://pkg.go.dev/github.com/go-vela/server/database/repo#RepoInterface repo.RepoInterface + // https://pkg.go.dev/github.com/go-vela/server/database/schedule#ScheduleInterface + schedule.ScheduleInterface // https://pkg.go.dev/github.com/go-vela/server/database/secret#SecretInterface secret.SecretInterface // https://pkg.go.dev/github.com/go-vela/server/database/service#ServiceInterface @@ -176,6 +179,9 @@ func NewTest() (*client, sqlmock.Sqlmock, error) { // ensure the mock expects the repo queries _mock.ExpectExec(repo.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(repo.CreateOrgNameIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the schedule queries + _mock.ExpectExec(schedule.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(schedule.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the secret queries _mock.ExpectExec(secret.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(secret.CreateTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -367,6 +373,18 @@ func createServices(c *client) error { return err } + // create the database agnostic engine for schedules + // + // https://pkg.go.dev/github.com/go-vela/server/database/schedule#New + c.ScheduleInterface, err = schedule.New( + schedule.WithClient(c.Postgres), + schedule.WithLogger(c.Logger), + schedule.WithSkipCreation(c.config.SkipCreation), + ) + if err != nil { + return err + } + // create the database agnostic engine for secrets // // https://pkg.go.dev/github.com/go-vela/server/database/secret#New diff --git a/database/postgres/postgres_test.go b/database/postgres/postgres_test.go index 6074c65ca..d61f443fd 100644 --- a/database/postgres/postgres_test.go +++ b/database/postgres/postgres_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/go-vela/server/database/schedule" + "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/server/database/hook" "github.com/go-vela/server/database/log" @@ -102,6 +104,9 @@ func TestPostgres_setupDatabase(t *testing.T) { // ensure the mock expects the repo queries _mock.ExpectExec(repo.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(repo.CreateOrgNameIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the schedule queries + _mock.ExpectExec(schedule.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(schedule.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the secret queries _mock.ExpectExec(secret.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(secret.CreateTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -269,6 +274,9 @@ func TestPostgres_createServices(t *testing.T) { // ensure the mock expects the repo queries _mock.ExpectExec(repo.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(repo.CreateOrgNameIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the schedule queries + _mock.ExpectExec(schedule.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(schedule.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the secret queries _mock.ExpectExec(secret.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(secret.CreateTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) diff --git a/database/schedule/count.go b/database/schedule/count.go new file mode 100644 index 000000000..c7ce5f3aa --- /dev/null +++ b/database/schedule/count.go @@ -0,0 +1,25 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "github.com/go-vela/types/constants" +) + +// CountSchedules gets the count of all schedules from the database. +func (e *engine) CountSchedules() (int64, error) { + e.logger.Tracef("getting count of all schedules from the database") + + // variable to store query results + var s int64 + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableSchedule). + Count(&s). + Error + + return s, err +} diff --git a/database/schedule/count_active.go b/database/schedule/count_active.go new file mode 100644 index 000000000..f3240f2f8 --- /dev/null +++ b/database/schedule/count_active.go @@ -0,0 +1,26 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "github.com/go-vela/types/constants" +) + +// CountActiveSchedules gets the count of all active schedules from the database. +func (e *engine) CountActiveSchedules() (int64, error) { + e.logger.Tracef("getting count of all active schedules from the database") + + // variable to store query results + var s int64 + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableSchedule). + Where("active = ?", true). + Count(&s). + Error + + return s, err +} diff --git a/database/schedule/count_active_test.go b/database/schedule/count_active_test.go new file mode 100644 index 000000000..aa417efaa --- /dev/null +++ b/database/schedule/count_active_test.go @@ -0,0 +1,102 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSchedule_Engine_CountActiveSchedules(t *testing.T) { + _scheduleOne := testSchedule() + _scheduleOne.SetID(1) + _scheduleOne.SetRepoID(1) + _scheduleOne.SetActive(true) + _scheduleOne.SetName("nightly") + _scheduleOne.SetEntry("0 0 * * *") + _scheduleOne.SetCreatedAt(1) + _scheduleOne.SetCreatedBy("user1") + _scheduleOne.SetUpdatedAt(1) + _scheduleOne.SetUpdatedBy("user2") + + _scheduleTwo := testSchedule() + _scheduleTwo.SetID(2) + _scheduleTwo.SetRepoID(2) + _scheduleTwo.SetActive(false) + _scheduleTwo.SetName("hourly") + _scheduleTwo.SetEntry("0 * * * *") + _scheduleTwo.SetCreatedAt(1) + _scheduleTwo.SetCreatedBy("user1") + _scheduleTwo.SetUpdatedAt(1) + _scheduleTwo.SetUpdatedBy("user2") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(1) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "schedules" WHERE active = $1`).WithArgs(true).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateSchedule(_scheduleOne) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + err = _sqlite.CreateSchedule(_scheduleTwo) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 1, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 1, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountActiveSchedules() + + if test.failure { + if err == nil { + t.Errorf("CountActiveSchedules for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountActiveSchedules for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountActiveSchedules for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/schedule/count_repo.go b/database/schedule/count_repo.go new file mode 100644 index 000000000..1ec5177cb --- /dev/null +++ b/database/schedule/count_repo.go @@ -0,0 +1,31 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// CountSchedulesForRepo gets the count of schedules by repo ID from the database. +func (e *engine) CountSchedulesForRepo(r *library.Repo) (int64, error) { + e.logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + }).Tracef("getting count of schedules for repo %s from the database", r.GetFullName()) + + // variable to store query results + var s int64 + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableSchedule). + Where("repo_id = ?", r.GetID()). + Count(&s). + Error + + return s, err +} diff --git a/database/schedule/count_repo_test.go b/database/schedule/count_repo_test.go new file mode 100644 index 000000000..792c21b1d --- /dev/null +++ b/database/schedule/count_repo_test.go @@ -0,0 +1,106 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSchedule_Engine_CountSchedulesForRepo(t *testing.T) { + _repo := testRepo() + _repo.SetID(1) + _repo.SetOrg("foo") + _repo.SetName("bar") + _repo.SetFullName("foo/bar") + + _scheduleOne := testSchedule() + _scheduleOne.SetID(1) + _scheduleOne.SetRepoID(1) + _scheduleOne.SetName("nightly") + _scheduleOne.SetEntry("0 0 * * *") + _scheduleOne.SetCreatedAt(1) + _scheduleOne.SetCreatedBy("user1") + _scheduleOne.SetUpdatedAt(1) + _scheduleOne.SetUpdatedBy("user2") + + _scheduleTwo := testSchedule() + _scheduleTwo.SetID(2) + _scheduleTwo.SetRepoID(2) + _scheduleTwo.SetName("hourly") + _scheduleTwo.SetEntry("0 * * * *") + _scheduleTwo.SetCreatedAt(1) + _scheduleTwo.SetCreatedBy("user1") + _scheduleTwo.SetUpdatedAt(1) + _scheduleTwo.SetUpdatedBy("user2") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(1) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "schedules" WHERE repo_id = $1`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateSchedule(_scheduleOne) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + err = _sqlite.CreateSchedule(_scheduleTwo) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 1, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 1, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountSchedulesForRepo(_repo) + + if test.failure { + if err == nil { + t.Errorf("CountSchedulesForRepo for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountSchedulesForRepo for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountSchedulesForRepo for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/schedule/count_test.go b/database/schedule/count_test.go new file mode 100644 index 000000000..01cfbbae4 --- /dev/null +++ b/database/schedule/count_test.go @@ -0,0 +1,100 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSchedule_Engine_CountSchedules(t *testing.T) { + _scheduleOne := testSchedule() + _scheduleOne.SetID(1) + _scheduleOne.SetRepoID(1) + _scheduleOne.SetName("nightly") + _scheduleOne.SetEntry("0 0 * * *") + _scheduleOne.SetCreatedAt(1) + _scheduleOne.SetCreatedBy("user1") + _scheduleOne.SetUpdatedAt(1) + _scheduleOne.SetUpdatedBy("user2") + + _scheduleTwo := testSchedule() + _scheduleTwo.SetID(2) + _scheduleTwo.SetRepoID(2) + _scheduleTwo.SetName("hourly") + _scheduleTwo.SetEntry("0 * * * *") + _scheduleTwo.SetCreatedAt(1) + _scheduleTwo.SetCreatedBy("user1") + _scheduleTwo.SetUpdatedAt(1) + _scheduleTwo.SetUpdatedBy("user2") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "schedules"`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateSchedule(_scheduleOne) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + err = _sqlite.CreateSchedule(_scheduleTwo) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 2, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 2, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountSchedules() + + if test.failure { + if err == nil { + t.Errorf("CountSchedules for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountSchedules for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountSchedules for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/schedule/create.go b/database/schedule/create.go new file mode 100644 index 000000000..a6e532e8c --- /dev/null +++ b/database/schedule/create.go @@ -0,0 +1,35 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with update.go +package schedule + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// CreateSchedule creates a new schedule in the database. +func (e *engine) CreateSchedule(s *library.Schedule) error { + e.logger.WithFields(logrus.Fields{ + "schedule": s.GetName(), + }).Tracef("creating schedule %s in the database", s.GetName()) + + // cast the library type to database type + schedule := database.ScheduleFromLibrary(s) + + // validate the necessary fields are populated + err := schedule.Validate() + if err != nil { + return err + } + + // send query to the database + return e.client. + Table(constants.TableSchedule). + Create(schedule). + Error +} diff --git a/database/schedule/create_test.go b/database/schedule/create_test.go new file mode 100644 index 000000000..015654151 --- /dev/null +++ b/database/schedule/create_test.go @@ -0,0 +1,76 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSchedule_Engine_CreateSchedule(t *testing.T) { + _schedule := testSchedule() + _schedule.SetID(1) + _schedule.SetRepoID(1) + _schedule.SetName("nightly") + _schedule.SetEntry("0 0 * * *") + _schedule.SetCreatedAt(1) + _schedule.SetCreatedBy("user1") + _schedule.SetUpdatedAt(1) + _schedule.SetUpdatedBy("user2") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) + + // ensure the mock expects the query + _mock.ExpectQuery(`INSERT INTO "schedules" +("repo_id","active","name","entry","created_at","created_by","updated_at","updated_by","scheduled_at","id") +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) RETURNING "id"`). + WithArgs(1, false, "nightly", "0 0 * * *", 1, "user1", 1, "user2", nil, 1). + WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateSchedule(_schedule) + + if test.failure { + if err == nil { + t.Errorf("CreateSchedule for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateSchedule for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/schedule/delete.go b/database/schedule/delete.go new file mode 100644 index 000000000..42ee4988d --- /dev/null +++ b/database/schedule/delete.go @@ -0,0 +1,28 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// DeleteSchedule deletes an existing schedule from the database. +func (e *engine) DeleteSchedule(s *library.Schedule) error { + e.logger.WithFields(logrus.Fields{ + "schedule": s.GetName(), + }).Tracef("deleting schedule %s in the database", s.GetName()) + + // cast the library type to database type + schedule := database.ScheduleFromLibrary(s) + + // send query to the database + return e.client. + Table(constants.TableSchedule). + Delete(schedule). + Error +} diff --git a/database/schedule/delete_test.go b/database/schedule/delete_test.go new file mode 100644 index 000000000..3d1a3cd48 --- /dev/null +++ b/database/schedule/delete_test.go @@ -0,0 +1,76 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSchedule_Engine_DeleteSchedule(t *testing.T) { + _schedule := testSchedule() + _schedule.SetID(1) + _schedule.SetRepoID(1) + _schedule.SetName("nightly") + _schedule.SetEntry("0 0 * * *") + _schedule.SetCreatedAt(1) + _schedule.SetCreatedBy("user1") + _schedule.SetUpdatedAt(1) + _schedule.SetUpdatedBy("user2") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`DELETE FROM "schedules" WHERE "schedules"."id" = $1`). + WithArgs(1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateSchedule(_schedule) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err = test.database.DeleteSchedule(_schedule) + + if test.failure { + if err == nil { + t.Errorf("DeleteSchedule for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("DeleteSchedule for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/schedule/get.go b/database/schedule/get.go new file mode 100644 index 000000000..73fcc162e --- /dev/null +++ b/database/schedule/get.go @@ -0,0 +1,31 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// GetSchedule gets a schedule by ID from the database. +func (e *engine) GetSchedule(id int64) (*library.Schedule, error) { + e.logger.Tracef("getting schedule %d from the database", id) + + // variable to store query results + s := new(database.Schedule) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableSchedule). + Where("id = ?", id). + Take(s). + Error + if err != nil { + return nil, err + } + + return s.ToLibrary(), nil +} diff --git a/database/schedule/get_repo.go b/database/schedule/get_repo.go new file mode 100644 index 000000000..ff5ab40e5 --- /dev/null +++ b/database/schedule/get_repo.go @@ -0,0 +1,37 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// GetScheduleForRepo gets a schedule by repo ID and name from the database. +func (e *engine) GetScheduleForRepo(r *library.Repo, name string) (*library.Schedule, error) { + e.logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + "schedule": name, + }).Tracef("getting schedule %s/%s from the database", r.GetFullName(), name) + + // variable to store query results + s := new(database.Schedule) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableSchedule). + Where("repo_id = ?", r.GetID()). + Where("name = ?", name). + Take(s). + Error + if err != nil { + return nil, err + } + + return s.ToLibrary(), nil +} diff --git a/database/schedule/get_repo_test.go b/database/schedule/get_repo_test.go new file mode 100644 index 000000000..fb29bea37 --- /dev/null +++ b/database/schedule/get_repo_test.go @@ -0,0 +1,94 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestSchedule_Engine_GetScheduleForRepo(t *testing.T) { + _repo := testRepo() + _repo.SetID(1) + _repo.SetOrg("foo") + _repo.SetName("bar") + _repo.SetFullName("foo/bar") + + _schedule := testSchedule() + _schedule.SetID(1) + _schedule.SetRepoID(1) + _schedule.SetName("nightly") + _schedule.SetEntry("0 0 * * *") + _schedule.SetCreatedAt(1) + _schedule.SetCreatedBy("user1") + _schedule.SetUpdatedAt(1) + _schedule.SetUpdatedBy("user2") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at"}, + ).AddRow(1, 1, false, "nightly", "0 0 * * *", 1, "user1", 1, "user2", nil) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "schedules" WHERE repo_id = $1 AND name = $2 LIMIT 1`).WithArgs(1, "nightly").WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateSchedule(_schedule) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.Schedule + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _schedule, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _schedule, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetScheduleForRepo(_repo, "nightly") + + if test.failure { + if err == nil { + t.Errorf("GetScheduleForRepo for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetScheduleForRepo for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetScheduleForRepo for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/schedule/get_test.go b/database/schedule/get_test.go new file mode 100644 index 000000000..acea35b2a --- /dev/null +++ b/database/schedule/get_test.go @@ -0,0 +1,88 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestSchedule_Engine_GetSchedule(t *testing.T) { + _schedule := testSchedule() + _schedule.SetID(1) + _schedule.SetRepoID(1) + _schedule.SetName("nightly") + _schedule.SetEntry("0 0 * * *") + _schedule.SetCreatedAt(1) + _schedule.SetCreatedBy("user1") + _schedule.SetUpdatedAt(1) + _schedule.SetUpdatedBy("user2") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at"}, + ).AddRow(1, 1, false, "nightly", "0 0 * * *", 1, "user1", 1, "user2", nil) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "schedules" WHERE id = $1 LIMIT 1`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateSchedule(_schedule) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.Schedule + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _schedule, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _schedule, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetSchedule(1) + + if test.failure { + if err == nil { + t.Errorf("GetSchedule for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetSchedule for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetSchedule for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/schedule/index.go b/database/schedule/index.go new file mode 100644 index 000000000..f14082cee --- /dev/null +++ b/database/schedule/index.go @@ -0,0 +1,24 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +const ( + // CreateRepoIDIndex represents a query to create an + // index on the schedules table for the repo_id column. + CreateRepoIDIndex = ` +CREATE INDEX +IF NOT EXISTS +schedules_repo_id +ON schedules (repo_id); +` +) + +// CreateScheduleIndexes creates the indexes for the schedules table in the database. +func (e *engine) CreateScheduleIndexes() error { + e.logger.Tracef("creating indexes for schedules table in the database") + + // create the repo_id column index for the schedules table + return e.client.Exec(CreateRepoIDIndex).Error +} diff --git a/database/schedule/index_test.go b/database/schedule/index_test.go new file mode 100644 index 000000000..b8d334366 --- /dev/null +++ b/database/schedule/index_test.go @@ -0,0 +1,59 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSchedule_Engine_CreateScheduleIndexes(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateScheduleIndexes() + + if test.failure { + if err == nil { + t.Errorf("CreateScheduleIndexes for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateScheduleIndexes for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/schedule/interface.go b/database/schedule/interface.go new file mode 100644 index 000000000..8aa14efb5 --- /dev/null +++ b/database/schedule/interface.go @@ -0,0 +1,49 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "github.com/go-vela/types/library" +) + +// ScheduleInterface represents the Vela interface for schedule +// functions with the supported Database backends. +// +//nolint:revive // ignore name stutter +type ScheduleInterface interface { + // Schedule Data Definition Language Functions + // + // https://en.wikipedia.org/wiki/Data_definition_language + + // CreateScheduleIndexes defines a function that creates the indexes for the schedules table. + CreateScheduleIndexes() error + // CreateScheduleTable defines a function that creates the schedules table. + CreateScheduleTable(string) error + + // Schedule Data Manipulation Language Functions + // + // https://en.wikipedia.org/wiki/Data_manipulation_language + + // CountSchedules defines a function that gets the count of all schedules. + CountSchedules() (int64, error) + // CountSchedulesForRepo defines a function that gets the count of schedules by repo ID. + CountSchedulesForRepo(*library.Repo) (int64, error) + // CreateSchedule defines a function that creates a new schedule. + CreateSchedule(*library.Schedule) error + // DeleteSchedule defines a function that deletes an existing schedule. + DeleteSchedule(*library.Schedule) error + // GetSchedule defines a function that gets a schedule by ID. + GetSchedule(int64) (*library.Schedule, error) + // GetScheduleForRepo defines a function that gets a schedule by repo ID and name. + GetScheduleForRepo(*library.Repo, string) (*library.Schedule, error) + // ListActiveSchedules defines a function that gets a list of all active schedules. + ListActiveSchedules() ([]*library.Schedule, error) + // ListSchedules defines a function that gets a list of all schedules. + ListSchedules() ([]*library.Schedule, error) + // ListSchedulesForRepo defines a function that gets a list of schedules by repo ID. + ListSchedulesForRepo(*library.Repo, int, int) ([]*library.Schedule, int64, error) + // UpdateSchedule defines a function that updates an existing schedule. + UpdateSchedule(*library.Schedule) error +} diff --git a/database/schedule/list.go b/database/schedule/list.go new file mode 100644 index 000000000..f4c5aa5e7 --- /dev/null +++ b/database/schedule/list.go @@ -0,0 +1,52 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// ListSchedules gets a list of all schedules from the database. +func (e *engine) ListSchedules() ([]*library.Schedule, error) { + e.logger.Trace("listing all schedules from the database") + + // variables to store query results and return value + count := int64(0) + s := new([]database.Schedule) + schedules := []*library.Schedule{} + + // count the results + count, err := e.CountSchedules() + if err != nil { + return nil, err + } + + // short-circuit if there are no results + if count == 0 { + return schedules, nil + } + + // send query to the database and store result in variable + err = e.client. + Table(constants.TableSchedule). + Find(&s). + Error + if err != nil { + return nil, err + } + + // iterate through all query results + for _, schedule := range *s { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := schedule + + // convert query result to API type + schedules = append(schedules, tmp.ToLibrary()) + } + + return schedules, nil +} diff --git a/database/schedule/list_active.go b/database/schedule/list_active.go new file mode 100644 index 000000000..d0efa5dde --- /dev/null +++ b/database/schedule/list_active.go @@ -0,0 +1,53 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// ListActiveSchedules gets a list of all active schedules from the database. +func (e *engine) ListActiveSchedules() ([]*library.Schedule, error) { + e.logger.Trace("listing all active schedules from the database") + + // variables to store query results and return value + count := int64(0) + s := new([]database.Schedule) + schedules := []*library.Schedule{} + + // count the results + count, err := e.CountActiveSchedules() + if err != nil { + return nil, err + } + + // short-circuit if there are no results + if count == 0 { + return schedules, nil + } + + // send query to the database and store result in variable + err = e.client. + Table(constants.TableSchedule). + Where("active = ?", true). + Find(&s). + Error + if err != nil { + return nil, err + } + + // iterate through all query results + for _, schedule := range *s { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := schedule + + // convert query result to API type + schedules = append(schedules, tmp.ToLibrary()) + } + + return schedules, nil +} diff --git a/database/schedule/list_active_test.go b/database/schedule/list_active_test.go new file mode 100644 index 000000000..69c690def --- /dev/null +++ b/database/schedule/list_active_test.go @@ -0,0 +1,111 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestSchedule_Engine_ListActiveSchedules(t *testing.T) { + _scheduleOne := testSchedule() + _scheduleOne.SetID(1) + _scheduleOne.SetRepoID(1) + _scheduleOne.SetActive(true) + _scheduleOne.SetName("nightly") + _scheduleOne.SetEntry("0 0 * * *") + _scheduleOne.SetCreatedAt(1) + _scheduleOne.SetCreatedBy("user1") + _scheduleOne.SetUpdatedAt(1) + _scheduleOne.SetUpdatedBy("user2") + + _scheduleTwo := testSchedule() + _scheduleTwo.SetID(2) + _scheduleTwo.SetRepoID(2) + _scheduleTwo.SetActive(false) + _scheduleTwo.SetName("hourly") + _scheduleTwo.SetEntry("0 * * * *") + _scheduleTwo.SetCreatedAt(1) + _scheduleTwo.SetCreatedBy("user1") + _scheduleTwo.SetUpdatedAt(1) + _scheduleTwo.SetUpdatedBy("user2") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "schedules" WHERE active = $1`).WithArgs(true).WillReturnRows(_rows) + + // create expected result in mock + _rows = sqlmock.NewRows( + []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at"}). + AddRow(1, 1, true, "nightly", "0 0 * * *", 1, "user1", 1, "user2", nil) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "schedules" WHERE active = $1`).WithArgs(true).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateSchedule(_scheduleOne) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + err = _sqlite.CreateSchedule(_scheduleTwo) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.Schedule + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.Schedule{_scheduleOne}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.Schedule{_scheduleOne}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.ListActiveSchedules() + + if test.failure { + if err == nil { + t.Errorf("ListActiveSchedules for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListActiveSchedules for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListActiveSchedules for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/schedule/list_repo.go b/database/schedule/list_repo.go new file mode 100644 index 000000000..1137e0c4b --- /dev/null +++ b/database/schedule/list_repo.go @@ -0,0 +1,63 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// ListSchedulesForRepo gets a list of schedules by repo ID from the database. +func (e *engine) ListSchedulesForRepo(r *library.Repo, page, perPage int) ([]*library.Schedule, int64, error) { + e.logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + }).Tracef("listing schedules for repo %s from the database", r.GetFullName()) + + // variables to store query results and return value + count := int64(0) + s := new([]database.Schedule) + schedules := []*library.Schedule{} + + // count the results + count, err := e.CountSchedulesForRepo(r) + if err != nil { + return nil, 0, err + } + + // short-circuit if there are no results + if count == 0 { + return schedules, 0, nil + } + + // calculate offset for pagination through results + offset := perPage * (page - 1) + + // send query to the database and store result in variable + err = e.client. + Table(constants.TableSchedule). + Where("repo_id = ?", r.GetID()). + Order("id DESC"). + Limit(perPage). + Offset(offset). + Find(&s). + Error + if err != nil { + return nil, count, err + } + + // iterate through all query results + for _, schedule := range *s { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := schedule + + // convert query result to library type + schedules = append(schedules, tmp.ToLibrary()) + } + + return schedules, count, nil +} diff --git a/database/schedule/list_repo_test.go b/database/schedule/list_repo_test.go new file mode 100644 index 000000000..df56c0ee4 --- /dev/null +++ b/database/schedule/list_repo_test.go @@ -0,0 +1,115 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestSchedule_Engine_ListSchedulesForRepo(t *testing.T) { + _repo := testRepo() + _repo.SetID(1) + _repo.SetOrg("foo") + _repo.SetName("bar") + _repo.SetFullName("foo/bar") + + _scheduleOne := testSchedule() + _scheduleOne.SetID(1) + _scheduleOne.SetRepoID(1) + _scheduleOne.SetName("nightly") + _scheduleOne.SetEntry("0 0 * * *") + _scheduleOne.SetCreatedAt(1) + _scheduleOne.SetCreatedBy("user1") + _scheduleOne.SetUpdatedAt(1) + _scheduleOne.SetUpdatedBy("user2") + + _scheduleTwo := testSchedule() + _scheduleTwo.SetID(2) + _scheduleTwo.SetRepoID(2) + _scheduleTwo.SetName("hourly") + _scheduleTwo.SetEntry("0 * * * *") + _scheduleTwo.SetCreatedAt(1) + _scheduleTwo.SetCreatedBy("user1") + _scheduleTwo.SetUpdatedAt(1) + _scheduleTwo.SetUpdatedBy("user2") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "schedules" WHERE repo_id = $1`).WithArgs(1).WillReturnRows(_rows) + + // create expected result in mock + _rows = sqlmock.NewRows( + []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at"}). + AddRow(1, 1, false, "nightly", "0 0 * * *", 1, "user1", 1, "user2", nil) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "schedules" WHERE repo_id = $1 ORDER BY id DESC LIMIT 10`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateSchedule(_scheduleOne) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + err = _sqlite.CreateSchedule(_scheduleTwo) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.Schedule + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.Schedule{_scheduleOne}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.Schedule{_scheduleOne}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, _, err := test.database.ListSchedulesForRepo(_repo, 1, 10) + + if test.failure { + if err == nil { + t.Errorf("ListSchedulesForRepo for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListSchedulesForRepo for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListSchedulesForRepo for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/schedule/list_test.go b/database/schedule/list_test.go new file mode 100644 index 000000000..b0c3bb417 --- /dev/null +++ b/database/schedule/list_test.go @@ -0,0 +1,110 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestSchedule_Engine_ListSchedules(t *testing.T) { + _scheduleOne := testSchedule() + _scheduleOne.SetID(1) + _scheduleOne.SetRepoID(1) + _scheduleOne.SetName("nightly") + _scheduleOne.SetEntry("0 0 * * *") + _scheduleOne.SetCreatedAt(1) + _scheduleOne.SetCreatedBy("user1") + _scheduleOne.SetUpdatedAt(1) + _scheduleOne.SetUpdatedBy("user2") + + _scheduleTwo := testSchedule() + _scheduleTwo.SetID(2) + _scheduleTwo.SetRepoID(2) + _scheduleTwo.SetName("hourly") + _scheduleTwo.SetEntry("0 * * * *") + _scheduleTwo.SetCreatedAt(1) + _scheduleTwo.SetCreatedBy("user1") + _scheduleTwo.SetUpdatedAt(1) + _scheduleTwo.SetUpdatedBy("user2") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "schedules"`).WillReturnRows(_rows) + + // create expected result in mock + _rows = sqlmock.NewRows( + []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at"}). + AddRow(1, 1, false, "nightly", "0 0 * * *", 1, "user1", 1, "user2", nil). + AddRow(2, 2, false, "hourly", "0 * * * *", 1, "user1", 1, "user2", nil) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "schedules"`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateSchedule(_scheduleOne) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + err = _sqlite.CreateSchedule(_scheduleTwo) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.Schedule + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.Schedule{_scheduleOne, _scheduleTwo}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.Schedule{_scheduleOne, _scheduleTwo}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.ListSchedules() + + if test.failure { + if err == nil { + t.Errorf("ListSchedules for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListSchedules for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListSchedules for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/schedule/opts.go b/database/schedule/opts.go new file mode 100644 index 000000000..7ac87bea1 --- /dev/null +++ b/database/schedule/opts.go @@ -0,0 +1,44 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +// EngineOpt represents a configuration option to initialize the database engine for Schedules. +type EngineOpt func(*engine) error + +// WithClient sets the gorm.io/gorm client in the database engine for Schedules. +func WithClient(client *gorm.DB) EngineOpt { + return func(e *engine) error { + // set the gorm.io/gorm client in the schedule engine + e.client = client + + return nil + } +} + +// WithLogger sets the github.com/sirupsen/logrus logger in the database engine for Schedules. +func WithLogger(logger *logrus.Entry) EngineOpt { + return func(e *engine) error { + // set the github.com/sirupsen/logrus logger in the schedule engine + e.logger = logger + + return nil + } +} + +// WithSkipCreation sets the skip creation logic in the database engine for Schedules. +func WithSkipCreation(skipCreation bool) EngineOpt { + return func(e *engine) error { + // set to skip creating tables and indexes in the schedule engine + e.config.SkipCreation = skipCreation + + return nil + } +} diff --git a/database/schedule/opts_test.go b/database/schedule/opts_test.go new file mode 100644 index 000000000..6159801e3 --- /dev/null +++ b/database/schedule/opts_test.go @@ -0,0 +1,161 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "reflect" + "testing" + + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +func TestSchedule_EngineOpt_WithClient(t *testing.T) { + // setup types + e := &engine{client: new(gorm.DB)} + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + want *gorm.DB + }{ + { + failure: false, + name: "client set to new database", + client: new(gorm.DB), + want: new(gorm.DB), + }, + { + failure: false, + name: "client set to nil", + client: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithClient(test.client)(e) + + if test.failure { + if err == nil { + t.Errorf("WithClient for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithClient returned err: %v", err) + } + + if !reflect.DeepEqual(e.client, test.want) { + t.Errorf("WithClient is %v, want %v", e.client, test.want) + } + }) + } +} + +func TestSchedule_EngineOpt_WithLogger(t *testing.T) { + // setup types + e := &engine{logger: new(logrus.Entry)} + + // setup tests + tests := []struct { + failure bool + name string + logger *logrus.Entry + want *logrus.Entry + }{ + { + failure: false, + name: "logger set to new entry", + logger: new(logrus.Entry), + want: new(logrus.Entry), + }, + { + failure: false, + name: "logger set to nil", + logger: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithLogger(test.logger)(e) + + if test.failure { + if err == nil { + t.Errorf("WithLogger for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithLogger returned err: %v", err) + } + + if !reflect.DeepEqual(e.logger, test.want) { + t.Errorf("WithLogger is %v, want %v", e.logger, test.want) + } + }) + } +} + +func TestSchedule_EngineOpt_WithSkipCreation(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + skipCreation bool + want bool + }{ + { + failure: false, + name: "skip creation set to true", + skipCreation: true, + want: true, + }, + { + failure: false, + name: "skip creation set to false", + skipCreation: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithSkipCreation(test.skipCreation)(e) + + if test.failure { + if err == nil { + t.Errorf("WithSkipCreation for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithSkipCreation returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.SkipCreation, test.want) { + t.Errorf("WithSkipCreation is %v, want %v", e.config.SkipCreation, test.want) + } + }) + } +} diff --git a/database/schedule/schedule.go b/database/schedule/schedule.go new file mode 100644 index 000000000..0517dcae9 --- /dev/null +++ b/database/schedule/schedule.go @@ -0,0 +1,80 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "fmt" + + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +type ( + // config represents the settings required to create the engine that implements the ScheduleInterface interface. + config struct { + // specifies to skip creating tables and indexes for the Schedule engine + SkipCreation bool + } + + // engine represents the schedule functionality that implements the ScheduleInterface interface. + engine struct { + // engine configuration settings used in schedule functions + config *config + + // gorm.io/gorm database client used in schedule functions + // + // https://pkg.go.dev/gorm.io/gorm#DB + client *gorm.DB + + // sirupsen/logrus logger used in schedule functions + // + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry + logger *logrus.Entry + } +) + +// New creates and returns a Vela service for integrating with schedules in the database. +// +//nolint:revive // ignore returning unexported engine +func New(opts ...EngineOpt) (*engine, error) { + // create new Schedule engine + e := new(engine) + + // create new fields + e.client = new(gorm.DB) + e.config = new(config) + e.logger = new(logrus.Entry) + + // apply all provided configuration options + for _, opt := range opts { + err := opt(e) + if err != nil { + return nil, err + } + } + + // check if we should skip creating schedule database objects + if e.config.SkipCreation { + e.logger.Warning("skipping creation of schedules table and indexes in the database") + + return e, nil + } + + // create the schedules table + err := e.CreateScheduleTable(e.client.Config.Dialector.Name()) + if err != nil { + return nil, fmt.Errorf("unable to create %s table: %w", constants.TableSchedule, err) + } + + // create the indexes for the schedules table + err = e.CreateScheduleIndexes() + if err != nil { + return nil, fmt.Errorf("unable to create indexes for %s table: %w", constants.TableSchedule, err) + } + + return e, nil +} diff --git a/database/schedule/schedule_test.go b/database/schedule/schedule_test.go new file mode 100644 index 000000000..c86f1f353 --- /dev/null +++ b/database/schedule/schedule_test.go @@ -0,0 +1,212 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" + + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestSchedule_New(t *testing.T) { + // setup types + logger := logrus.NewEntry(logrus.StandardLogger()) + + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + defer _sql.Close() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + _config := &gorm.Config{SkipDefaultTransaction: true} + + _postgres, err := gorm.Open(postgres.New(postgres.Config{Conn: _sql}), _config) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _sqlite, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), _config) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + defer func() { _sql, _ := _sqlite.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + key string + logger *logrus.Entry + skipCreation bool + want *engine + }{ + { + failure: false, + name: "postgres", + client: _postgres, + logger: logger, + skipCreation: false, + want: &engine{ + client: _postgres, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + { + failure: false, + name: "sqlite3", + client: _sqlite, + logger: logger, + skipCreation: false, + want: &engine{ + client: _sqlite, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := New( + WithClient(test.client), + WithLogger(test.logger), + WithSkipCreation(test.skipCreation), + ) + + if test.failure { + if err == nil { + t.Errorf("New for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("New for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("New for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} + +// testPostgres is a helper function to create a Postgres engine for testing. +func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { + // create the new mock sql database + // + // https://pkg.go.dev/github.com/DATA-DOG/go-sqlmock#New + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + // create the new mock Postgres database client + // + // https://pkg.go.dev/gorm.io/gorm#Open + _postgres, err := gorm.Open( + postgres.New(postgres.Config{Conn: _sql}), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _engine, err := New( + WithClient(_postgres), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new postgres schedule engine: %v", err) + } + + return _engine, _mock +} + +// testSqlite is a helper function to create a Sqlite engine for testing. +func testSqlite(t *testing.T) *engine { + _sqlite, err := gorm.Open( + sqlite.Open("file::memory:?cache=shared"), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + _engine, err := New( + WithClient(_sqlite), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new sqlite schedule engine: %v", err) + } + + return _engine +} + +// testSchedule is a test helper function to create an API Schedule type with all fields set to their zero values. +func testSchedule() *library.Schedule { + return &library.Schedule{ + ID: new(int64), + RepoID: new(int64), + Active: new(bool), + Name: new(string), + Entry: new(string), + CreatedAt: new(int64), + CreatedBy: new(string), + UpdatedAt: new(int64), + UpdatedBy: new(string), + ScheduledAt: new(int64), + } +} + +// testRepo is a test helper function to create a library Repo type with all fields set to their zero values. +func testRepo() *library.Repo { + return &library.Repo{ + ID: new(int64), + UserID: new(int64), + BuildLimit: new(int64), + Timeout: new(int64), + Counter: new(int), + PipelineType: new(string), + Hash: new(string), + Org: new(string), + Name: new(string), + FullName: new(string), + Link: new(string), + Clone: new(string), + Branch: new(string), + Visibility: new(string), + PreviousName: new(string), + Private: new(bool), + Trusted: new(bool), + Active: new(bool), + AllowPull: new(bool), + AllowPush: new(bool), + AllowDeploy: new(bool), + AllowTag: new(bool), + AllowComment: new(bool), + } +} diff --git a/database/schedule/table.go b/database/schedule/table.go new file mode 100644 index 000000000..05ffe8bad --- /dev/null +++ b/database/schedule/table.go @@ -0,0 +1,66 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "github.com/go-vela/types/constants" +) + +const ( + // CreatePostgresTable represents a query to create the Postgres schedules table. + CreatePostgresTable = ` +CREATE TABLE +IF NOT EXISTS +schedules ( + id SERIAL PRIMARY KEY, + repo_id INTEGER, + active BOOLEAN, + name VARCHAR(100), + entry VARCHAR(100), + created_at INTEGER, + created_by VARCHAR(250), + updated_at INTEGER, + updated_by VARCHAR(250), + scheduled_at INTEGER, + UNIQUE(repo_id, name) +); +` + + // CreateSqliteTable represents a query to create the Sqlite schedules table. + CreateSqliteTable = ` +CREATE TABLE +IF NOT EXISTS +schedules ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_id INTEGER, + active BOOLEAN, + name TEXT, + entry TEXT, + created_at INTEGER, + created_by TEXT, + updated_at INTEGER, + updated_by TEXT, + scheduled_at INTEGER, + UNIQUE(repo_id, name) +); +` +) + +// CreateScheduleTable creates the schedules table in the database. +func (e *engine) CreateScheduleTable(driver string) error { + e.logger.Tracef("creating schedules table in the database") + + // handle the driver provided to create the table + switch driver { + case constants.DriverPostgres: + // create the schedules table for Postgres + return e.client.Exec(CreatePostgresTable).Error + case constants.DriverSqlite: + fallthrough + default: + // create the schedules table for Sqlite + return e.client.Exec(CreateSqliteTable).Error + } +} diff --git a/database/schedule/table_test.go b/database/schedule/table_test.go new file mode 100644 index 000000000..c4dbf4e40 --- /dev/null +++ b/database/schedule/table_test.go @@ -0,0 +1,59 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSchedule_Engine_CreateScheduleTable(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateScheduleTable(test.name) + + if test.failure { + if err == nil { + t.Errorf("CreateScheduleTable for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateScheduleTable for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/schedule/update.go b/database/schedule/update.go new file mode 100644 index 000000000..b69f8eff0 --- /dev/null +++ b/database/schedule/update.go @@ -0,0 +1,35 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with create.go +package schedule + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// UpdateSchedule updates an existing schedule in the database. +func (e *engine) UpdateSchedule(s *library.Schedule) error { + e.logger.WithFields(logrus.Fields{ + "schedule": s.GetName(), + }).Tracef("updating schedule %s in the database", s.GetName()) + + // cast the library type to database type + schedule := database.ScheduleFromLibrary(s) + + // validate the necessary fields are populated + err := schedule.Validate() + if err != nil { + return err + } + + // send query to the database + return e.client. + Table(constants.TableSchedule). + Save(schedule). + Error +} diff --git a/database/schedule/update_test.go b/database/schedule/update_test.go new file mode 100644 index 000000000..05fff4286 --- /dev/null +++ b/database/schedule/update_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSchedule_Engine_UpdateSchedule(t *testing.T) { + _repo := testRepo() + _repo.SetID(1) + _repo.SetOrg("foo") + _repo.SetName("bar") + _repo.SetFullName("foo/bar") + + _schedule := testSchedule() + _schedule.SetID(1) + _schedule.SetRepoID(1) + _schedule.SetName("nightly") + _schedule.SetEntry("0 0 * * *") + _schedule.SetCreatedAt(1) + _schedule.SetCreatedBy("user1") + _schedule.SetUpdatedAt(1) + _schedule.SetUpdatedBy("user2") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`UPDATE "schedules" +SET "repo_id"=$1,"active"=$2,"name"=$3,"entry"=$4,"created_at"=$5,"created_by"=$6,"updated_at"=$7,"updated_by"=$8,"scheduled_at"=$9 +WHERE "id" = $10`). + WithArgs(1, false, "nightly", "0 0 * * *", 1, "user1", time.Now().UTC().Unix(), "user2", nil, 1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateSchedule(_schedule) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err = test.database.UpdateSchedule(_schedule) + + if test.failure { + if err == nil { + t.Errorf("UpdateSchedule for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("UpdateSchedule for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/sqlite/sqlite.go b/database/sqlite/sqlite.go index d16f7cf09..3faa555ba 100644 --- a/database/sqlite/sqlite.go +++ b/database/sqlite/sqlite.go @@ -12,6 +12,7 @@ import ( "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" "github.com/go-vela/server/database/repo" + "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" "github.com/go-vela/server/database/sqlite/ddl" @@ -57,6 +58,8 @@ type ( pipeline.PipelineInterface // https://pkg.go.dev/github.com/go-vela/server/database/repo#RepoInterface repo.RepoInterface + // https://pkg.go.dev/github.com/go-vela/server/database/schedule#ScheduleInterface + schedule.ScheduleInterface // https://pkg.go.dev/github.com/go-vela/server/database/secret#SecretInterface secret.SecretInterface // https://pkg.go.dev/github.com/go-vela/server/database/service#ServiceInterface @@ -338,6 +341,18 @@ func createServices(c *client) error { return err } + // create the database agnostic engine for schedules + // + // https://pkg.go.dev/github.com/go-vela/server/database/schedule#New + c.ScheduleInterface, err = schedule.New( + schedule.WithClient(c.Sqlite), + schedule.WithLogger(c.Logger), + schedule.WithSkipCreation(c.config.SkipCreation), + ) + if err != nil { + return err + } + // create the database agnostic engine for secrets // // https://pkg.go.dev/github.com/go-vela/server/database/secret#New diff --git a/go.mod b/go.mod index 7e26a5732..95e264713 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.0 github.com/go-playground/assert/v2 v2.2.0 - github.com/go-vela/types v0.19.2 + github.com/go-vela/types v0.19.3-0.20230516131722-f538de06bbf6 github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-cmp v0.5.9 github.com/google/go-github/v52 v52.0.0 @@ -45,6 +45,7 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/adhocore/gronx v1.6.2 // indirect github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -102,6 +103,7 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect diff --git a/go.sum b/go.sum index 478dcc750..e0f559ca4 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,8 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/adhocore/gronx v1.6.2 h1:/Pg6cuHFJmUGRIYWhRFjb6iL9fdzNmoMPj+/r6L01KU= +github.com/adhocore/gronx v1.6.2/go.mod h1:7oUY1WAU8rEJWmAxXR2DN0JaO4gi9khSgKjiRypqteg= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= @@ -136,8 +138,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= -github.com/go-vela/types v0.19.2 h1:xU61CX2jdMuBCtLOg8a7Z2aEWYM1zZt37Ygx1oHGbjM= -github.com/go-vela/types v0.19.2/go.mod h1:ZvDjYCKU36yJS3sLxPLCny/HLF1U6YtlOienzv/cXB4= +github.com/go-vela/types v0.19.3-0.20230516131722-f538de06bbf6 h1:WVmgeHuPN2WHTf/tJtseEMPxPoKdit2rD4nCZyPIias= +github.com/go-vela/types v0.19.3-0.20230516131722-f538de06bbf6/go.mod h1:0lsuPfGyVyTWJSi2h3NS6uaEW6DgnFvIzaZu1sXYKrs= github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -347,7 +349,8 @@ github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= From 5092d22e6df70ded2b2609e3bf18c577bf3e3f65 Mon Sep 17 00:00:00 2001 From: dave vader <48764154+plyr4@users.noreply.github.com> Date: Thu, 18 May 2023 15:45:08 -0500 Subject: [PATCH 08/53] fix: allow setting worker.active to false (#850) * fix: allow setting worker.active to false * fix: whitespace --- api/worker/update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/worker/update.go b/api/worker/update.go index 0c2f9c4fb..16d34f686 100644 --- a/api/worker/update.go +++ b/api/worker/update.go @@ -93,7 +93,7 @@ func UpdateWorker(c *gin.Context) { w.SetRoutes(input.GetRoutes()) } - if input.GetActive() { + if input.Active != nil { // update active if set w.SetActive(input.GetActive()) } From ca211753a9a03de39adfc837298d2c43fd71116c Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Fri, 19 May 2023 08:45:24 -0600 Subject: [PATCH 09/53] refactor(api): move service logic to separate package (#849) * refactor(api): move service logic to separate package * add doc.go * fix imports --- api/build.go | 56 ++++ api/service.go | 593 ------------------------------------------ api/service/create.go | 128 +++++++++ api/service/delete.go | 97 +++++++ api/service/doc.go | 10 + api/service/get.go | 86 ++++++ api/service/list.go | 146 +++++++++++ api/service/update.go | 149 +++++++++++ api/step.go | 4 - router/service.go | 20 +- 10 files changed, 682 insertions(+), 607 deletions(-) delete mode 100644 api/service.go create mode 100644 api/service/create.go create mode 100644 api/service/delete.go create mode 100644 api/service/doc.go create mode 100644 api/service/get.go create mode 100644 api/service/list.go create mode 100644 api/service/update.go diff --git a/api/build.go b/api/build.go index 60f7f8eb1..8519a8e53 100644 --- a/api/build.go +++ b/api/build.go @@ -1621,6 +1621,62 @@ func planBuild(database database.Interface, p *pipeline.Build, b *library.Build, return nil } +// planServices is a helper function to plan all services +// in the build for execution. This creates the services +// for the build in the configured backend. +func planServices(database database.Interface, p *pipeline.Build, b *library.Build) ([]*library.Service, error) { + // variable to store planned services + services := []*library.Service{} + + // iterate through all pipeline services + for _, service := range p.Services { + // create the service object + s := new(library.Service) + s.SetBuildID(b.GetID()) + s.SetRepoID(b.GetRepoID()) + s.SetName(service.Name) + s.SetImage(service.Image) + s.SetNumber(service.Number) + s.SetStatus(constants.StatusPending) + s.SetCreated(time.Now().UTC().Unix()) + + // send API call to create the service + err := database.CreateService(s) + if err != nil { + return services, fmt.Errorf("unable to create service %s: %w", s.GetName(), err) + } + + // send API call to capture the created service + s, err = database.GetServiceForBuild(b, s.GetNumber()) + if err != nil { + return services, fmt.Errorf("unable to get service %s: %w", s.GetName(), err) + } + + // populate environment variables from service library + // + // https://pkg.go.dev/github.com/go-vela/types/library#Service.Environment + err = service.MergeEnv(s.Environment()) + if err != nil { + return services, err + } + + // create the log object + l := new(library.Log) + l.SetServiceID(s.GetID()) + l.SetBuildID(b.GetID()) + l.SetRepoID(b.GetRepoID()) + l.SetData([]byte{}) + + // send API call to create the service logs + err = database.CreateLog(l) + if err != nil { + return services, fmt.Errorf("unable to create service logs for service %s: %w", s.GetName(), err) + } + } + + return services, nil +} + // cleanBuild is a helper function to kill the build // without execution. This will kill all resources, // like steps and services, for the build in the diff --git a/api/service.go b/api/service.go deleted file mode 100644 index 66b81b080..000000000 --- a/api/service.go +++ /dev/null @@ -1,593 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package api - -import ( - "fmt" - "net/http" - "strconv" - "time" - - "github.com/go-vela/server/router/middleware/org" - "github.com/go-vela/server/router/middleware/user" - - "github.com/go-vela/server/database" - "github.com/go-vela/server/router/middleware/build" - "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/service" - "github.com/go-vela/server/util" - - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" - "github.com/go-vela/types/pipeline" - - "github.com/gin-gonic/gin" - "github.com/sirupsen/logrus" -) - -// swagger:operation POST /api/v1/repos/{org}/{repo}/builds/{build}/services services CreateService -// -// Create a service for a build in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: body -// name: body -// description: Payload containing the service to create -// required: true -// schema: -// "$ref": "#/definitions/Service" -// security: -// - ApiKeyAuth: [] -// responses: -// '201': -// description: Successfully created the service -// schema: -// "$ref": "#/definitions/Service" -// '400': -// description: Unable to create the service -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to create the service -// schema: -// "$ref": "#/definitions/Error" - -// CreateService represents the API handler to create -// a service for a build in the configured backend. -// -//nolint:dupl // ignore similar code with step -func CreateService(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("creating new service for build %s", entry) - - // capture body from API request - input := new(library.Service) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for new service for build %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update fields in service object - input.SetRepoID(r.GetID()) - input.SetBuildID(b.GetID()) - - if len(input.GetStatus()) == 0 { - input.SetStatus(constants.StatusPending) - } - - if input.GetCreated() == 0 { - input.SetCreated(time.Now().UTC().Unix()) - } - - // send API call to create the service - err = database.FromContext(c).CreateService(input) - if err != nil { - retErr := fmt.Errorf("unable to create service for build %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the created service - s, _ := database.FromContext(c).GetServiceForBuild(b, input.GetNumber()) - - c.JSON(http.StatusCreated, s) -} - -// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/services services GetServices -// -// Get a list of all services for a build in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: query -// name: page -// description: The page of results to retrieve -// type: integer -// default: 1 -// - in: query -// name: per_page -// description: How many results per page to return -// type: integer -// maximum: 100 -// default: 10 -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the list of services -// schema: -// type: array -// items: -// "$ref": "#/definitions/Service" -// headers: -// X-Total-Count: -// description: Total number of results -// type: integer -// Link: -// description: see https://tools.ietf.org/html/rfc5988 -// type: string -// '400': -// description: Unable to retrieve the list of services -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to retrieve the list of services -// schema: -// "$ref": "#/definitions/Error" - -// GetServices represents the API handler to capture a list -// of services for a build from the configured backend. -func GetServices(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("reading services for build %s", entry) - - // capture page query parameter if present - page, err := strconv.Atoi(c.DefaultQuery("page", "1")) - if err != nil { - retErr := fmt.Errorf("unable to convert page query parameter for build %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // capture per_page query parameter if present - perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) - if err != nil { - retErr := fmt.Errorf("unable to convert per_page query parameter for build %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // ensure per_page isn't above or below allowed values - perPage = util.MaxInt(1, util.MinInt(100, perPage)) - - // send API call to capture the list of services for the build - s, t, err := database.FromContext(c).ListServicesForBuild(b, map[string]interface{}{}, page, perPage) - if err != nil { - retErr := fmt.Errorf("unable to get services for build %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // create pagination object - pagination := Pagination{ - Page: page, - PerPage: perPage, - Total: t, - } - // set pagination headers - pagination.SetHeaderLink(c) - - c.JSON(http.StatusOK, s) -} - -// -// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/services/{service} services GetService -// -// Get a service for a build in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: service -// description: Name of the service -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the service -// schema: -// "$ref": "#/definitions/Service" -// '400': -// description: Unable to retrieve the service -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to retrieve the service -// schema: -// "$ref": "#/definitions/Error" - -// GetService represents the API handler to capture a -// service for a build from the configured backend. -func GetService(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := service.Retrieve(c) - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "service": s.GetNumber(), - "user": u.GetName(), - }).Infof("reading service %s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - c.JSON(http.StatusOK, s) -} - -// -// swagger:operation PUT /api/v1/repos/{org}/{repo}/builds/{build}/services/{service} services UpdateService -// -// Update a service for a build in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: service -// description: Service number -// required: true -// type: integer -// - in: body -// name: body -// description: Payload containing the service to update -// required: true -// schema: -// "$ref": "#/definitions/Service" -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully updated the service -// schema: -// "$ref": "#/definitions/Service" -// '400': -// description: Unable to update the service -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to update the service -// schema: -// "$ref": "#/definitions/Error" - -// UpdateService represents the API handler to update -// a service for a build in the configured backend. -func UpdateService(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := service.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "service": s.GetNumber(), - "user": u.GetName(), - }).Infof("updating service %s", entry) - - // capture body from API request - input := new(library.Service) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for service %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update service fields if provided - if len(input.GetStatus()) > 0 { - // update status if set - s.SetStatus(input.GetStatus()) - } - - if len(input.GetError()) > 0 { - // update error if set - s.SetError(input.GetError()) - } - - if input.GetExitCode() > 0 { - // update exit_code if set - s.SetExitCode(input.GetExitCode()) - } - - if input.GetStarted() > 0 { - // update started if set - s.SetStarted(input.GetStarted()) - } - - if input.GetFinished() > 0 { - // update finished if set - s.SetFinished(input.GetFinished()) - } - - // send API call to update the service - err = database.FromContext(c).UpdateService(s) - if err != nil { - retErr := fmt.Errorf("unable to update service %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the updated service - s, _ = database.FromContext(c).GetServiceForBuild(b, s.GetNumber()) - - c.JSON(http.StatusOK, s) -} - -// -// swagger:operation DELETE /api/v1/repos/{org}/{repo}/builds/{build}/services/{service} services DeleteService -// -// Delete a service for a build in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: service -// description: Service Number -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully deleted the service -// schema: -// type: string -// '500': -// description: Unable to delete the service -// schema: -// "$ref": "#/definitions/Error" - -// DeleteService represents the API handler to remove -// a service for a build from the configured backend. -// -//nolint:dupl // ignore similar code with step -func DeleteService(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := service.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "service": s.GetNumber(), - "user": u.GetName(), - }).Infof("deleting service %s", entry) - - // send API call to remove the service - err := database.FromContext(c).DeleteService(s) - if err != nil { - retErr := fmt.Errorf("unable to delete service %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, fmt.Sprintf("service %s deleted", entry)) -} - -// planServices is a helper function to plan all services -// in the build for execution. This creates the services -// for the build in the configured backend. -func planServices(database database.Interface, p *pipeline.Build, b *library.Build) ([]*library.Service, error) { - // variable to store planned services - services := []*library.Service{} - - // iterate through all pipeline services - for _, service := range p.Services { - // create the service object - s := new(library.Service) - s.SetBuildID(b.GetID()) - s.SetRepoID(b.GetRepoID()) - s.SetName(service.Name) - s.SetImage(service.Image) - s.SetNumber(service.Number) - s.SetStatus(constants.StatusPending) - s.SetCreated(time.Now().UTC().Unix()) - - // send API call to create the service - err := database.CreateService(s) - if err != nil { - return services, fmt.Errorf("unable to create service %s: %w", s.GetName(), err) - } - - // send API call to capture the created service - s, err = database.GetServiceForBuild(b, s.GetNumber()) - if err != nil { - return services, fmt.Errorf("unable to get service %s: %w", s.GetName(), err) - } - - // populate environment variables from service library - // - // https://pkg.go.dev/github.com/go-vela/types/library#Service.Environment - err = service.MergeEnv(s.Environment()) - if err != nil { - return services, err - } - - // create the log object - l := new(library.Log) - l.SetServiceID(s.GetID()) - l.SetBuildID(b.GetID()) - l.SetRepoID(b.GetRepoID()) - l.SetData([]byte{}) - - // send API call to create the service logs - err = database.CreateLog(l) - if err != nil { - return services, fmt.Errorf("unable to create service logs for service %s: %w", s.GetName(), err) - } - } - - return services, nil -} diff --git a/api/service/create.go b/api/service/create.go new file mode 100644 index 000000000..426f13760 --- /dev/null +++ b/api/service/create.go @@ -0,0 +1,128 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "fmt" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation POST /api/v1/repos/{org}/{repo}/builds/{build}/services services CreateService +// +// Create a service for a build in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: body +// name: body +// description: Payload containing the service to create +// required: true +// schema: +// "$ref": "#/definitions/Service" +// security: +// - ApiKeyAuth: [] +// responses: +// '201': +// description: Successfully created the service +// schema: +// "$ref": "#/definitions/Service" +// '400': +// description: Unable to create the service +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to create the service +// schema: +// "$ref": "#/definitions/Error" + +// CreateService represents the API handler to create +// a service for a build in the configured backend. +func CreateService(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("creating new service for build %s", entry) + + // capture body from API request + input := new(library.Service) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for new service for build %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update fields in service object + input.SetRepoID(r.GetID()) + input.SetBuildID(b.GetID()) + + if len(input.GetStatus()) == 0 { + input.SetStatus(constants.StatusPending) + } + + if input.GetCreated() == 0 { + input.SetCreated(time.Now().UTC().Unix()) + } + + // send API call to create the service + err = database.FromContext(c).CreateService(input) + if err != nil { + retErr := fmt.Errorf("unable to create service for build %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the created service + s, _ := database.FromContext(c).GetServiceForBuild(b, input.GetNumber()) + + c.JSON(http.StatusCreated, s) +} diff --git a/api/service/delete.go b/api/service/delete.go new file mode 100644 index 000000000..d85aa0fe8 --- /dev/null +++ b/api/service/delete.go @@ -0,0 +1,97 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/service" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// +// swagger:operation DELETE /api/v1/repos/{org}/{repo}/builds/{build}/services/{service} services DeleteService +// +// Delete a service for a build in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: service +// description: Service Number +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully deleted the service +// schema: +// type: string +// '500': +// description: Unable to delete the service +// schema: +// "$ref": "#/definitions/Error" + +// DeleteService represents the API handler to remove +// a service for a build from the configured backend. +func DeleteService(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := service.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "service": s.GetNumber(), + "user": u.GetName(), + }).Infof("deleting service %s", entry) + + // send API call to remove the service + err := database.FromContext(c).DeleteService(s) + if err != nil { + retErr := fmt.Errorf("unable to delete service %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, fmt.Sprintf("service %s deleted", entry)) +} diff --git a/api/service/doc.go b/api/service/doc.go new file mode 100644 index 000000000..53dc07284 --- /dev/null +++ b/api/service/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package service provides the service handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/service" +package service diff --git a/api/service/get.go b/api/service/get.go new file mode 100644 index 000000000..b2165a69f --- /dev/null +++ b/api/service/get.go @@ -0,0 +1,86 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/service" + "github.com/go-vela/server/router/middleware/user" + "github.com/sirupsen/logrus" +) + +// +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/services/{service} services GetService +// +// Get a service for a build in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: service +// description: Name of the service +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the service +// schema: +// "$ref": "#/definitions/Service" +// '400': +// description: Unable to retrieve the service +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to retrieve the service +// schema: +// "$ref": "#/definitions/Error" + +// GetService represents the API handler to capture a +// service for a build from the configured backend. +func GetService(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := service.Retrieve(c) + u := user.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "service": s.GetNumber(), + "user": u.GetName(), + }).Infof("reading service %s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + c.JSON(http.StatusOK, s) +} diff --git a/api/service/list.go b/api/service/list.go new file mode 100644 index 000000000..22a7b8be7 --- /dev/null +++ b/api/service/list.go @@ -0,0 +1,146 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/api" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/services services ListServices +// +// Get a list of all services for a build in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: query +// name: page +// description: The page of results to retrieve +// type: integer +// default: 1 +// - in: query +// name: per_page +// description: How many results per page to return +// type: integer +// maximum: 100 +// default: 10 +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the list of services +// schema: +// type: array +// items: +// "$ref": "#/definitions/Service" +// headers: +// X-Total-Count: +// description: Total number of results +// type: integer +// Link: +// description: see https://tools.ietf.org/html/rfc5988 +// type: string +// '400': +// description: Unable to retrieve the list of services +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to retrieve the list of services +// schema: +// "$ref": "#/definitions/Error" + +// ListServices represents the API handler to capture a list +// of services for a build from the configured backend. +func ListServices(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("reading services for build %s", entry) + + // capture page query parameter if present + page, err := strconv.Atoi(c.DefaultQuery("page", "1")) + if err != nil { + retErr := fmt.Errorf("unable to convert page query parameter for build %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // capture per_page query parameter if present + perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) + if err != nil { + retErr := fmt.Errorf("unable to convert per_page query parameter for build %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // ensure per_page isn't above or below allowed values + perPage = util.MaxInt(1, util.MinInt(100, perPage)) + + // send API call to capture the list of services for the build + s, t, err := database.FromContext(c).ListServicesForBuild(b, map[string]interface{}{}, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to get services for build %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // create pagination object + pagination := api.Pagination{ + Page: page, + PerPage: perPage, + Total: t, + } + // set pagination headers + pagination.SetHeaderLink(c) + + c.JSON(http.StatusOK, s) +} diff --git a/api/service/update.go b/api/service/update.go new file mode 100644 index 000000000..bed781f88 --- /dev/null +++ b/api/service/update.go @@ -0,0 +1,149 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/service" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// +// swagger:operation PUT /api/v1/repos/{org}/{repo}/builds/{build}/services/{service} services UpdateService +// +// Update a service for a build in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: service +// description: Service number +// required: true +// type: integer +// - in: body +// name: body +// description: Payload containing the service to update +// required: true +// schema: +// "$ref": "#/definitions/Service" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully updated the service +// schema: +// "$ref": "#/definitions/Service" +// '400': +// description: Unable to update the service +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to update the service +// schema: +// "$ref": "#/definitions/Error" + +// UpdateService represents the API handler to update +// a service for a build in the configured backend. +func UpdateService(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := service.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "service": s.GetNumber(), + "user": u.GetName(), + }).Infof("updating service %s", entry) + + // capture body from API request + input := new(library.Service) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for service %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update service fields if provided + if len(input.GetStatus()) > 0 { + // update status if set + s.SetStatus(input.GetStatus()) + } + + if len(input.GetError()) > 0 { + // update error if set + s.SetError(input.GetError()) + } + + if input.GetExitCode() > 0 { + // update exit_code if set + s.SetExitCode(input.GetExitCode()) + } + + if input.GetStarted() > 0 { + // update started if set + s.SetStarted(input.GetStarted()) + } + + if input.GetFinished() > 0 { + // update finished if set + s.SetFinished(input.GetFinished()) + } + + // send API call to update the service + err = database.FromContext(c).UpdateService(s) + if err != nil { + retErr := fmt.Errorf("unable to update service %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the updated service + s, _ = database.FromContext(c).GetServiceForBuild(b, s.GetNumber()) + + c.JSON(http.StatusOK, s) +} diff --git a/api/step.go b/api/step.go index 23d72fac4..aab74067b 100644 --- a/api/step.go +++ b/api/step.go @@ -71,8 +71,6 @@ import ( // CreateStep represents the API handler to create // a step for a build in the configured backend. -// -//nolint:dupl // ignore similar code with service func CreateStep(c *gin.Context) { // capture middleware values b := build.Retrieve(c) @@ -501,8 +499,6 @@ func UpdateStep(c *gin.Context) { // DeleteStep represents the API handler to remove // a step for a build from the configured backend. -// -//nolint:dupl // ignore similar code with service func DeleteStep(c *gin.Context) { // capture middleware values b := build.Retrieve(c) diff --git a/router/service.go b/router/service.go index f344e03eb..7b81e7af7 100644 --- a/router/service.go +++ b/router/service.go @@ -7,10 +7,10 @@ package router import ( "github.com/gin-gonic/gin" - "github.com/go-vela/server/api" + "github.com/go-vela/server/api/service" "github.com/go-vela/server/router/middleware" "github.com/go-vela/server/router/middleware/perm" - "github.com/go-vela/server/router/middleware/service" + smiddleware "github.com/go-vela/server/router/middleware/service" ) // ServiceHandlers is a function that extends the provided base router group @@ -24,23 +24,23 @@ import ( // POST /api/v1/repos/:org/:repo/builds/:build/services/:service/logs // GET /api/v1/repos/:org/:repo/builds/:build/services/:service/logs // PUT /api/v1/repos/:org/:repo/builds/:build/services/:service/logs -// DELETE /api/v1/repos/:org/:repo/builds/:build/services/:service/logs +// DELETE /api/v1/repos/:org/:repo/builds/:build/services/:service/logs . func ServiceHandlers(base *gin.RouterGroup) { // Services endpoints services := base.Group("/services") { - services.POST("", perm.MustPlatformAdmin(), middleware.Payload(), api.CreateService) - services.GET("", perm.MustRead(), api.GetServices) + services.POST("", perm.MustPlatformAdmin(), middleware.Payload(), service.CreateService) + services.GET("", perm.MustRead(), service.ListServices) // Service endpoints - service := services.Group("/:service", service.Establish()) + s := services.Group("/:service", smiddleware.Establish()) { - service.GET("", perm.MustRead(), api.GetService) - service.PUT("", perm.MustBuildAccess(), middleware.Payload(), api.UpdateService) - service.DELETE("", perm.MustPlatformAdmin(), api.DeleteService) + s.GET("", perm.MustRead(), service.GetService) + s.PUT("", perm.MustBuildAccess(), middleware.Payload(), service.UpdateService) + s.DELETE("", perm.MustPlatformAdmin(), service.DeleteService) // Log endpoints - LogServiceHandlers(service) + LogServiceHandlers(s) } // end of service endpoints } // end of services endpoints } From 3919d6cd3c6e74ffbeae8489beef088e814a62d5 Mon Sep 17 00:00:00 2001 From: dave vader <48764154+plyr4@users.noreply.github.com> Date: Fri, 19 May 2023 12:07:15 -0500 Subject: [PATCH 10/53] chore: vscode gitignore from toptal (#845) --- .gitignore | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4ac8d000b..58613c5d3 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,29 @@ secrets.env .env.test # Files to be excluded. -.DS_Store \ No newline at end of file +.DS_Store + +# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix +__debug_bin + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode \ No newline at end of file From f276feeb42b72f5b31447792819d6c4b2ac21403 Mon Sep 17 00:00:00 2001 From: Jordan Sussman Date: Fri, 19 May 2023 16:07:40 -0500 Subject: [PATCH 11/53] feat(api): add support for schedules (#836) * feat(api/types): add support for schedules * feat(database/types): add support for schedules * feat(database): add support for schedules * chore: update go dependencies * feat(database): add schedule engine * feat(api): add support for schedules * add routes * fix: parse entry for schedules * more wip code * add schedule allowlist * fix tests * add validation for entry * add mocks w/o updated payloads * fix issues with create * update mock responses * use schedule mocks * make linter happy * use proper func * couple more updates * fix mock pathing * enhance: switch to adhocore/gronx * chore: update go deps * goimports * yet another goimports * sigh * wildcard goimport * chore: address linter feedback * chore: remove new types * chore: updates for removed types * chore: update go dependencies * chore: address review feedback * chore: remove new types * update go.mod * Update api/schedule/create.go Co-authored-by: Jordan Brockopp * Update api/schedule/create.go Co-authored-by: Jordan Brockopp * Update api/schedule/create.go Co-authored-by: Jordan Brockopp * Update api/schedule/create.go Co-authored-by: Jordan Brockopp * Update api/schedule/create.go Co-authored-by: Jordan Brockopp * Update api/schedule/create.go Co-authored-by: Jordan Brockopp * address feedback * goimports * address feedback * updated logic for updating inactive schedules * clarify error message --------- Co-authored-by: JordanBrockopp Co-authored-by: Jordan Brockopp --- .gitignore | 3 +- api/repo/create.go | 29 +-- api/schedule/create.go | 230 +++++++++++++++++++ api/schedule/create_test.go | 62 +++++ api/schedule/delete.go | 88 +++++++ api/schedule/get.go | 69 ++++++ api/schedule/list.go | 131 +++++++++++ api/schedule/update.go | 140 +++++++++++ cmd/vela-server/main.go | 13 ++ cmd/vela-server/server.go | 2 + docker-compose.yml | 1 + go.mod | 2 +- mock/server/schedule.go | 213 +++++++++++++++++ mock/server/server.go | 7 + router/middleware/allowlist_schedule.go | 18 ++ router/middleware/allowlist_schedule_test.go | 46 ++++ router/middleware/schedule/context.go | 39 ++++ router/middleware/schedule/context_test.go | 89 +++++++ router/middleware/schedule/schedule.go | 59 +++++ router/middleware/schedule_frequency.go | 20 ++ router/middleware/schedule_frequency_test.go | 47 ++++ router/router.go | 4 + router/schedule.go | 39 ++++ util/util.go | 29 +++ 24 files changed, 1350 insertions(+), 30 deletions(-) create mode 100644 api/schedule/create.go create mode 100644 api/schedule/create_test.go create mode 100644 api/schedule/delete.go create mode 100644 api/schedule/get.go create mode 100644 api/schedule/list.go create mode 100644 api/schedule/update.go create mode 100644 mock/server/schedule.go create mode 100644 router/middleware/allowlist_schedule.go create mode 100644 router/middleware/allowlist_schedule_test.go create mode 100644 router/middleware/schedule/context.go create mode 100644 router/middleware/schedule/context_test.go create mode 100644 router/middleware/schedule/schedule.go create mode 100644 router/middleware/schedule_frequency.go create mode 100644 router/middleware/schedule_frequency_test.go create mode 100644 router/schedule.go diff --git a/.gitignore b/.gitignore index 58613c5d3..f3be5762d 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ secrets.env # Files to be excluded. .DS_Store +api-spec.json # Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode # Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode @@ -66,4 +67,4 @@ __debug_bin .history .ionide -# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode \ No newline at end of file +# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode diff --git a/api/repo/create.go b/api/repo/create.go index d196c2a48..91d42064e 100644 --- a/api/repo/create.go +++ b/api/repo/create.go @@ -214,7 +214,7 @@ func CreateRepo(c *gin.Context) { ) // ensure repo is allowed to be activated - if !checkAllowlist(r, allowlist) { + if !util.CheckAllowlist(r, allowlist) { retErr := fmt.Errorf("unable to activate repo: %s is not on allowlist", r.GetFullName()) util.HandleError(c, http.StatusForbidden, retErr) @@ -328,30 +328,3 @@ func CreateRepo(c *gin.Context) { c.JSON(http.StatusCreated, r) } - -// checkAllowlist is a helper function to ensure only repos in the -// allowlist are allowed to enable repos. -// -// a single entry of '*' allows any repo to be enabled. -func checkAllowlist(r *library.Repo, allowlist []string) bool { - // check if all repos are allowed to be enabled - if len(allowlist) == 1 && allowlist[0] == "*" { - return true - } - - for _, repo := range allowlist { - // allow all repos in org - if strings.Contains(repo, "/*") { - if strings.HasPrefix(repo, r.GetOrg()) { - return true - } - } - - // allow specific repo within org - if repo == r.GetFullName() { - return true - } - } - - return false -} diff --git a/api/schedule/create.go b/api/schedule/create.go new file mode 100644 index 000000000..127c04909 --- /dev/null +++ b/api/schedule/create.go @@ -0,0 +1,230 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "fmt" + "net/http" + "time" + + "github.com/adhocore/gronx" + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation POST /api/v1/schedules/{org}/{repo} schedules CreateSchedule +// +// Create a schedule in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: body +// name: body +// description: Payload containing the schedule to create +// required: true +// schema: +// "$ref": "#/definitions/Schedule" +// security: +// - ApiKeyAuth: [] +// responses: +// '201': +// description: Successfully created the schedule +// schema: +// "$ref": "#/definitions/Schedule" +// '400': +// description: Unable to create the schedule +// schema: +// "$ref": "#/definitions/Error" +// '403': +// description: Unable to create the schedule +// schema: +// "$ref": "#/definitions/Error" +// '409': +// description: Unable to create the schedule +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to create the schedule +// schema: +// "$ref": "#/definitions/Error" +// '503': +// description: Unable to create the schedule +// schema: +// "$ref": "#/definitions/Error" + +// CreateSchedule represents the API handler to +// create a schedule in the configured backend. +func CreateSchedule(c *gin.Context) { + // capture middleware values + u := user.Retrieve(c) + r := repo.Retrieve(c) + allowlist := c.Value("allowlistschedule").([]string) + minimumFrequency := c.Value("scheduleminimumfrequency").(time.Duration) + + // capture body from API request + input := new(library.Schedule) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for new schedule: %w", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // ensure the entry is valid + err = validateEntry(minimumFrequency, input.GetEntry()) + if err != nil { + retErr := fmt.Errorf("schedule of %s with entry %s is invalid: %w", input.GetName(), input.GetEntry(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // ensure schedule name is defined + if input.GetName() == "" { + util.HandleError(c, http.StatusBadRequest, fmt.Errorf("schedule name must be set")) + } + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("creating new schedule %s", input.GetName()) + + // ensure repo is allowed to create new schedules + if !util.CheckAllowlist(r, allowlist) { + retErr := fmt.Errorf("unable to create schedule %s: %s is not on allowlist", input.GetName(), r.GetFullName()) + + util.HandleError(c, http.StatusForbidden, retErr) + + return + } + + s := new(library.Schedule) + + // update fields in schedule object + s.SetCreatedBy(u.GetName()) + s.SetRepoID(r.GetID()) + s.SetName(input.GetName()) + s.SetEntry(input.GetEntry()) + s.SetCreatedAt(time.Now().UTC().Unix()) + s.SetUpdatedAt(time.Now().UTC().Unix()) + s.SetUpdatedBy(u.GetName()) + + // set the active field based off the input provided + if input.Active == nil { + // default active field to true + s.SetActive(true) + } else { + s.SetActive(input.GetActive()) + } + + // send API call to capture the schedule from the database + dbSchedule, err := database.FromContext(c).GetScheduleForRepo(r, input.GetName()) + if err == nil && dbSchedule.GetActive() { + retErr := fmt.Errorf("unable to create schedule: %s is already active", input.GetName()) + + util.HandleError(c, http.StatusConflict, retErr) + + return + } + + if !r.GetActive() { + retErr := fmt.Errorf("unable to create schedule: %s repo %s is disabled", input.GetName(), r.GetFullName()) + + util.HandleError(c, http.StatusConflict, retErr) + + return + } + + // if the schedule exists but is inactive + if dbSchedule.GetID() != 0 && !dbSchedule.GetActive() && input.GetActive() { + // update the user who created the schedule + dbSchedule.SetUpdatedBy(u.GetName()) + // activate the schedule + dbSchedule.SetActive(true) + + // send API call to update the schedule + err = database.FromContext(c).UpdateSchedule(dbSchedule) + if err != nil { + retErr := fmt.Errorf("unable to set schedule %s to active: %w", dbSchedule.GetName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the updated schedule + s, _ = database.FromContext(c).GetScheduleForRepo(r, dbSchedule.GetName()) + } else { + // send API call to create the schedule + err = database.FromContext(c).CreateSchedule(s) + if err != nil { + retErr := fmt.Errorf("unable to create new schedule %s: %w", r.GetName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the created schedule + s, _ = database.FromContext(c).GetScheduleForRepo(r, input.GetName()) + } + + c.JSON(http.StatusCreated, s) +} + +// validateEntry validates the entry for a minimum frequency. +func validateEntry(minimum time.Duration, entry string) error { + gron := gronx.New() + + // check if expr is even valid + valid := gron.IsValid(entry) + if !valid { + return fmt.Errorf("invalid entry of %s", entry) + } + + // check the previous occurrence of the entry + prevTime, err := gronx.PrevTick(entry, true) + if err != nil { + return err + } + + // check the next occurrence of the entry + nextTime, err := gronx.NextTick(entry, true) + if err != nil { + return err + } + + // ensure the time between previous and next schedule exceeds the minimum duration + if nextTime.Sub(prevTime) < minimum { + return fmt.Errorf("entry needs to occur less frequently than every %s", minimum) + } + + return nil +} diff --git a/api/schedule/create_test.go b/api/schedule/create_test.go new file mode 100644 index 000000000..0ca425e32 --- /dev/null +++ b/api/schedule/create_test.go @@ -0,0 +1,62 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "testing" + "time" +) + +func Test_validateEntry(t *testing.T) { + type args struct { + minimum time.Duration + entry string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "exceeds minimum frequency", + args: args{ + minimum: 30 * time.Minute, + entry: "* * * * *", + }, + wantErr: true, + }, + { + name: "exceeds minimum frequency with tag", + args: args{ + minimum: 30 * time.Minute, + entry: "@15minutes", + }, + wantErr: true, + }, + { + name: "meets minimum frequency", + args: args{ + minimum: 30 * time.Second, + entry: "* * * * *", + }, + wantErr: false, + }, + { + name: "meets minimum frequency with tag", + args: args{ + minimum: 30 * time.Second, + entry: "@hourly", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := validateEntry(tt.args.minimum, tt.args.entry); (err != nil) != tt.wantErr { + t.Errorf("validateEntry() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/api/schedule/delete.go b/api/schedule/delete.go new file mode 100644 index 000000000..fd7c0715f --- /dev/null +++ b/api/schedule/delete.go @@ -0,0 +1,88 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/schedule" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation DELETE /api/v1/repos/{org}/{repo}/{schedule} schedules DeleteSchedule +// +// Delete a schedule in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: schedule +// description: Name of the schedule +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully deleted the schedule +// schema: +// type: string +// '500': +// description: Unable to delete the schedule +// schema: +// "$ref": "#/definitions/Error" +// '510': +// description: Unable to delete the schedule +// schema: +// "$ref": "#/definitions/Error" + +// DeleteSchedule represents the API handler to remove +// a schedule from the configured backend. +func DeleteSchedule(c *gin.Context) { + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + s := schedule.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("deleting schedule %s", s.GetName()) + + err := database.FromContext(c).DeleteSchedule(s) + if err != nil { + retErr := fmt.Errorf("unable to delete schedule %s: %w", s.GetName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, fmt.Sprintf("schedule %s deleted", s.GetName())) +} diff --git a/api/schedule/get.go b/api/schedule/get.go new file mode 100644 index 000000000..51a436bbe --- /dev/null +++ b/api/schedule/get.go @@ -0,0 +1,69 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/schedule" + "github.com/go-vela/server/router/middleware/user" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/schedules/{org}/{repo}/{schedule} schedules GetSchedule +// +// Get a schedule in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: schedule +// description: Name of the schedule +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the schedule +// schema: +// "$ref": "#/definitions/Schedule" + +// GetSchedule represents the API handler to +// capture a schedule from the configured backend. +func GetSchedule(c *gin.Context) { + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + s := schedule.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + "schedule": s.GetName(), + }).Infof("reading schedule %s", s.GetName()) + + c.JSON(http.StatusOK, s) +} diff --git a/api/schedule/list.go b/api/schedule/list.go new file mode 100644 index 000000000..c43188d86 --- /dev/null +++ b/api/schedule/list.go @@ -0,0 +1,131 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/api" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/schedules/{org}/{repo} schedules ListSchedules +// +// Get all schedules in the configured backend +// +// --- +// produces: +// - application/json +// security: +// - ApiKeyAuth: [] +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: query +// name: page +// description: The page of results to retrieve +// type: integer +// default: 1 +// - in: query +// name: per_page +// description: How many results per page to return +// type: integer +// maximum: 100 +// default: 10 +// responses: +// '200': +// description: Successfully retrieved the schedules +// schema: +// type: array +// items: +// "$ref": "#/definitions/Schedule" +// headers: +// X-Total-Count: +// description: Total number of results +// type: integer +// Link: +// description: see https://tools.ietf.org/html/rfc5988 +// type: string +// '400': +// description: Unable to retrieve the schedules +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to retrieve the schedules +// schema: +// "$ref": "#/definitions/Error" + +// ListSchedules represents the API handler to capture a list +// of schedules for a repo from the configured backend. +func ListSchedules(c *gin.Context) { + // capture middleware values + r := repo.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "repo": r.GetName(), + "org": r.GetOrg(), + }).Infof("listing schedules for repo %s", r.GetFullName()) + + // capture page query parameter if present + page, err := strconv.Atoi(c.DefaultQuery("page", "1")) + if err != nil { + retErr := fmt.Errorf("unable to convert page query parameter for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // capture per_page query parameter if present + perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) + if err != nil { + retErr := fmt.Errorf("unable to convert per_page query parameter for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // ensure per_page isn't above or below allowed values + perPage = util.MaxInt(1, util.MinInt(100, perPage)) + + // send API call to capture the list of schedules for the repo + s, t, err := database.FromContext(c).ListSchedulesForRepo(r, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to get schedules for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // create pagination object + pagination := api.Pagination{ + Page: page, + PerPage: perPage, + Total: t, + } + // set pagination headers + pagination.SetHeaderLink(c) + + c.JSON(http.StatusOK, s) +} diff --git a/api/schedule/update.go b/api/schedule/update.go new file mode 100644 index 000000000..bdadbdd2d --- /dev/null +++ b/api/schedule/update.go @@ -0,0 +1,140 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "fmt" + "net/http" + "time" + + "github.com/go-vela/server/router/middleware/schedule" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation PUT /api/v1/schedules/{org}/{repo}/{schedule} schedules UpdateSchedule +// +// Update a schedule for the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: schedule +// description: Name of the schedule +// required: true +// type: string +// - in: body +// name: body +// description: Payload containing the schedule to update +// required: true +// schema: +// "$ref": "#/definitions/Schedule" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully updated the schedule +// schema: +// "$ref": "#/definitions/Schedule" +// '400': +// description: Unable to update the schedule +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: Unable to update the schedule +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to update the schedule +// schema: +// "$ref": "#/definitions/Error" + +// UpdateSchedule represents the API handler to update +// a schedule in the configured backend. +func UpdateSchedule(c *gin.Context) { + // capture middleware values + r := repo.Retrieve(c) + s := schedule.Retrieve(c) + scheduleName := util.PathParameter(c, "schedule") + minimumFrequency := c.Value("scheduleminimumfrequency").(time.Duration) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "schedule": scheduleName, + "repo": r.GetName(), + "org": r.GetOrg(), + }).Infof("updating schedule %s", scheduleName) + + // capture body from API request + input := new(library.Schedule) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for schedule %s: %w", scheduleName, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update schedule fields if provided + if input.Active != nil { + // update active if set to true + s.SetActive(input.GetActive()) + } + + if input.GetName() != "" { + // update name if defined + s.SetName(input.GetName()) + } + + if input.GetEntry() != "" { + err = validateEntry(minimumFrequency, input.GetEntry()) + if err != nil { + retErr := fmt.Errorf("schedule entry of %s is invalid: %w", input.GetEntry(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update entry if defined + s.SetEntry(input.GetEntry()) + } + + // update the schedule within the database + err = database.FromContext(c).UpdateSchedule(s) + if err != nil { + retErr := fmt.Errorf("unable to update scheduled %s: %w", scheduleName, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // capture the updated scheduled + s, _ = database.FromContext(c).GetScheduleForRepo(r, scheduleName) + + c.JSON(http.StatusOK, s) +} diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 0dff41473..48b67af59 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -203,6 +203,19 @@ func main() { Usage: "interval at which workers will show as active within the /metrics endpoint", Value: 5 * time.Minute, }, + // schedule flags + &cli.DurationFlag{ + EnvVars: []string{"VELA_SCHEDULE_MINIMUM_FREQUENCY", "SCHEDULE_MINIMUM_FREQUENCY"}, + Name: "schedule-minimum-frequency", + Usage: "minimum time between each schedule entry", + Value: 1 * time.Hour, + }, + &cli.StringSliceFlag{ + EnvVars: []string{"VELA_SCHEDULE_ALLOWLIST"}, + Name: "vela-schedule-allowlist", + Usage: "limit which repos can be utilize the schedule feature within the system", + Value: &cli.StringSlice{}, + }, } // Add Database Flags app.Flags = append(app.Flags, database.Flags...) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 8ecb006e4..1f8bdbd9c 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -102,6 +102,8 @@ func server(c *cli.Context) error { middleware.SecureCookie(c.Bool("vela-enable-secure-cookie")), middleware.Worker(c.Duration("worker-active-interval")), middleware.DefaultRepoEvents(c.StringSlice("default-repo-events")), + middleware.AllowlistSchedule(c.StringSlice("vela-schedule-allowlist")), + middleware.ScheduleFrequency(c.Duration("schedule-minimum-frequency")), ) addr, err := url.Parse(c.String("server-addr")) diff --git a/docker-compose.yml b/docker-compose.yml index cb4ef0684..1394c317b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,6 +43,7 @@ services: VELA_DISABLE_WEBHOOK_VALIDATION: 'true' VELA_ENABLE_SECURE_COOKIE: 'false' VELA_REPO_ALLOWLIST: '*' + VELA_SCHEDULE_ALLOWLIST: '*' env_file: - .env restart: always diff --git a/go.mod b/go.mod index 95e264713..b2ab8d4c0 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 + github.com/adhocore/gronx v1.6.2 github.com/alicebob/miniredis/v2 v2.30.2 github.com/aws/aws-sdk-go v1.44.248 github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 @@ -45,7 +46,6 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/adhocore/gronx v1.6.2 // indirect github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/mock/server/schedule.go b/mock/server/schedule.go new file mode 100644 index 000000000..a1acc54f7 --- /dev/null +++ b/mock/server/schedule.go @@ -0,0 +1,213 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package server + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/go-vela/types" + "github.com/go-vela/types/library" +) + +const ( + // ScheduleResp represents a JSON return for a single schedule. + ScheduleResp = `{ + "id": 2, + "active": true, + "name": "foo", + "entry": "@weekly", + "created_at": 1683154980, + "created_by": "octocat", + "updated_at": 1683154980, + "updated_by": "octocat", + "scheduled_at": 0, + "repo": { + "id": 1, + "user_id": 1, + "org": "github", + "name": "octocat", + "full_name": "github/octocat", + "link": "https://github.com/github/octocat", + "clone": "https://github.com/github/octocat.git", + "branch": "main", + "topics": [], + "build_limit": 10, + "timeout": 30, + "counter": 0, + "visibility": "public", + "private": false, + "trusted": false, + "active": true, + "allow_pull": false, + "allow_push": true, + "allow_deploy": false, + "allow_tag": false, + "allow_comment": false, + "pipeline_type": "yaml", + "previous_name": "" + } +}` + SchedulesResp = `[ + { + "id": 2, + "active": true, + "name": "foo", + "entry": "@weekly", + "created_at": 1683154980, + "created_by": "octocat", + "updated_at": 1683154980, + "updated_by": "octocat", + "scheduled_at": 0, + "repo": { + "id": 1, + "user_id": 1, + "org": "github", + "name": "octokitty", + "full_name": "github/octokitty", + "link": "https://github.com/github/octokitty", + "clone": "https://github.com/github/octokitty.git", + "branch": "main", + "topics": [], + "build_limit": 10, + "timeout": 30, + "counter": 0, + "visibility": "public", + "private": false, + "trusted": false, + "active": true, + "allow_pull": false, + "allow_push": true, + "allow_deploy": false, + "allow_tag": false, + "allow_comment": false, + "pipeline_type": "yaml", + "previous_name": "" + } + }, + { + "id": 1, + "active": true, + "name": "bar", + "entry": "@weekly", + "created_at": 1683154974, + "created_by": "octocat", + "updated_at": 1683154974, + "updated_by": "octocat", + "scheduled_at": 0, + "repo": { + "id": 1, + "user_id": 1, + "org": "github", + "name": "octokitty", + "full_name": "github/octokitty", + "link": "https://github.com/github/octokitty", + "clone": "https://github.com/github/octokitty.git", + "branch": "main", + "topics": [], + "build_limit": 10, + "timeout": 30, + "counter": 0, + "visibility": "public", + "private": false, + "trusted": false, + "active": true, + "allow_pull": false, + "allow_push": true, + "allow_deploy": false, + "allow_tag": false, + "allow_comment": false, + "pipeline_type": "yaml", + "previous_name": "" + } + } +]` +) + +// getSchedules returns mock JSON for a http GET. +func getSchedules(c *gin.Context) { + data := []byte(SchedulesResp) + + var body []library.Schedule + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} + +// getSchedule has a param :schedule returns mock JSON for a http GET. +// +// Pass "not-found" to :schedule to test receiving a http 404 response. +func getSchedule(c *gin.Context) { + s := c.Param("schedule") + + if strings.Contains(s, "not-found") { + msg := fmt.Sprintf("Schedule %s does not exist", s) + + c.AbortWithStatusJSON(http.StatusNotFound, types.Error{Message: &msg}) + + return + } + + data := []byte(ScheduleResp) + + var body library.Schedule + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} + +// addSchedule returns mock JSON for a http POST. +func addSchedule(c *gin.Context) { + data := []byte(ScheduleResp) + + var body library.Schedule + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusCreated, body) +} + +// updateSchedule has a param :schedule returns mock JSON for a http PUT. +// +// Pass "not-found" to :schedule to test receiving a http 404 response. +func updateSchedule(c *gin.Context) { + if !strings.Contains(c.FullPath(), "admin") { + s := c.Param("schedule") + + if strings.Contains(s, "not-found") { + msg := fmt.Sprintf("Schedule %s does not exist", s) + + c.AbortWithStatusJSON(http.StatusNotFound, types.Error{Message: &msg}) + + return + } + } + + data := []byte(ScheduleResp) + + var body library.Schedule + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} + +// removeSchedule has a param :schedule returns mock JSON for a http DELETE. +// +// Pass "not-found" to :schedule to test receiving a http 404 response. +func removeSchedule(c *gin.Context) { + s := c.Param("schedule") + + if strings.Contains(s, "not-found") { + msg := fmt.Sprintf("Schedule %s does not exist", s) + + c.AbortWithStatusJSON(http.StatusNotFound, types.Error{Message: &msg}) + + return + } + + c.JSON(http.StatusOK, fmt.Sprintf("schedule %s deleted", s)) +} diff --git a/mock/server/server.go b/mock/server/server.go index 0eb70188d..8fb6b1061 100644 --- a/mock/server/server.go +++ b/mock/server/server.go @@ -122,6 +122,13 @@ func FakeHandler() http.Handler { e.POST("/api/v1/workers/:worker/refresh", refreshWorkerAuth) e.DELETE("/api/v1/workers/:worker", removeWorker) + // mock endpoints for schedule calls + e.GET("/api/v1/schedules/:org/:repo", getSchedules) + e.GET("/api/v1/schedules/:org/:repo/:schedule", getSchedule) + e.POST("/api/v1/schedules/:org/:repo", addSchedule) + e.PUT("/api/v1/schedules/:org/:repo/:schedule", updateSchedule) + e.DELETE("/api/v1/schedules/:org/:repo/:schedule", removeSchedule) + // mock endpoints for authentication calls e.GET("/token-refresh", getTokenRefresh) e.GET("/authenticate", getAuthenticate) diff --git a/router/middleware/allowlist_schedule.go b/router/middleware/allowlist_schedule.go new file mode 100644 index 000000000..8872d0361 --- /dev/null +++ b/router/middleware/allowlist_schedule.go @@ -0,0 +1,18 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package middleware + +import ( + "github.com/gin-gonic/gin" +) + +// AllowlistSchedule is a middleware function that attaches the allowlistschedule used +// to limit which repos can utilize the schedule feature within the system. +func AllowlistSchedule(allowlistschedule []string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("allowlistschedule", allowlistschedule) + c.Next() + } +} diff --git a/router/middleware/allowlist_schedule_test.go b/router/middleware/allowlist_schedule_test.go new file mode 100644 index 000000000..a9b03ae28 --- /dev/null +++ b/router/middleware/allowlist_schedule_test.go @@ -0,0 +1,46 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package middleware + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestMiddleware_AllowlistSchedule(t *testing.T) { + // setup types + got := []string{""} + want := []string{"foobar"} + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(AllowlistSchedule(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("allowlistschedule").([]string) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("AllowlistSchedule returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("AllowlistSchedule is %v, want %v", got, want) + } +} diff --git a/router/middleware/schedule/context.go b/router/middleware/schedule/context.go new file mode 100644 index 000000000..7ce62871c --- /dev/null +++ b/router/middleware/schedule/context.go @@ -0,0 +1,39 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "context" + + "github.com/go-vela/types/library" +) + +const key = "schedule" + +// Setter defines a context that enables setting values. +type Setter interface { + Set(string, interface{}) +} + +// FromContext returns the Schedule associated with this context. +func FromContext(c context.Context) *library.Schedule { + value := c.Value(key) + if value == nil { + return nil + } + + s, ok := value.(*library.Schedule) + if !ok { + return nil + } + + return s +} + +// ToContext adds the Schedule to this context if it supports +// the Setter interface. +func ToContext(c Setter, s *library.Schedule) { + c.Set(key, s) +} diff --git a/router/middleware/schedule/context_test.go b/router/middleware/schedule/context_test.go new file mode 100644 index 000000000..fb73d0b32 --- /dev/null +++ b/router/middleware/schedule/context_test.go @@ -0,0 +1,89 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "testing" + + "github.com/gin-gonic/gin" + "github.com/go-vela/types/library" +) + +func TestSchedule_FromContext(t *testing.T) { + // setup types + num := int64(1) + want := &library.Schedule{ID: &num} + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, want) + + // run test + got := FromContext(context) + + if got != want { + t.Errorf("FromContext is %v, want %v", got, want) + } +} + +func TestSchedule_FromContext_Bad(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, nil) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSchedule_FromContext_WrongType(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, 1) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSchedule_FromContext_Empty(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSchedule_ToContext(t *testing.T) { + // setup types + num := int64(1) + want := &library.Schedule{ID: &num} + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + ToContext(context, want) + + // run test + got := context.Value(key) + + if got != want { + t.Errorf("ToContext is %v, want %v", got, want) + } +} diff --git a/router/middleware/schedule/schedule.go b/router/middleware/schedule/schedule.go new file mode 100644 index 000000000..de06e0651 --- /dev/null +++ b/router/middleware/schedule/schedule.go @@ -0,0 +1,59 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package schedule + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// Retrieve gets the schedule in the given context. +func Retrieve(c *gin.Context) *library.Schedule { + return FromContext(c) +} + +// Establish sets the schedule in the given context. +func Establish() gin.HandlerFunc { + return func(c *gin.Context) { + r := repo.Retrieve(c) + u := user.Retrieve(c) + + sParam := util.PathParameter(c, "schedule") + if len(sParam) == 0 { + retErr := fmt.Errorf("no schedule parameter provided") + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + "user": u.GetName(), + }).Debugf("reading schedule %s for repo %s", sParam, r.GetFullName()) + + s, err := database.FromContext(c).GetScheduleForRepo(r, sParam) + if err != nil { + retErr := fmt.Errorf("unable to read schedule %s for repo %s: %w", sParam, r.GetFullName(), err) + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + ToContext(c, s) + c.Next() + } +} diff --git a/router/middleware/schedule_frequency.go b/router/middleware/schedule_frequency.go new file mode 100644 index 000000000..243c2ad06 --- /dev/null +++ b/router/middleware/schedule_frequency.go @@ -0,0 +1,20 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package middleware + +import ( + "time" + + "github.com/gin-gonic/gin" +) + +// ScheduleFrequency is a middleware function that attaches the scheduleminimumfrequency used +// to limit the frequency which schedules can be run within the system. +func ScheduleFrequency(scheduleFrequency time.Duration) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("scheduleminimumfrequency", scheduleFrequency) + c.Next() + } +} diff --git a/router/middleware/schedule_frequency_test.go b/router/middleware/schedule_frequency_test.go new file mode 100644 index 000000000..171f96ad3 --- /dev/null +++ b/router/middleware/schedule_frequency_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package middleware + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + "time" + + "github.com/gin-gonic/gin" +) + +func TestMiddleware_ScheduleFrequency(t *testing.T) { + // setup types + var got time.Duration + want := 30 * time.Minute + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(ScheduleFrequency(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("scheduleminimumfrequency").(time.Duration) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("ScheduleFrequency returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("ScheduleFrequency is %v, want %v", got, want) + } +} diff --git a/router/router.go b/router/router.go index f9d1f7b24..d1e88273a 100644 --- a/router/router.go +++ b/router/router.go @@ -115,6 +115,9 @@ func Load(options ...gin.HandlerFunc) *gin.Engine { // * Log endpoints RepoHandlers(baseAPI) + // Schedule endpoints + ScheduleHandler(baseAPI) + // Source code management endpoints ScmHandlers(baseAPI) @@ -132,6 +135,7 @@ func Load(options ...gin.HandlerFunc) *gin.Engine { // Pipeline endpoints PipelineHandlers(baseAPI) + } // end of api return r diff --git a/router/schedule.go b/router/schedule.go new file mode 100644 index 000000000..7c73e30c2 --- /dev/null +++ b/router/schedule.go @@ -0,0 +1,39 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package router + +import ( + "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/schedule" + "github.com/go-vela/server/router/middleware" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/perm" + "github.com/go-vela/server/router/middleware/repo" + sMiddleware "github.com/go-vela/server/router/middleware/schedule" +) + +// ScheduleHandler is a function that extends the provided base router group +// with the API handlers for schedule functionality. +// +// POST /api/v1/schedules/:org/:repo +// GET /api/v1/schedules/:org/:repo +// GET /api/v1/schedules/:org/:repo/:schedule +// PUT /api/v1/schedules/:org/:repo/:schedule +// DELETE /api/v1/schedules/:org/:repo/:schedule . +func ScheduleHandler(base *gin.RouterGroup) { + // Schedules endpoints + _schedules := base.Group("/schedules/:org/:repo", org.Establish(), repo.Establish()) + { + _schedules.POST("", perm.MustAdmin(), middleware.Payload(), schedule.CreateSchedule) + _schedules.GET("", perm.MustRead(), schedule.ListSchedules) + + s := _schedules.Group("/:schedule", sMiddleware.Establish()) + { + s.GET("", perm.MustRead(), schedule.GetSchedule) + s.PUT("", perm.MustAdmin(), middleware.Payload(), schedule.UpdateSchedule) + s.DELETE("", perm.MustAdmin(), schedule.DeleteSchedule) + } + } // end of schedules endpoints +} diff --git a/util/util.go b/util/util.go index 59b08121f..849ffdd95 100644 --- a/util/util.go +++ b/util/util.go @@ -8,6 +8,8 @@ import ( "html" "strings" + "github.com/go-vela/types/library" + "github.com/gin-gonic/gin" "github.com/go-vela/types" ) @@ -70,3 +72,30 @@ func EscapeValue(value string) string { // HTML escape the new line escaped value return html.EscapeString(escaped) } + +// CheckAllowlist is a helper function to ensure only repos in the +// allowlist are specified. +// +// a single entry of '*' allows any repo to be enabled. +func CheckAllowlist(r *library.Repo, allowlist []string) bool { + // check if all repos are allowed to be enabled + if len(allowlist) == 1 && allowlist[0] == "*" { + return true + } + + for _, repo := range allowlist { + // allow all repos in org + if strings.Contains(repo, "/*") { + if strings.HasPrefix(repo, r.GetOrg()) { + return true + } + } + + // allow specific repo within org + if repo == r.GetFullName() { + return true + } + } + + return false +} From a374e2b83bcac2f350f44e0308a8ba116ccb71d6 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Fri, 19 May 2023 15:51:34 -0600 Subject: [PATCH 12/53] refactor(api): move step logic to separate package (#851) * refactor(api): move step logic to separate package * get -> list --- api/build.go | 110 ++++++++ api/step.go | 644 --------------------------------------------- api/step/create.go | 128 +++++++++ api/step/delete.go | 96 +++++++ api/step/doc.go | 10 + api/step/get.go | 77 ++++++ api/step/list.go | 146 ++++++++++ api/step/update.go | 163 ++++++++++++ router/step.go | 20 +- 9 files changed, 740 insertions(+), 654 deletions(-) delete mode 100644 api/step.go create mode 100644 api/step/create.go create mode 100644 api/step/delete.go create mode 100644 api/step/doc.go create mode 100644 api/step/get.go create mode 100644 api/step/list.go create mode 100644 api/step/update.go diff --git a/api/build.go b/api/build.go index 8519a8e53..ce755027a 100644 --- a/api/build.go +++ b/api/build.go @@ -1677,6 +1677,116 @@ func planServices(database database.Interface, p *pipeline.Build, b *library.Bui return services, nil } +// planSteps is a helper function to plan all steps +// in the build for execution. This creates the steps +// for the build in the configured backend. +func planSteps(database database.Interface, p *pipeline.Build, b *library.Build) ([]*library.Step, error) { + // variable to store planned steps + steps := []*library.Step{} + + // iterate through all pipeline stages + for _, stage := range p.Stages { + // iterate through all steps for each pipeline stage + for _, step := range stage.Steps { + // create the step object + s := new(library.Step) + s.SetBuildID(b.GetID()) + s.SetRepoID(b.GetRepoID()) + s.SetNumber(step.Number) + s.SetName(step.Name) + s.SetImage(step.Image) + s.SetStage(stage.Name) + s.SetStatus(constants.StatusPending) + s.SetCreated(time.Now().UTC().Unix()) + + // send API call to create the step + err := database.CreateStep(s) + if err != nil { + return steps, fmt.Errorf("unable to create step %s: %w", s.GetName(), err) + } + + // send API call to capture the created step + s, err = database.GetStepForBuild(b, s.GetNumber()) + if err != nil { + return steps, fmt.Errorf("unable to get step %s: %w", s.GetName(), err) + } + + // populate environment variables from step library + // + // https://pkg.go.dev/github.com/go-vela/types/library#step.Environment + err = step.MergeEnv(s.Environment()) + if err != nil { + return steps, err + } + + // create the log object + l := new(library.Log) + l.SetStepID(s.GetID()) + l.SetBuildID(b.GetID()) + l.SetRepoID(b.GetRepoID()) + l.SetData([]byte{}) + + // send API call to create the step logs + err = database.CreateLog(l) + if err != nil { + return nil, fmt.Errorf("unable to create logs for step %s: %w", s.GetName(), err) + } + + steps = append(steps, s) + } + } + + // iterate through all pipeline steps + for _, step := range p.Steps { + // create the step object + s := new(library.Step) + s.SetBuildID(b.GetID()) + s.SetRepoID(b.GetRepoID()) + s.SetNumber(step.Number) + s.SetName(step.Name) + s.SetImage(step.Image) + s.SetStatus(constants.StatusPending) + s.SetCreated(time.Now().UTC().Unix()) + + // send API call to create the step + err := database.CreateStep(s) + if err != nil { + return steps, fmt.Errorf("unable to create step %s: %w", s.GetName(), err) + } + + // send API call to capture the created step + s, err = database.GetStepForBuild(b, s.GetNumber()) + if err != nil { + return steps, fmt.Errorf("unable to get step %s: %w", s.GetName(), err) + } + + // populate environment variables from step library + // + // https://pkg.go.dev/github.com/go-vela/types/library#step.Environment + err = step.MergeEnv(s.Environment()) + if err != nil { + return steps, err + } + + // create the log object + l := new(library.Log) + l.SetStepID(s.GetID()) + l.SetBuildID(b.GetID()) + l.SetRepoID(b.GetRepoID()) + l.SetData([]byte{}) + + // send API call to create the step logs + err = database.CreateLog(l) + if err != nil { + return steps, fmt.Errorf("unable to create logs for step %s: %w", s.GetName(), err) + } + + steps = append(steps, s) + } + + return steps, nil +} + // cleanBuild is a helper function to kill the build // without execution. This will kill all resources, // like steps and services, for the build in the diff --git a/api/step.go b/api/step.go deleted file mode 100644 index aab74067b..000000000 --- a/api/step.go +++ /dev/null @@ -1,644 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package api - -import ( - "fmt" - "net/http" - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/go-vela/server/database" - "github.com/go-vela/server/router/middleware/build" - "github.com/go-vela/server/router/middleware/org" - "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/step" - "github.com/go-vela/server/router/middleware/user" - "github.com/go-vela/server/util" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" - "github.com/go-vela/types/pipeline" - "github.com/sirupsen/logrus" -) - -// swagger:operation POST /api/v1/repos/{org}/{repo}/builds/{build}/steps steps CreateStep -// -// Create a step for a build -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: body -// name: body -// description: Payload containing the step to create -// required: true -// schema: -// "$ref": "#/definitions/Step" -// security: -// - ApiKeyAuth: [] -// responses: -// '201': -// description: Successfully created the step -// schema: -// "$ref": "#/definitions/Step" -// '400': -// description: Unable to create the step -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to create the step -// schema: -// "$ref": "#/definitions/Error" - -// CreateStep represents the API handler to create -// a step for a build in the configured backend. -func CreateStep(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("creating new step for build %s", entry) - - // capture body from API request - input := new(library.Step) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for new step for build %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update fields in step object - input.SetRepoID(r.GetID()) - input.SetBuildID(b.GetID()) - - if len(input.GetStatus()) == 0 { - input.SetStatus(constants.StatusPending) - } - - if input.GetCreated() == 0 { - input.SetCreated(time.Now().UTC().Unix()) - } - - // send API call to create the step - err = database.FromContext(c).CreateStep(input) - if err != nil { - retErr := fmt.Errorf("unable to create step for build %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the created step - s, _ := database.FromContext(c).GetStepForBuild(b, input.GetNumber()) - - c.JSON(http.StatusCreated, s) -} - -// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/steps steps GetSteps -// -// Retrieve a list of steps for a build -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: query -// name: page -// description: The page of results to retrieve -// type: integer -// default: 1 -// - in: query -// name: per_page -// description: How many results per page to return -// type: integer -// maximum: 100 -// default: 10 -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the list of steps -// schema: -// type: array -// items: -// "$ref": "#/definitions/Step" -// headers: -// X-Total-Count: -// description: Total number of results -// type: integer -// Link: -// description: see https://tools.ietf.org/html/rfc5988 -// type: string -// '400': -// description: Unable to retrieve the list of steps -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to retrieve the list of steps -// schema: -// "$ref": "#/definitions/Error" - -// GetSteps represents the API handler to capture a list -// of steps for a build from the configured backend. -func GetSteps(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("reading steps for build %s", entry) - - // capture page query parameter if present - page, err := strconv.Atoi(c.DefaultQuery("page", "1")) - if err != nil { - retErr := fmt.Errorf("unable to convert page query parameter for build %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // capture per_page query parameter if present - perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) - if err != nil { - retErr := fmt.Errorf("unable to convert per_page query parameter for build %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // ensure per_page isn't above or below allowed values - perPage = util.MaxInt(1, util.MinInt(100, perPage)) - - // send API call to capture the list of steps for the build - s, t, err := database.FromContext(c).ListStepsForBuild(b, map[string]interface{}{}, page, perPage) - if err != nil { - retErr := fmt.Errorf("unable to get steps for build %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // create pagination object - pagination := Pagination{ - Page: page, - PerPage: perPage, - Total: t, - } - // set pagination headers - pagination.SetHeaderLink(c) - - c.JSON(http.StatusOK, s) -} - -// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step} steps GetStep -// -// Retrieve a step for a build -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: step -// description: Step number -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the step -// schema: -// "$ref": "#/definitions/Step" - -// GetStep represents the API handler to capture a -// step for a build from the configured backend. -func GetStep(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := step.Retrieve(c) - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "step": s.GetNumber(), - "user": u.GetName(), - }).Infof("reading step %s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - c.JSON(http.StatusOK, s) -} - -// swagger:operation PUT /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step} steps UpdateStep -// -// Update a step for a build -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: step -// description: Step number -// required: true -// type: integer -// - in: body -// name: body -// description: Payload containing the step to update -// required: true -// schema: -// "$ref": "#/definitions/Step" -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully updated the step -// schema: -// "$ref": "#/definitions/Step" -// '400': -// description: Unable to update the step -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to update the step -// schema: -// "$ref": "#/definitions/Error" - -// UpdateStep represents the API handler to update -// a step for a build in the configured backend. -func UpdateStep(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := step.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "step": s.GetNumber(), - "user": u.GetName(), - }).Infof("updating step %s", entry) - - // capture body from API request - input := new(library.Step) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for step %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update step fields if provided - if len(input.GetStatus()) > 0 { - // update status if set - s.SetStatus(input.GetStatus()) - } - - if len(input.GetError()) > 0 { - // update error if set - s.SetError(input.GetError()) - } - - if input.GetExitCode() > 0 { - // update exit_code if set - s.SetExitCode(input.GetExitCode()) - } - - if input.GetStarted() > 0 { - // update started if set - s.SetStarted(input.GetStarted()) - } - - if input.GetFinished() > 0 { - // update finished if set - s.SetFinished(input.GetFinished()) - } - - if len(input.GetHost()) > 0 { - // update host if set - s.SetHost(input.GetHost()) - } - - if len(input.GetRuntime()) > 0 { - // update runtime if set - s.SetRuntime(input.GetRuntime()) - } - - if len(input.GetDistribution()) > 0 { - // update distribution if set - s.SetDistribution(input.GetDistribution()) - } - - // send API call to update the step - err = database.FromContext(c).UpdateStep(s) - if err != nil { - retErr := fmt.Errorf("unable to update step %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the updated step - s, _ = database.FromContext(c).GetStepForBuild(b, s.GetNumber()) - - c.JSON(http.StatusOK, s) -} - -// swagger:operation DELETE /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step} steps DeleteStep -// -// Delete a step for a build -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: step -// description: Step number -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully deleted the step -// schema: -// type: string -// '500': -// description: Successfully deleted the step -// schema: -// "$ref": "#/definitions/Error" - -// DeleteStep represents the API handler to remove -// a step for a build from the configured backend. -func DeleteStep(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := step.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "step": s.GetNumber(), - "user": u.GetName(), - }).Infof("deleting step %s", entry) - - // send API call to remove the step - err := database.FromContext(c).DeleteStep(s) - if err != nil { - retErr := fmt.Errorf("unable to delete step %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, fmt.Sprintf("step %s deleted", entry)) -} - -// planSteps is a helper function to plan all steps -// in the build for execution. This creates the steps -// for the build in the configured backend. -func planSteps(database database.Interface, p *pipeline.Build, b *library.Build) ([]*library.Step, error) { - // variable to store planned steps - steps := []*library.Step{} - - // iterate through all pipeline stages - for _, stage := range p.Stages { - // iterate through all steps for each pipeline stage - for _, step := range stage.Steps { - // create the step object - s := new(library.Step) - s.SetBuildID(b.GetID()) - s.SetRepoID(b.GetRepoID()) - s.SetNumber(step.Number) - s.SetName(step.Name) - s.SetImage(step.Image) - s.SetStage(stage.Name) - s.SetStatus(constants.StatusPending) - s.SetCreated(time.Now().UTC().Unix()) - - // send API call to create the step - err := database.CreateStep(s) - if err != nil { - return steps, fmt.Errorf("unable to create step %s: %w", s.GetName(), err) - } - - // send API call to capture the created step - s, err = database.GetStepForBuild(b, s.GetNumber()) - if err != nil { - return steps, fmt.Errorf("unable to get step %s: %w", s.GetName(), err) - } - - // populate environment variables from step library - // - // https://pkg.go.dev/github.com/go-vela/types/library#step.Environment - err = step.MergeEnv(s.Environment()) - if err != nil { - return steps, err - } - - // create the log object - l := new(library.Log) - l.SetStepID(s.GetID()) - l.SetBuildID(b.GetID()) - l.SetRepoID(b.GetRepoID()) - l.SetData([]byte{}) - - // send API call to create the step logs - err = database.CreateLog(l) - if err != nil { - return nil, fmt.Errorf("unable to create logs for step %s: %w", s.GetName(), err) - } - - steps = append(steps, s) - } - } - - // iterate through all pipeline steps - for _, step := range p.Steps { - // create the step object - s := new(library.Step) - s.SetBuildID(b.GetID()) - s.SetRepoID(b.GetRepoID()) - s.SetNumber(step.Number) - s.SetName(step.Name) - s.SetImage(step.Image) - s.SetStatus(constants.StatusPending) - s.SetCreated(time.Now().UTC().Unix()) - - // send API call to create the step - err := database.CreateStep(s) - if err != nil { - return steps, fmt.Errorf("unable to create step %s: %w", s.GetName(), err) - } - - // send API call to capture the created step - s, err = database.GetStepForBuild(b, s.GetNumber()) - if err != nil { - return steps, fmt.Errorf("unable to get step %s: %w", s.GetName(), err) - } - - // populate environment variables from step library - // - // https://pkg.go.dev/github.com/go-vela/types/library#step.Environment - err = step.MergeEnv(s.Environment()) - if err != nil { - return steps, err - } - - // create the log object - l := new(library.Log) - l.SetStepID(s.GetID()) - l.SetBuildID(b.GetID()) - l.SetRepoID(b.GetRepoID()) - l.SetData([]byte{}) - - // send API call to create the step logs - err = database.CreateLog(l) - if err != nil { - return steps, fmt.Errorf("unable to create logs for step %s: %w", s.GetName(), err) - } - - steps = append(steps, s) - } - - return steps, nil -} diff --git a/api/step/create.go b/api/step/create.go new file mode 100644 index 000000000..d87bfaae0 --- /dev/null +++ b/api/step/create.go @@ -0,0 +1,128 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package step + +import ( + "fmt" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation POST /api/v1/repos/{org}/{repo}/builds/{build}/steps steps CreateStep +// +// Create a step for a build +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: body +// name: body +// description: Payload containing the step to create +// required: true +// schema: +// "$ref": "#/definitions/Step" +// security: +// - ApiKeyAuth: [] +// responses: +// '201': +// description: Successfully created the step +// schema: +// "$ref": "#/definitions/Step" +// '400': +// description: Unable to create the step +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to create the step +// schema: +// "$ref": "#/definitions/Error" + +// CreateStep represents the API handler to create +// a step for a build in the configured backend. +func CreateStep(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("creating new step for build %s", entry) + + // capture body from API request + input := new(library.Step) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for new step for build %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update fields in step object + input.SetRepoID(r.GetID()) + input.SetBuildID(b.GetID()) + + if len(input.GetStatus()) == 0 { + input.SetStatus(constants.StatusPending) + } + + if input.GetCreated() == 0 { + input.SetCreated(time.Now().UTC().Unix()) + } + + // send API call to create the step + err = database.FromContext(c).CreateStep(input) + if err != nil { + retErr := fmt.Errorf("unable to create step for build %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the created step + s, _ := database.FromContext(c).GetStepForBuild(b, input.GetNumber()) + + c.JSON(http.StatusCreated, s) +} diff --git a/api/step/delete.go b/api/step/delete.go new file mode 100644 index 000000000..1a43c30b3 --- /dev/null +++ b/api/step/delete.go @@ -0,0 +1,96 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package step + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/step" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation DELETE /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step} steps DeleteStep +// +// Delete a step for a build +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: step +// description: Step number +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully deleted the step +// schema: +// type: string +// '500': +// description: Successfully deleted the step +// schema: +// "$ref": "#/definitions/Error" + +// DeleteStep represents the API handler to remove +// a step for a build from the configured backend. +func DeleteStep(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := step.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "step": s.GetNumber(), + "user": u.GetName(), + }).Infof("deleting step %s", entry) + + // send API call to remove the step + err := database.FromContext(c).DeleteStep(s) + if err != nil { + retErr := fmt.Errorf("unable to delete step %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, fmt.Sprintf("step %s deleted", entry)) +} diff --git a/api/step/doc.go b/api/step/doc.go new file mode 100644 index 000000000..fad7dce79 --- /dev/null +++ b/api/step/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package step provides the step handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/step" +package step diff --git a/api/step/get.go b/api/step/get.go new file mode 100644 index 000000000..cf55c1bb5 --- /dev/null +++ b/api/step/get.go @@ -0,0 +1,77 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package step + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/step" + "github.com/go-vela/server/router/middleware/user" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step} steps GetStep +// +// Retrieve a step for a build +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: step +// description: Step number +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the step +// schema: +// "$ref": "#/definitions/Step" + +// GetStep represents the API handler to capture a +// step for a build from the configured backend. +func GetStep(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := step.Retrieve(c) + u := user.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "step": s.GetNumber(), + "user": u.GetName(), + }).Infof("reading step %s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + c.JSON(http.StatusOK, s) +} diff --git a/api/step/list.go b/api/step/list.go new file mode 100644 index 000000000..5d76fb33c --- /dev/null +++ b/api/step/list.go @@ -0,0 +1,146 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package step + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/api" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/steps steps ListSteps +// +// Retrieve a list of steps for a build +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: query +// name: page +// description: The page of results to retrieve +// type: integer +// default: 1 +// - in: query +// name: per_page +// description: How many results per page to return +// type: integer +// maximum: 100 +// default: 10 +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the list of steps +// schema: +// type: array +// items: +// "$ref": "#/definitions/Step" +// headers: +// X-Total-Count: +// description: Total number of results +// type: integer +// Link: +// description: see https://tools.ietf.org/html/rfc5988 +// type: string +// '400': +// description: Unable to retrieve the list of steps +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to retrieve the list of steps +// schema: +// "$ref": "#/definitions/Error" + +// ListSteps represents the API handler to capture a list +// of steps for a build from the configured backend. +func ListSteps(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("listing steps for build %s", entry) + + // capture page query parameter if present + page, err := strconv.Atoi(c.DefaultQuery("page", "1")) + if err != nil { + retErr := fmt.Errorf("unable to convert page query parameter for build %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // capture per_page query parameter if present + perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) + if err != nil { + retErr := fmt.Errorf("unable to convert per_page query parameter for build %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // ensure per_page isn't above or below allowed values + perPage = util.MaxInt(1, util.MinInt(100, perPage)) + + // send API call to capture the list of steps for the build + s, t, err := database.FromContext(c).ListStepsForBuild(b, map[string]interface{}{}, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to list steps for build %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // create pagination object + pagination := api.Pagination{ + Page: page, + PerPage: perPage, + Total: t, + } + // set pagination headers + pagination.SetHeaderLink(c) + + c.JSON(http.StatusOK, s) +} diff --git a/api/step/update.go b/api/step/update.go new file mode 100644 index 000000000..8a64fbb12 --- /dev/null +++ b/api/step/update.go @@ -0,0 +1,163 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package step + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/step" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation PUT /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step} steps UpdateStep +// +// Update a step for a build +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: step +// description: Step number +// required: true +// type: integer +// - in: body +// name: body +// description: Payload containing the step to update +// required: true +// schema: +// "$ref": "#/definitions/Step" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully updated the step +// schema: +// "$ref": "#/definitions/Step" +// '400': +// description: Unable to update the step +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to update the step +// schema: +// "$ref": "#/definitions/Error" + +// UpdateStep represents the API handler to update +// a step for a build in the configured backend. +func UpdateStep(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := step.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "step": s.GetNumber(), + "user": u.GetName(), + }).Infof("updating step %s", entry) + + // capture body from API request + input := new(library.Step) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for step %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update step fields if provided + if len(input.GetStatus()) > 0 { + // update status if set + s.SetStatus(input.GetStatus()) + } + + if len(input.GetError()) > 0 { + // update error if set + s.SetError(input.GetError()) + } + + if input.GetExitCode() > 0 { + // update exit_code if set + s.SetExitCode(input.GetExitCode()) + } + + if input.GetStarted() > 0 { + // update started if set + s.SetStarted(input.GetStarted()) + } + + if input.GetFinished() > 0 { + // update finished if set + s.SetFinished(input.GetFinished()) + } + + if len(input.GetHost()) > 0 { + // update host if set + s.SetHost(input.GetHost()) + } + + if len(input.GetRuntime()) > 0 { + // update runtime if set + s.SetRuntime(input.GetRuntime()) + } + + if len(input.GetDistribution()) > 0 { + // update distribution if set + s.SetDistribution(input.GetDistribution()) + } + + // send API call to update the step + err = database.FromContext(c).UpdateStep(s) + if err != nil { + retErr := fmt.Errorf("unable to update step %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the updated step + s, _ = database.FromContext(c).GetStepForBuild(b, s.GetNumber()) + + c.JSON(http.StatusOK, s) +} diff --git a/router/step.go b/router/step.go index 82719ddf7..9a53e4e97 100644 --- a/router/step.go +++ b/router/step.go @@ -7,10 +7,10 @@ package router import ( "github.com/gin-gonic/gin" - "github.com/go-vela/server/api" + "github.com/go-vela/server/api/step" "github.com/go-vela/server/router/middleware" "github.com/go-vela/server/router/middleware/perm" - "github.com/go-vela/server/router/middleware/step" + smiddleware "github.com/go-vela/server/router/middleware/step" ) // StepHandlers is a function that extends the provided base router group @@ -24,23 +24,23 @@ import ( // POST /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs // GET /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs // PUT /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs -// DELETE /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs +// DELETE /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs . func StepHandlers(base *gin.RouterGroup) { // Steps endpoints steps := base.Group("/steps") { - steps.POST("", perm.MustPlatformAdmin(), middleware.Payload(), api.CreateStep) - steps.GET("", perm.MustRead(), api.GetSteps) + steps.POST("", perm.MustPlatformAdmin(), middleware.Payload(), step.CreateStep) + steps.GET("", perm.MustRead(), step.ListSteps) // Step endpoints - step := steps.Group("/:step", step.Establish()) + s := steps.Group("/:step", smiddleware.Establish()) { - step.GET("", perm.MustRead(), api.GetStep) - step.PUT("", perm.MustBuildAccess(), middleware.Payload(), api.UpdateStep) - step.DELETE("", perm.MustPlatformAdmin(), api.DeleteStep) + s.GET("", perm.MustRead(), step.GetStep) + s.PUT("", perm.MustBuildAccess(), middleware.Payload(), step.UpdateStep) + s.DELETE("", perm.MustPlatformAdmin(), step.DeleteStep) // Log endpoints - LogStepHandlers(step) + LogStepHandlers(s) } // end of step endpoints } // end of steps endpoints } From 1884ca00af1c52452991d33f0aab467690fabbc1 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 22 May 2023 09:48:13 -0500 Subject: [PATCH 13/53] refactor(api): move auth logic to separate package (#853) * add api/auth package * move api/login.go -> api/auth/login.go * move api/logout.go -> api/auth/logout.go * move api/token.go -> api/auth/token.go * move api/authenticate.go -> api/auth/authenticate.go * move api/auth/get * minimize * move api/auth/redirect * minimize * move api/auth/post_token * minimize * move api/auth/refresh * minimize * move api/auth/validate * minimize * update copyright * rename some auth/api funcs for clarity * Update api/auth/validate.go Co-authored-by: Easton Crupper <65553218+ecrupper@users.noreply.github.com> --------- Co-authored-by: Easton Crupper <65553218+ecrupper@users.noreply.github.com> --- api/auth/doc.go | 10 + api/auth/get_token.go | 168 +++++++++++++++ api/{ => auth}/login.go | 4 +- api/{ => auth}/logout.go | 4 +- api/auth/post_token.go | 92 +++++++++ api/auth/redirect.go | 102 +++++++++ api/{token.go => auth/refresh.go} | 45 +--- api/auth/validate.go | 50 +++++ api/authenticate.go | 332 ------------------------------ router/router.go | 20 +- 10 files changed, 439 insertions(+), 388 deletions(-) create mode 100644 api/auth/doc.go create mode 100644 api/auth/get_token.go rename api/{ => auth}/login.go (96%) rename api/{ => auth}/logout.go (96%) create mode 100644 api/auth/post_token.go create mode 100644 api/auth/redirect.go rename api/{token.go => auth/refresh.go} (62%) create mode 100644 api/auth/validate.go delete mode 100644 api/authenticate.go diff --git a/api/auth/doc.go b/api/auth/doc.go new file mode 100644 index 000000000..c200b5faf --- /dev/null +++ b/api/auth/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package auth provides the auth handlers (authenticate, login, ...) for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/auth" +package auth diff --git a/api/auth/get_token.go b/api/auth/get_token.go new file mode 100644 index 000000000..29829f072 --- /dev/null +++ b/api/auth/get_token.go @@ -0,0 +1,168 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package auth + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/internal/token" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" +) + +// swagger:operation GET /authenticate authenticate GetAuthToken +// +// Start OAuth flow or exchange tokens +// +// --- +// produces: +// - application/json +// parameters: +// - in: query +// name: code +// description: the code received after identity confirmation +// type: string +// - in: query +// name: state +// description: a random string +// type: string +// - in: query +// name: redirect_uri +// description: the url where the user will be sent after authorization +// type: string +// responses: +// '200': +// description: Successfully authenticated +// headers: +// Set-Cookie: +// type: string +// schema: +// "$ref": "#/definitions/Token" +// '307': +// description: Redirected for authentication +// '401': +// description: Unable to authenticate +// schema: +// "$ref": "#/definitions/Error" +// '503': +// description: Service unavailable +// schema: +// "$ref": "#/definitions/Error" + +// GetAuthToken represents the API handler to +// process a user logging in to Vela from +// the API or UI. +func GetAuthToken(c *gin.Context) { + var err error + + tm := c.MustGet("token-manager").(*token.Manager) + + // capture the OAuth state if present + oAuthState := c.Request.FormValue("state") + + // capture the OAuth code if present + code := c.Request.FormValue("code") + if len(code) == 0 { + // start the initial OAuth workflow + oAuthState, err = scm.FromContext(c).Login(c.Writer, c.Request) + if err != nil { + retErr := fmt.Errorf("unable to login user: %w", err) + + util.HandleError(c, http.StatusUnauthorized, retErr) + + return + } + } + + // complete the OAuth workflow and authenticates the user + newUser, err := scm.FromContext(c).Authenticate(c.Writer, c.Request, oAuthState) + if err != nil { + retErr := fmt.Errorf("unable to authenticate user: %w", err) + + util.HandleError(c, http.StatusUnauthorized, retErr) + + return + } + + // this will happen if the user is redirected by the + // source provider as part of the authorization workflow. + if newUser == nil { + return + } + + // send API call to capture the user logging in + u, err := database.FromContext(c).GetUserForName(newUser.GetName()) + // create a new user account + if len(u.GetName()) == 0 || err != nil { + // create the user account + u := new(library.User) + u.SetName(newUser.GetName()) + u.SetToken(newUser.GetToken()) + u.SetActive(true) + u.SetAdmin(false) + + // compose jwt tokens for user + rt, at, err := tm.Compose(c, u) + if err != nil { + retErr := fmt.Errorf("unable to compose token for user %s: %w", u.GetName(), err) + + util.HandleError(c, http.StatusServiceUnavailable, retErr) + + return + } + + // store the refresh token with the user object + u.SetRefreshToken(rt) + + // send API call to create the user in the database + err = database.FromContext(c).CreateUser(u) + if err != nil { + retErr := fmt.Errorf("unable to create user %s: %w", u.GetName(), err) + + util.HandleError(c, http.StatusServiceUnavailable, retErr) + + return + } + + // return the jwt access token + c.JSON(http.StatusOK, library.Token{Token: &at}) + + return + } + + // update the user account + u.SetToken(newUser.GetToken()) + u.SetActive(true) + + // compose jwt tokens for user + rt, at, err := tm.Compose(c, u) + if err != nil { + retErr := fmt.Errorf("unable to compose token for user %s: %w", u.GetName(), err) + + util.HandleError(c, http.StatusServiceUnavailable, retErr) + + return + } + + // store the refresh token with the user object + u.SetRefreshToken(rt) + + // send API call to update the user in the database + err = database.FromContext(c).UpdateUser(u) + if err != nil { + retErr := fmt.Errorf("unable to update user %s: %w", u.GetName(), err) + + util.HandleError(c, http.StatusServiceUnavailable, retErr) + + return + } + + // return the user with their jwt access token + c.JSON(http.StatusOK, library.Token{Token: &at}) +} diff --git a/api/login.go b/api/auth/login.go similarity index 96% rename from api/login.go rename to api/auth/login.go index 74ac91226..36b977009 100644 --- a/api/login.go +++ b/api/auth/login.go @@ -1,8 +1,8 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. -package api +package auth import ( "fmt" diff --git a/api/logout.go b/api/auth/logout.go similarity index 96% rename from api/logout.go rename to api/auth/logout.go index 5d5d39556..8accdf42e 100644 --- a/api/logout.go +++ b/api/auth/logout.go @@ -1,8 +1,8 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. -package api +package auth import ( "fmt" diff --git a/api/auth/post_token.go b/api/auth/post_token.go new file mode 100644 index 000000000..8b386bfcf --- /dev/null +++ b/api/auth/post_token.go @@ -0,0 +1,92 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package auth + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/internal/token" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" +) + +// swagger:operation POST /authenticate/token authenticate PostAuthToken +// +// Authenticate to Vela via personal access token +// +// --- +// produces: +// - application/json +// parameters: +// - in: header +// name: Token +// type: string +// required: true +// description: > +// scopes: repo, repo:status, user:email, read:user, and read:org +// responses: +// '200': +// description: Successfully authenticated +// schema: +// "$ref": "#/definitions/Token" +// '401': +// description: Unable to authenticate +// schema: +// "$ref": "#/definitions/Error" +// '503': +// description: Service unavailable +// schema: +// "$ref": "#/definitions/Error" + +// PostAuthToken represents the API handler to +// process a user logging in using PAT to Vela from +// the API. +func PostAuthToken(c *gin.Context) { + // attempt to get user from source + u, err := scm.FromContext(c).AuthenticateToken(c.Request) + if err != nil { + retErr := fmt.Errorf("unable to authenticate user: %w", err) + + util.HandleError(c, http.StatusUnauthorized, retErr) + + return + } + + // check if the user exists + u, err = database.FromContext(c).GetUserForName(u.GetName()) + if err != nil { + retErr := fmt.Errorf("user %s not found", u.GetName()) + + util.HandleError(c, http.StatusUnauthorized, retErr) + + return + } + + // We don't need refresh token for this scenario + // We only need access token and are configured based on the config defined + tm := c.MustGet("token-manager").(*token.Manager) + + // mint token options for access token + amto := &token.MintTokenOpts{ + User: u, + TokenType: constants.UserAccessTokenType, + TokenDuration: tm.UserAccessTokenDuration, + } + at, err := tm.MintToken(amto) + + if err != nil { + retErr := fmt.Errorf("unable to compose token for user %s: %w", u.GetName(), err) + + util.HandleError(c, http.StatusServiceUnavailable, retErr) + } + + // return the user with their jwt access token + c.JSON(http.StatusOK, library.Token{Token: &at}) +} diff --git a/api/auth/redirect.go b/api/auth/redirect.go new file mode 100644 index 000000000..32b486393 --- /dev/null +++ b/api/auth/redirect.go @@ -0,0 +1,102 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package auth + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/util" + "github.com/go-vela/types" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /authenticate/web authenticate GetAuthenticateTypeWeb +// +// Authentication entrypoint that builds the right post-auth +// redirect URL for web authentication requests +// and redirects to /authenticate after +// +// --- +// produces: +// - application/json +// parameters: +// - in: query +// name: code +// description: the code received after identity confirmation +// type: string +// - in: query +// name: state +// description: a random string +// type: string +// responses: +// '307': +// description: Redirected for authentication + +// swagger:operation GET /authenticate/cli/{port} authenticate GetAuthenticateTypeCLI +// +// Authentication entrypoint that builds the right post-auth +// redirect URL for CLI authentication requests +// and redirects to /authenticate after +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: port +// required: true +// description: the port number +// type: integer +// - in: query +// name: code +// description: the code received after identity confirmation +// type: string +// - in: query +// name: state +// description: a random string +// type: string +// responses: +// '307': +// description: Redirected for authentication + +// GetAuthRedirect handles cases where the OAuth callback was +// overridden by supplying a redirect_uri in the login process. +// It will send the user to the destination to handle the last leg +// in the auth flow - exchanging "code" and "state" for a token. +// This will only handle non-headless flows (ie. web or cli). +func GetAuthRedirect(c *gin.Context) { + // load the metadata + m := c.MustGet("metadata").(*types.Metadata) + + logrus.Info("redirecting for final auth flow destination") + + // capture the path elements + t := util.PathParameter(c, "type") + p := util.PathParameter(c, "port") + + // capture the current query parameters - + // they should contain the "code" and "state" values + q := c.Request.URL.Query() + + // default redirect location if a user ended up here + // by providing an unsupported type + r := fmt.Sprintf("%s/authenticate", m.Vela.Address) + + switch t { + // cli auth flow + case "cli": + r = fmt.Sprintf("http://127.0.0.1:%s", p) + // web auth flow + case "web": + r = fmt.Sprintf("%s%s", m.Vela.WebAddress, m.Vela.WebOauthCallbackPath) + } + + // append the code and state values + r = fmt.Sprintf("%s?%s", r, q.Encode()) + + c.Redirect(http.StatusTemporaryRedirect, r) +} diff --git a/api/token.go b/api/auth/refresh.go similarity index 62% rename from api/token.go rename to api/auth/refresh.go index 6c76439ec..0c12425cc 100644 --- a/api/token.go +++ b/api/auth/refresh.go @@ -1,22 +1,18 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. -package api +package auth import ( "fmt" "net/http" - "strings" + "github.com/gin-gonic/gin" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/router/middleware/auth" - "github.com/go-vela/server/router/middleware/claims" "github.com/go-vela/server/util" - "github.com/go-vela/types/library" - - "github.com/gin-gonic/gin" ) // swagger:operation GET /token-refresh authenticate GetRefreshAccessToken @@ -67,38 +63,3 @@ func RefreshAccessToken(c *gin.Context) { c.JSON(http.StatusOK, library.Token{Token: &newAccessToken}) } - -// swagger:operation GET /validate-token authenticate ValidateServerToken -// -// Validate a server token -// -// --- -// produces: -// - application/json -// security: -// - CookieAuth: [] -// responses: -// '200': -// description: Successfully validated a token -// schema: -// type: string -// '401': -// description: Unauthorized -// schema: -// "$ref": "#/definitions/Error" - -// ValidateServerToken will return the claims of a valid server token -// if it is provided in the auth header. -func ValidateServerToken(c *gin.Context) { - cl := claims.Retrieve(c) - - if !strings.EqualFold(cl.Subject, "vela-server") { - retErr := fmt.Errorf("token is not a valid server token") - - util.HandleError(c, http.StatusUnauthorized, retErr) - - return - } - - c.JSON(http.StatusOK, "valid server token") -} diff --git a/api/auth/validate.go b/api/auth/validate.go new file mode 100644 index 000000000..d78057193 --- /dev/null +++ b/api/auth/validate.go @@ -0,0 +1,50 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package auth + +import ( + "fmt" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/router/middleware/claims" + "github.com/go-vela/server/util" +) + +// swagger:operation GET /validate-token authenticate ValidateServerToken +// +// Validate a server token +// +// --- +// produces: +// - application/json +// security: +// - CookieAuth: [] +// responses: +// '200': +// description: Successfully validated a token +// schema: +// type: string +// '401': +// description: Unauthorized +// schema: +// "$ref": "#/definitions/Error" + +// ValidateServerToken will validate if a token was issued by the server +// if it is provided in the auth header. +func ValidateServerToken(c *gin.Context) { + cl := claims.Retrieve(c) + + if !strings.EqualFold(cl.Subject, "vela-server") { + retErr := fmt.Errorf("token is not a valid server token") + + util.HandleError(c, http.StatusUnauthorized, retErr) + + return + } + + c.JSON(http.StatusOK, "valid server token") +} diff --git a/api/authenticate.go b/api/authenticate.go deleted file mode 100644 index 6d89d7611..000000000 --- a/api/authenticate.go +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package api - -import ( - "fmt" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/go-vela/server/database" - "github.com/go-vela/server/internal/token" - "github.com/go-vela/server/scm" - "github.com/go-vela/server/util" - "github.com/go-vela/types" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" -) - -// swagger:operation GET /authenticate authenticate GetAuthenticate -// -// Start OAuth flow or exchange tokens -// -// --- -// produces: -// - application/json -// parameters: -// - in: query -// name: code -// description: the code received after identity confirmation -// type: string -// - in: query -// name: state -// description: a random string -// type: string -// - in: query -// name: redirect_uri -// description: the url where the user will be sent after authorization -// type: string -// responses: -// '200': -// description: Successfully authenticated -// headers: -// Set-Cookie: -// type: string -// schema: -// "$ref": "#/definitions/Token" -// '307': -// description: Redirected for authentication -// '401': -// description: Unable to authenticate -// schema: -// "$ref": "#/definitions/Error" -// '503': -// description: Service unavailable -// schema: -// "$ref": "#/definitions/Error" - -// Authenticate represents the API handler to -// process a user logging in to Vela from -// the API or UI. -func Authenticate(c *gin.Context) { - var err error - - tm := c.MustGet("token-manager").(*token.Manager) - - // capture the OAuth state if present - oAuthState := c.Request.FormValue("state") - - // capture the OAuth code if present - code := c.Request.FormValue("code") - if len(code) == 0 { - // start the initial OAuth workflow - oAuthState, err = scm.FromContext(c).Login(c.Writer, c.Request) - if err != nil { - retErr := fmt.Errorf("unable to login user: %w", err) - - util.HandleError(c, http.StatusUnauthorized, retErr) - - return - } - } - - // complete the OAuth workflow and authenticates the user - newUser, err := scm.FromContext(c).Authenticate(c.Writer, c.Request, oAuthState) - if err != nil { - retErr := fmt.Errorf("unable to authenticate user: %w", err) - - util.HandleError(c, http.StatusUnauthorized, retErr) - - return - } - - // this will happen if the user is redirected by the - // source provider as part of the authorization workflow. - if newUser == nil { - return - } - - // send API call to capture the user logging in - u, err := database.FromContext(c).GetUserForName(newUser.GetName()) - // create a new user account - if len(u.GetName()) == 0 || err != nil { - // create the user account - u := new(library.User) - u.SetName(newUser.GetName()) - u.SetToken(newUser.GetToken()) - u.SetActive(true) - u.SetAdmin(false) - - // compose jwt tokens for user - rt, at, err := tm.Compose(c, u) - if err != nil { - retErr := fmt.Errorf("unable to compose token for user %s: %w", u.GetName(), err) - - util.HandleError(c, http.StatusServiceUnavailable, retErr) - - return - } - - // store the refresh token with the user object - u.SetRefreshToken(rt) - - // send API call to create the user in the database - err = database.FromContext(c).CreateUser(u) - if err != nil { - retErr := fmt.Errorf("unable to create user %s: %w", u.GetName(), err) - - util.HandleError(c, http.StatusServiceUnavailable, retErr) - - return - } - - // return the jwt access token - c.JSON(http.StatusOK, library.Token{Token: &at}) - - return - } - - // update the user account - u.SetToken(newUser.GetToken()) - u.SetActive(true) - - // compose jwt tokens for user - rt, at, err := tm.Compose(c, u) - if err != nil { - retErr := fmt.Errorf("unable to compose token for user %s: %w", u.GetName(), err) - - util.HandleError(c, http.StatusServiceUnavailable, retErr) - - return - } - - // store the refresh token with the user object - u.SetRefreshToken(rt) - - // send API call to update the user in the database - err = database.FromContext(c).UpdateUser(u) - if err != nil { - retErr := fmt.Errorf("unable to update user %s: %w", u.GetName(), err) - - util.HandleError(c, http.StatusServiceUnavailable, retErr) - - return - } - - // return the user with their jwt access token - c.JSON(http.StatusOK, library.Token{Token: &at}) -} - -// swagger:operation GET /authenticate/web authenticate GetAuthenticateTypeWeb -// -// Authentication entrypoint that builds the right post-auth -// redirect URL for web authentication requests -// and redirects to /authenticate after -// -// --- -// produces: -// - application/json -// parameters: -// - in: query -// name: code -// description: the code received after identity confirmation -// type: string -// - in: query -// name: state -// description: a random string -// type: string -// responses: -// '307': -// description: Redirected for authentication - -// swagger:operation GET /authenticate/cli/{port} authenticate GetAuthenticateTypeCLI -// -// Authentication entrypoint that builds the right post-auth -// redirect URL for CLI authentication requests -// and redirects to /authenticate after -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: port -// required: true -// description: the port number -// type: integer -// - in: query -// name: code -// description: the code received after identity confirmation -// type: string -// - in: query -// name: state -// description: a random string -// type: string -// responses: -// '307': -// description: Redirected for authentication - -// AuthenticateType handles cases where the OAuth callback was -// overridden by supplying a redirect_uri in the login process. -// It will send the user to the destination to handle the last leg -// in the auth flow - exchanging "code" and "state" for a token. -// This will only handle non-headless flows (ie. web or cli). -func AuthenticateType(c *gin.Context) { - // load the metadata - m := c.MustGet("metadata").(*types.Metadata) - - logrus.Info("redirecting for final auth flow destination") - - // capture the path elements - t := util.PathParameter(c, "type") - p := util.PathParameter(c, "port") - - // capture the current query parameters - - // they should contain the "code" and "state" values - q := c.Request.URL.Query() - - // default redirect location if a user ended up here - // by providing an unsupported type - r := fmt.Sprintf("%s/authenticate", m.Vela.Address) - - switch t { - // cli auth flow - case "cli": - r = fmt.Sprintf("http://127.0.0.1:%s", p) - // web auth flow - case "web": - r = fmt.Sprintf("%s%s", m.Vela.WebAddress, m.Vela.WebOauthCallbackPath) - } - - // append the code and state values - r = fmt.Sprintf("%s?%s", r, q.Encode()) - - c.Redirect(http.StatusTemporaryRedirect, r) -} - -// swagger:operation POST /authenticate/token authenticate PostAuthenticateToken -// -// Authenticate to Vela via personal access token -// -// --- -// produces: -// - application/json -// parameters: -// - in: header -// name: Token -// type: string -// required: true -// description: > -// scopes: repo, repo:status, user:email, read:user, and read:org -// responses: -// '200': -// description: Successfully authenticated -// schema: -// "$ref": "#/definitions/Token" -// '401': -// description: Unable to authenticate -// schema: -// "$ref": "#/definitions/Error" -// '503': -// description: Service unavailable -// schema: -// "$ref": "#/definitions/Error" - -// AuthenticateToken represents the API handler to -// process a user logging in using PAT to Vela from -// the API. -func AuthenticateToken(c *gin.Context) { - // attempt to get user from source - u, err := scm.FromContext(c).AuthenticateToken(c.Request) - if err != nil { - retErr := fmt.Errorf("unable to authenticate user: %w", err) - - util.HandleError(c, http.StatusUnauthorized, retErr) - - return - } - - // check if the user exists - u, err = database.FromContext(c).GetUserForName(u.GetName()) - if err != nil { - retErr := fmt.Errorf("user %s not found", u.GetName()) - - util.HandleError(c, http.StatusUnauthorized, retErr) - - return - } - - // We don't need refresh token for this scenario - // We only need access token and are configured based on the config defined - tm := c.MustGet("token-manager").(*token.Manager) - - // mint token options for access token - amto := &token.MintTokenOpts{ - User: u, - TokenType: constants.UserAccessTokenType, - TokenDuration: tm.UserAccessTokenDuration, - } - at, err := tm.MintToken(amto) - - if err != nil { - retErr := fmt.Errorf("unable to compose token for user %s: %w", u.GetName(), err) - - util.HandleError(c, http.StatusServiceUnavailable, retErr) - } - - // return the user with their jwt access token - c.JSON(http.StatusOK, library.Token{Token: &at}) -} diff --git a/router/router.go b/router/router.go index d1e88273a..f756f9f4e 100644 --- a/router/router.go +++ b/router/router.go @@ -32,14 +32,14 @@ package router import ( + "github.com/gin-gonic/gin" "github.com/go-vela/server/api" + "github.com/go-vela/server/api/auth" "github.com/go-vela/server/router/middleware" "github.com/go-vela/server/router/middleware/claims" "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/user" - - "github.com/gin-gonic/gin" ) const ( @@ -66,19 +66,19 @@ func Load(options ...gin.HandlerFunc) *gin.Engine { r.GET("/health", api.Health) // Login endpoint - r.GET("/login", api.Login) + r.GET("/login", auth.Login) // Logout endpoint - r.GET("/logout", user.Establish(), api.Logout) + r.GET("/logout", user.Establish(), auth.Logout) // Refresh Access Token endpoint - r.GET("/token-refresh", api.RefreshAccessToken) + r.GET("/token-refresh", auth.RefreshAccessToken) // Metric endpoint r.GET("/metrics", api.CustomMetrics, gin.WrapH(api.BaseMetrics())) // Validate Server Token endpoint - r.GET("/validate-token", claims.Establish(), api.ValidateServerToken) + r.GET("/validate-token", claims.Establish(), auth.ValidateServerToken) // Version endpoint r.GET("/version", api.Version) @@ -89,10 +89,10 @@ func Load(options ...gin.HandlerFunc) *gin.Engine { // Authentication endpoints authenticate := r.Group("/authenticate") { - authenticate.GET("", api.Authenticate) - authenticate.GET("/:type", api.AuthenticateType) - authenticate.GET("/:type/:port", api.AuthenticateType) - authenticate.POST("/token", api.AuthenticateToken) + authenticate.GET("", auth.GetAuthToken) + authenticate.GET("/:type", auth.GetAuthRedirect) + authenticate.GET("/:type/:port", auth.GetAuthRedirect) + authenticate.POST("/token", auth.PostAuthToken) } // API endpoints From c1a661a626a20cd420b7166d47c638d8ca948dea Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 22 May 2023 09:07:31 -0600 Subject: [PATCH 14/53] refactor(api): move log logic into separate package (#855) * refactor(api): move log logic into separate package * service number, import alpha, listing over reading --- api/log.go | 925 -------------------------------------- api/log/create_service.go | 129 ++++++ api/log/create_step.go | 129 ++++++ api/log/delete_service.go | 107 +++++ api/log/delete_step.go | 107 +++++ api/log/doc.go | 10 + api/log/get_service.go | 97 ++++ api/log/get_step.go | 98 ++++ api/log/list_build.go | 134 ++++++ api/log/update_service.go | 140 ++++++ api/log/update_step.go | 140 ++++++ router/build.go | 3 +- router/log.go | 18 +- 13 files changed, 1102 insertions(+), 935 deletions(-) delete mode 100644 api/log.go create mode 100644 api/log/create_service.go create mode 100644 api/log/create_step.go create mode 100644 api/log/delete_service.go create mode 100644 api/log/delete_step.go create mode 100644 api/log/doc.go create mode 100644 api/log/get_service.go create mode 100644 api/log/get_step.go create mode 100644 api/log/list_build.go create mode 100644 api/log/update_service.go create mode 100644 api/log/update_step.go diff --git a/api/log.go b/api/log.go deleted file mode 100644 index 9cb02b0d9..000000000 --- a/api/log.go +++ /dev/null @@ -1,925 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package api - -import ( - "fmt" - "net/http" - "strconv" - - "github.com/go-vela/server/database" - "github.com/go-vela/server/router/middleware/build" - "github.com/go-vela/server/router/middleware/org" - "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/service" - "github.com/go-vela/server/router/middleware/step" - "github.com/go-vela/server/router/middleware/user" - "github.com/go-vela/server/util" - - "github.com/go-vela/types/library" - - "github.com/gin-gonic/gin" - "github.com/sirupsen/logrus" -) - -// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/logs builds GetBuildLogs -// -// Get logs for a build in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: query -// name: page -// description: The page of results to retrieve -// type: integer -// default: 1 -// - in: query -// name: per_page -// description: How many results per page to return -// type: integer -// maximum: 100 -// default: 10 -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved logs for the build -// schema: -// type: array -// items: -// "$ref": "#/definitions/Log" -// '500': -// description: Unable to retrieve logs for the build -// schema: -// "$ref": "#/definitions/Error" - -// GetBuildLogs represents the API handler to capture a -// list of logs for a build from the configured backend. -func GetBuildLogs(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("reading logs for build %s", entry) - - // capture page query parameter if present - page, err := strconv.Atoi(c.DefaultQuery("page", "1")) - if err != nil { - //nolint:lll // ignore long line length due to error message - retErr := fmt.Errorf("unable to convert page query parameter for build %s: %w", entry, err) - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // capture per_page query parameter if present - perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) - if err != nil { - retErr := fmt.Errorf("unable to convert per_page query parameter for build %s: %w", entry, err) - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // ensure per_page isn't above or below allowed values - perPage = util.MaxInt(1, util.MinInt(100, perPage)) - - // send API call to capture the list of logs for the build - l, t, err := database.FromContext(c).ListLogsForBuild(b, page, perPage) - if err != nil { - retErr := fmt.Errorf("unable to get logs for build %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // create pagination object - pagination := Pagination{ - Page: page, - PerPage: perPage, - Total: t, - } - // set pagination headers - pagination.SetHeaderLink(c) - - c.JSON(http.StatusOK, l) -} - -// -// swagger:operation POST /api/v1/repos/{org}/{repo}/builds/{build}/services/{service}/logs services CreateServiceLogs -// -// Create the logs for a service -// -// --- -// deprecated: true -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: service -// description: ID of the service -// required: true -// type: integer -// - in: body -// name: body -// description: Payload containing the log to create -// required: true -// schema: -// "$ref": "#/definitions/Log" -// security: -// - ApiKeyAuth: [] -// responses: -// '201': -// description: Successfully created the service logs -// schema: -// "$ref": "#/definitions/Log" -// '400': -// description: Unable to create the service logs -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to create the service logs -// schema: -// "$ref": "#/definitions/Error" - -// CreateServiceLog represents the API handler to create -// the logs for a service in the configured backend. -// -//nolint:dupl // ignore similar code with step -func CreateServiceLog(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := service.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "service": s.GetNumber(), - "user": u.GetName(), - }).Infof("creating logs for service %s", entry) - - // capture body from API request - input := new(library.Log) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for service %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update fields in log object - input.SetServiceID(s.GetID()) - input.SetBuildID(b.GetID()) - input.SetRepoID(r.GetID()) - - // send API call to create the logs - err = database.FromContext(c).CreateLog(input) - if err != nil { - retErr := fmt.Errorf("unable to create logs for service %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the created log - l, _ := database.FromContext(c).GetLogForService(s) - - c.JSON(http.StatusCreated, l) -} - -// -// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/services/{service}/logs services GetServiceLogs -// -// Retrieve the logs for a service -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: service -// description: ID of the service -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the service logs -// schema: -// "$ref": "#/definitions/Log" -// '500': -// description: Unable to retrieve the service logs -// schema: -// "$ref": "#/definitions/Error" - -// GetServiceLog represents the API handler to capture -// the logs for a service from the configured backend. -func GetServiceLog(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := service.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "service": s.GetNumber(), - "user": u.GetName(), - }).Infof("reading logs for service %s", entry) - - // send API call to capture the service logs - l, err := database.FromContext(c).GetLogForService(s) - if err != nil { - retErr := fmt.Errorf("unable to get logs for service %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, l) -} - -// -// swagger:operation PUT /api/v1/repos/{org}/{repo}/builds/{build}/services/{service}/logs services UpdateServiceLog -// -// Update the logs for a service -// -// --- -// deprecated: true -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: service -// description: ID of the service -// required: true -// type: integer -// - in: body -// name: body -// description: Payload containing the log to update -// required: true -// schema: -// "$ref": "#/definitions/Log" -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully updated the service logs -// schema: -// "$ref": "#/definitions/Log" -// '400': -// description: Unable to updated the service logs -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to updates the service logs -// schema: -// "$ref": "#/definitions/Error" - -// UpdateServiceLog represents the API handler to update -// the logs for a service in the configured backend. -func UpdateServiceLog(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := service.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "service": s.GetNumber(), - "user": u.GetName(), - }).Infof("updating logs for service %s", entry) - - // send API call to capture the service logs - l, err := database.FromContext(c).GetLogForService(s) - if err != nil { - retErr := fmt.Errorf("unable to get logs for service %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // capture body from API request - input := new(library.Log) - - err = c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for service %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update log fields if provided - if len(input.GetData()) > 0 { - // update data if set - l.SetData(input.GetData()) - } - - // send API call to update the log - err = database.FromContext(c).UpdateLog(l) - if err != nil { - retErr := fmt.Errorf("unable to update logs for service %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the updated log - l, _ = database.FromContext(c).GetLogForService(s) - - c.JSON(http.StatusOK, l) -} - -// -// swagger:operation DELETE /api/v1/repos/{org}/{repo}/builds/{build}/services/{service}/logs services DeleteServiceLogs -// -// Delete the logs for a service -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: service -// description: ID of the service -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully deleted the service logs -// schema: -// type: string -// '500': -// description: Unable to delete the service logs -// schema: -// "$ref": "#/definitions/Error" - -// DeleteServiceLog represents the API handler to remove -// the logs for a service from the configured backend. -func DeleteServiceLog(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := service.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "service": s.GetNumber(), - "user": u.GetName(), - }).Infof("deleting logs for service %s", entry) - - // send API call to capture the service logs - l, err := database.FromContext(c).GetLogForService(s) - if err != nil { - retErr := fmt.Errorf("unable to get logs for service %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to remove the log - err = database.FromContext(c).DeleteLog(l) - if err != nil { - retErr := fmt.Errorf("unable to delete logs for service %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, fmt.Sprintf("logs deleted for service %s", entry)) -} - -// -// swagger:operation POST /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step}/logs steps CreateStepLog -// -// Create the logs for a step -// -// --- -// deprecated: true -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: step -// description: Step number -// required: true -// type: integer -// - in: body -// name: body -// description: Payload containing the log to create -// required: true -// schema: -// "$ref": "#/definitions/Log" -// security: -// - ApiKeyAuth: [] -// responses: -// '201': -// description: Successfully created the logs for step -// schema: -// "$ref": "#/definitions/Log" -// '400': -// description: Unable to create the logs for a step -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to create the logs for a step -// schema: -// "$ref": "#/definitions/Error" - -// CreateStepLog represents the API handler to create -// the logs for a step in the configured backend. -// -//nolint:dupl // ignore similar code with service -func CreateStepLog(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := step.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "step": s.GetNumber(), - "user": u.GetName(), - }).Infof("creating logs for step %s", entry) - - // capture body from API request - input := new(library.Log) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for step %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update fields in log object - input.SetStepID(s.GetID()) - input.SetBuildID(b.GetID()) - input.SetRepoID(r.GetID()) - - // send API call to create the logs - err = database.FromContext(c).CreateLog(input) - if err != nil { - retErr := fmt.Errorf("unable to create logs for step %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the created log - l, _ := database.FromContext(c).GetLogForStep(s) - - c.JSON(http.StatusCreated, l) -} - -// -// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step}/logs steps GetStepLog -// -// Retrieve the logs for a step -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: step -// description: Step number -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the logs for step -// type: json -// schema: -// "$ref": "#/definitions/Log" -// '500': -// description: Unable to retrieve the logs for a step -// schema: -// "$ref": "#/definitions/Error" - -// GetStepLog represents the API handler to capture -// the logs for a step from the configured backend. -func GetStepLog(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := step.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "step": s.GetNumber(), - "user": u.GetName(), - }).Infof("reading logs for step %s", entry) - - // send API call to capture the step logs - l, err := database.FromContext(c).GetLogForStep(s) - if err != nil { - retErr := fmt.Errorf("unable to get logs for step %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, l) -} - -// -// swagger:operation PUT /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step}/logs steps UpdateStepLog -// -// Update the logs for a step -// -// --- -// deprecated: true -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: step -// description: Step number -// required: true -// type: integer -// - in: body -// name: body -// description: Payload containing the log to update -// required: true -// schema: -// "$ref": "#/definitions/Log" -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully updated the logs for step -// schema: -// "$ref": "#/definitions/Log" -// '400': -// description: Unable to update the logs for a step -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to update the logs for a step -// schema: -// "$ref": "#/definitions/Error" - -// UpdateStepLog represents the API handler to update -// the logs for a step in the configured backend. -func UpdateStepLog(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := step.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "step": s.GetNumber(), - "user": u.GetName(), - }).Infof("updating logs for step %s", entry) - - // send API call to capture the step logs - l, err := database.FromContext(c).GetLogForStep(s) - if err != nil { - retErr := fmt.Errorf("unable to get logs for step %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // capture body from API request - input := new(library.Log) - - err = c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for step %s: %w", entry, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update log fields if provided - if len(input.GetData()) > 0 { - // update data if set - l.SetData(input.GetData()) - } - - // send API call to update the log - err = database.FromContext(c).UpdateLog(l) - if err != nil { - retErr := fmt.Errorf("unable to update logs for step %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the updated log - l, _ = database.FromContext(c).GetLogForStep(s) - - c.JSON(http.StatusOK, l) -} - -// -// swagger:operation DELETE /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step}/logs steps DeleteStepLog -// -// Delete the logs for a step -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// - in: path -// name: step -// description: Step number -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully deleted the logs for the step -// schema: -// type: string -// '500': -// description: Unable to delete the logs for the step -// schema: -// "$ref": "#/definitions/Error" - -// DeleteStepLog represents the API handler to remove -// the logs for a step from the configured backend. -func DeleteStepLog(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - s := step.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "step": s.GetNumber(), - "user": u.GetName(), - }).Infof("deleting logs for step %s", entry) - - // send API call to capture the step logs - l, err := database.FromContext(c).GetLogForStep(s) - if err != nil { - retErr := fmt.Errorf("unable to get logs for step %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to remove the log - err = database.FromContext(c).DeleteLog(l) - if err != nil { - retErr := fmt.Errorf("unable to delete logs for step %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, fmt.Sprintf("logs deleted for step %s", entry)) -} diff --git a/api/log/create_service.go b/api/log/create_service.go new file mode 100644 index 000000000..dd559073c --- /dev/null +++ b/api/log/create_service.go @@ -0,0 +1,129 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code to step +package log + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/service" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation POST /api/v1/repos/{org}/{repo}/builds/{build}/services/{service}/logs services CreateServiceLog +// +// Create the logs for a service +// +// --- +// deprecated: true +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: service +// description: Service number +// required: true +// type: integer +// - in: body +// name: body +// description: Payload containing the log to create +// required: true +// schema: +// "$ref": "#/definitions/Log" +// security: +// - ApiKeyAuth: [] +// responses: +// '201': +// description: Successfully created the service logs +// schema: +// "$ref": "#/definitions/Log" +// '400': +// description: Unable to create the service logs +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to create the service logs +// schema: +// "$ref": "#/definitions/Error" + +// CreateServiceLog represents the API handler to create +// the logs for a service in the configured backend. +func CreateServiceLog(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := service.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "service": s.GetNumber(), + "user": u.GetName(), + }).Infof("creating logs for service %s", entry) + + // capture body from API request + input := new(library.Log) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for service %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update fields in log object + input.SetServiceID(s.GetID()) + input.SetBuildID(b.GetID()) + input.SetRepoID(r.GetID()) + + // send API call to create the logs + err = database.FromContext(c).CreateLog(input) + if err != nil { + retErr := fmt.Errorf("unable to create logs for service %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the created log + l, _ := database.FromContext(c).GetLogForService(s) + + c.JSON(http.StatusCreated, l) +} diff --git a/api/log/create_step.go b/api/log/create_step.go new file mode 100644 index 000000000..0f8a1ca5a --- /dev/null +++ b/api/log/create_step.go @@ -0,0 +1,129 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code to service +package log + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/step" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation POST /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step}/logs steps CreateStepLog +// +// Create the logs for a step +// +// --- +// deprecated: true +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: step +// description: Step number +// required: true +// type: integer +// - in: body +// name: body +// description: Payload containing the log to create +// required: true +// schema: +// "$ref": "#/definitions/Log" +// security: +// - ApiKeyAuth: [] +// responses: +// '201': +// description: Successfully created the logs for step +// schema: +// "$ref": "#/definitions/Log" +// '400': +// description: Unable to create the logs for a step +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to create the logs for a step +// schema: +// "$ref": "#/definitions/Error" + +// CreateStepLog represents the API handler to create +// the logs for a step in the configured backend. +func CreateStepLog(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := step.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "step": s.GetNumber(), + "user": u.GetName(), + }).Infof("creating logs for step %s", entry) + + // capture body from API request + input := new(library.Log) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for step %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update fields in log object + input.SetStepID(s.GetID()) + input.SetBuildID(b.GetID()) + input.SetRepoID(r.GetID()) + + // send API call to create the logs + err = database.FromContext(c).CreateLog(input) + if err != nil { + retErr := fmt.Errorf("unable to create logs for step %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the created log + l, _ := database.FromContext(c).GetLogForStep(s) + + c.JSON(http.StatusCreated, l) +} diff --git a/api/log/delete_service.go b/api/log/delete_service.go new file mode 100644 index 000000000..c87d87b1b --- /dev/null +++ b/api/log/delete_service.go @@ -0,0 +1,107 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with step +package log + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/service" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation DELETE /api/v1/repos/{org}/{repo}/builds/{build}/services/{service}/logs services DeleteServiceLog +// +// Delete the logs for a service +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: service +// description: Service number +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully deleted the service logs +// schema: +// type: string +// '500': +// description: Unable to delete the service logs +// schema: +// "$ref": "#/definitions/Error" + +// DeleteServiceLog represents the API handler to remove +// the logs for a service from the configured backend. +func DeleteServiceLog(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := service.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "service": s.GetNumber(), + "user": u.GetName(), + }).Infof("deleting logs for service %s", entry) + + // send API call to capture the service logs + l, err := database.FromContext(c).GetLogForService(s) + if err != nil { + retErr := fmt.Errorf("unable to get logs for service %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to remove the log + err = database.FromContext(c).DeleteLog(l) + if err != nil { + retErr := fmt.Errorf("unable to delete logs for service %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, fmt.Sprintf("logs deleted for service %s", entry)) +} diff --git a/api/log/delete_step.go b/api/log/delete_step.go new file mode 100644 index 000000000..90e7fc7ac --- /dev/null +++ b/api/log/delete_step.go @@ -0,0 +1,107 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with service +package log + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/step" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation DELETE /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step}/logs steps DeleteStepLog +// +// Delete the logs for a step +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: step +// description: Step number +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully deleted the logs for the step +// schema: +// type: string +// '500': +// description: Unable to delete the logs for the step +// schema: +// "$ref": "#/definitions/Error" + +// DeleteStepLog represents the API handler to remove +// the logs for a step from the configured backend. +func DeleteStepLog(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := step.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "step": s.GetNumber(), + "user": u.GetName(), + }).Infof("deleting logs for step %s", entry) + + // send API call to capture the step logs + l, err := database.FromContext(c).GetLogForStep(s) + if err != nil { + retErr := fmt.Errorf("unable to get logs for step %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to remove the log + err = database.FromContext(c).DeleteLog(l) + if err != nil { + retErr := fmt.Errorf("unable to delete logs for step %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, fmt.Sprintf("logs deleted for step %s", entry)) +} diff --git a/api/log/doc.go b/api/log/doc.go new file mode 100644 index 000000000..9f619b01c --- /dev/null +++ b/api/log/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package log provides the log handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/log" +package log diff --git a/api/log/get_service.go b/api/log/get_service.go new file mode 100644 index 000000000..2a15ad7e5 --- /dev/null +++ b/api/log/get_service.go @@ -0,0 +1,97 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with step +package log + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/service" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/services/{service}/logs services GetServiceLog +// +// Retrieve the logs for a service +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: service +// description: Service number +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the service logs +// schema: +// "$ref": "#/definitions/Log" +// '500': +// description: Unable to retrieve the service logs +// schema: +// "$ref": "#/definitions/Error" + +// GetServiceLog represents the API handler to capture +// the logs for a service from the configured backend. +func GetServiceLog(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := service.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "service": s.GetNumber(), + "user": u.GetName(), + }).Infof("reading logs for service %s", entry) + + // send API call to capture the service logs + l, err := database.FromContext(c).GetLogForService(s) + if err != nil { + retErr := fmt.Errorf("unable to get logs for service %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, l) +} diff --git a/api/log/get_step.go b/api/log/get_step.go new file mode 100644 index 000000000..39859f1dd --- /dev/null +++ b/api/log/get_step.go @@ -0,0 +1,98 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with service +package log + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/step" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step}/logs steps GetStepLog +// +// Retrieve the logs for a step +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: step +// description: Step number +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the logs for step +// type: json +// schema: +// "$ref": "#/definitions/Log" +// '500': +// description: Unable to retrieve the logs for a step +// schema: +// "$ref": "#/definitions/Error" + +// GetStepLog represents the API handler to capture +// the logs for a step from the configured backend. +func GetStepLog(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := step.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "step": s.GetNumber(), + "user": u.GetName(), + }).Infof("reading logs for step %s", entry) + + // send API call to capture the step logs + l, err := database.FromContext(c).GetLogForStep(s) + if err != nil { + retErr := fmt.Errorf("unable to get logs for step %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, l) +} diff --git a/api/log/list_build.go b/api/log/list_build.go new file mode 100644 index 000000000..fe93947d0 --- /dev/null +++ b/api/log/list_build.go @@ -0,0 +1,134 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/api" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/logs builds ListLogsForBuild +// +// List logs for a build in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: query +// name: page +// description: The page of results to retrieve +// type: integer +// default: 1 +// - in: query +// name: per_page +// description: How many results per page to return +// type: integer +// maximum: 100 +// default: 10 +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved logs for the build +// schema: +// type: array +// items: +// "$ref": "#/definitions/Log" +// '500': +// description: Unable to retrieve logs for the build +// schema: +// "$ref": "#/definitions/Error" + +// ListLogsForBuild represents the API handler to capture a +// list of logs for a build from the configured backend. +func ListLogsForBuild(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("listing logs for build %s", entry) + + // capture page query parameter if present + page, err := strconv.Atoi(c.DefaultQuery("page", "1")) + if err != nil { + //nolint:lll // ignore long line length due to error message + retErr := fmt.Errorf("unable to convert page query parameter for build %s: %w", entry, err) + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // capture per_page query parameter if present + perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) + if err != nil { + retErr := fmt.Errorf("unable to convert per_page query parameter for build %s: %w", entry, err) + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // ensure per_page isn't above or below allowed values + perPage = util.MaxInt(1, util.MinInt(100, perPage)) + + // send API call to capture the list of logs for the build + l, t, err := database.FromContext(c).ListLogsForBuild(b, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to list logs for build %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // create pagination object + pagination := api.Pagination{ + Page: page, + PerPage: perPage, + Total: t, + } + // set pagination headers + pagination.SetHeaderLink(c) + + c.JSON(http.StatusOK, l) +} diff --git a/api/log/update_service.go b/api/log/update_service.go new file mode 100644 index 000000000..746fd63e5 --- /dev/null +++ b/api/log/update_service.go @@ -0,0 +1,140 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with step +package log + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/service" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation PUT /api/v1/repos/{org}/{repo}/builds/{build}/services/{service}/logs services UpdateServiceLog +// +// Update the logs for a service +// +// --- +// deprecated: true +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: service +// description: Service number +// required: true +// type: integer +// - in: body +// name: body +// description: Payload containing the log to update +// required: true +// schema: +// "$ref": "#/definitions/Log" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully updated the service logs +// schema: +// "$ref": "#/definitions/Log" +// '400': +// description: Unable to updated the service logs +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to updates the service logs +// schema: +// "$ref": "#/definitions/Error" + +// UpdateServiceLog represents the API handler to update +// the logs for a service in the configured backend. +func UpdateServiceLog(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := service.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "service": s.GetNumber(), + "user": u.GetName(), + }).Infof("updating logs for service %s", entry) + + // send API call to capture the service logs + l, err := database.FromContext(c).GetLogForService(s) + if err != nil { + retErr := fmt.Errorf("unable to get logs for service %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // capture body from API request + input := new(library.Log) + + err = c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for service %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update log fields if provided + if len(input.GetData()) > 0 { + // update data if set + l.SetData(input.GetData()) + } + + // send API call to update the log + err = database.FromContext(c).UpdateLog(l) + if err != nil { + retErr := fmt.Errorf("unable to update logs for service %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the updated log + l, _ = database.FromContext(c).GetLogForService(s) + + c.JSON(http.StatusOK, l) +} diff --git a/api/log/update_step.go b/api/log/update_step.go new file mode 100644 index 000000000..a86a74113 --- /dev/null +++ b/api/log/update_step.go @@ -0,0 +1,140 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with service +package log + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/step" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation PUT /api/v1/repos/{org}/{repo}/builds/{build}/steps/{step}/logs steps UpdateStepLog +// +// Update the logs for a step +// +// --- +// deprecated: true +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// - in: path +// name: step +// description: Step number +// required: true +// type: integer +// - in: body +// name: body +// description: Payload containing the log to update +// required: true +// schema: +// "$ref": "#/definitions/Log" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully updated the logs for step +// schema: +// "$ref": "#/definitions/Log" +// '400': +// description: Unable to update the logs for a step +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to update the logs for a step +// schema: +// "$ref": "#/definitions/Error" + +// UpdateStepLog represents the API handler to update +// the logs for a step in the configured backend. +func UpdateStepLog(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + s := step.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d/%d", r.GetFullName(), b.GetNumber(), s.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "step": s.GetNumber(), + "user": u.GetName(), + }).Infof("updating logs for step %s", entry) + + // send API call to capture the step logs + l, err := database.FromContext(c).GetLogForStep(s) + if err != nil { + retErr := fmt.Errorf("unable to get logs for step %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // capture body from API request + input := new(library.Log) + + err = c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for step %s: %w", entry, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update log fields if provided + if len(input.GetData()) > 0 { + // update data if set + l.SetData(input.GetData()) + } + + // send API call to update the log + err = database.FromContext(c).UpdateLog(l) + if err != nil { + retErr := fmt.Errorf("unable to update logs for step %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the updated log + l, _ = database.FromContext(c).GetLogForStep(s) + + c.JSON(http.StatusOK, l) +} diff --git a/router/build.go b/router/build.go index 4c922bcb0..4aab6a9c7 100644 --- a/router/build.go +++ b/router/build.go @@ -7,6 +7,7 @@ package router import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/api" + "github.com/go-vela/server/api/log" "github.com/go-vela/server/router/middleware" "github.com/go-vela/server/router/middleware/build" "github.com/go-vela/server/router/middleware/executors" @@ -58,7 +59,7 @@ func BuildHandlers(base *gin.RouterGroup) { build.PUT("", perm.MustBuildAccess(), middleware.Payload(), api.UpdateBuild) build.DELETE("", perm.MustPlatformAdmin(), api.DeleteBuild) build.DELETE("/cancel", executors.Establish(), perm.MustWrite(), api.CancelBuild) - build.GET("/logs", perm.MustRead(), api.GetBuildLogs) + build.GET("/logs", perm.MustRead(), log.ListLogsForBuild) build.GET("/token", perm.MustWorkerAuthToken(), api.GetBuildToken) // Service endpoints diff --git a/router/log.go b/router/log.go index fc7498b48..71c6674c8 100644 --- a/router/log.go +++ b/router/log.go @@ -6,7 +6,7 @@ package router import ( "github.com/gin-gonic/gin" - "github.com/go-vela/server/api" + "github.com/go-vela/server/api/log" "github.com/go-vela/server/router/middleware/perm" ) @@ -21,10 +21,10 @@ func LogServiceHandlers(base *gin.RouterGroup) { // Logs endpoints logs := base.Group("/logs") { - logs.POST("", perm.MustAdmin(), api.CreateServiceLog) - logs.GET("", perm.MustRead(), api.GetServiceLog) - logs.PUT("", perm.MustBuildAccess(), api.UpdateServiceLog) - logs.DELETE("", perm.MustPlatformAdmin(), api.DeleteServiceLog) + logs.POST("", perm.MustAdmin(), log.CreateServiceLog) + logs.GET("", perm.MustRead(), log.GetServiceLog) + logs.PUT("", perm.MustBuildAccess(), log.UpdateServiceLog) + logs.DELETE("", perm.MustPlatformAdmin(), log.DeleteServiceLog) } // end of logs endpoints } @@ -39,9 +39,9 @@ func LogStepHandlers(base *gin.RouterGroup) { // Logs endpoints logs := base.Group("/logs") { - logs.POST("", perm.MustAdmin(), api.CreateStepLog) - logs.GET("", perm.MustRead(), api.GetStepLog) - logs.PUT("", perm.MustBuildAccess(), api.UpdateStepLog) - logs.DELETE("", perm.MustPlatformAdmin(), api.DeleteStepLog) + logs.POST("", perm.MustAdmin(), log.CreateStepLog) + logs.GET("", perm.MustRead(), log.GetStepLog) + logs.PUT("", perm.MustBuildAccess(), log.UpdateStepLog) + logs.DELETE("", perm.MustPlatformAdmin(), log.DeleteStepLog) } // end of logs endpoints } From 4ffd2f1b7ea7bfe99dd6c1c6fb87985701aa2503 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 22 May 2023 10:16:02 -0600 Subject: [PATCH 15/53] fix(router): establish claims before user on logout (#857) * fix(router): establish claims before user on logout * add a basic nil check in user establish --- router/middleware/user/user.go | 4 ++-- router/router.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/router/middleware/user/user.go b/router/middleware/user/user.go index 1dd894a10..ad94a5e0d 100644 --- a/router/middleware/user/user.go +++ b/router/middleware/user/user.go @@ -29,8 +29,8 @@ func Establish() gin.HandlerFunc { return func(c *gin.Context) { cl := claims.Retrieve(c) - // if token is not a user token, establish empty user to better handle nil checks - if !strings.EqualFold(cl.TokenType, constants.UserAccessTokenType) { + // if token is not a user token or claims were not retrieved, establish empty user to better handle nil checks + if cl == nil || !strings.EqualFold(cl.TokenType, constants.UserAccessTokenType) { u := new(library.User) ToContext(c, u) diff --git a/router/router.go b/router/router.go index f756f9f4e..2ed598034 100644 --- a/router/router.go +++ b/router/router.go @@ -69,7 +69,7 @@ func Load(options ...gin.HandlerFunc) *gin.Engine { r.GET("/login", auth.Login) // Logout endpoint - r.GET("/logout", user.Establish(), auth.Logout) + r.GET("/logout", claims.Establish(), user.Establish(), auth.Logout) // Refresh Access Token endpoint r.GET("/token-refresh", auth.RefreshAccessToken) From afd00ce206085f6e03ae86722061fc85c573cbb8 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 22 May 2023 12:55:22 -0600 Subject: [PATCH 16/53] refactor(api): move secret logic to separate package (#856) * refactor(api): move secret logic to separate package * Update api/secret/list.go --------- Co-authored-by: Jacob Floyd --- api/secret.go | 822 ------------------------------------------- api/secret/create.go | 244 +++++++++++++ api/secret/delete.go | 121 +++++++ api/secret/doc.go | 10 + api/secret/get.go | 130 +++++++ api/secret/list.go | 210 +++++++++++ api/secret/update.go | 177 ++++++++++ router/secret.go | 12 +- util/util.go | 17 + 9 files changed, 915 insertions(+), 828 deletions(-) delete mode 100644 api/secret.go create mode 100644 api/secret/create.go create mode 100644 api/secret/delete.go create mode 100644 api/secret/doc.go create mode 100644 api/secret/get.go create mode 100644 api/secret/list.go create mode 100644 api/secret/update.go diff --git a/api/secret.go b/api/secret.go deleted file mode 100644 index 6e1534d01..000000000 --- a/api/secret.go +++ /dev/null @@ -1,822 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package api - -import ( - "fmt" - "net/http" - "strconv" - "strings" - "time" - - "github.com/gin-gonic/gin" - "github.com/go-vela/server/router/middleware/claims" - "github.com/go-vela/server/router/middleware/user" - "github.com/go-vela/server/scm" - "github.com/go-vela/server/secret" - "github.com/go-vela/server/util" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" -) - -// -// swagger:operation POST /api/v1/secrets/{engine}/{type}/{org}/{name} secrets CreateSecret -// -// Create a secret -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: engine -// description: Secret engine to create a secret in, eg. "native" -// required: true -// type: string -// - in: path -// name: type -// description: Secret type to create -// enum: -// - org -// - repo -// - shared -// required: true -// type: string -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: name -// description: Name of the repo if a repo secret, team name if a shared secret, or '*' if an org secret -// required: true -// type: string -// - in: body -// name: body -// description: Payload containing the secret to create -// required: true -// schema: -// "$ref": "#/definitions/Secret" -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully created the secret -// schema: -// "$ref": "#/definitions/Secret" -// '400': -// description: Unable to create the secret -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to create the secret -// schema: -// "$ref": "#/definitions/Error" - -// CreateSecret represents the API handler to -// create a secret in the configured backend. -// -//nolint:funlen // suppress long function error -func CreateSecret(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - e := util.PathParameter(c, "engine") - t := util.PathParameter(c, "type") - o := util.PathParameter(c, "org") - n := util.PathParameter(c, "name") - - entry := fmt.Sprintf("%s/%s/%s", t, o, n) - - // create log fields from API metadata - fields := logrus.Fields{ - "engine": e, - "org": o, - "repo": n, - "type": t, - "user": u.GetName(), - } - - // check if secret is a shared secret - if strings.EqualFold(t, constants.SecretShared) { - // update log fields from API metadata - fields = logrus.Fields{ - "engine": e, - "org": o, - "team": n, - "type": t, - "user": u.GetName(), - } - } - - if strings.EqualFold(t, constants.SecretOrg) { - // retrieve org name from SCM - // - // SCM can be case insensitive, causing access retrieval to work - // but Org/Repo != org/repo in Vela. So this check ensures that - // what a user inputs matches the casing we expect in Vela since - // the SCM will have the source of truth for casing. - org, err := scm.FromContext(c).GetOrgName(u, o) - if err != nil { - retErr := fmt.Errorf("unable to retrieve organization %s", o) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - // check if casing is accurate - if org != o { - retErr := fmt.Errorf("unable to retrieve organization %s. Did you mean %s?", o, org) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - } - - if strings.EqualFold(t, constants.SecretRepo) { - // retrieve org and repo name from SCM - // - // same story as org secret. SCM has accurate casing. - scmOrg, scmRepo, err := scm.FromContext(c).GetOrgAndRepoName(u, o, n) - if err != nil { - retErr := fmt.Errorf("unable to retrieve repository %s/%s", o, n) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - // check if casing is accurate for org entry - if scmOrg != o { - retErr := fmt.Errorf("unable to retrieve org %s. Did you mean %s?", o, scmOrg) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - // check if casing is accurate for repo entry - if scmRepo != n { - retErr := fmt.Errorf("unable to retrieve repository %s. Did you mean %s?", n, scmRepo) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - } - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(fields).Infof("creating new secret %s for %s service", entry, e) - - // capture body from API request - input := new(library.Secret) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for secret %s for %s service: %w", entry, e, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // reject secrets with solely whitespace characters as its value - trimmed := strings.TrimSpace(input.GetValue()) - if len(trimmed) == 0 { - retErr := fmt.Errorf("secret value must contain non-whitespace characters") - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update fields in secret object - input.SetOrg(o) - input.SetRepo(n) - input.SetType(t) - input.SetCreatedAt(time.Now().UTC().Unix()) - input.SetCreatedBy(u.GetName()) - input.SetUpdatedAt(time.Now().UTC().Unix()) - input.SetUpdatedBy(u.GetName()) - - if len(input.GetImages()) > 0 { - input.SetImages(unique(input.GetImages())) - } - - if len(input.GetEvents()) > 0 { - input.SetEvents(unique(input.GetEvents())) - } - - if len(input.GetEvents()) == 0 { - // set default events to enable for the secret - input.SetEvents([]string{constants.EventPush, constants.EventTag, constants.EventDeploy}) - } - - if input.AllowCommand == nil { - input.SetAllowCommand(true) - } - - // check if secret is a shared secret - if strings.EqualFold(t, constants.SecretShared) { - // update the team instead of repo - input.SetTeam(n) - input.Repo = nil - } - - // send API call to create the secret - err = secret.FromContext(c, e).Create(t, o, n, input) - if err != nil { - retErr := fmt.Errorf("unable to create secret %s for %s service: %w", entry, e, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - s, _ := secret.FromContext(c, e).Get(t, o, n, input.GetName()) - - c.JSON(http.StatusOK, s.Sanitize()) -} - -// -// swagger:operation GET /api/v1/secrets/{engine}/{type}/{org}/{name} secrets GetSecrets -// -// Retrieve a list of secrets from the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: engine -// description: Secret engine to create a secret in, eg. "native" -// required: true -// type: string -// - in: path -// name: type -// description: Secret type to create -// enum: -// - org -// - repo -// - shared -// required: true -// type: string -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: name -// description: Name of the repo if a repo secret, team name if a shared secret, or '*' if an org secret -// required: true -// type: string -// - in: query -// name: page -// description: The page of results to retrieve -// type: integer -// default: 1 -// - in: query -// name: per_page -// description: How many results per page to return -// type: integer -// maximum: 100 -// default: 10 -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the list of secrets -// schema: -// type: array -// items: -// "$ref": "#/definitions/Secret" -// headers: -// X-Total-Count: -// description: Total number of results -// type: integer -// Link: -// description: see https://tools.ietf.org/html/rfc5988 -// type: string -// '400': -// description: Unable to retrieve the list of secrets -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to retrieve the list of secrets -// schema: -// "$ref": "#/definitions/Error" - -// GetSecrets represents the API handler to capture -// a list of secrets from the configured backend. -func GetSecrets(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - e := util.PathParameter(c, "engine") - t := util.PathParameter(c, "type") - o := util.PathParameter(c, "org") - n := util.PathParameter(c, "name") - - var teams []string - // get list of user's teams if type is shared secret and team is '*' - if t == constants.SecretShared && n == "*" { - var err error - - teams, err = scm.FromContext(c).ListUsersTeamsForOrg(u, o) - if err != nil { - retErr := fmt.Errorf("unable to get users %s teams for org %s: %w", u.GetName(), o, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - } - - entry := fmt.Sprintf("%s/%s/%s", t, o, n) - - // create log fields from API metadata - fields := logrus.Fields{ - "engine": e, - "org": o, - "repo": n, - "type": t, - "user": u.GetName(), - } - - // check if secret is a shared secret - if strings.EqualFold(t, constants.SecretShared) { - // update log fields from API metadata - fields = logrus.Fields{ - "engine": e, - "org": o, - "team": n, - "type": t, - "user": u.GetName(), - } - } - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(fields).Infof("reading secrets %s from %s service", entry, e) - - // capture page query parameter if present - page, err := strconv.Atoi(c.DefaultQuery("page", "1")) - if err != nil { - retErr := fmt.Errorf("unable to convert page query parameter for %s from %s service: %w", entry, e, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // capture per_page query parameter if present - perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) - if err != nil { - retErr := fmt.Errorf("unable to convert per_page query parameter for %s from %s service: %w", entry, e, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // send API call to capture the total number of secrets - total, err := secret.FromContext(c, e).Count(t, o, n, teams) - if err != nil { - retErr := fmt.Errorf("unable to get secret count for %s from %s service: %w", entry, e, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // ensure per_page isn't above or below allowed values - perPage = util.MaxInt(1, util.MinInt(100, perPage)) - - // send API call to capture the list of secrets - s, err := secret.FromContext(c, e).List(t, o, n, page, perPage, teams) - if err != nil { - retErr := fmt.Errorf("unable to get secrets for %s from %s service: %w", entry, e, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // create pagination object - pagination := Pagination{ - Page: page, - PerPage: perPage, - Total: total, - } - // set pagination headers - pagination.SetHeaderLink(c) - - // variable we want to return - secrets := []*library.Secret{} - // iterate through all secrets - for _, secret := range s { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := secret - - // sanitize secret to ensure no value is provided - secrets = append(secrets, tmp.Sanitize()) - } - - c.JSON(http.StatusOK, secrets) -} - -// -// swagger:operation GET /api/v1/secrets/{engine}/{type}/{org}/{name}/{secret} secrets GetSecret -// -// Retrieve a secret from the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: engine -// description: Secret engine to create a secret in, eg. "native" -// required: true -// type: string -// - in: path -// name: type -// description: Secret type to create -// enum: -// - org -// - repo -// - shared -// required: true -// type: string -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: name -// description: Name of the repo if a repo secret, team name if a shared secret, or '*' if an org secret -// required: true -// type: string -// - in: path -// name: secret -// description: Name of the secret -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the secret -// schema: -// "$ref": "#/definitions/Secret" -// '500': -// description: Unable to retrieve the secret -// schema: -// "$ref": "#/definitions/Error" - -// GetSecret gets a secret from the provided secrets service. -func GetSecret(c *gin.Context) { - // capture middleware values - cl := claims.Retrieve(c) - u := user.Retrieve(c) - e := util.PathParameter(c, "engine") - t := util.PathParameter(c, "type") - o := util.PathParameter(c, "org") - n := util.PathParameter(c, "name") - s := strings.TrimPrefix(util.PathParameter(c, "secret"), "/") - - entry := fmt.Sprintf("%s/%s/%s/%s", t, o, n, s) - - // create log fields from API metadata - fields := logrus.Fields{ - "engine": e, - "org": o, - "repo": n, - "secret": s, - "type": t, - "user": u.GetName(), - } - - // check if secret is a shared secret - if strings.EqualFold(t, constants.SecretShared) { - // update log fields from API metadata - fields = logrus.Fields{ - "engine": e, - "org": o, - "secret": s, - "team": n, - "type": t, - "user": u.GetName(), - } - } - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(fields).Infof("reading secret %s from %s service", entry, e) - - // send API call to capture the secret - secret, err := secret.FromContext(c, e).Get(t, o, n, s) - if err != nil { - retErr := fmt.Errorf("unable to get secret %s from %s service: %w", entry, e, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // only allow workers to access the full secret with the value - if strings.EqualFold(cl.TokenType, constants.WorkerBuildTokenType) { - c.JSON(http.StatusOK, secret) - - return - } - - c.JSON(http.StatusOK, secret.Sanitize()) -} - -// -// swagger:operation PUT /api/v1/secrets/{engine}/{type}/{org}/{name}/{secret} secrets UpdateSecrets -// -// Update a secret on the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: engine -// description: Secret engine to update the secret in, eg. "native" -// required: true -// type: string -// - in: path -// name: type -// description: Secret type to update -// enum: -// - org -// - repo -// - shared -// required: true -// type: string -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: name -// description: Name of the repo if a repo secret, team name if a shared secret, or '*' if an org secret -// required: true -// type: string -// - in: path -// name: secret -// description: Name of the secret -// required: true -// type: string -// - in: body -// name: body -// description: Payload containing the secret to create -// required: true -// schema: -// "$ref": "#/definitions/Secret" -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully updated the secret -// schema: -// "$ref": "#/definitions/Secret" -// '400': -// description: Unable to update the secret -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to update the secret -// schema: -// "$ref": "#/definitions/Error" - -// UpdateSecret updates a secret for the provided secrets service. -func UpdateSecret(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - e := util.PathParameter(c, "engine") - t := util.PathParameter(c, "type") - o := util.PathParameter(c, "org") - n := util.PathParameter(c, "name") - s := strings.TrimPrefix(util.PathParameter(c, "secret"), "/") - - entry := fmt.Sprintf("%s/%s/%s/%s", t, o, n, s) - - // create log fields from API metadata - fields := logrus.Fields{ - "engine": e, - "org": o, - "repo": n, - "secret": s, - "type": t, - "user": u.GetName(), - } - - // check if secret is a shared secret - if strings.EqualFold(t, constants.SecretShared) { - // update log fields from API metadata - fields = logrus.Fields{ - "engine": e, - "org": o, - "secret": s, - "team": n, - "type": t, - "user": u.GetName(), - } - } - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(fields).Infof("updating secret %s for %s service", entry, e) - - // capture body from API request - input := new(library.Secret) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for secret %s for %s service: %w", entry, e, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update secret fields if provided - input.SetName(s) - input.SetOrg(o) - input.SetRepo(n) - input.SetType(t) - input.SetUpdatedAt(time.Now().UTC().Unix()) - input.SetUpdatedBy(u.GetName()) - - if input.Images != nil { - // update images if set - input.SetImages(unique(input.GetImages())) - } - - if len(input.GetEvents()) > 0 { - input.SetEvents(unique(input.GetEvents())) - } - - if input.AllowCommand != nil { - // update allow_command if set - input.SetAllowCommand(input.GetAllowCommand()) - } - - // check if secret is a shared secret - if strings.EqualFold(t, constants.SecretShared) { - // update the team instead of repo - input.SetTeam(n) - input.Repo = nil - } - - // send API call to update the secret - err = secret.FromContext(c, e).Update(t, o, n, input) - if err != nil { - retErr := fmt.Errorf("unable to update secret %s for %s service: %w", entry, e, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the updated secret - secret, _ := secret.FromContext(c, e).Get(t, o, n, input.GetName()) - - c.JSON(http.StatusOK, secret.Sanitize()) -} - -// -// swagger:operation DELETE /api/v1/secrets/{engine}/{type}/{org}/{name}/{secret} secrets DeleteSecret -// -// Delete a secret from the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: engine -// description: Secret engine to delete the secret from, eg. "native" -// required: true -// type: string -// - in: path -// name: type -// description: Secret type to delete -// enum: -// - org -// - repo -// - shared -// required: true -// type: string -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: name -// description: Name of the repo if a repo secret, team name if a shared secret, or '*' if an org secret -// required: true -// type: string -// - in: path -// name: secret -// description: Name of the secret -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully deleted the secret -// schema: -// type: string -// '500': -// description: Unable to delete the secret -// schema: -// "$ref": "#/definitions/Error" - -// DeleteSecret deletes a secret from the provided secrets service. -func DeleteSecret(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - e := util.PathParameter(c, "engine") - t := util.PathParameter(c, "type") - o := util.PathParameter(c, "org") - n := util.PathParameter(c, "name") - s := strings.TrimPrefix(util.PathParameter(c, "secret"), "/") - - entry := fmt.Sprintf("%s/%s/%s/%s", t, o, n, s) - - // create log fields from API metadata - fields := logrus.Fields{ - "engine": e, - "org": o, - "repo": n, - "secret": s, - "type": t, - "user": u.GetName(), - } - - // check if secret is a shared secret - if strings.EqualFold(t, constants.SecretShared) { - // update log fields from API metadata - fields = logrus.Fields{ - "engine": e, - "org": o, - "secret": s, - "team": n, - "type": t, - "user": u.GetName(), - } - } - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(fields).Infof("deleting secret %s from %s service", entry, e) - - // send API call to remove the secret - err := secret.FromContext(c, e).Delete(t, o, n, s) - if err != nil { - retErr := fmt.Errorf("unable to delete secret %s from %s service: %w", entry, e, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, fmt.Sprintf("secret %s deleted from %s service", entry, e)) -} - -// unique is a helper function that takes a slice and -// validates that there are no duplicate entries. -func unique(stringSlice []string) []string { - keys := make(map[string]bool) - list := []string{} - - for _, entry := range stringSlice { - if _, value := keys[entry]; !value { - keys[entry] = true - - list = append(list, entry) - } - } - - return list -} diff --git a/api/secret/create.go b/api/secret/create.go new file mode 100644 index 000000000..3ba5061f9 --- /dev/null +++ b/api/secret/create.go @@ -0,0 +1,244 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package secret + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/secret" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// +// swagger:operation POST /api/v1/secrets/{engine}/{type}/{org}/{name} secrets CreateSecret +// +// Create a secret +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: engine +// description: Secret engine to create a secret in, eg. "native" +// required: true +// type: string +// - in: path +// name: type +// description: Secret type to create +// enum: +// - org +// - repo +// - shared +// required: true +// type: string +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: name +// description: Name of the repo if a repo secret, team name if a shared secret, or '*' if an org secret +// required: true +// type: string +// - in: body +// name: body +// description: Payload containing the secret to create +// required: true +// schema: +// "$ref": "#/definitions/Secret" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully created the secret +// schema: +// "$ref": "#/definitions/Secret" +// '400': +// description: Unable to create the secret +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to create the secret +// schema: +// "$ref": "#/definitions/Error" + +// CreateSecret represents the API handler to +// create a secret in the configured backend. +// +//nolint:funlen // suppress long function error +func CreateSecret(c *gin.Context) { + // capture middleware values + u := user.Retrieve(c) + e := util.PathParameter(c, "engine") + t := util.PathParameter(c, "type") + o := util.PathParameter(c, "org") + n := util.PathParameter(c, "name") + + entry := fmt.Sprintf("%s/%s/%s", t, o, n) + + // create log fields from API metadata + fields := logrus.Fields{ + "engine": e, + "org": o, + "repo": n, + "type": t, + "user": u.GetName(), + } + + // check if secret is a shared secret + if strings.EqualFold(t, constants.SecretShared) { + // update log fields from API metadata + fields = logrus.Fields{ + "engine": e, + "org": o, + "team": n, + "type": t, + "user": u.GetName(), + } + } + + if strings.EqualFold(t, constants.SecretOrg) { + // retrieve org name from SCM + // + // SCM can be case insensitive, causing access retrieval to work + // but Org/Repo != org/repo in Vela. So this check ensures that + // what a user inputs matches the casing we expect in Vela since + // the SCM will have the source of truth for casing. + org, err := scm.FromContext(c).GetOrgName(u, o) + if err != nil { + retErr := fmt.Errorf("unable to retrieve organization %s", o) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // check if casing is accurate + if org != o { + retErr := fmt.Errorf("unable to retrieve organization %s. Did you mean %s?", o, org) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + } + + if strings.EqualFold(t, constants.SecretRepo) { + // retrieve org and repo name from SCM + // + // same story as org secret. SCM has accurate casing. + scmOrg, scmRepo, err := scm.FromContext(c).GetOrgAndRepoName(u, o, n) + if err != nil { + retErr := fmt.Errorf("unable to retrieve repository %s/%s", o, n) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // check if casing is accurate for org entry + if scmOrg != o { + retErr := fmt.Errorf("unable to retrieve org %s. Did you mean %s?", o, scmOrg) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // check if casing is accurate for repo entry + if scmRepo != n { + retErr := fmt.Errorf("unable to retrieve repository %s. Did you mean %s?", n, scmRepo) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + } + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(fields).Infof("creating new secret %s for %s service", entry, e) + + // capture body from API request + input := new(library.Secret) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for secret %s for %s service: %w", entry, e, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // reject secrets with solely whitespace characters as its value + trimmed := strings.TrimSpace(input.GetValue()) + if len(trimmed) == 0 { + retErr := fmt.Errorf("secret value must contain non-whitespace characters") + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update fields in secret object + input.SetOrg(o) + input.SetRepo(n) + input.SetType(t) + input.SetCreatedAt(time.Now().UTC().Unix()) + input.SetCreatedBy(u.GetName()) + input.SetUpdatedAt(time.Now().UTC().Unix()) + input.SetUpdatedBy(u.GetName()) + + if len(input.GetImages()) > 0 { + input.SetImages(util.Unique(input.GetImages())) + } + + if len(input.GetEvents()) > 0 { + input.SetEvents(util.Unique(input.GetEvents())) + } + + if len(input.GetEvents()) == 0 { + // set default events to enable for the secret + input.SetEvents([]string{constants.EventPush, constants.EventTag, constants.EventDeploy}) + } + + if input.AllowCommand == nil { + input.SetAllowCommand(true) + } + + // check if secret is a shared secret + if strings.EqualFold(t, constants.SecretShared) { + // update the team instead of repo + input.SetTeam(n) + input.Repo = nil + } + + // send API call to create the secret + err = secret.FromContext(c, e).Create(t, o, n, input) + if err != nil { + retErr := fmt.Errorf("unable to create secret %s for %s service: %w", entry, e, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + s, _ := secret.FromContext(c, e).Get(t, o, n, input.GetName()) + + c.JSON(http.StatusOK, s.Sanitize()) +} diff --git a/api/secret/delete.go b/api/secret/delete.go new file mode 100644 index 000000000..f8134ba5f --- /dev/null +++ b/api/secret/delete.go @@ -0,0 +1,121 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package secret + +import ( + "fmt" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/secret" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" +) + +// +// swagger:operation DELETE /api/v1/secrets/{engine}/{type}/{org}/{name}/{secret} secrets DeleteSecret +// +// Delete a secret from the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: engine +// description: Secret engine to delete the secret from, eg. "native" +// required: true +// type: string +// - in: path +// name: type +// description: Secret type to delete +// enum: +// - org +// - repo +// - shared +// required: true +// type: string +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: name +// description: Name of the repo if a repo secret, team name if a shared secret, or '*' if an org secret +// required: true +// type: string +// - in: path +// name: secret +// description: Name of the secret +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully deleted the secret +// schema: +// type: string +// '500': +// description: Unable to delete the secret +// schema: +// "$ref": "#/definitions/Error" + +// DeleteSecret deletes a secret from the provided secrets service. +func DeleteSecret(c *gin.Context) { + // capture middleware values + u := user.Retrieve(c) + e := util.PathParameter(c, "engine") + t := util.PathParameter(c, "type") + o := util.PathParameter(c, "org") + n := util.PathParameter(c, "name") + s := strings.TrimPrefix(util.PathParameter(c, "secret"), "/") + + entry := fmt.Sprintf("%s/%s/%s/%s", t, o, n, s) + + // create log fields from API metadata + fields := logrus.Fields{ + "engine": e, + "org": o, + "repo": n, + "secret": s, + "type": t, + "user": u.GetName(), + } + + // check if secret is a shared secret + if strings.EqualFold(t, constants.SecretShared) { + // update log fields from API metadata + fields = logrus.Fields{ + "engine": e, + "org": o, + "secret": s, + "team": n, + "type": t, + "user": u.GetName(), + } + } + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(fields).Infof("deleting secret %s from %s service", entry, e) + + // send API call to remove the secret + err := secret.FromContext(c, e).Delete(t, o, n, s) + if err != nil { + retErr := fmt.Errorf("unable to delete secret %s from %s service: %w", entry, e, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, fmt.Sprintf("secret %s deleted from %s service", entry, e)) +} diff --git a/api/secret/doc.go b/api/secret/doc.go new file mode 100644 index 000000000..db89d7b55 --- /dev/null +++ b/api/secret/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package secret provides the secret handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/secret" +package secret diff --git a/api/secret/get.go b/api/secret/get.go new file mode 100644 index 000000000..0f24b95c0 --- /dev/null +++ b/api/secret/get.go @@ -0,0 +1,130 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package secret + +import ( + "fmt" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/router/middleware/claims" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/secret" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" +) + +// +// swagger:operation GET /api/v1/secrets/{engine}/{type}/{org}/{name}/{secret} secrets GetSecret +// +// Retrieve a secret from the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: engine +// description: Secret engine to create a secret in, eg. "native" +// required: true +// type: string +// - in: path +// name: type +// description: Secret type to create +// enum: +// - org +// - repo +// - shared +// required: true +// type: string +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: name +// description: Name of the repo if a repo secret, team name if a shared secret, or '*' if an org secret +// required: true +// type: string +// - in: path +// name: secret +// description: Name of the secret +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the secret +// schema: +// "$ref": "#/definitions/Secret" +// '500': +// description: Unable to retrieve the secret +// schema: +// "$ref": "#/definitions/Error" + +// GetSecret gets a secret from the provided secrets service. +func GetSecret(c *gin.Context) { + // capture middleware values + cl := claims.Retrieve(c) + u := user.Retrieve(c) + e := util.PathParameter(c, "engine") + t := util.PathParameter(c, "type") + o := util.PathParameter(c, "org") + n := util.PathParameter(c, "name") + s := strings.TrimPrefix(util.PathParameter(c, "secret"), "/") + + entry := fmt.Sprintf("%s/%s/%s/%s", t, o, n, s) + + // create log fields from API metadata + fields := logrus.Fields{ + "engine": e, + "org": o, + "repo": n, + "secret": s, + "type": t, + "user": u.GetName(), + } + + // check if secret is a shared secret + if strings.EqualFold(t, constants.SecretShared) { + // update log fields from API metadata + fields = logrus.Fields{ + "engine": e, + "org": o, + "secret": s, + "team": n, + "type": t, + "user": u.GetName(), + } + } + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(fields).Infof("reading secret %s from %s service", entry, e) + + // send API call to capture the secret + secret, err := secret.FromContext(c, e).Get(t, o, n, s) + if err != nil { + retErr := fmt.Errorf("unable to get secret %s from %s service: %w", entry, e, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // only allow workers to access the full secret with the value + if strings.EqualFold(cl.TokenType, constants.WorkerBuildTokenType) { + c.JSON(http.StatusOK, secret) + + return + } + + c.JSON(http.StatusOK, secret.Sanitize()) +} diff --git a/api/secret/list.go b/api/secret/list.go new file mode 100644 index 000000000..e603c473f --- /dev/null +++ b/api/secret/list.go @@ -0,0 +1,210 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package secret + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/api" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/secret" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// +// swagger:operation GET /api/v1/secrets/{engine}/{type}/{org}/{name} secrets ListSecrets +// +// Retrieve a list of secrets from the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: engine +// description: Secret engine to create a secret in, eg. "native" +// required: true +// type: string +// - in: path +// name: type +// description: Secret type to create +// enum: +// - org +// - repo +// - shared +// required: true +// type: string +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: name +// description: Name of the repo if a repo secret, team name if a shared secret, or '*' if an org secret +// required: true +// type: string +// - in: query +// name: page +// description: The page of results to retrieve +// type: integer +// default: 1 +// - in: query +// name: per_page +// description: How many results per page to return +// type: integer +// maximum: 100 +// default: 10 +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the list of secrets +// schema: +// type: array +// items: +// "$ref": "#/definitions/Secret" +// headers: +// X-Total-Count: +// description: Total number of results +// type: integer +// Link: +// description: see https://tools.ietf.org/html/rfc5988 +// type: string +// '400': +// description: Unable to retrieve the list of secrets +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to retrieve the list of secrets +// schema: +// "$ref": "#/definitions/Error" + +// ListSecrets represents the API handler to capture +// a list of secrets from the configured backend. +func ListSecrets(c *gin.Context) { + // capture middleware values + u := user.Retrieve(c) + e := util.PathParameter(c, "engine") + t := util.PathParameter(c, "type") + o := util.PathParameter(c, "org") + n := util.PathParameter(c, "name") + + var teams []string + // get list of user's teams if type is shared secret and team is '*' + if t == constants.SecretShared && n == "*" { + var err error + + teams, err = scm.FromContext(c).ListUsersTeamsForOrg(u, o) + if err != nil { + retErr := fmt.Errorf("unable to list users %s teams for org %s: %w", u.GetName(), o, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + } + + entry := fmt.Sprintf("%s/%s/%s", t, o, n) + + // create log fields from API metadata + fields := logrus.Fields{ + "engine": e, + "org": o, + "repo": n, + "type": t, + "user": u.GetName(), + } + + // check if secret is a shared secret + if strings.EqualFold(t, constants.SecretShared) { + // update log fields from API metadata + fields = logrus.Fields{ + "engine": e, + "org": o, + "team": n, + "type": t, + "user": u.GetName(), + } + } + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(fields).Infof("listing secrets %s from %s service", entry, e) + + // capture page query parameter if present + page, err := strconv.Atoi(c.DefaultQuery("page", "1")) + if err != nil { + retErr := fmt.Errorf("unable to convert page query parameter for %s from %s service: %w", entry, e, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // capture per_page query parameter if present + perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) + if err != nil { + retErr := fmt.Errorf("unable to convert per_page query parameter for %s from %s service: %w", entry, e, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to capture the total number of secrets + total, err := secret.FromContext(c, e).Count(t, o, n, teams) + if err != nil { + retErr := fmt.Errorf("unable to get secret count for %s from %s service: %w", entry, e, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // ensure per_page isn't above or below allowed values + perPage = util.MaxInt(1, util.MinInt(100, perPage)) + + // send API call to capture the list of secrets + s, err := secret.FromContext(c, e).List(t, o, n, page, perPage, teams) + if err != nil { + retErr := fmt.Errorf("unable to list secrets for %s from %s service: %w", entry, e, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // create pagination object + pagination := api.Pagination{ + Page: page, + PerPage: perPage, + Total: total, + } + // set pagination headers + pagination.SetHeaderLink(c) + + // variable we want to return + secrets := []*library.Secret{} + // iterate through all secrets + for _, secret := range s { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := secret + + // sanitize secret to ensure no value is provided + secrets = append(secrets, tmp.Sanitize()) + } + + c.JSON(http.StatusOK, secrets) +} diff --git a/api/secret/update.go b/api/secret/update.go new file mode 100644 index 000000000..f71889701 --- /dev/null +++ b/api/secret/update.go @@ -0,0 +1,177 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package secret + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/secret" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// +// swagger:operation PUT /api/v1/secrets/{engine}/{type}/{org}/{name}/{secret} secrets UpdateSecret +// +// Update a secret on the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: engine +// description: Secret engine to update the secret in, eg. "native" +// required: true +// type: string +// - in: path +// name: type +// description: Secret type to update +// enum: +// - org +// - repo +// - shared +// required: true +// type: string +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: name +// description: Name of the repo if a repo secret, team name if a shared secret, or '*' if an org secret +// required: true +// type: string +// - in: path +// name: secret +// description: Name of the secret +// required: true +// type: string +// - in: body +// name: body +// description: Payload containing the secret to create +// required: true +// schema: +// "$ref": "#/definitions/Secret" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully updated the secret +// schema: +// "$ref": "#/definitions/Secret" +// '400': +// description: Unable to update the secret +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to update the secret +// schema: +// "$ref": "#/definitions/Error" + +// UpdateSecret updates a secret for the provided secrets service. +func UpdateSecret(c *gin.Context) { + // capture middleware values + u := user.Retrieve(c) + e := util.PathParameter(c, "engine") + t := util.PathParameter(c, "type") + o := util.PathParameter(c, "org") + n := util.PathParameter(c, "name") + s := strings.TrimPrefix(util.PathParameter(c, "secret"), "/") + + entry := fmt.Sprintf("%s/%s/%s/%s", t, o, n, s) + + // create log fields from API metadata + fields := logrus.Fields{ + "engine": e, + "org": o, + "repo": n, + "secret": s, + "type": t, + "user": u.GetName(), + } + + // check if secret is a shared secret + if strings.EqualFold(t, constants.SecretShared) { + // update log fields from API metadata + fields = logrus.Fields{ + "engine": e, + "org": o, + "secret": s, + "team": n, + "type": t, + "user": u.GetName(), + } + } + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(fields).Infof("updating secret %s for %s service", entry, e) + + // capture body from API request + input := new(library.Secret) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for secret %s for %s service: %w", entry, e, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update secret fields if provided + input.SetName(s) + input.SetOrg(o) + input.SetRepo(n) + input.SetType(t) + input.SetUpdatedAt(time.Now().UTC().Unix()) + input.SetUpdatedBy(u.GetName()) + + if input.Images != nil { + // update images if set + input.SetImages(util.Unique(input.GetImages())) + } + + if len(input.GetEvents()) > 0 { + input.SetEvents(util.Unique(input.GetEvents())) + } + + if input.AllowCommand != nil { + // update allow_command if set + input.SetAllowCommand(input.GetAllowCommand()) + } + + // check if secret is a shared secret + if strings.EqualFold(t, constants.SecretShared) { + // update the team instead of repo + input.SetTeam(n) + input.Repo = nil + } + + // send API call to update the secret + err = secret.FromContext(c, e).Update(t, o, n, input) + if err != nil { + retErr := fmt.Errorf("unable to update secret %s for %s service: %w", entry, e, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the updated secret + secret, _ := secret.FromContext(c, e).Get(t, o, n, input.GetName()) + + c.JSON(http.StatusOK, secret.Sanitize()) +} diff --git a/router/secret.go b/router/secret.go index dbc18e6b6..0ce94d982 100644 --- a/router/secret.go +++ b/router/secret.go @@ -5,7 +5,7 @@ package router import ( - "github.com/go-vela/server/api" + "github.com/go-vela/server/api/secret" "github.com/go-vela/server/router/middleware/perm" "github.com/gin-gonic/gin" @@ -23,10 +23,10 @@ func SecretHandlers(base *gin.RouterGroup) { // Secrets endpoints secrets := base.Group("/secrets/:engine/:type/:org/:name", perm.MustSecretAdmin()) { - secrets.POST("", api.CreateSecret) - secrets.GET("", api.GetSecrets) - secrets.GET("/*secret", api.GetSecret) - secrets.PUT("/*secret", api.UpdateSecret) - secrets.DELETE("/*secret", api.DeleteSecret) + secrets.POST("", secret.CreateSecret) + secrets.GET("", secret.ListSecrets) + secrets.GET("/*secret", secret.GetSecret) + secrets.PUT("/*secret", secret.UpdateSecret) + secrets.DELETE("/*secret", secret.DeleteSecret) } // end of secrets endpoints } diff --git a/util/util.go b/util/util.go index 849ffdd95..667653fbe 100644 --- a/util/util.go +++ b/util/util.go @@ -73,6 +73,23 @@ func EscapeValue(value string) string { return html.EscapeString(escaped) } +// Unique is a helper function that takes a slice and +// validates that there are no duplicate entries. +func Unique(stringSlice []string) []string { + keys := make(map[string]bool) + list := []string{} + + for _, entry := range stringSlice { + if _, value := keys[entry]; !value { + keys[entry] = true + + list = append(list, entry) + } + } + + return list +} + // CheckAllowlist is a helper function to ensure only repos in the // allowlist are specified. // From ffac676b781bab77d57caccbc12edd33d26e66b4 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Tue, 23 May 2023 08:31:08 -0600 Subject: [PATCH 17/53] fix(compiler): add commit field to compiler engine to handle file type template lite compilation (#859) --- api/build.go | 2 ++ api/pipeline/compile.go | 2 +- api/pipeline/expand.go | 2 +- api/pipeline/template.go | 2 +- api/pipeline/validate.go | 2 +- api/webhook.go | 1 + compiler/engine.go | 3 +++ compiler/native/expand.go | 2 +- compiler/native/expand_test.go | 7 +------ compiler/native/native.go | 10 ++++++++++ router/middleware/pipeline/pipeline.go | 1 + 11 files changed, 23 insertions(+), 11 deletions(-) diff --git a/api/build.go b/api/build.go index ce755027a..fed9184da 100644 --- a/api/build.go +++ b/api/build.go @@ -271,6 +271,7 @@ func CreateBuild(c *gin.Context) { p, compiled, err = compiler.FromContext(c). Duplicate(). WithBuild(input). + WithCommit(input.GetCommit()). WithFiles(files). WithMetadata(m). WithRepo(r). @@ -1200,6 +1201,7 @@ func RestartBuild(c *gin.Context) { p, compiled, err = compiler.FromContext(c). Duplicate(). WithBuild(b). + WithCommit(b.GetCommit()). WithFiles(files). WithMetadata(m). WithRepo(r). diff --git a/api/pipeline/compile.go b/api/pipeline/compile.go index 6e05121bc..723bf2522 100644 --- a/api/pipeline/compile.go +++ b/api/pipeline/compile.go @@ -94,7 +94,7 @@ func CompilePipeline(c *gin.Context) { r.SetPipelineType(p.GetType()) // create the compiler object - compiler := compiler.FromContext(c).Duplicate().WithMetadata(m).WithRepo(r).WithUser(u) + compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) // compile the pipeline pipeline, _, err := compiler.CompileLite(p.GetData(), true, true, nil) diff --git a/api/pipeline/expand.go b/api/pipeline/expand.go index 9e0c44466..05e5c7dd8 100644 --- a/api/pipeline/expand.go +++ b/api/pipeline/expand.go @@ -95,7 +95,7 @@ func ExpandPipeline(c *gin.Context) { r.SetPipelineType(p.GetType()) // create the compiler object - compiler := compiler.FromContext(c).Duplicate().WithMetadata(m).WithRepo(r).WithUser(u) + compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) // expand the templates in the pipeline pipeline, _, err := compiler.CompileLite(p.GetData(), true, false, nil) diff --git a/api/pipeline/template.go b/api/pipeline/template.go index 4bb01cb19..54233b691 100644 --- a/api/pipeline/template.go +++ b/api/pipeline/template.go @@ -96,7 +96,7 @@ func GetTemplates(c *gin.Context) { }).Infof("reading templates from pipeline %s", entry) // create the compiler object - compiler := compiler.FromContext(c).Duplicate().WithMetadata(m).WithRepo(r).WithUser(u) + compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) // parse the pipeline configuration pipeline, _, err := compiler.Parse(p.GetData(), p.GetType(), new(yaml.Template)) diff --git a/api/pipeline/validate.go b/api/pipeline/validate.go index 01fa574d3..42e49e2ae 100644 --- a/api/pipeline/validate.go +++ b/api/pipeline/validate.go @@ -94,7 +94,7 @@ func ValidatePipeline(c *gin.Context) { r.SetPipelineType(p.GetType()) // create the compiler object - compiler := compiler.FromContext(c).Duplicate().WithMetadata(m).WithRepo(r).WithUser(u) + compiler := compiler.FromContext(c).Duplicate().WithCommit(p.GetCommit()).WithMetadata(m).WithRepo(r).WithUser(u) // capture optional template query parameter template, err := strconv.ParseBool(c.DefaultQuery("template", "true")) diff --git a/api/webhook.go b/api/webhook.go index 501caffa6..0c8c5bd73 100644 --- a/api/webhook.go +++ b/api/webhook.go @@ -511,6 +511,7 @@ func PostWebhook(c *gin.Context) { Duplicate(). WithBuild(b). WithComment(webhook.Comment). + WithCommit(b.GetCommit()). WithFiles(files). WithMetadata(m). WithRepo(repo). diff --git a/compiler/engine.go b/compiler/engine.go index f2ff2a33e..dfe0716c4 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -121,6 +121,9 @@ type Engine interface { // WithComment defines a function that sets // the comment in the Engine. WithComment(string) Engine + // WithCommit defines a function that sets + // the commit in the Engine. + WithCommit(string) Engine // WithFiles defines a function that sets // the changeset files in the Engine. WithFiles([]string) Engine diff --git a/compiler/native/expand.go b/compiler/native/expand.go index 4256de7e8..102ae9774 100644 --- a/compiler/native/expand.go +++ b/compiler/native/expand.go @@ -235,7 +235,7 @@ func (c *client) getTemplate(tmpl *yaml.Template, name string) ([]byte, error) { Org: c.repo.GetOrg(), Repo: c.repo.GetName(), Name: tmpl.Source, - Ref: c.build.GetCommit(), + Ref: c.commit, } if !c.UsePrivateGithub { diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index a8c232a71..c89d23d50 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -192,11 +192,6 @@ func TestNative_ExpandSteps(t *testing.T) { set.String("github-token", "", "doc") c := cli.NewContext(nil, set, nil) - testBuild := new(library.Build) - - testBuild.SetID(1) - testBuild.SetCommit("123abc456def") - testRepo := new(library.Repo) testRepo.SetID(1) @@ -318,7 +313,7 @@ func TestNative_ExpandSteps(t *testing.T) { t.Errorf("Creating new compiler returned err: %v", err) } - compiler.WithBuild(testBuild).WithRepo(testRepo) + compiler.WithCommit("123abc456def").WithRepo(testRepo) for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/compiler/native/native.go b/compiler/native/native.go index e7e549b58..61b411838 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -35,6 +35,7 @@ type client struct { build *library.Build comment string + commit string files []string local bool metadata *types.Metadata @@ -131,6 +132,15 @@ func (c *client) WithComment(cmt string) compiler.Engine { return c } +// WithCommit sets the comment in the Engine. +func (c *client) WithCommit(cmt string) compiler.Engine { + if cmt != "" { + c.commit = cmt + } + + return c +} + // WithFiles sets the changeset files in the Engine. func (c *client) WithFiles(f []string) compiler.Engine { if f != nil { diff --git a/router/middleware/pipeline/pipeline.go b/router/middleware/pipeline/pipeline.go index ecf08767b..1910e2c0f 100644 --- a/router/middleware/pipeline/pipeline.go +++ b/router/middleware/pipeline/pipeline.go @@ -77,6 +77,7 @@ func Establish() gin.HandlerFunc { // parse and compile the pipeline configuration file _, pipeline, err = compiler.FromContext(c). Duplicate(). + WithCommit(p). WithMetadata(c.MustGet("metadata").(*types.Metadata)). WithRepo(r). WithUser(u). From 5a9a92d279e058b75b37e589e1eef2ec63f550a6 Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Tue, 23 May 2023 14:10:14 -0500 Subject: [PATCH 18/53] feat: add support for processing schedules (#846) * feat(api/types): add support for schedules * feat(database/types): add support for schedules * feat(database): add support for schedules * chore: update go dependencies * feat(database): add schedule engine * feat(api): add support for schedules * add routes * fix: parse entry for schedules * more wip code * add schedule allowlist * fix tests * add validation for entry * add mocks w/o updated payloads * fix issues with create * update mock responses * use schedule mocks * make linter happy * use proper func * couple more updates * fix mock pathing * enhance: switch to adhocore/gronx * chore: update go deps * goimports * yet another goimports * sigh * wildcard goimport * chore: address linter feedback * chore: save work * chore: remove new types * chore: updates for removed types * chore: update go dependencies * chore: address review feedback * chore: remove new types * feat: initial code for scheduler * chore: misc updates * chore: update go dependencies * chore: updates for local testing * chore: save work * fix: introduce jitter * chore: address review feedback * chore: address review feedback * chore: update go dependencies * chore: address review feedback * fix(scheduler): use WithCommit in compiler * chore: address review feedback --------- Co-authored-by: Jordan Sussman Co-authored-by: JordanSussman --- .gitignore | 2 +- api/build.go | 31 ++- api/build_test.go | 8 +- api/schedule/create.go | 2 +- api/schedule/delete.go | 2 +- api/schedule/get.go | 2 +- api/schedule/list.go | 2 +- api/schedule/update.go | 3 +- api/webhook.go | 10 +- cmd/vela-server/schedule.go | 381 ++++++++++++++++++++++++++++++++ cmd/vela-server/server.go | 120 +++++++--- docker-compose.yml | 2 +- go.mod | 5 +- go.sum | 9 +- scm/github/repo.go | 19 ++ scm/github/repo_test.go | 49 ++++ scm/github/testdata/branch.json | 101 +++++++++ scm/service.go | 3 + 18 files changed, 675 insertions(+), 76 deletions(-) create mode 100644 cmd/vela-server/schedule.go create mode 100644 scm/github/testdata/branch.json diff --git a/.gitignore b/.gitignore index f3be5762d..c557bc18b 100644 --- a/.gitignore +++ b/.gitignore @@ -67,4 +67,4 @@ __debug_bin .history .ionide -# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode +# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode \ No newline at end of file diff --git a/api/build.go b/api/build.go index fed9184da..445d4cc8a 100644 --- a/api/build.go +++ b/api/build.go @@ -14,26 +14,23 @@ import ( "strings" "time" - "github.com/go-vela/server/internal/token" - "github.com/go-vela/server/router/middleware/claims" - "github.com/go-vela/server/router/middleware/org" - + "github.com/gin-gonic/gin" "github.com/go-vela/server/compiler" "github.com/go-vela/server/database" + "github.com/go-vela/server/internal/token" "github.com/go-vela/server/queue" "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/claims" "github.com/go-vela/server/router/middleware/executors" + "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" - "github.com/go-vela/types" "github.com/go-vela/types/constants" "github.com/go-vela/types/library" "github.com/go-vela/types/pipeline" - - "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" ) @@ -293,7 +290,7 @@ func CreateBuild(c *gin.Context) { r.SetPipelineType(pipelineType) // skip the build if only the init or clone steps are found - skip := skipEmptyBuild(p) + skip := SkipEmptyBuild(p) if skip != "" { // set build to successful status input.SetStatus(constants.StatusSuccess) @@ -343,7 +340,7 @@ func CreateBuild(c *gin.Context) { input.SetPipelineID(pipeline.GetID()) // create the objects from the pipeline in the database - err = planBuild(database.FromContext(c), p, input, r) + err = PlanBuild(database.FromContext(c), p, input, r) if err != nil { util.HandleError(c, http.StatusInternalServerError, err) @@ -372,7 +369,7 @@ func CreateBuild(c *gin.Context) { } // publish the build to the queue - go publishToQueue( + go PublishToQueue( queue.FromGinContext(c), database.FromContext(c), p, @@ -382,11 +379,11 @@ func CreateBuild(c *gin.Context) { ) } -// skipEmptyBuild checks if the build should be skipped due to it +// SkipEmptyBuild checks if the build should be skipped due to it // not containing any steps besides init or clone. // //nolint:goconst // ignore init and clone constants -func skipEmptyBuild(p *pipeline.Build) string { +func SkipEmptyBuild(p *pipeline.Build) string { if len(p.Stages) == 1 { if p.Stages[0].Name == "init" { return "skipping build since only init stage found" @@ -1223,7 +1220,7 @@ func RestartBuild(c *gin.Context) { r.SetPipelineType(pipelineType) // skip the build if only the init or clone steps are found - skip := skipEmptyBuild(p) + skip := SkipEmptyBuild(p) if skip != "" { // set build to successful status b.SetStatus(constants.StatusSkipped) @@ -1273,7 +1270,7 @@ func RestartBuild(c *gin.Context) { b.SetPipelineID(pipeline.GetID()) // create the objects from the pipeline in the database - err = planBuild(database.FromContext(c), p, b, r) + err = PlanBuild(database.FromContext(c), p, b, r) if err != nil { util.HandleError(c, http.StatusInternalServerError, err) @@ -1301,7 +1298,7 @@ func RestartBuild(c *gin.Context) { } // publish the build to the queue - go publishToQueue( + go PublishToQueue( queue.FromGinContext(c), database.FromContext(c), p, @@ -1568,12 +1565,12 @@ func getPRNumberFromBuild(b *library.Build) (int, error) { return strconv.Atoi(parts[2]) } -// planBuild is a helper function to plan the build for +// PlanBuild is a helper function to plan the build for // execution. This creates all resources, like steps // and services, for the build in the configured backend. // TODO: // - return build and error. -func planBuild(database database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo) error { +func PlanBuild(database database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo) error { // update fields in build object b.SetCreated(time.Now().UTC().Unix()) diff --git a/api/build_test.go b/api/build_test.go index b92802303..1fb395d7d 100644 --- a/api/build_test.go +++ b/api/build_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. @@ -10,7 +10,7 @@ import ( "github.com/go-vela/types/pipeline" ) -func Test_skipEmptyBuild(t *testing.T) { +func Test_SkipEmptyBuild(t *testing.T) { type args struct { p *pipeline.Build } @@ -72,8 +72,8 @@ func Test_skipEmptyBuild(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := skipEmptyBuild(tt.args.p); got != tt.want { - t.Errorf("skipEmptyBuild() = %v, want %v", got, tt.want) + if got := SkipEmptyBuild(tt.args.p); got != tt.want { + t.Errorf("SkipEmptyBuild() = %v, want %v", got, tt.want) } }) } diff --git a/api/schedule/create.go b/api/schedule/create.go index 127c04909..c8eb92741 100644 --- a/api/schedule/create.go +++ b/api/schedule/create.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. diff --git a/api/schedule/delete.go b/api/schedule/delete.go index fd7c0715f..a697954d8 100644 --- a/api/schedule/delete.go +++ b/api/schedule/delete.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. diff --git a/api/schedule/get.go b/api/schedule/get.go index 51a436bbe..727de5b84 100644 --- a/api/schedule/get.go +++ b/api/schedule/get.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. diff --git a/api/schedule/list.go b/api/schedule/list.go index c43188d86..cb64ada74 100644 --- a/api/schedule/list.go +++ b/api/schedule/list.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. diff --git a/api/schedule/update.go b/api/schedule/update.go index bdadbdd2d..646e55fc2 100644 --- a/api/schedule/update.go +++ b/api/schedule/update.go @@ -9,11 +9,10 @@ import ( "net/http" "time" - "github.com/go-vela/server/router/middleware/schedule" - "github.com/gin-gonic/gin" "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/schedule" "github.com/go-vela/server/util" "github.com/go-vela/types/library" "github.com/sirupsen/logrus" diff --git a/api/webhook.go b/api/webhook.go index 0c8c5bd73..5f4ddc5bf 100644 --- a/api/webhook.go +++ b/api/webhook.go @@ -542,7 +542,7 @@ func PostWebhook(c *gin.Context) { repo.SetPipelineType(pipelineType) // skip the build if only the init or clone steps are found - skip := skipEmptyBuild(p) + skip := SkipEmptyBuild(p) if skip != "" { // set build to successful status b.SetStatus(constants.StatusSkipped) @@ -609,7 +609,7 @@ func PostWebhook(c *gin.Context) { // using the same Number and thus create a constraint // conflict; consider deleting the partially created // build object in the database - err = planBuild(database.FromContext(c), p, b, repo) + err = PlanBuild(database.FromContext(c), p, b, repo) if err != nil { retErr := fmt.Errorf("%s: %w", baseErr, err) @@ -696,7 +696,7 @@ func PostWebhook(c *gin.Context) { } // publish the build to the queue - go publishToQueue( + go PublishToQueue( queue.FromGinContext(c), database.FromContext(c), p, @@ -706,9 +706,9 @@ func PostWebhook(c *gin.Context) { ) } -// publishToQueue is a helper function that creates +// PublishToQueue is a helper function that creates // a build item and publishes it to the queue. -func publishToQueue(queue queue.Service, db database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo, u *library.User) { +func PublishToQueue(queue queue.Service, db database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo, u *library.User) { item := types.ToItem(p, b, r, u) logrus.Infof("Converting queue item to json for build %d for %s", b.GetNumber(), r.GetFullName()) diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go new file mode 100644 index 000000000..172b5c251 --- /dev/null +++ b/cmd/vela-server/schedule.go @@ -0,0 +1,381 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package main + +import ( + "fmt" + "strings" + "time" + + "github.com/adhocore/gronx" + "github.com/go-vela/server/api" + "github.com/go-vela/server/compiler" + "github.com/go-vela/server/database" + "github.com/go-vela/server/queue" + "github.com/go-vela/server/scm" + "github.com/go-vela/types" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + "github.com/sirupsen/logrus" + + "k8s.io/apimachinery/pkg/util/wait" +) + +const baseErr = "unable to schedule build" + +func processSchedules(compiler compiler.Engine, database database.Interface, metadata *types.Metadata, queue queue.Service, scm scm.Service) error { + logrus.Infof("processing active schedules to create builds") + + // send API call to capture the list of active schedules + schedules, err := database.ListActiveSchedules() + if err != nil { + return err + } + + // iterate through the list of active schedules + for _, s := range schedules { + // send API call to capture the schedule + // + // This is needed to ensure we are not dealing with a stale schedule since we fetch + // all schedules once and iterate through that list which can take a significant + // amount of time to get to the end of the list. + schedule, err := database.GetSchedule(s.GetID()) + if err != nil { + logrus.WithError(err).Warnf("%s for %s", baseErr, schedule.GetName()) + + continue + } + + // create a variable to track if a build should be triggered based off the schedule + trigger := false + + // check if a build has already been triggered for the schedule + if schedule.GetScheduledAt() == 0 { + // trigger a build for the schedule since one has not already been scheduled + trigger = true + } else { + // parse the previous occurrence of the entry for the schedule + prevTime, err := gronx.PrevTick(schedule.GetEntry(), true) + if err != nil { + logrus.WithError(err).Warnf("%s for %s", baseErr, schedule.GetName()) + + continue + } + + // parse the next occurrence of the entry for the schedule + nextTime, err := gronx.NextTick(schedule.GetEntry(), true) + if err != nil { + logrus.WithError(err).Warnf("%s for %s", baseErr, schedule.GetName()) + + continue + } + + // parse the UNIX timestamp from when the last build was triggered for the schedule + t := time.Unix(schedule.GetScheduledAt(), 0).UTC() + + // check if the time since the last triggered build is greater than the entry duration for the schedule + if time.Since(t) > nextTime.Sub(prevTime) { + // trigger a build for the schedule since it has not previously ran + trigger = true + } + } + + if trigger && schedule.GetActive() { + err = processSchedule(schedule, compiler, database, metadata, queue, scm) + if err != nil { + logrus.WithError(err).Warnf("%s for %s", baseErr, schedule.GetName()) + + continue + } + } + } + + return nil +} + +//nolint:funlen // ignore function length and number of statements +func processSchedule(s *library.Schedule, compiler compiler.Engine, database database.Interface, metadata *types.Metadata, queue queue.Service, scm scm.Service) error { + // sleep for 1s - 3s before processing the schedule + // + // This should prevent multiple servers from processing a schedule at the same time by + // leveraging a base duration along with a standard deviation of randomness a.k.a. + // "jitter". To create the jitter, we use a base duration of 1s with a scale factor of 3.0. + time.Sleep(wait.Jitter(time.Second, 3.0)) + + // send API call to capture the repo for the schedule + r, err := database.GetRepo(s.GetRepoID()) + if err != nil { + return fmt.Errorf("unable to fetch repo: %w", err) + } + + logrus.Tracef("processing schedule %s/%s", r.GetFullName(), s.GetName()) + + // check if the repo is active + if !r.GetActive() { + return fmt.Errorf("repo %s is not active", r.GetFullName()) + } + + // check if the repo has a valid owner + if r.GetUserID() == 0 { + return fmt.Errorf("repo %s does not have a valid owner", r.GetFullName()) + } + + // send API call to capture the owner for the repo + u, err := database.GetUser(r.GetUserID()) + if err != nil { + return fmt.Errorf("unable to get owner for repo %s: %w", r.GetFullName(), err) + } + + // send API call to confirm repo owner has at least write access to repo + _, err = scm.RepoAccess(u, u.GetToken(), r.GetOrg(), r.GetName()) + if err != nil { + return fmt.Errorf("%s does not have at least write access for repo %s", u.GetName(), r.GetFullName()) + } + + // create SQL filters for querying pending and running builds for repo + filters := map[string]interface{}{ + "status": []string{constants.StatusPending, constants.StatusRunning}, + } + + // send API call to capture the number of pending or running builds for the repo + builds, err := database.GetRepoBuildCount(r, filters) + if err != nil { + return fmt.Errorf("unable to get count of builds for repo %s: %w", r.GetFullName(), err) + } + + // check if the number of pending and running builds exceeds the limit for the repo + if builds >= r.GetBuildLimit() { + return fmt.Errorf("repo %s has excceded the concurrent build limit of %d", r.GetFullName(), r.GetBuildLimit()) + } + + // send API call to capture the commit sha for the branch + _, commit, err := scm.GetBranch(u, r) + if err != nil { + return fmt.Errorf("failed to get commit for repo %s on %s branch: %w", r.GetFullName(), r.GetBranch(), err) + } + + url := strings.TrimSuffix(r.GetClone(), ".git") + + b := new(library.Build) + b.SetAuthor(s.GetCreatedBy()) + b.SetBranch(r.GetBranch()) + b.SetClone(r.GetClone()) + b.SetCommit(commit) + b.SetDeploy(s.GetName()) + b.SetEvent(constants.EventSchedule) + b.SetMessage(fmt.Sprintf("triggered for %s schedule with %s entry", s.GetName(), s.GetEntry())) + b.SetRef(fmt.Sprintf("refs/heads/%s", b.GetBranch())) + b.SetRepoID(r.GetID()) + b.SetSender(s.GetUpdatedBy()) + b.SetSource(fmt.Sprintf("%s/tree/%s", url, b.GetBranch())) + b.SetStatus(constants.StatusPending) + b.SetTitle(fmt.Sprintf("%s received from %s", constants.EventSchedule, url)) + + // populate the build link if a web address is provided + if len(metadata.Vela.WebAddress) > 0 { + b.SetLink(fmt.Sprintf("%s/%s/%d", metadata.Vela.WebAddress, r.GetFullName(), b.GetNumber())) + } + + var ( + // variable to store the raw pipeline configuration + config []byte + // variable to store executable pipeline + p *pipeline.Build + // variable to store pipeline configuration + pipeline *library.Pipeline + // variable to control number of times to retry processing pipeline + retryLimit = 5 + // variable to store the pipeline type for the repository + pipelineType = r.GetPipelineType() + ) + + // implement a loop to process asynchronous operations with a retry limit + // + // Some operations taken during this workflow can lead to race conditions failing to successfully process + // the request. This logic ensures we attempt our best efforts to handle these cases gracefully. + for i := 0; i < retryLimit; i++ { + logrus.Debugf("compilation loop - attempt %d", i+1) + // check if we're on the first iteration of the loop + if i > 0 { + // incrementally sleep in between retries + time.Sleep(time.Duration(i) * time.Second) + } + + // send API call to attempt to capture the pipeline + pipeline, err = database.GetPipelineForRepo(b.GetCommit(), r) + if err != nil { // assume the pipeline doesn't exist in the database yet + // send API call to capture the pipeline configuration file + config, err = scm.ConfigBackoff(u, r, b.GetCommit()) + if err != nil { + return fmt.Errorf("unable to get pipeline config for %s/%s: %w", r.GetFullName(), b.GetCommit(), err) + } + } else { + config = pipeline.GetData() + } + + // send API call to capture repo for the counter (grabbing repo again to ensure counter is correct) + r, err = database.GetRepoForOrg(r.GetOrg(), r.GetName()) + if err != nil { + err = fmt.Errorf("unable to get repo %s: %w", r.GetFullName(), err) + + // check if the retry limit has been exceeded + if i < retryLimit-1 { + logrus.WithError(err).Warningf("retrying #%d", i+1) + + // continue to the next iteration of the loop + continue + } + + return err + } + + // set the build numbers based off repo counter + r.SetCounter(r.GetCounter() + 1) + b.SetNumber(r.GetCounter() + 1) + // set the parent equal to the current repo counter + b.SetParent(r.GetCounter()) + // check if the parent is set to 0 + if b.GetParent() == 0 { + // parent should be "1" if it's the first build ran + b.SetParent(1) + } + + // set the build link if a web address is provided + if len(metadata.Vela.WebAddress) > 0 { + b.SetLink(fmt.Sprintf("%s/%s/%d", metadata.Vela.WebAddress, r.GetFullName(), b.GetNumber())) + } + + // ensure we use the expected pipeline type when compiling + // + // The pipeline type for a repo can change at any time which can break compiling + // existing pipelines in the system for that repo. To account for this, we update + // the repo pipeline type to match what was defined for the existing pipeline + // before compiling. After we're done compiling, we reset the pipeline type. + if len(pipeline.GetType()) > 0 { + r.SetPipelineType(pipeline.GetType()) + } + + var compiled *library.Pipeline + // parse and compile the pipeline configuration file + p, compiled, err = compiler. + Duplicate(). + WithBuild(b). + WithCommit(b.GetCommit()). + WithMetadata(metadata). + WithRepo(r). + WithUser(u). + Compile(config) + if err != nil { + return fmt.Errorf("unable to compile pipeline config for %s/%s: %w", r.GetFullName(), b.GetCommit(), err) + } + + // reset the pipeline type for the repo + // + // The pipeline type for a repo can change at any time which can break compiling + // existing pipelines in the system for that repo. To account for this, we update + // the repo pipeline type to match what was defined for the existing pipeline + // before compiling. After we're done compiling, we reset the pipeline type. + r.SetPipelineType(pipelineType) + + // skip the build if only the init or clone steps are found + skip := api.SkipEmptyBuild(p) + if skip != "" { + return nil + } + + // check if the pipeline did not already exist in the database + if pipeline == nil { + pipeline = compiled + pipeline.SetRepoID(r.GetID()) + pipeline.SetCommit(b.GetCommit()) + pipeline.SetRef(b.GetRef()) + + // send API call to create the pipeline + err = database.CreatePipeline(pipeline) + if err != nil { + err = fmt.Errorf("failed to create pipeline for %s: %w", r.GetFullName(), err) + + // check if the retry limit has been exceeded + if i < retryLimit-1 { + logrus.WithError(err).Warningf("retrying #%d", i+1) + + // continue to the next iteration of the loop + continue + } + + return err + } + + // send API call to capture the created pipeline + pipeline, err = database.GetPipelineForRepo(pipeline.GetCommit(), r) + if err != nil { + return fmt.Errorf("unable to get new pipeline %s/%s: %w", r.GetFullName(), pipeline.GetCommit(), err) + } + } + + b.SetPipelineID(pipeline.GetID()) + + // create the objects from the pipeline in the database + // TODO: + // - if a build gets created and something else fails midway, + // the next loop will attempt to create the same build, + // using the same Number and thus create a constraint + // conflict; consider deleting the partially created + // build object in the database + err = api.PlanBuild(database, p, b, r) + if err != nil { + // check if the retry limit has been exceeded + if i < retryLimit-1 { + logrus.WithError(err).Warningf("retrying #%d", i+1) + + // reset fields set by cleanBuild for retry + b.SetError("") + b.SetStatus(constants.StatusPending) + b.SetFinished(0) + + // continue to the next iteration of the loop + continue + } + + return err + } + + s.SetScheduledAt(time.Now().UTC().Unix()) + + // break the loop because everything was successful + break + } // end of retry loop + + // send API call to update repo for ensuring counter is incremented + err = database.UpdateRepo(r) + if err != nil { + return fmt.Errorf("unable to update repo %s: %w", r.GetFullName(), err) + } + + // send API call to update schedule for ensuring scheduled_at field is set + err = database.UpdateSchedule(s) + if err != nil { + return fmt.Errorf("unable to update schedule %s/%s: %w", r.GetFullName(), s.GetName(), err) + } + + // send API call to capture the triggered build + b, err = database.GetBuild(b.GetNumber(), r) + if err != nil { + return fmt.Errorf("unable to get new build %s/%d: %w", r.GetFullName(), b.GetNumber(), err) + } + + // publish the build to the queue + go api.PublishToQueue( + queue, + database, + p, + b, + r, + u, + ) + + return nil +} diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 1f8bdbd9c..80508e4b0 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -9,16 +9,19 @@ import ( "fmt" "net/http" "net/url" + "os" + "os/signal" + "syscall" "time" + "github.com/gin-gonic/gin" "github.com/go-vela/server/router" "github.com/go-vela/server/router/middleware" - - "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" - "gopkg.in/tomb.v2" + "golang.org/x/sync/errgroup" + + "k8s.io/apimachinery/pkg/util/wait" ) func server(c *cli.Context) error { @@ -111,47 +114,92 @@ func server(c *cli.Context) error { return err } - var tomb tomb.Tomb - // start http server - tomb.Go(func() error { - port := addr.Port() + port := addr.Port() + // check if a port is part of the address + if len(port) == 0 { + port = c.String("server-port") + } + + // gin expects the address to be ":" ie ":8080" + srv := &http.Server{ + Addr: fmt.Sprintf(":%s", port), + Handler: router, + ReadHeaderTimeout: 60 * time.Second, + } - // check if a port is part of the address - if len(port) == 0 { - port = c.String("server-port") + // create the context for controlling the worker subprocesses + ctx, done := context.WithCancel(context.Background()) + // create the errgroup for managing worker subprocesses + // + // https://pkg.go.dev/golang.org/x/sync/errgroup?tab=doc#Group + g, gctx := errgroup.WithContext(ctx) + + // spawn goroutine to check for signals to gracefully shutdown + g.Go(func() error { + signalChannel := make(chan os.Signal, 1) + signal.Notify(signalChannel, os.Interrupt, syscall.SIGTERM) + + select { + case sig := <-signalChannel: + logrus.Infof("received signal: %s", sig) + err := srv.Shutdown(ctx) + if err != nil { + logrus.Error(err) + } + done() + case <-gctx.Done(): + logrus.Info("closing signal goroutine") + err := srv.Shutdown(ctx) + if err != nil { + logrus.Error(err) + } + return gctx.Err() } - // gin expects the address to be ":" ie ":8080" - srv := &http.Server{ - Addr: fmt.Sprintf(":%s", port), - Handler: router, - ReadHeaderTimeout: 60 * time.Second, + return nil + }) + + // spawn goroutine for starting the server + g.Go(func() error { + logrus.Infof("starting server on %s", addr.Host) + err = srv.ListenAndServe() + if err != nil { + // log a message indicating the failure of the server + logrus.Errorf("failing server: %v", err) } - logrus.Infof("running server on %s", addr.Host) - go func() { - logrus.Info("Starting HTTP server...") - err := srv.ListenAndServe() - if err != nil { - tomb.Kill(err) - } - }() + return err + }) - //nolint:gosimple // ignore this for now + // spawn goroutine for starting the scheduler + g.Go(func() error { + logrus.Info("starting scheduler") for { - select { - case <-tomb.Dying(): - logrus.Info("Stopping HTTP server...") - return srv.Shutdown(context.Background()) + // cut the configured minimum frequency duration for schedules in half + // + // We need to sleep for some amount of time before we attempt to process schedules + // setup in the database. Since the minimum frequency is configurable, we cut it in + // half and use that as the base duration to determine how long to sleep for. + base := c.Duration("schedule-minimum-frequency") / 2 + logrus.Infof("sleeping for %v before scheduling builds", base) + + // sleep for a duration of time before processing schedules + // + // This should prevent multiple servers from processing schedules at the same time by + // leveraging a base duration along with a standard deviation of randomness a.k.a. + // "jitter". To create the jitter, we use the configured minimum frequency duration + // along with a scale factor of 0.1. + time.Sleep(wait.Jitter(base, 0.1)) + + err = processSchedules(compiler, database, metadata, queue, scm) + if err != nil { + logrus.WithError(err).Warn("unable to process schedules") + } else { + logrus.Trace("successfully processed schedules") } } }) - // Wait for stuff and watch for errors - err = tomb.Wait() - if err != nil { - return err - } - - return tomb.Err() + // wait for errors from server subprocesses + return g.Wait() } diff --git a/docker-compose.yml b/docker-compose.yml index 1394c317b..d030743e6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -157,4 +157,4 @@ services: - IPC_LOCK networks: - vela: + vela: \ No newline at end of file diff --git a/go.mod b/go.mod index b2ab8d4c0..5ff15db7b 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.0 github.com/go-playground/assert/v2 v2.2.0 - github.com/go-vela/types v0.19.3-0.20230516131722-f538de06bbf6 + github.com/go-vela/types v0.19.3-0.20230519215217-0da8c8b5e90f github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-cmp v0.5.9 github.com/google/go-github/v52 v52.0.0 @@ -33,8 +33,8 @@ require ( github.com/urfave/cli/v2 v2.25.1 go.starlark.net v0.0.0-20230302034142-4b1e35fe2254 golang.org/x/oauth2 v0.7.0 + golang.org/x/sync v0.1.0 gopkg.in/square/go-jose.v2 v2.6.0 - gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 gorm.io/driver/postgres v1.5.0 gorm.io/driver/sqlite v1.4.4 gorm.io/gorm v1.25.0 @@ -123,4 +123,5 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect ) diff --git a/go.sum b/go.sum index e0f559ca4..e988c342d 100644 --- a/go.sum +++ b/go.sum @@ -138,8 +138,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= -github.com/go-vela/types v0.19.3-0.20230516131722-f538de06bbf6 h1:WVmgeHuPN2WHTf/tJtseEMPxPoKdit2rD4nCZyPIias= -github.com/go-vela/types v0.19.3-0.20230516131722-f538de06bbf6/go.mod h1:0lsuPfGyVyTWJSi2h3NS6uaEW6DgnFvIzaZu1sXYKrs= +github.com/go-vela/types v0.19.3-0.20230519215217-0da8c8b5e90f h1:13H381Djx9iFC3BSj2f/ac57HlaI3mQL0el9vM7a3+k= +github.com/go-vela/types v0.19.3-0.20230519215217-0da8c8b5e90f/go.mod h1:0lsuPfGyVyTWJSi2h3NS6uaEW6DgnFvIzaZu1sXYKrs= github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -510,6 +510,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -736,8 +737,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 h1:yiW+nvdHb9LVqSHQBXfZCieqV4fzYhNBql77zY0ykqs= -gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -765,6 +764,8 @@ k8s.io/apimachinery v0.27.1 h1:EGuZiLI95UQQcClhanryclaQE6xjg1Bts6/L3cD7zyc= k8s.io/apimachinery v0.27.1/go.mod h1:5ikh59fK3AJ287GUvpUsryoMFtH9zj/ARfWCo3AyXTM= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= +k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/scm/github/repo.go b/scm/github/repo.go index d82c2aeac..b8fa31c3f 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -537,3 +537,22 @@ func (c *client) GetHTMLURL(u *library.User, org, repo, name, ref string) (strin return "", fmt.Errorf("no valid repository contents found") } + +// GetBranch defines a function that retrieves a branch for a repo. +func (c *client) GetBranch(u *library.User, r *library.Repo) (string, string, error) { + c.Logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + "user": u.GetName(), + }).Tracef("retrieving branch %s for repo %s", r.GetBranch(), r.GetFullName()) + + // create GitHub OAuth client with user's token + client := c.newClientToken(u.GetToken()) + + data, _, err := client.Repositories.GetBranch(ctx, r.GetOrg(), r.GetName(), r.GetBranch(), true) + if err != nil { + return "", "", err + } + + return data.GetName(), data.GetCommit().GetSHA(), nil +} diff --git a/scm/github/repo_test.go b/scm/github/repo_test.go index 3dba1bac3..7482c71ce 100644 --- a/scm/github/repo_test.go +++ b/scm/github/repo_test.go @@ -1302,3 +1302,52 @@ func TestGithub_GetPullRequest(t *testing.T) { t.Errorf("HeadRef is %v, want %v", gotHeadRef, wantHeadRef) } } + +func TestGithub_GetBranch(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.GET("/api/v3/repos/:owner/:repo/branches/:branch", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/branch.json") + }) + + s := httptest.NewServer(engine) + defer s.Close() + + // setup types + u := new(library.User) + u.SetName("foo") + u.SetToken("bar") + + r := new(library.Repo) + r.SetOrg("octocat") + r.SetName("Hello-World") + r.SetFullName("octocat/Hello-World") + r.SetBranch("main") + + wantBranch := "main" + wantCommit := "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d" + + client, _ := NewTest(s.URL) + + // run test + gotBranch, gotCommit, err := client.GetBranch(u, r) + + if err != nil { + t.Errorf("Status returned err: %v", err) + } + + if !strings.EqualFold(gotBranch, wantBranch) { + t.Errorf("Branch is %v, want %v", gotBranch, wantBranch) + } + + if !strings.EqualFold(gotCommit, wantCommit) { + t.Errorf("Commit is %v, want %v", gotCommit, wantCommit) + } +} diff --git a/scm/github/testdata/branch.json b/scm/github/testdata/branch.json new file mode 100644 index 000000000..b133e7b38 --- /dev/null +++ b/scm/github/testdata/branch.json @@ -0,0 +1,101 @@ +{ + "name": "main", + "commit": { + "sha": "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d", + "node_id": "MDY6Q29tbWl0MTI5NjI2OTo3ZmQxYTYwYjAxZjkxYjMxNGY1OTk1NWE0ZTRkNGU4MGQ4ZWRmMTFk", + "commit": { + "author": { + "name": "The Octocat", + "email": "octocat@nowhere.com", + "date": "2012-03-06T23:06:50Z" + }, + "committer": { + "name": "The Octocat", + "email": "octocat@nowhere.com", + "date": "2012-03-06T23:06:50Z" + }, + "message": "Merge pull request #6 from Spaceghost/patch-1\n\nNew line at end of file.", + "tree": { + "sha": "b4eecafa9be2f2006ce1b709d6857b07069b4608", + "url": "https://api.github.com/repos/octocat/Hello-World/git/trees/b4eecafa9be2f2006ce1b709d6857b07069b4608" + }, + "url": "https://api.github.com/repos/octocat/Hello-World/git/commits/7fd1a60b01f91b314f59955a4e4d4e80d8edf11d", + "comment_count": 77, + "verification": { + "verified": false, + "reason": "unsigned", + "signature": null, + "payload": null + } + }, + "url": "https://api.github.com/repos/octocat/Hello-World/commits/7fd1a60b01f91b314f59955a4e4d4e80d8edf11d", + "html_url": "https://github.com/octocat/Hello-World/commit/7fd1a60b01f91b314f59955a4e4d4e80d8edf11d", + "comments_url": "https://api.github.com/repos/octocat/Hello-World/commits/7fd1a60b01f91b314f59955a4e4d4e80d8edf11d/comments", + "author": { + "login": "octocat", + "id": 583231, + "node_id": "MDQ6VXNlcjU4MzIzMQ==", + "avatar_url": "https://avatars.githubusercontent.com/u/583231?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "committer": { + "login": "octocat", + "id": 583231, + "node_id": "MDQ6VXNlcjU4MzIzMQ==", + "avatar_url": "https://avatars.githubusercontent.com/u/583231?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "parents": [ + { + "sha": "553c2077f0edc3d5dc5d17262f6aa498e69d6f8e", + "url": "https://api.github.com/repos/octocat/Hello-World/commits/553c2077f0edc3d5dc5d17262f6aa498e69d6f8e", + "html_url": "https://github.com/octocat/Hello-World/commit/553c2077f0edc3d5dc5d17262f6aa498e69d6f8e" + }, + { + "sha": "762941318ee16e59dabbacb1b4049eec22f0d303", + "url": "https://api.github.com/repos/octocat/Hello-World/commits/762941318ee16e59dabbacb1b4049eec22f0d303", + "html_url": "https://github.com/octocat/Hello-World/commit/762941318ee16e59dabbacb1b4049eec22f0d303" + } + ] + }, + "_links": { + "self": "https://api.github.com/repos/octocat/Hello-World/branches/main", + "html": "https://github.com/octocat/Hello-World/tree/main" + }, + "protected": false, + "protection": { + "enabled": false, + "required_status_checks": { + "enforcement_level": "off", + "contexts": [], + "checks": [] + } + }, + "protection_url": "https://api.github.com/repos/octocat/Hello-World/branches/main/protection" +} \ No newline at end of file diff --git a/scm/service.go b/scm/service.go index bb0c0e275..c92cbedbf 100644 --- a/scm/service.go +++ b/scm/service.go @@ -108,6 +108,9 @@ type Service interface { // ListUserRepos defines a function that retrieves // all repos with admin rights for the user. ListUserRepos(*library.User) ([]*library.Repo, error) + // GetBranch defines a function that retrieves + // a branch for a repo. + GetBranch(*library.User, *library.Repo) (string, string, error) // GetPullRequest defines a function that retrieves // a pull request for a repo. GetPullRequest(*library.User, *library.Repo, int) (string, string, string, string, error) From 5f5bfe395886077e5edd1996f3dd30f4196378eb Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Fri, 26 May 2023 10:45:32 -0500 Subject: [PATCH 19/53] refactor(database): move build logic into separate package (#858) * chore: save work * chore: save work * feat(database): add build engine * chore(database): remove old service logic * chore: updates for database build engine * chore: address review feedback * fix: updates for scheduler --------- Co-authored-by: Easton Crupper <65553218+ecrupper@users.noreply.github.com> --- api/admin/build.go | 2 +- api/badge.go | 2 +- api/build.go | 20 +- api/deployment/list.go | 2 +- api/metrics.go | 14 +- api/webhook.go | 12 +- cmd/vela-server/schedule.go | 4 +- database/build/build.go | 80 +++ database/build/build_test.go | 268 +++++++++ database/build/count.go | 25 + database/build/count_deployment.go | 32 + database/build/count_deployment_test.go | 102 ++++ database/build/count_org.go | 31 + database/build/count_org_test.go | 159 +++++ database/build/count_repo.go | 32 + database/build/count_repo_test.go | 104 ++++ database/build/count_status.go | 27 + database/build/count_status_test.go | 97 ++++ database/build/count_test.go | 93 +++ database/build/create.go | 39 ++ database/build/create_test.go | 73 +++ database/build/delete.go | 30 + database/build/delete_test.go | 73 +++ database/build/get.go | 31 + database/build/get_repo.go | 37 ++ database/build/get_repo_test.go | 94 +++ database/build/get_test.go | 85 +++ database/build/index.go | 69 +++ database/build/index_test.go | 62 ++ database/build/interface.go | 61 ++ database/build/last_repo.go | 47 ++ database/build/last_repo_test.go | 95 +++ database/build/list.go | 54 ++ database/build/list_deployment.go | 66 +++ database/build/list_deployment_test.go | 112 ++++ database/build/list_org.go | 69 +++ database/build/list_org_test.go | 204 +++++++ database/build/list_pending_running.go | 46 ++ database/build/list_pending_running_test.go | 131 +++++ database/build/list_repo.go | 69 +++ database/build/list_repo_test.go | 117 ++++ database/build/list_test.go | 103 ++++ database/build/opts.go | 44 ++ database/build/opts_test.go | 161 ++++++ database/build/table.go | 108 ++++ database/build/table_test.go | 59 ++ database/build/update.go | 39 ++ database/build/update_test.go | 75 +++ database/interface.go | 84 +-- database/postgres/build.go | 197 ------- database/postgres/build_count.go | 83 --- database/postgres/build_count_test.go | 345 ----------- database/postgres/build_list.go | 174 ------ database/postgres/build_list_test.go | 459 --------------- database/postgres/build_test.go | 591 ------------------- database/postgres/ddl/build.go | 85 --- database/postgres/ddl/doc.go | 12 - database/postgres/dml/build.go | 90 --- database/postgres/dml/doc.go | 12 - database/postgres/postgres.go | 82 +-- database/postgres/postgres_test.go | 155 +---- database/sqlite/build.go | 197 ------- database/sqlite/build_count.go | 83 --- database/sqlite/build_count_test.go | 439 -------------- database/sqlite/build_list.go | 176 ------ database/sqlite/build_list_test.go | 532 ----------------- database/sqlite/build_test.go | 611 -------------------- database/sqlite/ddl/build.go | 85 --- database/sqlite/ddl/doc.go | 12 - database/sqlite/dml/build.go | 89 --- database/sqlite/dml/doc.go | 12 - database/sqlite/sqlite.go | 82 +-- database/sqlite/sqlite_test.go | 109 ---- router/middleware/build/build.go | 2 +- 74 files changed, 3393 insertions(+), 4764 deletions(-) create mode 100644 database/build/build.go create mode 100644 database/build/build_test.go create mode 100644 database/build/count.go create mode 100644 database/build/count_deployment.go create mode 100644 database/build/count_deployment_test.go create mode 100644 database/build/count_org.go create mode 100644 database/build/count_org_test.go create mode 100644 database/build/count_repo.go create mode 100644 database/build/count_repo_test.go create mode 100644 database/build/count_status.go create mode 100644 database/build/count_status_test.go create mode 100644 database/build/count_test.go create mode 100644 database/build/create.go create mode 100644 database/build/create_test.go create mode 100644 database/build/delete.go create mode 100644 database/build/delete_test.go create mode 100644 database/build/get.go create mode 100644 database/build/get_repo.go create mode 100644 database/build/get_repo_test.go create mode 100644 database/build/get_test.go create mode 100644 database/build/index.go create mode 100644 database/build/index_test.go create mode 100644 database/build/interface.go create mode 100644 database/build/last_repo.go create mode 100644 database/build/last_repo_test.go create mode 100644 database/build/list.go create mode 100644 database/build/list_deployment.go create mode 100644 database/build/list_deployment_test.go create mode 100644 database/build/list_org.go create mode 100644 database/build/list_org_test.go create mode 100644 database/build/list_pending_running.go create mode 100644 database/build/list_pending_running_test.go create mode 100644 database/build/list_repo.go create mode 100644 database/build/list_repo_test.go create mode 100644 database/build/list_test.go create mode 100644 database/build/opts.go create mode 100644 database/build/opts_test.go create mode 100644 database/build/table.go create mode 100644 database/build/table_test.go create mode 100644 database/build/update.go create mode 100644 database/build/update_test.go delete mode 100644 database/postgres/build.go delete mode 100644 database/postgres/build_count.go delete mode 100644 database/postgres/build_count_test.go delete mode 100644 database/postgres/build_list.go delete mode 100644 database/postgres/build_list_test.go delete mode 100644 database/postgres/build_test.go delete mode 100644 database/postgres/ddl/build.go delete mode 100644 database/postgres/ddl/doc.go delete mode 100644 database/postgres/dml/build.go delete mode 100644 database/postgres/dml/doc.go delete mode 100644 database/sqlite/build.go delete mode 100644 database/sqlite/build_count.go delete mode 100644 database/sqlite/build_count_test.go delete mode 100644 database/sqlite/build_list.go delete mode 100644 database/sqlite/build_list_test.go delete mode 100644 database/sqlite/build_test.go delete mode 100644 database/sqlite/ddl/build.go delete mode 100644 database/sqlite/ddl/doc.go delete mode 100644 database/sqlite/dml/build.go delete mode 100644 database/sqlite/dml/doc.go diff --git a/api/admin/build.go b/api/admin/build.go index 28ca14ddf..71a343e88 100644 --- a/api/admin/build.go +++ b/api/admin/build.go @@ -56,7 +56,7 @@ func AllBuildsQueue(c *gin.Context) { after := c.DefaultQuery("after", strconv.FormatInt(time.Now().UTC().Add(-24*time.Hour).Unix(), 10)) // send API call to capture pending and running builds - b, err := database.FromContext(c).GetPendingAndRunningBuilds(after) + b, err := database.FromContext(c).ListPendingAndRunningBuilds(after) if err != nil { retErr := fmt.Errorf("unable to capture all running and pending builds: %w", err) diff --git a/api/badge.go b/api/badge.go index 799de69ab..cdc028e28 100644 --- a/api/badge.go +++ b/api/badge.go @@ -57,7 +57,7 @@ func GetBadge(c *gin.Context) { }).Infof("creating latest build badge for repo %s on branch %s", r.GetFullName(), branch) // send API call to capture the last build for the repo and branch - b, err := database.FromContext(c).GetLastBuildByBranch(r, branch) + b, err := database.FromContext(c).LastBuildForRepo(r, branch) if err != nil { c.String(http.StatusOK, constants.BadgeUnknown) return diff --git a/api/build.go b/api/build.go index 445d4cc8a..cd6c8b86f 100644 --- a/api/build.go +++ b/api/build.go @@ -144,7 +144,7 @@ func CreateBuild(c *gin.Context) { } // send API call to capture the number of pending or running builds for the repo - builds, err := database.FromContext(c).GetRepoBuildCount(r, filters) + builds, err := database.FromContext(c).CountBuildsForRepo(r, filters) if err != nil { retErr := fmt.Errorf("unable to create new build: unable to get count of builds for repo %s", r.GetFullName()) @@ -358,7 +358,7 @@ func CreateBuild(c *gin.Context) { } // send API call to capture the created build - input, _ = database.FromContext(c).GetBuild(input.GetNumber(), r) + input, _ = database.FromContext(c).GetBuildForRepo(r, input.GetNumber()) c.JSON(http.StatusCreated, input) @@ -472,7 +472,7 @@ func GetBuildByID(c *gin.Context) { }).Infof("reading build %d", id) // Get build from database - b, err = database.FromContext(c).GetBuildByID(id) + b, err = database.FromContext(c).GetBuild(id) if err != nil { retErr := fmt.Errorf("unable to get build: %w", err) @@ -725,7 +725,7 @@ func GetBuilds(c *gin.Context) { return } - b, t, err = database.FromContext(c).GetRepoBuildList(r, filters, before, after, page, perPage) + b, t, err = database.FromContext(c).ListBuildsForRepo(r, filters, before, after, page, perPage) if err != nil { retErr := fmt.Errorf("unable to get builds for repo %s: %w", r.GetFullName(), err) @@ -898,7 +898,7 @@ func GetOrgBuilds(c *gin.Context) { } // send API call to capture the list of builds for the org (and event type if passed in) - b, t, err = database.FromContext(c).GetOrgBuildList(o, filters, page, perPage) + b, t, err = database.FromContext(c).ListBuildsForOrg(o, filters, page, perPage) if err != nil { retErr := fmt.Errorf("unable to get builds for org %s: %w", o, err) @@ -1063,7 +1063,7 @@ func RestartBuild(c *gin.Context) { } // send API call to capture the number of pending or running builds for the repo - builds, err := database.FromContext(c).GetRepoBuildCount(r, filters) + builds, err := database.FromContext(c).CountBuildsForRepo(r, filters) if err != nil { retErr := fmt.Errorf("unable to restart build: unable to get count of builds for repo %s", r.GetFullName()) @@ -1287,7 +1287,7 @@ func RestartBuild(c *gin.Context) { } // send API call to capture the restarted build - b, _ = database.FromContext(c).GetBuild(b.GetNumber(), r) + b, _ = database.FromContext(c).GetBuildForRepo(r, b.GetNumber()) c.JSON(http.StatusCreated, b) @@ -1448,7 +1448,7 @@ func UpdateBuild(c *gin.Context) { } // send API call to capture the updated build - b, _ = database.FromContext(c).GetBuild(b.GetNumber(), r) + b, _ = database.FromContext(c).GetBuildForRepo(r, b.GetNumber()) c.JSON(http.StatusOK, b) @@ -1533,7 +1533,7 @@ func DeleteBuild(c *gin.Context) { }).Infof("deleting build %s", entry) // send API call to remove the build - err := database.FromContext(c).DeleteBuild(b.GetID()) + err := database.FromContext(c).DeleteBuild(b) if err != nil { retErr := fmt.Errorf("unable to delete build %s: %w", entry, err) @@ -1594,7 +1594,7 @@ func PlanBuild(database database.Interface, p *pipeline.Build, b *library.Build, // send API call to capture the created build // TODO: this can be dropped once we return // the created build above - b, err = database.GetBuild(b.GetNumber(), r) + b, err = database.GetBuildForRepo(r, b.GetNumber()) if err != nil { return fmt.Errorf("unable to get new build for %s: %w", r.GetFullName(), err) } diff --git a/api/deployment/list.go b/api/deployment/list.go index bc5f9e97a..b3745aeca 100644 --- a/api/deployment/list.go +++ b/api/deployment/list.go @@ -138,7 +138,7 @@ func ListDeployments(c *gin.Context) { dWithBs := []*library.Deployment{} for _, deployment := range d { - b, err := database.FromContext(c).GetDeploymentBuildList(*deployment.URL) + b, _, err := database.FromContext(c).ListBuildsForDeployment(deployment, nil, 1, 3) if err != nil { retErr := fmt.Errorf("unable to get builds for deployment %d: %w", deployment.GetID(), err) diff --git a/api/metrics.go b/api/metrics.go index d68ab35e7..d005481b3 100644 --- a/api/metrics.go +++ b/api/metrics.go @@ -236,7 +236,7 @@ func recordGauges(c *gin.Context) { // build_count if q.BuildCount { // send API call to capture the total number of builds - b, err := database.FromContext(c).GetBuildCount() + b, err := database.FromContext(c).CountBuilds() if err != nil { logrus.Errorf("unable to get count of all builds: %v", err) } @@ -247,7 +247,7 @@ func recordGauges(c *gin.Context) { // running_build_count if q.RunningBuildCount { // send API call to capture the total number of running builds - bRun, err := database.FromContext(c).GetBuildCountByStatus("running") + bRun, err := database.FromContext(c).CountBuildsForStatus("running", nil) if err != nil { logrus.Errorf("unable to get count of all running builds: %v", err) } @@ -258,7 +258,7 @@ func recordGauges(c *gin.Context) { // pending_build_count if q.PendingBuildCount { // send API call to capture the total number of pending builds - bPen, err := database.FromContext(c).GetBuildCountByStatus("pending") + bPen, err := database.FromContext(c).CountBuildsForStatus("pending", nil) if err != nil { logrus.Errorf("unable to get count of all pending builds: %v", err) } @@ -280,7 +280,7 @@ func recordGauges(c *gin.Context) { // failure_build_count if q.FailureBuildCount { // send API call to capture the total number of failure builds - bFail, err := database.FromContext(c).GetBuildCountByStatus("failure") + bFail, err := database.FromContext(c).CountBuildsForStatus("failure", nil) if err != nil { logrus.Errorf("unable to get count of all failure builds: %v", err) } @@ -291,7 +291,7 @@ func recordGauges(c *gin.Context) { // killed_build_count if q.KilledBuildCount { // send API call to capture the total number of killed builds - bKill, err := database.FromContext(c).GetBuildCountByStatus("killed") + bKill, err := database.FromContext(c).CountBuildsForStatus("killed", nil) if err != nil { logrus.Errorf("unable to get count of all killed builds: %v", err) } @@ -302,7 +302,7 @@ func recordGauges(c *gin.Context) { // success_build_count if q.SuccessBuildCount { // send API call to capture the total number of success builds - bSucc, err := database.FromContext(c).GetBuildCountByStatus("success") + bSucc, err := database.FromContext(c).CountBuildsForStatus("success", nil) if err != nil { logrus.Errorf("unable to get count of all success builds: %v", err) } @@ -313,7 +313,7 @@ func recordGauges(c *gin.Context) { // error_build_count if q.ErrorBuildCount { // send API call to capture the total number of error builds - bErr, err := database.FromContext(c).GetBuildCountByStatus("error") + bErr, err := database.FromContext(c).CountBuildsForStatus("error", nil) if err != nil { logrus.Errorf("unable to get count of all error builds: %v", err) } diff --git a/api/webhook.go b/api/webhook.go index 5f4ddc5bf..6881a03b7 100644 --- a/api/webhook.go +++ b/api/webhook.go @@ -15,18 +15,16 @@ import ( "strings" "time" + "github.com/gin-gonic/gin" "github.com/go-vela/server/compiler" "github.com/go-vela/server/database" "github.com/go-vela/server/queue" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" - "github.com/go-vela/types" "github.com/go-vela/types/constants" "github.com/go-vela/types/library" "github.com/go-vela/types/pipeline" - - "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" ) @@ -318,7 +316,7 @@ func PostWebhook(c *gin.Context) { } // send API call to capture the number of pending or running builds for the repo - builds, err := database.FromContext(c).GetRepoBuildCount(repo, filters) + builds, err := database.FromContext(c).CountBuildsForRepo(repo, filters) if err != nil { retErr := fmt.Errorf("%s: unable to get count of builds for repo %s", baseErr, repo.GetFullName()) util.HandleError(c, http.StatusBadRequest, retErr) @@ -673,7 +671,7 @@ func PostWebhook(c *gin.Context) { } // send API call to capture the triggered build - b, err = database.FromContext(c).GetBuild(b.GetNumber(), repo) + b, err = database.FromContext(c).GetBuildForRepo(repo, b.GetNumber()) if err != nil { retErr := fmt.Errorf("%s: failed to get new build %s/%d: %w", baseErr, repo.GetFullName(), b.GetNumber(), err) util.HandleError(c, http.StatusInternalServerError, retErr) @@ -942,7 +940,7 @@ func renameRepository(h *library.Hook, r *library.Repo, c *gin.Context, m *types } // get total number of builds associated with repository - t, err = database.FromContext(c).GetRepoBuildCount(dbR, nil) + t, err = database.FromContext(c).CountBuildsForRepo(dbR, nil) if err != nil { return nil, fmt.Errorf("unable to get build count for repo %s: %w", dbR.GetFullName(), err) } @@ -951,7 +949,7 @@ func renameRepository(h *library.Hook, r *library.Repo, c *gin.Context, m *types page = 1 // capture all builds belonging to repo in database for build := int64(0); build < t; build += 100 { - b, _, err := database.FromContext(c).GetRepoBuildList(dbR, nil, time.Now().Unix(), 0, page, 100) + b, _, err := database.FromContext(c).ListBuildsForRepo(dbR, nil, time.Now().Unix(), 0, page, 100) if err != nil { return nil, fmt.Errorf("unable to get build list for repo %s: %w", dbR.GetFullName(), err) } diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index 172b5c251..cb607c59b 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -141,7 +141,7 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat } // send API call to capture the number of pending or running builds for the repo - builds, err := database.GetRepoBuildCount(r, filters) + builds, err := database.CountBuildsForRepo(r, filters) if err != nil { return fmt.Errorf("unable to get count of builds for repo %s: %w", r.GetFullName(), err) } @@ -362,7 +362,7 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat } // send API call to capture the triggered build - b, err = database.GetBuild(b.GetNumber(), r) + b, err = database.GetBuildForRepo(r, b.GetNumber()) if err != nil { return fmt.Errorf("unable to get new build %s/%d: %w", r.GetFullName(), b.GetNumber(), err) } diff --git a/database/build/build.go b/database/build/build.go new file mode 100644 index 000000000..6deb13892 --- /dev/null +++ b/database/build/build.go @@ -0,0 +1,80 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "fmt" + + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +type ( + // config represents the settings required to create the engine that implements the BuildInterface interface. + config struct { + // specifies to skip creating tables and indexes for the Build engine + SkipCreation bool + } + + // engine represents the build functionality that implements the BuildInterface interface. + engine struct { + // engine configuration settings used in build functions + config *config + + // gorm.io/gorm database client used in build functions + // + // https://pkg.go.dev/gorm.io/gorm#DB + client *gorm.DB + + // sirupsen/logrus logger used in build functions + // + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry + logger *logrus.Entry + } +) + +// New creates and returns a Vela service for integrating with builds in the database. +// +//nolint:revive // ignore returning unexported engine +func New(opts ...EngineOpt) (*engine, error) { + // create new Build engine + e := new(engine) + + // create new fields + e.client = new(gorm.DB) + e.config = new(config) + e.logger = new(logrus.Entry) + + // apply all provided configuration options + for _, opt := range opts { + err := opt(e) + if err != nil { + return nil, err + } + } + + // check if we should skip creating build database objects + if e.config.SkipCreation { + e.logger.Warning("skipping creation of builds table and indexes in the database") + + return e, nil + } + + // create the builds table + err := e.CreateBuildTable(e.client.Config.Dialector.Name()) + if err != nil { + return nil, fmt.Errorf("unable to create %s table: %w", constants.TableBuild, err) + } + + // create the indexes for the builds table + err = e.CreateBuildIndexes() + if err != nil { + return nil, fmt.Errorf("unable to create indexes for %s table: %w", constants.TableBuild, err) + } + + return e, nil +} diff --git a/database/build/build_test.go b/database/build/build_test.go new file mode 100644 index 000000000..6304cd167 --- /dev/null +++ b/database/build/build_test.go @@ -0,0 +1,268 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "database/sql/driver" + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" + + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestBuild_New(t *testing.T) { + // setup types + logger := logrus.NewEntry(logrus.StandardLogger()) + + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + defer _sql.Close() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + _config := &gorm.Config{SkipDefaultTransaction: true} + + _postgres, err := gorm.Open(postgres.New(postgres.Config{Conn: _sql}), _config) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _sqlite, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), _config) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + defer func() { _sql, _ := _sqlite.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + logger *logrus.Entry + skipCreation bool + want *engine + }{ + { + failure: false, + name: "postgres", + client: _postgres, + logger: logger, + skipCreation: false, + want: &engine{ + client: _postgres, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + { + failure: false, + name: "sqlite3", + client: _sqlite, + logger: logger, + skipCreation: false, + want: &engine{ + client: _sqlite, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := New( + WithClient(test.client), + WithLogger(test.logger), + WithSkipCreation(test.skipCreation), + ) + + if test.failure { + if err == nil { + t.Errorf("New for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("New for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("New for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} + +// testPostgres is a helper function to create a Postgres engine for testing. +func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { + // create the new mock sql database + // + // https://pkg.go.dev/github.com/DATA-DOG/go-sqlmock#New + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + // create the new mock Postgres database client + // + // https://pkg.go.dev/gorm.io/gorm#Open + _postgres, err := gorm.Open( + postgres.New(postgres.Config{Conn: _sql}), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _engine, err := New( + WithClient(_postgres), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new postgres build engine: %v", err) + } + + return _engine, _mock +} + +// testSqlite is a helper function to create a Sqlite engine for testing. +func testSqlite(t *testing.T) *engine { + _sqlite, err := gorm.Open( + sqlite.Open("file::memory:?cache=shared"), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + _engine, err := New( + WithClient(_sqlite), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new sqlite build engine: %v", err) + } + + return _engine +} + +// testBuild is a test helper function to create a library +// Build type with all fields set to their zero values. +func testBuild() *library.Build { + return &library.Build{ + ID: new(int64), + RepoID: new(int64), + PipelineID: new(int64), + Number: new(int), + Parent: new(int), + Event: new(string), + EventAction: new(string), + Status: new(string), + Error: new(string), + Enqueued: new(int64), + Created: new(int64), + Started: new(int64), + Finished: new(int64), + Deploy: new(string), + Clone: new(string), + Source: new(string), + Title: new(string), + Message: new(string), + Commit: new(string), + Sender: new(string), + Author: new(string), + Email: new(string), + Link: new(string), + Branch: new(string), + Ref: new(string), + BaseRef: new(string), + HeadRef: new(string), + Host: new(string), + Runtime: new(string), + Distribution: new(string), + } +} + +// testDeployment is a test helper function to create a library +// Repo type with all fields set to their zero values. +func testDeployment() *library.Deployment { + return &library.Deployment{ + ID: new(int64), + RepoID: new(int64), + URL: new(string), + User: new(string), + Commit: new(string), + Ref: new(string), + Task: new(string), + Target: new(string), + Description: new(string), + } +} + +// testRepo is a test helper function to create a library +// Repo type with all fields set to their zero values. +func testRepo() *library.Repo { + return &library.Repo{ + ID: new(int64), + UserID: new(int64), + BuildLimit: new(int64), + Timeout: new(int64), + Counter: new(int), + PipelineType: new(string), + Hash: new(string), + Org: new(string), + Name: new(string), + FullName: new(string), + Link: new(string), + Clone: new(string), + Branch: new(string), + Visibility: new(string), + PreviousName: new(string), + Private: new(bool), + Trusted: new(bool), + Active: new(bool), + AllowPull: new(bool), + AllowPush: new(bool), + AllowDeploy: new(bool), + AllowTag: new(bool), + AllowComment: new(bool), + } +} + +// This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values +// that are otherwise not easily compared. These typically would be values generated +// before adding or updating them in the database. +// +// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime +type AnyArgument struct{} + +// Match satisfies sqlmock.Argument interface. +func (a AnyArgument) Match(v driver.Value) bool { + return true +} diff --git a/database/build/count.go b/database/build/count.go new file mode 100644 index 000000000..7700c5786 --- /dev/null +++ b/database/build/count.go @@ -0,0 +1,25 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" +) + +// CountBuilds gets the count of all builds from the database. +func (e *engine) CountBuilds() (int64, error) { + e.logger.Tracef("getting count of all builds from the database") + + // variable to store query results + var b int64 + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableBuild). + Count(&b). + Error + + return b, err +} diff --git a/database/build/count_deployment.go b/database/build/count_deployment.go new file mode 100644 index 000000000..6203f196b --- /dev/null +++ b/database/build/count_deployment.go @@ -0,0 +1,32 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// CountBuildsForDeployment gets the count of builds by deployment URL from the database. +func (e *engine) CountBuildsForDeployment(d *library.Deployment, filters map[string]interface{}) (int64, error) { + e.logger.WithFields(logrus.Fields{ + "deployment": d.GetURL(), + }).Tracef("getting count of builds for deployment %s from the database", d.GetURL()) + + // variable to store query results + var b int64 + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableBuild). + Where("source = ?", d.GetURL()). + Where(filters). + Order("number DESC"). + Count(&b). + Error + + return b, err +} diff --git a/database/build/count_deployment_test.go b/database/build/count_deployment_test.go new file mode 100644 index 000000000..53c0c1df6 --- /dev/null +++ b/database/build/count_deployment_test.go @@ -0,0 +1,102 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestBuild_Engine_CountBuildsForDeployment(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetDeployPayload(nil) + _buildOne.SetSource("https://github.com/github/octocat/deployments/1") + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(1) + _buildTwo.SetNumber(2) + _buildTwo.SetDeployPayload(nil) + _buildTwo.SetSource("https://github.com/github/octocat/deployments/1") + + _deployment := testDeployment() + _deployment.SetID(1) + _deployment.SetRepoID(1) + _deployment.SetURL("https://github.com/github/octocat/deployments/1") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "builds" WHERE source = $1`).WithArgs("https://github.com/github/octocat/deployments/1").WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_buildOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 2, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 2, + }, + } + + filters := map[string]interface{}{} + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountBuildsForDeployment(_deployment, filters) + + if test.failure { + if err == nil { + t.Errorf("CountBuildsForDeployment for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountBuildsForDeployment for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountBuildsForDeployment for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/count_org.go b/database/build/count_org.go new file mode 100644 index 000000000..37825bad6 --- /dev/null +++ b/database/build/count_org.go @@ -0,0 +1,31 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" +) + +// CountBuildsForOrg gets the count of builds by org name from the database. +func (e *engine) CountBuildsForOrg(org string, filters map[string]interface{}) (int64, error) { + e.logger.WithFields(logrus.Fields{ + "org": org, + }).Tracef("getting count of builds for org %s from the database", org) + + // variable to store query results + var b int64 + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableBuild). + Joins("JOIN repos ON builds.repo_id = repos.id"). + Where("repos.org = ?", org). + Where(filters). + Count(&b). + Error + + return b, err +} diff --git a/database/build/count_org_test.go b/database/build/count_org_test.go new file mode 100644 index 000000000..429149e54 --- /dev/null +++ b/database/build/count_org_test.go @@ -0,0 +1,159 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" +) + +func TestBuild_Engine_CountBuildsForOrg(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetDeployPayload(nil) + _buildOne.SetEvent("push") + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(2) + _buildTwo.SetNumber(2) + _buildTwo.SetDeployPayload(nil) + _buildTwo.SetEvent("push") + + _repoOne := testRepo() + _repoOne.SetID(1) + _repoOne.SetUserID(1) + _repoOne.SetHash("baz") + _repoOne.SetOrg("foo") + _repoOne.SetName("bar") + _repoOne.SetFullName("foo/bar") + _repoOne.SetVisibility("public") + _repoOne.SetPipelineType("yaml") + _repoOne.SetTopics([]string{}) + + _repoTwo := testRepo() + _repoTwo.SetID(2) + _repoTwo.SetUserID(1) + _repoTwo.SetHash("bar") + _repoTwo.SetOrg("foo") + _repoTwo.SetName("baz") + _repoTwo.SetFullName("foo/baz") + _repoTwo.SetVisibility("public") + _repoTwo.SetPipelineType("yaml") + _repoTwo.SetTopics([]string{}) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result without filters in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + // ensure the mock expects the query without filters + _mock.ExpectQuery(`SELECT count(*) FROM "builds" JOIN repos ON builds.repo_id = repos.id WHERE repos.org = $1`).WithArgs("foo").WillReturnRows(_rows) + + // create expected result with event filter in mock + _rows = sqlmock.NewRows([]string{"count"}).AddRow(2) + // ensure the mock expects the query with event filter + _mock.ExpectQuery(`SELECT count(*) FROM "builds" JOIN repos ON builds.repo_id = repos.id WHERE repos.org = $1 AND "event" = $2`).WithArgs("foo", "push").WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_buildOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.client.AutoMigrate(&database.Repo{}) + if err != nil { + t.Errorf("unable to create repo table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableRepo).Create(database.RepoFromLibrary(_repoOne)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableRepo).Create(database.RepoFromLibrary(_repoTwo)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + filters map[string]interface{} + want int64 + }{ + { + failure: false, + name: "postgres without filters", + database: _postgres, + filters: map[string]interface{}{}, + want: 2, + }, + { + failure: false, + name: "postgres with event filter", + database: _postgres, + filters: map[string]interface{}{ + "event": "push", + }, + want: 2, + }, + { + failure: false, + name: "sqlite3 without filters", + database: _sqlite, + filters: map[string]interface{}{}, + want: 2, + }, + { + failure: false, + name: "sqlite3 with event filter", + database: _sqlite, + filters: map[string]interface{}{ + "event": "push", + }, + want: 2, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountBuildsForOrg("foo", test.filters) + + if test.failure { + if err == nil { + t.Errorf("CountBuildsForOrg for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountBuildsForOrg for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountBuildsForOrg for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/count_repo.go b/database/build/count_repo.go new file mode 100644 index 000000000..b49a415e0 --- /dev/null +++ b/database/build/count_repo.go @@ -0,0 +1,32 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// CountBuildsForRepo gets the count of builds by repo ID from the database. +func (e *engine) CountBuildsForRepo(r *library.Repo, filters map[string]interface{}) (int64, error) { + e.logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + }).Tracef("getting count of builds for repo %s from the database", r.GetFullName()) + + // variable to store query results + var b int64 + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableBuild). + Where("repo_id = ?", r.GetID()). + Where(filters). + Count(&b). + Error + + return b, err +} diff --git a/database/build/count_repo_test.go b/database/build/count_repo_test.go new file mode 100644 index 000000000..673887183 --- /dev/null +++ b/database/build/count_repo_test.go @@ -0,0 +1,104 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestBuild_Engine_CountBuildsForRepo(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetDeployPayload(nil) + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(1) + _buildTwo.SetNumber(2) + _buildTwo.SetDeployPayload(nil) + + _repo := testRepo() + _repo.SetID(1) + _repo.SetUserID(1) + _repo.SetHash("baz") + _repo.SetOrg("foo") + _repo.SetName("bar") + _repo.SetFullName("foo/bar") + _repo.SetVisibility("public") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "builds" WHERE repo_id = $1`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_buildOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 2, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 2, + }, + } + + filters := map[string]interface{}{} + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountBuildsForRepo(_repo, filters) + + if test.failure { + if err == nil { + t.Errorf("CountBuildsForRepo for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountBuildsForRepo for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountBuildsForRepo for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/count_status.go b/database/build/count_status.go new file mode 100644 index 000000000..af6a9f957 --- /dev/null +++ b/database/build/count_status.go @@ -0,0 +1,27 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" +) + +// CountBuildsForStatus gets the count of builds by status from the database. +func (e *engine) CountBuildsForStatus(status string, filters map[string]interface{}) (int64, error) { + e.logger.Tracef("getting count of builds for status %s from the database", status) + + // variable to store query results + var b int64 + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableBuild). + Where("status = ?", status). + Where(filters). + Count(&b). + Error + + return b, err +} diff --git a/database/build/count_status_test.go b/database/build/count_status_test.go new file mode 100644 index 000000000..f88759d1c --- /dev/null +++ b/database/build/count_status_test.go @@ -0,0 +1,97 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestBuild_Engine_CountBuildsForStatus(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetDeployPayload(nil) + _buildOne.SetStatus("running") + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(1) + _buildTwo.SetNumber(2) + _buildTwo.SetDeployPayload(nil) + _buildTwo.SetStatus("running") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "builds" WHERE status = $1`).WithArgs("running").WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_buildOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 2, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 2, + }, + } + + filters := map[string]interface{}{} + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountBuildsForStatus("running", filters) + + if test.failure { + if err == nil { + t.Errorf("CountBuildsForStatus for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountBuildsForStatus for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountBuildsForStatus for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/count_test.go b/database/build/count_test.go new file mode 100644 index 000000000..6254648ee --- /dev/null +++ b/database/build/count_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestBuild_Engine_CountBuilds(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetDeployPayload(nil) + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(1) + _buildTwo.SetNumber(2) + _buildTwo.SetDeployPayload(nil) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "builds"`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_buildOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 2, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 2, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountBuilds() + + if test.failure { + if err == nil { + t.Errorf("CountBuilds for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountBuilds for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountBuilds for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/create.go b/database/build/create.go new file mode 100644 index 000000000..911c6f148 --- /dev/null +++ b/database/build/create.go @@ -0,0 +1,39 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with update.go +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// CreateBuild creates a new build in the database. +func (e *engine) CreateBuild(b *library.Build) error { + e.logger.WithFields(logrus.Fields{ + "build": b.GetNumber(), + }).Tracef("creating build %d in the database", b.GetNumber()) + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#BuildFromLibrary + build := database.BuildFromLibrary(b) + + // validate the necessary fields are populated + // + // https://pkg.go.dev/github.com/go-vela/types/database#Build.Validate + err := build.Validate() + if err != nil { + return err + } + + // send query to the database + return e.client. + Table(constants.TableBuild). + Create(build.Crop()). + Error +} diff --git a/database/build/create_test.go b/database/build/create_test.go new file mode 100644 index 000000000..3cfe1fdf9 --- /dev/null +++ b/database/build/create_test.go @@ -0,0 +1,73 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestBuild_Engine_CreateBuild(t *testing.T) { + // setup types + _build := testBuild() + _build.SetID(1) + _build.SetRepoID(1) + _build.SetNumber(1) + _build.SetDeployPayload(nil) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) + + // ensure the mock expects the query + _mock.ExpectQuery(`INSERT INTO "builds" +("repo_id","pipeline_id","number","parent","event","event_action","status","error","enqueued","created","started","finished","deploy","deploy_payload","clone","source","title","message","commit","sender","author","email","link","branch","ref","base_ref","head_ref","host","runtime","distribution","id") +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31) RETURNING "id"`). + WithArgs(1, nil, 1, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, AnyArgument{}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 1). + WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateBuild(_build) + + if test.failure { + if err == nil { + t.Errorf("CreateBuild for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateBuild for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/build/delete.go b/database/build/delete.go new file mode 100644 index 000000000..76643eed1 --- /dev/null +++ b/database/build/delete.go @@ -0,0 +1,30 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// DeleteBuild deletes an existing build from the database. +func (e *engine) DeleteBuild(b *library.Build) error { + e.logger.WithFields(logrus.Fields{ + "build": b.GetNumber(), + }).Tracef("deleting build %d from the database", b.GetNumber()) + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#BuildFromLibrary + build := database.BuildFromLibrary(b) + + // send query to the database + return e.client. + Table(constants.TableBuild). + Delete(build). + Error +} diff --git a/database/build/delete_test.go b/database/build/delete_test.go new file mode 100644 index 000000000..91dd1c9c7 --- /dev/null +++ b/database/build/delete_test.go @@ -0,0 +1,73 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestBuild_Engine_DeleteBuild(t *testing.T) { + // setup types + _build := testBuild() + _build.SetID(1) + _build.SetRepoID(1) + _build.SetNumber(1) + _build.SetDeployPayload(nil) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`DELETE FROM "builds" WHERE "builds"."id" = $1`). + WithArgs(1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_build) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err = test.database.DeleteBuild(_build) + + if test.failure { + if err == nil { + t.Errorf("DeleteBuild for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("DeleteBuild for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/build/get.go b/database/build/get.go new file mode 100644 index 000000000..236d62cca --- /dev/null +++ b/database/build/get.go @@ -0,0 +1,31 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// GetBuild gets a build by ID from the database. +func (e *engine) GetBuild(id int64) (*library.Build, error) { + e.logger.Tracef("getting build %d from the database", id) + + // variable to store query results + b := new(database.Build) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableBuild). + Where("id = ?", id). + Take(b). + Error + if err != nil { + return nil, err + } + + return b.ToLibrary(), nil +} diff --git a/database/build/get_repo.go b/database/build/get_repo.go new file mode 100644 index 000000000..8d93dcb3c --- /dev/null +++ b/database/build/get_repo.go @@ -0,0 +1,37 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// GetBuildForRepo gets a build by repo ID and number from the database. +func (e *engine) GetBuildForRepo(r *library.Repo, number int) (*library.Build, error) { + e.logger.WithFields(logrus.Fields{ + "build": number, + "org": r.GetOrg(), + "repo": r.GetName(), + }).Tracef("getting build %s/%d from the database", r.GetFullName(), number) + + // variable to store query results + b := new(database.Build) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableBuild). + Where("repo_id = ?", r.GetID()). + Where("number = ?", number). + Take(b). + Error + if err != nil { + return nil, err + } + + return b.ToLibrary(), nil +} diff --git a/database/build/get_repo_test.go b/database/build/get_repo_test.go new file mode 100644 index 000000000..f1341d960 --- /dev/null +++ b/database/build/get_repo_test.go @@ -0,0 +1,94 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestBuild_Engine_GetBuildForRepo(t *testing.T) { + // setup types + _build := testBuild() + _build.SetID(1) + _build.SetRepoID(1) + _build.SetNumber(1) + _build.SetDeployPayload(nil) + + _repo := testRepo() + _repo.SetID(1) + _repo.SetUserID(1) + _repo.SetHash("baz") + _repo.SetOrg("foo") + _repo.SetName("bar") + _repo.SetFullName("foo/bar") + _repo.SetVisibility("public") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}). + AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "builds" WHERE repo_id = $1 AND number = $2 LIMIT 1`).WithArgs(1, 1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_build) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.Build + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _build, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _build, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetBuildForRepo(_repo, 1) + + if test.failure { + if err == nil { + t.Errorf("GetBuildForRepo for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetBuildForRepo for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetBuildForRepo for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/get_test.go b/database/build/get_test.go new file mode 100644 index 000000000..7aec15f13 --- /dev/null +++ b/database/build/get_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestBuild_Engine_GetBuild(t *testing.T) { + // setup types + _build := testBuild() + _build.SetID(1) + _build.SetRepoID(1) + _build.SetNumber(1) + _build.SetDeployPayload(nil) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}). + AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "builds" WHERE id = $1 LIMIT 1`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_build) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.Build + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _build, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _build, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetBuild(1) + + if test.failure { + if err == nil { + t.Errorf("GetBuild for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetBuild for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetBuild for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/index.go b/database/build/index.go new file mode 100644 index 000000000..77e15e451 --- /dev/null +++ b/database/build/index.go @@ -0,0 +1,69 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +const ( + // CreateCreatedIndex represents a query to create an + // index on the builds table for the created column. + CreateCreatedIndex = ` +CREATE INDEX +IF NOT EXISTS +builds_created +ON builds (created); +` + + // CreateRepoIDIndex represents a query to create an + // index on the builds table for the repo_id column. + CreateRepoIDIndex = ` +CREATE INDEX +IF NOT EXISTS +builds_repo_id +ON builds (repo_id); +` + + // CreateSourceIndex represents a query to create an + // index on the builds table for the source column. + CreateSourceIndex = ` +CREATE INDEX +IF NOT EXISTS +builds_source +ON builds (source); +` + + // CreateStatusIndex represents a query to create an + // index on the builds table for the status column. + CreateStatusIndex = ` +CREATE INDEX +IF NOT EXISTS +builds_status +ON builds (status); +` +) + +// CreateBuildIndexes creates the indexes for the builds table in the database. +func (e *engine) CreateBuildIndexes() error { + e.logger.Tracef("creating indexes for builds table in the database") + + // create the created column index for the builds table + err := e.client.Exec(CreateCreatedIndex).Error + if err != nil { + return err + } + + // create the repo_id column index for the builds table + err = e.client.Exec(CreateRepoIDIndex).Error + if err != nil { + return err + } + + // create the source column index for the builds table + err = e.client.Exec(CreateSourceIndex).Error + if err != nil { + return err + } + + // create the status column index for the builds table + return e.client.Exec(CreateStatusIndex).Error +} diff --git a/database/build/index_test.go b/database/build/index_test.go new file mode 100644 index 000000000..bd61ff5d2 --- /dev/null +++ b/database/build/index_test.go @@ -0,0 +1,62 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestBuild_Engine_CreateBuildIndexes(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateBuildIndexes() + + if test.failure { + if err == nil { + t.Errorf("CreateBuildIndexes for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateBuildIndexes for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/build/interface.go b/database/build/interface.go new file mode 100644 index 000000000..e4264f6a8 --- /dev/null +++ b/database/build/interface.go @@ -0,0 +1,61 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/library" +) + +// BuildInterface represents the Vela interface for build +// functions with the supported Database backends. +// +//nolint:revive // ignore name stutter +type BuildInterface interface { + // Build Data Definition Language Functions + // + // https://en.wikipedia.org/wiki/Data_definition_language + + // CreateBuildIndexes defines a function that creates the indexes for the builds table. + CreateBuildIndexes() error + // CreateBuildTable defines a function that creates the builds table. + CreateBuildTable(string) error + + // Build Data Manipulation Language Functions + // + // https://en.wikipedia.org/wiki/Data_manipulation_language + + // CountBuilds defines a function that gets the count of all builds. + CountBuilds() (int64, error) + // CountBuildsForDeployment defines a function that gets the count of builds by deployment url. + CountBuildsForDeployment(*library.Deployment, map[string]interface{}) (int64, error) + // CountBuildsForOrg defines a function that gets the count of builds by org name. + CountBuildsForOrg(string, map[string]interface{}) (int64, error) + // CountBuildsForRepo defines a function that gets the count of builds by repo ID. + CountBuildsForRepo(*library.Repo, map[string]interface{}) (int64, error) + // CountBuildsForStatus defines a function that gets the count of builds by status. + CountBuildsForStatus(string, map[string]interface{}) (int64, error) + // CreateBuild defines a function that creates a new build. + CreateBuild(*library.Build) error + // DeleteBuild defines a function that deletes an existing build. + DeleteBuild(*library.Build) error + // GetBuild defines a function that gets a build by ID. + GetBuild(int64) (*library.Build, error) + // GetBuildForRepo defines a function that gets a build by repo ID and number. + GetBuildForRepo(*library.Repo, int) (*library.Build, error) + // LastBuildForRepo defines a function that gets the last build ran by repo ID and branch. + LastBuildForRepo(*library.Repo, string) (*library.Build, error) + // ListBuilds defines a function that gets a list of all builds. + ListBuilds() ([]*library.Build, error) + // ListBuildsForDeployment defines a function that gets a list of builds by deployment url. + ListBuildsForDeployment(*library.Deployment, map[string]interface{}, int, int) ([]*library.Build, int64, error) + // ListBuildsForOrg defines a function that gets a list of builds by org name. + ListBuildsForOrg(string, map[string]interface{}, int, int) ([]*library.Build, int64, error) + // ListBuildsForRepo defines a function that gets a list of builds by repo ID. + ListBuildsForRepo(*library.Repo, map[string]interface{}, int64, int64, int, int) ([]*library.Build, int64, error) + // ListPendingAndRunningBuilds defines a function that gets a list of pending and running builds. + ListPendingAndRunningBuilds(string) ([]*library.BuildQueue, error) + // UpdateBuild defines a function that updates an existing build. + UpdateBuild(*library.Build) error +} diff --git a/database/build/last_repo.go b/database/build/last_repo.go new file mode 100644 index 000000000..ed0adebbf --- /dev/null +++ b/database/build/last_repo.go @@ -0,0 +1,47 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "errors" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +// LastBuildForRepo gets the last build by repo ID and branch from the database. +func (e *engine) LastBuildForRepo(r *library.Repo, branch string) (*library.Build, error) { + e.logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + }).Tracef("getting last build for repo %s from the database", r.GetFullName()) + + // variable to store query results + b := new(database.Build) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableBuild). + Where("repo_id = ?", r.GetID()). + Where("branch = ?", branch). + Order("number DESC"). + Take(b). + Error + if err != nil { + // check if the query returned a record not found error + if errors.Is(err, gorm.ErrRecordNotFound) { + // the record will not exist if it is a new repo + return nil, nil + } + + return nil, err + } + + return b.ToLibrary(), nil +} diff --git a/database/build/last_repo_test.go b/database/build/last_repo_test.go new file mode 100644 index 000000000..48cb32c01 --- /dev/null +++ b/database/build/last_repo_test.go @@ -0,0 +1,95 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestBuild_Engine_LastBuildForRepo(t *testing.T) { + // setup types + _build := testBuild() + _build.SetID(1) + _build.SetRepoID(1) + _build.SetNumber(1) + _build.SetDeployPayload(nil) + _build.SetBranch("master") + + _repo := testRepo() + _repo.SetID(1) + _repo.SetUserID(1) + _repo.SetHash("baz") + _repo.SetOrg("foo") + _repo.SetName("bar") + _repo.SetFullName("foo/bar") + _repo.SetVisibility("public") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}). + AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "master", "", "", "", "", "", "", 0) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "builds" WHERE repo_id = $1 AND branch = $2 ORDER BY number DESC LIMIT 1`).WithArgs(1, "master").WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_build) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.Build + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _build, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _build, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.LastBuildForRepo(_repo, "master") + + if test.failure { + if err == nil { + t.Errorf("LastBuildForRepo for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("LastBuildForRepo for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("LastBuildForRepo for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/list.go b/database/build/list.go new file mode 100644 index 000000000..1139cf490 --- /dev/null +++ b/database/build/list.go @@ -0,0 +1,54 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// ListBuilds gets a list of all builds from the database. +func (e *engine) ListBuilds() ([]*library.Build, error) { + e.logger.Trace("listing all builds from the database") + + // variables to store query results and return value + count := int64(0) + b := new([]database.Build) + builds := []*library.Build{} + + // count the results + count, err := e.CountBuilds() + if err != nil { + return nil, err + } + + // short-circuit if there are no results + if count == 0 { + return builds, nil + } + + // send query to the database and store result in variable + err = e.client. + Table(constants.TableBuild). + Find(&b). + Error + if err != nil { + return nil, err + } + + // iterate through all query results + for _, build := range *b { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := build + + // convert query result to library type + // + // https://pkg.go.dev/github.com/go-vela/types/database#Build.ToLibrary + builds = append(builds, tmp.ToLibrary()) + } + + return builds, nil +} diff --git a/database/build/list_deployment.go b/database/build/list_deployment.go new file mode 100644 index 000000000..ff397f2ae --- /dev/null +++ b/database/build/list_deployment.go @@ -0,0 +1,66 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// ListBuildsForDeployment gets a list of builds by deployment url from the database. +// +//nolint:lll // ignore long line length due to variable names +func (e *engine) ListBuildsForDeployment(d *library.Deployment, filters map[string]interface{}, page, perPage int) ([]*library.Build, int64, error) { + e.logger.WithFields(logrus.Fields{ + "deployment": d.GetURL(), + }).Tracef("listing builds for deployment %s from the database", d.GetURL()) + + // variables to store query results and return values + count := int64(0) + b := new([]database.Build) + builds := []*library.Build{} + + // count the results + count, err := e.CountBuildsForDeployment(d, filters) + if err != nil { + return builds, 0, err + } + + // short-circuit if there are no results + if count == 0 { + return builds, 0, nil + } + + // calculate offset for pagination through results + offset := perPage * (page - 1) + + err = e.client. + Table(constants.TableBuild). + Where("source = ?", d.GetURL()). + Where(filters). + Order("number DESC"). + Limit(perPage). + Offset(offset). + Find(&b). + Error + if err != nil { + return nil, count, err + } + + // iterate through all query results + for _, build := range *b { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := build + + // convert query result to library type + // + // https://pkg.go.dev/github.com/go-vela/types/database#Build.ToLibrary + builds = append(builds, tmp.ToLibrary()) + } + + return builds, count, nil +} diff --git a/database/build/list_deployment_test.go b/database/build/list_deployment_test.go new file mode 100644 index 000000000..94983c56e --- /dev/null +++ b/database/build/list_deployment_test.go @@ -0,0 +1,112 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestBuild_Engine_ListBuildsForDeployment(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetDeployPayload(nil) + _buildOne.SetSource("https://github.com/github/octocat/deployments/1") + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(1) + _buildTwo.SetNumber(2) + _buildTwo.SetDeployPayload(nil) + _buildTwo.SetSource("https://github.com/github/octocat/deployments/1") + + _deployment := testDeployment() + _deployment.SetID(1) + _deployment.SetRepoID(1) + _deployment.SetURL("https://github.com/github/octocat/deployments/1") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected count query result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the count query + _mock.ExpectQuery(`SELECT count(*) FROM "builds" WHERE source = $1`).WithArgs("https://github.com/github/octocat/deployments/1").WillReturnRows(_rows) + + // create expected query result in mock + _rows = sqlmock.NewRows( + []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}). + AddRow(2, 1, nil, 2, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "https://github.com/github/octocat/deployments/1", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0). + AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "https://github.com/github/octocat/deployments/1", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "builds" WHERE source = $1 ORDER BY number DESC LIMIT 10`).WithArgs("https://github.com/github/octocat/deployments/1").WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_buildOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.Build + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.Build{_buildTwo, _buildOne}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.Build{_buildTwo, _buildOne}, + }, + } + + filters := map[string]interface{}{} + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, _, err := test.database.ListBuildsForDeployment(_deployment, filters, 1, 10) + + if test.failure { + if err == nil { + t.Errorf("ListBuildsForDeployment for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListBuildsForDeployment for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListBuildsForDeployment for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/list_org.go b/database/build/list_org.go new file mode 100644 index 000000000..8f4e62e58 --- /dev/null +++ b/database/build/list_org.go @@ -0,0 +1,69 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// ListBuildsForOrg gets a list of builds by org name from the database. +// +//nolint:lll // ignore long line length due to variable names +func (e *engine) ListBuildsForOrg(org string, filters map[string]interface{}, page, perPage int) ([]*library.Build, int64, error) { + e.logger.WithFields(logrus.Fields{ + "org": org, + }).Tracef("listing builds for org %s from the database", org) + + // variables to store query results and return values + count := int64(0) + b := new([]database.Build) + builds := []*library.Build{} + + // count the results + count, err := e.CountBuildsForOrg(org, filters) + if err != nil { + return builds, 0, err + } + + // short-circuit if there are no results + if count == 0 { + return builds, 0, nil + } + + // calculate offset for pagination through results + offset := perPage * (page - 1) + + err = e.client. + Table(constants.TableBuild). + Select("builds.*"). + Joins("JOIN repos ON builds.repo_id = repos.id"). + Where("repos.org = ?", org). + Where(filters). + Order("created DESC"). + Order("id"). + Limit(perPage). + Offset(offset). + Find(&b). + Error + if err != nil { + return nil, count, err + } + + // iterate through all query results + for _, build := range *b { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := build + + // convert query result to library type + // + // https://pkg.go.dev/github.com/go-vela/types/database#Build.ToLibrary + builds = append(builds, tmp.ToLibrary()) + } + + return builds, count, nil +} diff --git a/database/build/list_org_test.go b/database/build/list_org_test.go new file mode 100644 index 000000000..e9121e024 --- /dev/null +++ b/database/build/list_org_test.go @@ -0,0 +1,204 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +func TestBuild_Engine_ListBuildsForOrg(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetDeployPayload(nil) + _buildOne.SetEvent("push") + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(2) + _buildTwo.SetNumber(2) + _buildTwo.SetDeployPayload(nil) + _buildTwo.SetEvent("push") + + _repoOne := testRepo() + _repoOne.SetID(1) + _repoOne.SetUserID(1) + _repoOne.SetHash("baz") + _repoOne.SetOrg("foo") + _repoOne.SetName("bar") + _repoOne.SetFullName("foo/bar") + _repoOne.SetVisibility("public") + _repoOne.SetPipelineType("yaml") + _repoOne.SetTopics([]string{}) + + _repoTwo := testRepo() + _repoTwo.SetID(2) + _repoTwo.SetUserID(1) + _repoTwo.SetHash("bar") + _repoTwo.SetOrg("foo") + _repoTwo.SetName("baz") + _repoTwo.SetFullName("foo/baz") + _repoTwo.SetVisibility("public") + _repoTwo.SetPipelineType("yaml") + _repoTwo.SetTopics([]string{}) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected count query without filters result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + // ensure the mock expects the count query without filters + _mock.ExpectQuery(`SELECT count(*) FROM "builds" JOIN repos ON builds.repo_id = repos.id WHERE repos.org = $1`).WithArgs("foo").WillReturnRows(_rows) + // create expected query without filters result in mock + _rows = sqlmock.NewRows( + []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}). + AddRow(1, 1, nil, 1, 0, "push", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0). + AddRow(2, 2, nil, 2, 0, "push", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) + // ensure the mock expects the query without filters + _mock.ExpectQuery(`SELECT builds.* FROM "builds" JOIN repos ON builds.repo_id = repos.id WHERE repos.org = $1 ORDER BY created DESC,id LIMIT 10`).WithArgs("foo").WillReturnRows(_rows) + + // create expected count query with event filter result in mock + _rows = sqlmock.NewRows([]string{"count"}).AddRow(2) + // ensure the mock expects the count query with event filter + _mock.ExpectQuery(`SELECT count(*) FROM "builds" JOIN repos ON builds.repo_id = repos.id WHERE repos.org = $1 AND "event" = $2`).WithArgs("foo", "push").WillReturnRows(_rows) + // create expected query with event filter result in mock + _rows = sqlmock.NewRows( + []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}). + AddRow(1, 1, nil, 1, 0, "push", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0). + AddRow(2, 2, nil, 2, 0, "push", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) + // ensure the mock expects the query with event filter + _mock.ExpectQuery(`SELECT builds.* FROM "builds" JOIN repos ON builds.repo_id = repos.id WHERE repos.org = $1 AND "event" = $2 ORDER BY created DESC,id LIMIT 10`).WithArgs("foo", "push").WillReturnRows(_rows) + + // create expected count query with visibility filter result in mock + _rows = sqlmock.NewRows([]string{"count"}).AddRow(2) + // ensure the mock expects the count query with visibility filter + _mock.ExpectQuery(`SELECT count(*) FROM "builds" JOIN repos ON builds.repo_id = repos.id WHERE repos.org = $1 AND "visibility" = $2`).WithArgs("foo", "public").WillReturnRows(_rows) + // create expected query with visibility filter result in mock + _rows = sqlmock.NewRows( + []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}). + AddRow(1, 1, nil, 1, 0, "push", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0). + AddRow(2, 2, nil, 2, 0, "push", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) + // ensure the mock expects the query with visibility filter + _mock.ExpectQuery(`SELECT builds.* FROM "builds" JOIN repos ON builds.repo_id = repos.id WHERE repos.org = $1 AND "visibility" = $2 ORDER BY created DESC,id LIMIT 10`).WithArgs("foo", "public").WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_buildOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.client.AutoMigrate(&database.Repo{}) + if err != nil { + t.Errorf("unable to create repo table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableRepo).Create(database.RepoFromLibrary(_repoOne)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableRepo).Create(database.RepoFromLibrary(_repoTwo)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + filters map[string]interface{} + want []*library.Build + }{ + { + failure: false, + name: "postgres without filters", + database: _postgres, + filters: map[string]interface{}{}, + want: []*library.Build{_buildOne, _buildTwo}, + }, + { + failure: false, + name: "postgres with event filter", + database: _postgres, + filters: map[string]interface{}{ + "event": "push", + }, + want: []*library.Build{_buildOne, _buildTwo}, + }, + { + failure: false, + name: "postgres with visibility filter", + database: _postgres, + filters: map[string]interface{}{ + "visibility": "public", + }, + want: []*library.Build{_buildOne, _buildTwo}, + }, + { + failure: false, + name: "sqlite3 without filters", + database: _sqlite, + filters: map[string]interface{}{}, + want: []*library.Build{_buildOne, _buildTwo}, + }, + { + failure: false, + name: "sqlite3 with event filter", + database: _sqlite, + filters: map[string]interface{}{ + "event": "push", + }, + want: []*library.Build{_buildOne, _buildTwo}, + }, + { + failure: false, + name: "sqlite3 with visibility filter", + database: _sqlite, + filters: map[string]interface{}{ + "visibility": "public", + }, + want: []*library.Build{_buildOne, _buildTwo}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, _, err := test.database.ListBuildsForOrg("foo", test.filters, 1, 10) + + if test.failure { + if err == nil { + t.Errorf("ListBuildsForOrg for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListBuildsForOrg for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListBuildsForOrg for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/list_pending_running.go b/database/build/list_pending_running.go new file mode 100644 index 000000000..bd2a219b5 --- /dev/null +++ b/database/build/list_pending_running.go @@ -0,0 +1,46 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// ListPendingAndRunningBuilds gets a list of all pending and running builds in the provided timeframe from the database. +func (e *engine) ListPendingAndRunningBuilds(after string) ([]*library.BuildQueue, error) { + e.logger.Trace("listing all pending and running builds from the database") + + // variables to store query results and return value + b := new([]database.BuildQueue) + builds := []*library.BuildQueue{} + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableBuild). + Select("builds.created, builds.number, builds.status, repos.full_name"). + InnerJoins("INNER JOIN repos ON builds.repo_id = repos.id"). + Where("builds.created > ?", after). + Where("builds.status = 'running' OR builds.status = 'pending'"). + Find(&b). + Error + if err != nil { + return nil, err + } + + // iterate through all query results + for _, build := range *b { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := build + + // convert query result to library type + // + // https://pkg.go.dev/github.com/go-vela/types/database#Build.ToLibrary + builds = append(builds, tmp.ToLibrary()) + } + + return builds, nil +} diff --git a/database/build/list_pending_running_test.go b/database/build/list_pending_running_test.go new file mode 100644 index 000000000..8d47dc63b --- /dev/null +++ b/database/build/list_pending_running_test.go @@ -0,0 +1,131 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +func TestBuild_Engine_ListPendingAndRunningBuilds(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetStatus("running") + _buildOne.SetCreated(1) + _buildOne.SetDeployPayload(nil) + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(1) + _buildTwo.SetNumber(2) + _buildTwo.SetStatus("pending") + _buildTwo.SetCreated(1) + _buildTwo.SetDeployPayload(nil) + + _queueOne := new(library.BuildQueue) + _queueOne.SetCreated(1) + _queueOne.SetFullName("foo/bar") + _queueOne.SetNumber(1) + _queueOne.SetStatus("running") + + _queueTwo := new(library.BuildQueue) + _queueTwo.SetCreated(1) + _queueTwo.SetFullName("foo/bar") + _queueTwo.SetNumber(2) + _queueTwo.SetStatus("pending") + + _repo := testRepo() + _repo.SetID(1) + _repo.SetUserID(1) + _repo.SetHash("baz") + _repo.SetOrg("foo") + _repo.SetName("bar") + _repo.SetFullName("foo/bar") + _repo.SetVisibility("public") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected name query result in mock + _rows := sqlmock.NewRows([]string{"created", "full_name", "number", "status"}).AddRow(1, "foo/bar", 2, "pending").AddRow(1, "foo/bar", 1, "running") + + // ensure the mock expects the name query + _mock.ExpectQuery(`SELECT builds.created, builds.number, builds.status, repos.full_name FROM "builds" INNER JOIN repos ON builds.repo_id = repos.id WHERE builds.created > $1 AND (builds.status = 'running' OR builds.status = 'pending')`).WithArgs("0").WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_buildOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.client.AutoMigrate(&database.Repo{}) + if err != nil { + t.Errorf("unable to create repo table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableRepo).Create(database.RepoFromLibrary(_repo)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.BuildQueue + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.BuildQueue{_queueTwo, _queueOne}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.BuildQueue{_queueTwo, _queueOne}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.ListPendingAndRunningBuilds("0") + + if test.failure { + if err == nil { + t.Errorf("ListPendingAndRunningBuilds for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListPendingAndRunningBuilds for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListPendingAndRunningBuilds for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/list_repo.go b/database/build/list_repo.go new file mode 100644 index 000000000..83ebd4adb --- /dev/null +++ b/database/build/list_repo.go @@ -0,0 +1,69 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// ListBuildsForRepo gets a list of builds by repo ID from the database. +// +//nolint:lll // ignore long line length due to variable names +func (e *engine) ListBuildsForRepo(r *library.Repo, filters map[string]interface{}, before, after int64, page, perPage int) ([]*library.Build, int64, error) { + e.logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + }).Tracef("listing builds for repo %s from the database", r.GetFullName()) + + // variables to store query results and return values + count := int64(0) + b := new([]database.Build) + builds := []*library.Build{} + + // count the results + count, err := e.CountBuildsForRepo(r, filters) + if err != nil { + return builds, 0, err + } + + // short-circuit if there are no results + if count == 0 { + return builds, 0, nil + } + + // calculate offset for pagination through results + offset := perPage * (page - 1) + + err = e.client. + Table(constants.TableBuild). + Where("repo_id = ?", r.GetID()). + Where("created < ?", before). + Where("created > ?", after). + Where(filters). + Order("number DESC"). + Limit(perPage). + Offset(offset). + Find(&b). + Error + if err != nil { + return nil, count, err + } + + // iterate through all query results + for _, build := range *b { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := build + + // convert query result to library type + // + // https://pkg.go.dev/github.com/go-vela/types/database#Build.ToLibrary + builds = append(builds, tmp.ToLibrary()) + } + + return builds, count, nil +} diff --git a/database/build/list_repo_test.go b/database/build/list_repo_test.go new file mode 100644 index 000000000..ac367e414 --- /dev/null +++ b/database/build/list_repo_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestBuild_Engine_ListBuildsForRepo(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetDeployPayload(nil) + _buildOne.SetCreated(1) + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(1) + _buildTwo.SetNumber(2) + _buildTwo.SetDeployPayload(nil) + _buildTwo.SetCreated(2) + + _repo := testRepo() + _repo.SetID(1) + _repo.SetUserID(1) + _repo.SetHash("baz") + _repo.SetOrg("foo") + _repo.SetName("bar") + _repo.SetFullName("foo/bar") + _repo.SetVisibility("public") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected count query result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the count query + _mock.ExpectQuery(`SELECT count(*) FROM "builds" WHERE repo_id = $1`).WithArgs(1).WillReturnRows(_rows) + + // create expected query result in mock + _rows = sqlmock.NewRows( + []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}). + AddRow(2, 1, nil, 2, 0, "", "", "", "", 0, 2, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0). + AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 1, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "builds" WHERE repo_id = $1 AND created < $2 AND created > $3 ORDER BY number DESC LIMIT 10`).WithArgs(1, AnyArgument{}, 0).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_buildOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.Build + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.Build{_buildTwo, _buildOne}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.Build{_buildTwo, _buildOne}, + }, + } + + filters := map[string]interface{}{} + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, _, err := test.database.ListBuildsForRepo(_repo, filters, time.Now().UTC().Unix(), 0, 1, 10) + + if test.failure { + if err == nil { + t.Errorf("ListBuildsForRepo for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListBuildsForRepo for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListBuildsForRepo for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/list_test.go b/database/build/list_test.go new file mode 100644 index 000000000..b0e441387 --- /dev/null +++ b/database/build/list_test.go @@ -0,0 +1,103 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestBuild_Engine_ListBuilds(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetDeployPayload(nil) + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(1) + _buildTwo.SetNumber(2) + _buildTwo.SetDeployPayload(nil) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "builds"`).WillReturnRows(_rows) + + // create expected result in mock + _rows = sqlmock.NewRows( + []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}). + AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0). + AddRow(2, 1, nil, 2, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "builds"`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_buildOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.Build + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.Build{_buildOne, _buildTwo}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.Build{_buildOne, _buildTwo}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.ListBuilds() + + if test.failure { + if err == nil { + t.Errorf("ListBuilds for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListBuilds for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListBuilds for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/opts.go b/database/build/opts.go new file mode 100644 index 000000000..d5a5d4137 --- /dev/null +++ b/database/build/opts.go @@ -0,0 +1,44 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +// EngineOpt represents a configuration option to initialize the database engine for Builds. +type EngineOpt func(*engine) error + +// WithClient sets the gorm.io/gorm client in the database engine for Builds. +func WithClient(client *gorm.DB) EngineOpt { + return func(e *engine) error { + // set the gorm.io/gorm client in the build engine + e.client = client + + return nil + } +} + +// WithLogger sets the github.com/sirupsen/logrus logger in the database engine for Builds. +func WithLogger(logger *logrus.Entry) EngineOpt { + return func(e *engine) error { + // set the github.com/sirupsen/logrus logger in the build engine + e.logger = logger + + return nil + } +} + +// WithSkipCreation sets the skip creation logic in the database engine for Builds. +func WithSkipCreation(skipCreation bool) EngineOpt { + return func(e *engine) error { + // set to skip creating tables and indexes in the build engine + e.config.SkipCreation = skipCreation + + return nil + } +} diff --git a/database/build/opts_test.go b/database/build/opts_test.go new file mode 100644 index 000000000..2d41cc676 --- /dev/null +++ b/database/build/opts_test.go @@ -0,0 +1,161 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +func TestBuild_EngineOpt_WithClient(t *testing.T) { + // setup types + e := &engine{client: new(gorm.DB)} + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + want *gorm.DB + }{ + { + failure: false, + name: "client set to new database", + client: new(gorm.DB), + want: new(gorm.DB), + }, + { + failure: false, + name: "client set to nil", + client: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithClient(test.client)(e) + + if test.failure { + if err == nil { + t.Errorf("WithClient for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithClient returned err: %v", err) + } + + if !reflect.DeepEqual(e.client, test.want) { + t.Errorf("WithClient is %v, want %v", e.client, test.want) + } + }) + } +} + +func TestBuild_EngineOpt_WithLogger(t *testing.T) { + // setup types + e := &engine{logger: new(logrus.Entry)} + + // setup tests + tests := []struct { + failure bool + name string + logger *logrus.Entry + want *logrus.Entry + }{ + { + failure: false, + name: "logger set to new entry", + logger: new(logrus.Entry), + want: new(logrus.Entry), + }, + { + failure: false, + name: "logger set to nil", + logger: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithLogger(test.logger)(e) + + if test.failure { + if err == nil { + t.Errorf("WithLogger for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithLogger returned err: %v", err) + } + + if !reflect.DeepEqual(e.logger, test.want) { + t.Errorf("WithLogger is %v, want %v", e.logger, test.want) + } + }) + } +} + +func TestBuild_EngineOpt_WithSkipCreation(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + skipCreation bool + want bool + }{ + { + failure: false, + name: "skip creation set to true", + skipCreation: true, + want: true, + }, + { + failure: false, + name: "skip creation set to false", + skipCreation: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithSkipCreation(test.skipCreation)(e) + + if test.failure { + if err == nil { + t.Errorf("WithSkipCreation for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithSkipCreation returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.SkipCreation, test.want) { + t.Errorf("WithSkipCreation is %v, want %v", e.config.SkipCreation, test.want) + } + }) + } +} diff --git a/database/build/table.go b/database/build/table.go new file mode 100644 index 000000000..faf76746f --- /dev/null +++ b/database/build/table.go @@ -0,0 +1,108 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import "github.com/go-vela/types/constants" + +const ( + // CreatePostgresTable represents a query to create the Postgres builds table. + CreatePostgresTable = ` +CREATE TABLE +IF NOT EXISTS +builds ( + id SERIAL PRIMARY KEY, + repo_id INTEGER, + pipeline_id INTEGER, + number INTEGER, + parent INTEGER, + event VARCHAR(250), + event_action VARCHAR(250), + status VARCHAR(250), + error VARCHAR(1000), + enqueued INTEGER, + created INTEGER, + started INTEGER, + finished INTEGER, + deploy VARCHAR(500), + deploy_payload VARCHAR(2000), + clone VARCHAR(1000), + source VARCHAR(1000), + title VARCHAR(1000), + message VARCHAR(2000), + commit VARCHAR(500), + sender VARCHAR(250), + author VARCHAR(250), + email VARCHAR(500), + link VARCHAR(1000), + branch VARCHAR(500), + ref VARCHAR(500), + base_ref VARCHAR(500), + head_ref VARCHAR(500), + host VARCHAR(250), + runtime VARCHAR(250), + distribution VARCHAR(250), + timestamp INTEGER, + UNIQUE(repo_id, number) +); +` + + // CreateSqliteTable represents a query to create the Sqlite builds table. + CreateSqliteTable = ` +CREATE TABLE +IF NOT EXISTS +builds ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_id INTEGER, + pipeline_id INTEGER, + number INTEGER, + parent INTEGER, + event TEXT, + event_action TEXT, + status TEXT, + error TEXT, + enqueued INTEGER, + created INTEGER, + started INTEGER, + finished INTEGER, + deploy TEXT, + deploy_payload TEXT, + clone TEXT, + source TEXT, + title TEXT, + message TEXT, + 'commit' TEXT, + sender TEXT, + author TEXT, + email TEXT, + link TEXT, + branch TEXT, + ref TEXT, + base_ref TEXT, + head_ref TEXT, + host TEXT, + runtime TEXT, + distribution TEXT, + timestamp INTEGER, + UNIQUE(repo_id, number) +); +` +) + +// CreateBuildTable creates the builds table in the database. +func (e *engine) CreateBuildTable(driver string) error { + e.logger.Tracef("creating builds table in the database") + + // handle the driver provided to create the table + switch driver { + case constants.DriverPostgres: + // create the builds table for Postgres + return e.client.Exec(CreatePostgresTable).Error + case constants.DriverSqlite: + fallthrough + default: + // create the builds table for Sqlite + return e.client.Exec(CreateSqliteTable).Error + } +} diff --git a/database/build/table_test.go b/database/build/table_test.go new file mode 100644 index 000000000..394a1287e --- /dev/null +++ b/database/build/table_test.go @@ -0,0 +1,59 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestBuild_Engine_CreateBuildTable(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateBuildTable(test.name) + + if test.failure { + if err == nil { + t.Errorf("CreateBuildTable for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateBuildTable for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/build/update.go b/database/build/update.go new file mode 100644 index 000000000..d16329419 --- /dev/null +++ b/database/build/update.go @@ -0,0 +1,39 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with create.go +package build + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// UpdateBuild updates an existing build in the database. +func (e *engine) UpdateBuild(b *library.Build) error { + e.logger.WithFields(logrus.Fields{ + "build": b.GetNumber(), + }).Tracef("updating build %d in the database", b.GetNumber()) + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#BuildFromLibrary + build := database.BuildFromLibrary(b) + + // validate the necessary fields are populated + // + // https://pkg.go.dev/github.com/go-vela/types/database#Build.Validate + err := build.Validate() + if err != nil { + return err + } + + // send query to the database + return e.client. + Table(constants.TableBuild). + Save(build.Crop()). + Error +} diff --git a/database/build/update_test.go b/database/build/update_test.go new file mode 100644 index 000000000..98ac9ae16 --- /dev/null +++ b/database/build/update_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestBuild_Engine_UpdateBuild(t *testing.T) { + // setup types + _build := testBuild() + _build.SetID(1) + _build.SetRepoID(1) + _build.SetNumber(1) + _build.SetDeployPayload(nil) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`UPDATE "builds" +SET "repo_id"=$1,"pipeline_id"=$2,"number"=$3,"parent"=$4,"event"=$5,"event_action"=$6,"status"=$7,"error"=$8,"enqueued"=$9,"created"=$10,"started"=$11,"finished"=$12,"deploy"=$13,"deploy_payload"=$14,"clone"=$15,"source"=$16,"title"=$17,"message"=$18,"commit"=$19,"sender"=$20,"author"=$21,"email"=$22,"link"=$23,"branch"=$24,"ref"=$25,"base_ref"=$26,"head_ref"=$27,"host"=$28,"runtime"=$29,"distribution"=$30 +WHERE "id" = $31`). + WithArgs(1, nil, 1, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, AnyArgument{}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_build) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err = test.database.UpdateBuild(_build) + + if test.failure { + if err == nil { + t.Errorf("UpdateBuild for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("UpdateBuild for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/interface.go b/database/interface.go index 9a2177b14..975206c93 100644 --- a/database/interface.go +++ b/database/interface.go @@ -5,6 +5,7 @@ package database import ( + "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/hook" "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" @@ -15,7 +16,6 @@ import ( "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" - "github.com/go-vela/types/library" ) // Interface represents the interface for Vela integrating @@ -27,94 +27,36 @@ type Interface interface { // the configured database driver. Driver() string - // Build Database Interface Functions + // BuildInterface defines the interface for builds stored in the database. + build.BuildInterface - // GetBuild defines a function that - // gets a build by number and repo ID. - GetBuild(int, *library.Repo) (*library.Build, error) - // GetBuildByID defines a function that - // gets a build by its id. - GetBuildByID(int64) (*library.Build, error) - // GetLastBuild defines a function that - // gets the last build ran by repo ID. - GetLastBuild(*library.Repo) (*library.Build, error) - // GetLastBuildByBranch defines a function that - // gets the last build ran by repo ID and branch. - GetLastBuildByBranch(*library.Repo, string) (*library.Build, error) - // GetBuildCount defines a function that - // gets the count of builds. - GetBuildCount() (int64, error) - // GetBuildCountByStatus defines a function that - // gets a the count of builds by status. - GetBuildCountByStatus(string) (int64, error) - // GetBuildList defines a function that gets - // a list of all builds. - GetBuildList() ([]*library.Build, error) - // GetDeploymentBuildList defines a function that gets - // a list of builds related to a deployment. - GetDeploymentBuildList(string) ([]*library.Build, error) - // GetRepoBuildList defines a function that - // gets a list of builds by repo ID. - GetRepoBuildList(*library.Repo, map[string]interface{}, int64, int64, int, int) ([]*library.Build, int64, error) - // GetOrgBuildList defines a function that - // gets a list of builds by org. - GetOrgBuildList(string, map[string]interface{}, int, int) ([]*library.Build, int64, error) - // GetRepoBuildCount defines a function that - // gets the count of builds by repo ID. - GetRepoBuildCount(*library.Repo, map[string]interface{}) (int64, error) - // GetOrgBuildCount defines a function that - // gets the count of builds by org. - GetOrgBuildCount(string, map[string]interface{}) (int64, error) - // GetPendingAndRunningBuilds defines a function that - // gets the list of pending and running builds. - GetPendingAndRunningBuilds(string) ([]*library.BuildQueue, error) - // CreateBuild defines a function that - // creates a new build. - CreateBuild(*library.Build) error - // UpdateBuild defines a function that - // updates a build. - UpdateBuild(*library.Build) error - // DeleteBuild defines a function that - // deletes a build by unique ID. - DeleteBuild(int64) error - - // HookInterface provides the interface for functionality - // related to hooks stored in the database. + // HookInterface defines the interface for hooks stored in the database. hook.HookInterface - // LogInterface provides the interface for functionality - // related to logs stored in the database. + // LogInterface defines the interface for logs stored in the database. log.LogInterface - // PipelineInterface provides the interface for functionality - // related to pipelines stored in the database. + // PipelineInterface defines the interface for pipelines stored in the database. pipeline.PipelineInterface - // RepoInterface provides the interface for functionality - // related to repos stored in the database. + // RepoInterface defines the interface for repos stored in the database. repo.RepoInterface - // ScheduleInterface provides the interface for functionality - // related to schedules stored in the database. + // ScheduleInterface defines the interface for schedules stored in the database. schedule.ScheduleInterface - // SecretInterface provides the interface for functionality - // related to secrets stored in the database. + // SecretInterface defines the interface for secrets stored in the database. secret.SecretInterface - // ServiceInterface provides the interface for functionality - // related to services stored in the database. + // ServiceInterface defines the interface for services stored in the database. service.ServiceInterface - // StepInterface provides the interface for functionality - // related to steps stored in the database. + // StepInterface defines the interface for steps stored in the database. step.StepInterface - // UserInterface provides the interface for functionality - // related to users stored in the database. + // UserInterface defines the interface for users stored in the database. user.UserInterface - // WorkerInterface provides the interface for functionality - // related to workers stored in the database. + // WorkerInterface defines the interface for workers stored in the database. worker.WorkerInterface } diff --git a/database/postgres/build.go b/database/postgres/build.go deleted file mode 100644 index c8b7b66df..000000000 --- a/database/postgres/build.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "errors" - - "github.com/sirupsen/logrus" - - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -// GetBuild gets a build by number and repo ID from the database. -func (c *client) GetBuild(number int, r *library.Repo) (*library.Build, error) { - c.Logger.WithFields(logrus.Fields{ - "build": number, - "org": r.GetOrg(), - "repo": r.GetName(), - }).Tracef("getting build %s/%d from the database", r.GetFullName(), number) - - // variable to store query results - b := new(database.Build) - - // send query to the database and store result in variable - result := c.Postgres. - Table(constants.TableBuild). - Raw(dml.SelectRepoBuild, r.GetID(), number). - Scan(b) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - return b.ToLibrary(), result.Error -} - -// GetBuildByID gets a build by id from the database. -func (c *client) GetBuildByID(id int64) (*library.Build, error) { - c.Logger.WithFields(logrus.Fields{ - "build": id, - }).Tracef("getting build %d from the database", id) - - // variable to store query result - b := new(database.Build) - - // send query to the database and store result in variable - result := c.Postgres. - Table(constants.TableBuild). - Raw(dml.SelectBuildByID, id). - Scan(b) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - return b.ToLibrary(), result.Error -} - -// GetLastBuild gets the last build by repo ID from the database. -func (c *client) GetLastBuild(r *library.Repo) (*library.Build, error) { - c.Logger.WithFields(logrus.Fields{ - "org": r.GetOrg(), - "repo": r.GetName(), - }).Tracef("getting last build for repo %s from the database", r.GetFullName()) - - // variable to store query results - b := new(database.Build) - - // send query to the database and store result in variable - result := c.Postgres. - Table(constants.TableBuild). - Raw(dml.SelectLastRepoBuild, r.GetID()). - Scan(b) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - // the record will not exist if it's a new repo - return nil, nil - } - - return b.ToLibrary(), result.Error -} - -// GetLastBuildByBranch gets the last build by repo ID and branch from the database. -func (c *client) GetLastBuildByBranch(r *library.Repo, branch string) (*library.Build, error) { - c.Logger.WithFields(logrus.Fields{ - "org": r.GetOrg(), - "repo": r.GetName(), - }).Tracef("getting last build for repo %s on branch %s from the database", r.GetFullName(), branch) - - // variable to store query results - b := new(database.Build) - - // send query to the database and store result in variable - result := c.Postgres. - Table(constants.TableBuild). - Raw(dml.SelectLastRepoBuildByBranch, r.GetID(), branch). - Scan(b) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - // the record will not exist if it's a new repo - return nil, nil - } - - return b.ToLibrary(), result.Error -} - -// GetPendingAndRunningBuilds returns the list of pending -// and running builds within the given timeframe. -func (c *client) GetPendingAndRunningBuilds(after string) ([]*library.BuildQueue, error) { - c.Logger.Trace("getting pending and running builds from the database") - - // variable to store query results - b := new([]database.BuildQueue) - - // send query to the database and store result in variable - result := c.Postgres. - Table(constants.TableBuild). - Raw(dml.SelectPendingAndRunningBuilds, after). - Scan(b) - - // variable we want to return - builds := []*library.BuildQueue{} - - // iterate through all query results - for _, build := range *b { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := build - - // convert query result to library type - builds = append(builds, tmp.ToLibrary()) - } - - return builds, result.Error -} - -// CreateBuild creates a new build in the database. -func (c *client) CreateBuild(b *library.Build) error { - c.Logger.WithFields(logrus.Fields{ - "build": b.GetNumber(), - }).Tracef("creating build %d in the database", b.GetNumber()) - - // cast to database type - build := database.BuildFromLibrary(b) - - // validate the necessary fields are populated - err := build.Validate() - if err != nil { - return err - } - - // send query to the database - return c.Postgres. - Table(constants.TableBuild). - Create(build.Crop()).Error -} - -// UpdateBuild updates a build in the database. -func (c *client) UpdateBuild(b *library.Build) error { - c.Logger.WithFields(logrus.Fields{ - "build": b.GetNumber(), - }).Tracef("updating build %d in the database", b.GetNumber()) - - // cast to database type - build := database.BuildFromLibrary(b) - - // validate the necessary fields are populated - err := build.Validate() - if err != nil { - return err - } - - // send query to the database - return c.Postgres. - Table(constants.TableBuild). - Save(build.Crop()).Error -} - -// DeleteBuild deletes a build by unique ID from the database. -func (c *client) DeleteBuild(id int64) error { - c.Logger.Tracef("deleting build %d in the database", id) - - // send query to the database - return c.Postgres. - Table(constants.TableBuild). - Exec(dml.DeleteBuild, id).Error -} diff --git a/database/postgres/build_count.go b/database/postgres/build_count.go deleted file mode 100644 index 0d7af8f5f..000000000 --- a/database/postgres/build_count.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" -) - -// GetBuildCount gets a count of all builds from the database. -func (c *client) GetBuildCount() (int64, error) { - c.Logger.Trace("getting count of builds from the database") - - // variable to store query results - var b int64 - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableBuild). - Raw(dml.SelectBuildsCount). - Pluck("count", &b).Error - - return b, err -} - -// GetBuildCountByStatus gets a count of all builds by status from the database. -func (c *client) GetBuildCountByStatus(status string) (int64, error) { - c.Logger.Tracef("getting count of builds by status %s from the database", status) - - // variable to store query results - var b int64 - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableBuild). - Raw(dml.SelectBuildsCountByStatus, status). - Pluck("count", &b).Error - - return b, err -} - -// GetOrgBuildCount gets the count of all builds by repo ID from the database. -func (c *client) GetOrgBuildCount(org string, filters map[string]interface{}) (int64, error) { - c.Logger.WithFields(logrus.Fields{ - "org": org, - }).Tracef("getting count of builds for org %s from the database", org) - - // variable to store query results - var b int64 - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableBuild). - Joins("JOIN repos ON builds.repo_id = repos.id and repos.org = ?", org). - Where(filters). - Count(&b).Error - - return b, err -} - -// GetRepoBuildCount gets the count of all builds by repo ID from the database. -func (c *client) GetRepoBuildCount(r *library.Repo, filters map[string]interface{}) (int64, error) { - c.Logger.WithFields(logrus.Fields{ - "org": r.GetOrg(), - "name": r.GetName(), - }).Tracef("getting count of builds for repo %s from the database", r.GetFullName()) - - // variable to store query results - var b int64 - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableBuild). - Where("repo_id = ?", r.GetID()). - Where(filters). - Count(&b).Error - - return b, err -} diff --git a/database/postgres/build_count_test.go b/database/postgres/build_count_test.go deleted file mode 100644 index 0c4522798..000000000 --- a/database/postgres/build_count_test.go +++ /dev/null @@ -1,345 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "reflect" - "testing" - - "github.com/DATA-DOG/go-sqlmock" - - "github.com/go-vela/server/database/postgres/dml" - - "gorm.io/gorm" -) - -func TestPostgres_Client_GetBuildCount(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectBuildsCount).Statement - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) - - // ensure the mock expects the query - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetBuildCount() - - if test.failure { - if err == nil { - t.Errorf("GetBuildCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildCount is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetBuildCountByStatus(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectBuildsCountByStatus, "running").Statement - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) - - // ensure the mock expects the query - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetBuildCountByStatus("running") - - if test.failure { - if err == nil { - t.Errorf("GetBuildCountByStatus should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildCountByStatus returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildCountByStatus is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetOrgBuildCount(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) - - // ensure the mock expects the query - _mock.ExpectQuery("SELECT count(*) FROM \"builds\" JOIN repos ON builds.repo_id = repos.id and repos.org = $1").WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - filters := map[string]interface{}{} - // run tests - for _, test := range tests { - got, err := _database.GetOrgBuildCount("foo", filters) - - if test.failure { - if err == nil { - t.Errorf("GetOrgBuildCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetOrgBuildCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetOrgBuildCount is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetOrgBuildCountByEvent(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) - - // ensure the mock expects the query - _mock.ExpectQuery("SELECT count(*) FROM \"builds\" JOIN repos ON builds.repo_id = repos.id and repos.org = $1 WHERE \"event\" = $2").WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - filters := map[string]interface{}{ - "event": "push", - } - - // run tests - for _, test := range tests { - got, err := _database.GetOrgBuildCount("foo", filters) - - if test.failure { - if err == nil { - t.Errorf("GetOrgBuildCountByEvent should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetOrgBuildCountByEvent returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetOrgBuildCountByEvent is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetRepoBuildCount(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) - - // ensure the mock expects the query - _mock.ExpectQuery(`SELECT count(*) FROM "builds" WHERE repo_id = $1`).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - - filters := map[string]interface{}{} - - // run tests - for _, test := range tests { - got, err := _database.GetRepoBuildCount(_repo, filters) - - if test.failure { - if err == nil { - t.Errorf("GetRepoBuildCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetRepoBuildCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetRepoBuildCount is %v, want %v", got, test.want) - } - } -} diff --git a/database/postgres/build_list.go b/database/postgres/build_list.go deleted file mode 100644 index 76fe1dce4..000000000 --- a/database/postgres/build_list.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" -) - -// GetBuildList gets a list of all builds from the database. -func (c *client) GetBuildList() ([]*library.Build, error) { - c.Logger.Trace("listing builds from the database") - - // variable to store query results - b := new([]database.Build) - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableBuild). - Raw(dml.ListBuilds). - Scan(b).Error - - // variable we want to return - builds := []*library.Build{} - // iterate through all query results - for _, build := range *b { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := build - - // convert query result to library type - builds = append(builds, tmp.ToLibrary()) - } - - return builds, err -} - -// GetDeploymentBuildList gets a list of all builds from the database. -func (c *client) GetDeploymentBuildList(deployment string) ([]*library.Build, error) { - c.Logger.WithFields(logrus.Fields{ - "deployment": deployment, - }).Tracef("listing builds for deployment %s from the database", deployment) - - // variable to store query results - b := new([]database.Build) - - filters := map[string]string{} - if len(deployment) > 0 { - filters["source"] = deployment - } - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableBuild). - Select("*"). - Where(filters). - Limit(3). - Order("number DESC"). - Scan(b).Error - - // variable we want to return - builds := []*library.Build{} - // iterate through all query results - for _, build := range *b { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := build - - // convert query result to library type - builds = append(builds, tmp.ToLibrary()) - } - - return builds, err -} - -// GetOrgBuildList gets a list of all builds by org name and allows filters from the database. -func (c *client) GetOrgBuildList(org string, filters map[string]interface{}, page, perPage int) ([]*library.Build, int64, error) { - c.Logger.WithFields(logrus.Fields{ - "org": org, - }).Tracef("listing builds for org %s from the database", org) - - // variables to store query results - b := new([]database.Build) - builds := []*library.Build{} - count := int64(0) - - // count the results - count, err := c.GetOrgBuildCount(org, filters) - if err != nil { - return builds, 0, err - } - - // short-circuit if there are no results - if count == 0 { - return builds, 0, nil - } - - // calculate offset for pagination through results - offset := perPage * (page - 1) - - // send query to the database and store result in variable - err = c.Postgres. - Table(constants.TableBuild). - Select("builds.*"). - Joins("JOIN repos ON builds.repo_id = repos.id and repos.org = ?", org). - Where(filters). - Order("created DESC"). - Order("id"). - Limit(perPage). - Offset(offset). - Scan(b).Error - - // iterate through all query results - for _, build := range *b { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := build - - // convert query result to library type - builds = append(builds, tmp.ToLibrary()) - } - - return builds, count, err -} - -// GetRepoBuildList gets a list of all builds by repo ID from the database. -func (c *client) GetRepoBuildList(r *library.Repo, filters map[string]interface{}, before, after int64, page, perPage int) ([]*library.Build, int64, error) { - c.Logger.WithFields(logrus.Fields{ - "org": r.GetOrg(), - "repo": r.GetName(), - }).Tracef("listing builds for repo %s from the database", r.GetFullName()) - - // variable to store query results - b := new([]database.Build) - builds := []*library.Build{} - count := int64(0) - - // count the results - count, err := c.GetRepoBuildCount(r, filters) - if err != nil { - return builds, 0, err - } - - // short-circuit if there are no results - if count == 0 { - return builds, 0, nil - } - - // calculate offset for pagination through results - offset := perPage * (page - 1) - - err = c.Postgres. - Table(constants.TableBuild). - Where("repo_id = ?", r.GetID()). - Where("created < ?", before). - Where("created > ?", after). - Where(filters). - Order("number DESC"). - Limit(perPage). - Offset(offset). - Scan(b).Error - - // iterate through all query results - for _, build := range *b { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := build - - // convert query result to library type - builds = append(builds, tmp.ToLibrary()) - } - - return builds, count, err -} diff --git a/database/postgres/build_list_test.go b/database/postgres/build_list_test.go deleted file mode 100644 index bd2e34878..000000000 --- a/database/postgres/build_list_test.go +++ /dev/null @@ -1,459 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "reflect" - "testing" - "time" - - "github.com/DATA-DOG/go-sqlmock" - - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -func TestPostgres_Client_GetBuildList(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.ListBuilds).Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}, - ).AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0). - AddRow(2, 1, nil, 2, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) - - // ensure the mock expects the query - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want []*library.Build - }{ - { - failure: false, - want: []*library.Build{_buildOne, _buildTwo}, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetBuildList() - - if test.failure { - if err == nil { - t.Errorf("GetBuildList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildList is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetDeploymentBuildList(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - _buildOne.SetSource("https://github.com/github/octocat/deployments/1") - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - _buildTwo.SetSource("https://github.com/github/octocat/deployments/1") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}, - ).AddRow(2, 1, nil, 2, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "https://github.com/github/octocat/deployments/1", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0). - AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "https://github.com/github/octocat/deployments/1", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) - - // ensure the mock expects the query - _mock.ExpectQuery("SELECT * FROM \"builds\" WHERE \"source\" = $1 ORDER BY number DESC LIMIT 3").WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want []*library.Build - }{ - { - failure: false, - want: []*library.Build{_buildTwo, _buildOne}, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetDeploymentBuildList("https://github.com/github/octocat/deployments/1") - - if test.failure { - if err == nil { - t.Errorf("GetDeploymentBuildList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetDeploymentBuildList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetDeploymentBuildList is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetOrgBuildList(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) - - // ensure the mock expects the query - _mock.ExpectQuery("SELECT count(*) FROM \"builds\" JOIN repos ON builds.repo_id = repos.id and repos.org = $1").WillReturnRows(_rows) - - // create expected return in mock - _rows = sqlmock.NewRows( - []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}, - ).AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0). - AddRow(2, 1, nil, 2, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) - - // ensure the mock expects the query - _mock.ExpectQuery("SELECT builds.* FROM \"builds\" JOIN repos ON builds.repo_id = repos.id and repos.org = $1 ORDER BY created DESC,id LIMIT 10").WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want []*library.Build - }{ - { - failure: false, - want: []*library.Build{_buildOne, _buildTwo}, - }, - } - - filters := map[string]interface{}{} - - // run tests - for _, test := range tests { - got, _, err := _database.GetOrgBuildList("foo", filters, 1, 10) - - if test.failure { - if err == nil { - t.Errorf("GetOrgBuildList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetOrgBuildList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetOrgBuildList is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetOrgBuildList_NonAdmin(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) - - // ensure the mock expects the query - _mock.ExpectQuery("SELECT count(*) FROM \"builds\" JOIN repos ON builds.repo_id = repos.id and repos.org = $1 WHERE \"visibility\" = $2").WillReturnRows(_rows) - - // create expected return in mock - _rows = sqlmock.NewRows( - []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}, - ).AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) - - // ensure the mock expects the query - _mock.ExpectQuery("SELECT builds.* FROM \"builds\" JOIN repos ON builds.repo_id = repos.id and repos.org = $1 WHERE \"visibility\" = $2 ORDER BY created DESC,id LIMIT 10").WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want []*library.Build - }{ - { - failure: false, - want: []*library.Build{_buildOne}, - }, - } - - filters := map[string]interface{}{ - "visibility": "public", - } - - // run tests - for _, test := range tests { - got, _, err := _database.GetOrgBuildList("foo", filters, 1, 10) - - if test.failure { - if err == nil { - t.Errorf("GetOrgBuildList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetOrgBuildList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetOrgBuildList is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetOrgBuildListByEvent(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) - - // ensure the mock expects the query - _mock.ExpectQuery("SELECT count(*) FROM \"builds\" JOIN repos ON builds.repo_id = repos.id and repos.org = $1 WHERE \"event\" = $2").WillReturnRows(_rows) - - // create expected return in mock - _rows = sqlmock.NewRows( - []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}, - ).AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0). - AddRow(2, 1, nil, 2, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) - - // ensure the mock expects the query - _mock.ExpectQuery("SELECT builds.* FROM \"builds\" JOIN repos ON builds.repo_id = repos.id and repos.org = $1 WHERE \"event\" = $2 ORDER BY created DESC,id LIMIT 10").WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want []*library.Build - }{ - { - failure: false, - want: []*library.Build{_buildOne, _buildTwo}, - }, - } - - filters := map[string]interface{}{ - "event": "push", - } - - // run tests - for _, test := range tests { - got, _, err := _database.GetOrgBuildList("foo", filters, 1, 10) - - if test.failure { - if err == nil { - t.Errorf("GetOrgBuildListByEvent should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetOrgBuildListByEvent returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetOrgBuildListByEvent is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetRepoBuildList(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - _buildOne.SetCreated(1) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - _buildTwo.SetCreated(2) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(3) - - // ensure the mock expects the query - _mock.ExpectQuery(`SELECT count(*) FROM "builds" WHERE repo_id = $1`).WillReturnRows(_rows) - - // create expected return in mock - _rows = sqlmock.NewRows( - []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}, - ).AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 1, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0). - AddRow(2, 1, nil, 2, 0, "", "", "", "", 0, 2, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) - - // ensure the mock expects the query - _mock.ExpectQuery(`SELECT * FROM "builds" WHERE repo_id = $1 AND created < $2 AND created > $3 ORDER BY number DESC LIMIT 10`).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - before int64 - after int64 - want []*library.Build - }{ - { - failure: false, - before: time.Now().UTC().Unix(), - after: 0, - want: []*library.Build{_buildOne, _buildTwo}, - }, - } - - filters := map[string]interface{}{} - - // run tests - for _, test := range tests { - got, _, err := _database.GetRepoBuildList(_repo, filters, test.before, test.after, 1, 10) - - if test.failure { - if err == nil { - t.Errorf("GetRepoBuildList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetRepoBuildList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetRepoBuildList is %v, want %v", got, test.want) - } - } -} diff --git a/database/postgres/build_test.go b/database/postgres/build_test.go deleted file mode 100644 index 06d6238b2..000000000 --- a/database/postgres/build_test.go +++ /dev/null @@ -1,591 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "reflect" - "testing" - - "github.com/DATA-DOG/go-sqlmock" - - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -func TestPostgres_Client_GetBuild(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - _build.SetDeployPayload(nil) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectRepoBuild, 1, 1).Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}, - ).AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) - - // ensure the mock expects the query for test case 1 - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - // ensure the mock expects the error for test case 2 - _mock.ExpectQuery(_query.SQL.String()).WillReturnError(gorm.ErrRecordNotFound) - - // setup tests - tests := []struct { - failure bool - want *library.Build - }{ - { - failure: false, - want: _build, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetBuild(1, _repo) - - if test.failure { - if err == nil { - t.Errorf("GetBuild should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuild returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuild is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetBuildByID(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - _build.SetDeployPayload(nil) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectBuildByID, 1).Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}, - ).AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) - - // ensure the mock expects the query for test case 1 - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - // ensure the mock expects the error for test case 2 - _mock.ExpectQuery(_query.SQL.String()).WillReturnError(gorm.ErrRecordNotFound) - - // setup tests - tests := []struct { - failure bool - want *library.Build - }{ - { - failure: false, - want: _build, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetBuildByID(1) - - if test.failure { - if err == nil { - t.Errorf("GetBuildByID should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildByID returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildByID is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetLastBuild(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - _build.SetDeployPayload(nil) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectLastRepoBuild, 1).Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}, - ).AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) - - // ensure the mock expects the query for test case 1 - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - // ensure the mock expects the error for test case 2 - _mock.ExpectQuery(_query.SQL.String()).WillReturnError(gorm.ErrRecordNotFound) - - // setup tests - tests := []struct { - failure bool - want *library.Build - }{ - { - failure: false, - want: _build, - }, - { - failure: false, - want: nil, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetLastBuild(_repo) - - if test.failure { - if err == nil { - t.Errorf("GetLastBuild should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetLastBuild returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetLastBuild is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetLastBuildByBranch(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - _build.SetDeployPayload(nil) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectLastRepoBuildByBranch, 1, "master").Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}, - ).AddRow(1, 1, nil, 1, 0, "", "", "", "", 0, 0, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) - - // ensure the mock expects the query for test case 1 - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - // ensure the mock expects the error for test case 2 - _mock.ExpectQuery(_query.SQL.String()).WillReturnError(gorm.ErrRecordNotFound) - - // setup tests - tests := []struct { - failure bool - want *library.Build - }{ - { - failure: false, - want: _build, - }, - { - failure: false, - want: nil, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetLastBuildByBranch(_repo, "master") - - if test.failure { - if err == nil { - t.Errorf("GetLastBuildByBranch should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetLastBuildByBranch returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetLastBuildByBranch is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetPendingAndRunningBuilds(t *testing.T) { - // setup types - _buildOne := new(library.BuildQueue) - _buildOne.SetCreated(0) - _buildOne.SetFullName("") - _buildOne.SetNumber(1) - _buildOne.SetStatus("") - - _buildTwo := new(library.BuildQueue) - _buildTwo.SetCreated(0) - _buildTwo.SetFullName("") - _buildTwo.SetNumber(2) - _buildTwo.SetStatus("") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectPendingAndRunningBuilds, "").Statement - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"created", "full_name", "number", "status"}). - AddRow(0, "", 1, "").AddRow(0, "", 2, "") - - // ensure the mock expects the query for test case 1 - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - // ensure the mock expects the error for test case 2 - _mock.ExpectQuery(_query.SQL.String()).WillReturnError(gorm.ErrRecordNotFound) - - // setup tests - tests := []struct { - failure bool - want []*library.BuildQueue - }{ - { - failure: false, - want: []*library.BuildQueue{_buildOne, _buildTwo}, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetPendingAndRunningBuilds("") - - if test.failure { - if err == nil { - t.Errorf("GetPendingAndRunningBuilds should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetPendingAndRunningBuilds returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetPendingAndRunningBuilds is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_CreateBuild(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) - - // ensure the mock expects the query - _mock.ExpectQuery(`INSERT INTO "builds" ("repo_id","pipeline_id","number","parent","event","event_action","status","error","enqueued","created","started","finished","deploy","deploy_payload","clone","source","title","message","commit","sender","author","email","link","branch","ref","base_ref","head_ref","host","runtime","distribution","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31) RETURNING "id"`). - WithArgs(1, nil, 1, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, AnyArgument{}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 1). - WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := _database.CreateBuild(_build) - - if test.failure { - if err == nil { - t.Errorf("CreateBuild should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("CreateBuild returned err: %v", err) - } - } -} - -func TestPostgres_Client_UpdateBuild(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // ensure the mock expects the query - _mock.ExpectExec(`UPDATE "builds" SET "repo_id"=$1,"pipeline_id"=$2,"number"=$3,"parent"=$4,"event"=$5,"event_action"=$6,"status"=$7,"error"=$8,"enqueued"=$9,"created"=$10,"started"=$11,"finished"=$12,"deploy"=$13,"deploy_payload"=$14,"clone"=$15,"source"=$16,"title"=$17,"message"=$18,"commit"=$19,"sender"=$20,"author"=$21,"email"=$22,"link"=$23,"branch"=$24,"ref"=$25,"base_ref"=$26,"head_ref"=$27,"host"=$28,"runtime"=$29,"distribution"=$30 WHERE "id" = $31`). - WithArgs(1, nil, 1, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, AnyArgument{}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, 1). - WillReturnResult(sqlmock.NewResult(1, 1)) - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := _database.UpdateBuild(_build) - - if test.failure { - if err == nil { - t.Errorf("UpdateBuild should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("UpdateBuild returned err: %v", err) - } - } -} - -func TestPostgres_Client_DeleteBuild(t *testing.T) { - // setup types - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Exec(dml.DeleteBuild, 1).Statement - - // ensure the mock expects the query - _mock.ExpectExec(_query.SQL.String()).WillReturnResult(sqlmock.NewResult(1, 1)) - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := _database.DeleteBuild(1) - - if test.failure { - if err == nil { - t.Errorf("DeleteBuild should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("DeleteBuild returned err: %v", err) - } - } -} - -// testBuild is a test helper function to create a -// library Build type with all fields set to their -// zero values. -func testBuild() *library.Build { - i64 := int64(0) - i := 0 - str := "" - - return &library.Build{ - ID: &i64, - RepoID: &i64, - PipelineID: &i64, - Number: &i, - Parent: &i, - Event: &str, - EventAction: &str, - Status: &str, - Error: &str, - Enqueued: &i64, - Created: &i64, - Started: &i64, - Finished: &i64, - Deploy: &str, - Clone: &str, - Source: &str, - Title: &str, - Message: &str, - Commit: &str, - Sender: &str, - Author: &str, - Email: &str, - Link: &str, - Branch: &str, - Ref: &str, - BaseRef: &str, - HeadRef: &str, - Host: &str, - Runtime: &str, - Distribution: &str, - } -} diff --git a/database/postgres/ddl/build.go b/database/postgres/ddl/build.go deleted file mode 100644 index 58eacf15b..000000000 --- a/database/postgres/ddl/build.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package ddl - -const ( - // CreateBuildTable represents a query to - // create the builds table for Vela. - CreateBuildTable = ` -CREATE TABLE -IF NOT EXISTS -builds ( - id SERIAL PRIMARY KEY, - repo_id INTEGER, - pipeline_id INTEGER, - number INTEGER, - parent INTEGER, - event VARCHAR(250), - event_action VARCHAR(250), - status VARCHAR(250), - error VARCHAR(1000), - enqueued INTEGER, - created INTEGER, - started INTEGER, - finished INTEGER, - deploy VARCHAR(500), - deploy_payload VARCHAR(2000), - clone VARCHAR(1000), - source VARCHAR(1000), - title VARCHAR(1000), - message VARCHAR(2000), - commit VARCHAR(500), - sender VARCHAR(250), - author VARCHAR(250), - email VARCHAR(500), - link VARCHAR(1000), - branch VARCHAR(500), - ref VARCHAR(500), - base_ref VARCHAR(500), - head_ref VARCHAR(500), - host VARCHAR(250), - runtime VARCHAR(250), - distribution VARCHAR(250), - timestamp INTEGER, - UNIQUE(repo_id, number) -); -` - - // CreateBuildRepoIDIndex represents a query to create an - // index on the builds table for the repo_id column. - CreateBuildRepoIDIndex = ` -CREATE INDEX -IF NOT EXISTS -builds_repo_id -ON builds (repo_id); -` - - // CreateBuildStatusIndex represents a query to create an - // index on the builds table for the status column. - CreateBuildStatusIndex = ` -CREATE INDEX -IF NOT EXISTS -builds_status -ON builds (status); -` - - // CreateBuildCreatedIndex represents a query to create an - // index on the builds table for the created column. - CreateBuildCreatedIndex = ` -CREATE INDEX CONCURRENTLY -IF NOT EXISTS -builds_created -ON builds (created); -` - - // CreateBuildSourceIndex represents a query to create an - // index on the builds table for the source column. - CreateBuildSourceIndex = ` -CREATE INDEX CONCURRENTLY -IF NOT EXISTS -builds_source -ON builds (source); -` -) diff --git a/database/postgres/ddl/doc.go b/database/postgres/ddl/doc.go deleted file mode 100644 index 3e21c882f..000000000 --- a/database/postgres/ddl/doc.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -// Package ddl provides the Postgres data definition language (DDL) for Vela. -// -// https://en.wikipedia.org/wiki/Data_definition_language -// -// Usage: -// -// import "github.com/go-vela/server/database/postgres/ddl" -package ddl diff --git a/database/postgres/dml/build.go b/database/postgres/dml/build.go deleted file mode 100644 index 9080157c6..000000000 --- a/database/postgres/dml/build.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package dml - -const ( - // ListBuilds represents a query to - // list all builds in the database. - ListBuilds = ` -SELECT * -FROM builds; -` - - // SelectBuildByID represents a query to select - // a build for its id in the database. - SelectBuildByID = ` -SELECT * -FROM builds -WHERE id = ? -LIMIT 1; -` - - // SelectRepoBuild represents a query to select - // a build for a repo_id in the database. - SelectRepoBuild = ` -SELECT * -FROM builds -WHERE repo_id = ? -AND number = ? -LIMIT 1; -` - - // SelectLastRepoBuild represents a query to select - // the last build for a repo_id in the database. - SelectLastRepoBuild = ` -SELECT * -FROM builds -WHERE repo_id = ? -ORDER BY number DESC -LIMIT 1; -` - - // SelectLastRepoBuildByBranch represents a query to - // select the last build for a repo_id and branch name - // in the database. - SelectLastRepoBuildByBranch = ` -SELECT * -FROM builds -WHERE repo_id = ? -AND branch = ? -ORDER BY number DESC -LIMIT 1; -` - - // SelectBuildsCount represents a query to select - // the count of builds in the database. - SelectBuildsCount = ` -SELECT count(*) as count -FROM builds; -` - - // SelectBuildsCountByStatus represents a query to select - // the count of builds for a status in the database. - SelectBuildsCountByStatus = ` -SELECT count(*) as count -FROM builds -WHERE status = ?; -` - - // DeleteBuild represents a query to - // remove a build from the database. - DeleteBuild = ` -DELETE -FROM builds -WHERE id = ?; -` - - // SelectPendingAndRunningBuilds represents a joined query - // between the builds & repos table to select - // the created builds that are in pending or running builds status - // since the specified timeframe. - SelectPendingAndRunningBuilds = ` -SELECT builds.created, builds.number, builds.status, repos.full_name -FROM builds INNER JOIN repos -ON builds.repo_id = repos.id -WHERE builds.created > ? -AND (builds.status = 'running' OR builds.status = 'pending'); -` -) diff --git a/database/postgres/dml/doc.go b/database/postgres/dml/doc.go deleted file mode 100644 index f7e517d36..000000000 --- a/database/postgres/dml/doc.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -// Package dml provides the Postgres data manipulation language (DML) for Vela. -// -// https://en.wikipedia.org/wiki/Data_manipulation_language -// -// Usage: -// -// import "github.com/go-vela/server/database/postgres/dml" -package dml diff --git a/database/postgres/postgres.go b/database/postgres/postgres.go index 0c316531b..fea05037d 100644 --- a/database/postgres/postgres.go +++ b/database/postgres/postgres.go @@ -5,14 +5,13 @@ package postgres import ( - "fmt" "time" "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/hook" "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" - "github.com/go-vela/server/database/postgres/ddl" "github.com/go-vela/server/database/repo" "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" @@ -20,7 +19,6 @@ import ( "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" - "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" "gorm.io/driver/postgres" @@ -51,6 +49,8 @@ type ( Postgres *gorm.DB // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry + // https://pkg.go.dev/github.com/go-vela/server/database/build#BuildInterface + build.BuildInterface // https://pkg.go.dev/github.com/go-vela/server/database/hook#HookInterface hook.HookInterface // https://pkg.go.dev/github.com/go-vela/server/database/log#LogInterface @@ -167,6 +167,12 @@ func NewTest() (*client, sqlmock.Sqlmock, error) { return nil, nil, err } + // ensure the mock expects the build queries + _mock.ExpectExec(build.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the hook queries _mock.ExpectExec(hook.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(hook.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -257,64 +263,6 @@ func setupDatabase(c *client) error { return nil } - // create the tables in the database - err = createTables(c) - if err != nil { - return err - } - - // create the indexes in the database - err = createIndexes(c) - if err != nil { - return err - } - - return nil -} - -// createTables is a helper function to setup -// the database with the necessary tables. -func createTables(c *client) error { - c.Logger.Trace("creating data tables in the postgres database") - - // create the builds table - err := c.Postgres.Exec(ddl.CreateBuildTable).Error - if err != nil { - return fmt.Errorf("unable to create %s table: %w", constants.TableBuild, err) - } - - return nil -} - -// createIndexes is a helper function to setup -// the database with the necessary indexes. -func createIndexes(c *client) error { - c.Logger.Trace("creating data indexes in the postgres database") - - // create the builds_repo_id index for the builds table - err := c.Postgres.Exec(ddl.CreateBuildRepoIDIndex).Error - if err != nil { - return fmt.Errorf("unable to create builds_repo_id index for the %s table: %w", constants.TableBuild, err) - } - - // create the builds_status index for the builds table - err = c.Postgres.Exec(ddl.CreateBuildStatusIndex).Error - if err != nil { - return fmt.Errorf("unable to create builds_status index for the %s table: %w", constants.TableBuild, err) - } - - // create the builds_created index for the builds table - err = c.Postgres.Exec(ddl.CreateBuildCreatedIndex).Error - if err != nil { - return fmt.Errorf("unable to create builds_created index for the %s table: %w", constants.TableBuild, err) - } - - // create the builds_source index for the builds table - err = c.Postgres.Exec(ddl.CreateBuildSourceIndex).Error - if err != nil { - return fmt.Errorf("unable to create builds_source index for the %s table: %w", constants.TableBuild, err) - } - return nil } @@ -322,6 +270,18 @@ func createIndexes(c *client) error { func createServices(c *client) error { var err error + // create the database agnostic engine for builds + // + // https://pkg.go.dev/github.com/go-vela/server/database/build#New + c.BuildInterface, err = build.New( + build.WithClient(c.Postgres), + build.WithLogger(c.Logger), + build.WithSkipCreation(c.config.SkipCreation), + ) + if err != nil { + return err + } + // create the database agnostic engine for hooks // // https://pkg.go.dev/github.com/go-vela/server/database/hook#New diff --git a/database/postgres/postgres_test.go b/database/postgres/postgres_test.go index d61f443fd..a8f74d1ff 100644 --- a/database/postgres/postgres_test.go +++ b/database/postgres/postgres_test.go @@ -5,24 +5,22 @@ package postgres import ( - "database/sql/driver" "testing" "time" "github.com/go-vela/server/database/schedule" "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/hook" "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" - "github.com/go-vela/server/database/postgres/ddl" "github.com/go-vela/server/database/repo" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" - "github.com/go-vela/types/library" ) func TestPostgres_New(t *testing.T) { @@ -83,15 +81,12 @@ func TestPostgres_setupDatabase(t *testing.T) { // ensure the mock expects the ping _mock.ExpectPing() - // ensure the mock expects the table queries - _mock.ExpectExec(ddl.CreateBuildTable).WillReturnResult(sqlmock.NewResult(1, 1)) - - // ensure the mock expects the index queries - _mock.ExpectExec(ddl.CreateBuildRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateBuildStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateBuildCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateBuildSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - + // ensure the mock expects the build queries + _mock.ExpectExec(build.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the hook queries _mock.ExpectExec(hook.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(hook.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -171,87 +166,6 @@ func TestPostgres_setupDatabase(t *testing.T) { } } -func TestPostgres_createTables(t *testing.T) { - // setup types - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // ensure the mock expects the table queries - _mock.ExpectExec(ddl.CreateBuildTable).WillReturnResult(sqlmock.NewResult(1, 1)) - - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := createTables(_database) - - if test.failure { - if err == nil { - t.Errorf("createTables should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("createTables returned err: %v", err) - } - } -} - -func TestPostgres_createIndexes(t *testing.T) { - // setup types - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // ensure the mock expects the index queries - _mock.ExpectExec(ddl.CreateBuildRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateBuildStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateBuildCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateBuildSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := createIndexes(_database) - - if test.failure { - if err == nil { - t.Errorf("createIndexes should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("createIndexes returned err: %v", err) - } - } -} - func TestPostgres_createServices(t *testing.T) { // setup types // setup the test database client @@ -262,6 +176,12 @@ func TestPostgres_createServices(t *testing.T) { defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() + // ensure the mock expects the build queries + _mock.ExpectExec(build.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the hook queries _mock.ExpectExec(hook.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(hook.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -318,52 +238,3 @@ func TestPostgres_createServices(t *testing.T) { } } } - -// This will be used with the github.com/DATA-DOG/go-sqlmock -// library to compare values that are otherwise not easily -// compared. These typically would be values generated before -// adding or updating them in the database. -// -// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime -type AnyArgument struct{} - -// Match satisfies sqlmock.Argument interface. -func (a AnyArgument) Match(v driver.Value) bool { - return true -} - -// testRepo is a test helper function to create a -// library Repo type with all fields set to their -// zero values. -func testRepo() *library.Repo { - i64 := int64(0) - i := 0 - str := "" - b := false - - return &library.Repo{ - ID: &i64, - PipelineType: &str, - UserID: &i64, - Hash: &str, - Org: &str, - Name: &str, - FullName: &str, - Link: &str, - Clone: &str, - Branch: &str, - BuildLimit: &i64, - Timeout: &i64, - Counter: &i, - Visibility: &str, - Private: &b, - Trusted: &b, - Active: &b, - AllowPull: &b, - AllowPush: &b, - AllowDeploy: &b, - AllowTag: &b, - AllowComment: &b, - PreviousName: &str, - } -} diff --git a/database/sqlite/build.go b/database/sqlite/build.go deleted file mode 100644 index 974f619b0..000000000 --- a/database/sqlite/build.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "errors" - - "github.com/sirupsen/logrus" - - "github.com/go-vela/server/database/sqlite/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -// GetBuild gets a build by number and repo ID from the database. -func (c *client) GetBuild(number int, r *library.Repo) (*library.Build, error) { - c.Logger.WithFields(logrus.Fields{ - "build": number, - "org": r.GetOrg(), - "repo": r.GetName(), - }).Tracef("getting build %s/%d from the database", r.GetFullName(), number) - - // variable to store query results - b := new(database.Build) - - // send query to the database and store result in variable - result := c.Sqlite. - Table(constants.TableBuild). - Raw(dml.SelectRepoBuild, r.GetID(), number). - Scan(b) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - return b.ToLibrary(), result.Error -} - -// GetBuildByID gets a build by its ID from the database. -func (c *client) GetBuildByID(id int64) (*library.Build, error) { - c.Logger.WithFields(logrus.Fields{ - "build": id, - }).Tracef("getting build %d from the database", id) - - // variable to store query result - b := new(database.Build) - - // send query to the database and store result in variable - result := c.Sqlite. - Table(constants.TableBuild). - Raw(dml.SelectBuildByID, id). - Scan(b) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - return b.ToLibrary(), result.Error -} - -// GetLastBuild gets the last build by repo ID from the database. -func (c *client) GetLastBuild(r *library.Repo) (*library.Build, error) { - c.Logger.WithFields(logrus.Fields{ - "org": r.GetOrg(), - "repo": r.GetName(), - }).Tracef("getting last build for repo %s from the database", r.GetFullName()) - - // variable to store query results - b := new(database.Build) - - // send query to the database and store result in variable - result := c.Sqlite. - Table(constants.TableBuild). - Raw(dml.SelectLastRepoBuild, r.GetID()). - Scan(b) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - // the record will not exist if it's a new repo - return nil, nil - } - - return b.ToLibrary(), result.Error -} - -// GetLastBuildByBranch gets the last build by repo ID and branch from the database. -func (c *client) GetLastBuildByBranch(r *library.Repo, branch string) (*library.Build, error) { - c.Logger.WithFields(logrus.Fields{ - "org": r.GetOrg(), - "repo": r.GetName(), - }).Tracef("getting last build for repo %s on branch %s from the database", r.GetFullName(), branch) - - // variable to store query results - b := new(database.Build) - - // send query to the database and store result in variable - result := c.Sqlite. - Table(constants.TableBuild). - Raw(dml.SelectLastRepoBuildByBranch, r.GetID(), branch). - Scan(b) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - // the record will not exist if it's a new repo - return nil, nil - } - - return b.ToLibrary(), result.Error -} - -// GetPendingAndRunningBuilds returns the list of pending -// and running builds within the given timeframe. -func (c *client) GetPendingAndRunningBuilds(after string) ([]*library.BuildQueue, error) { - c.Logger.Trace("getting pending and running builds from the database") - - // variable to store query results - b := new([]database.BuildQueue) - - // send query to the database and store result in variable - result := c.Sqlite. - Table(constants.TableBuild). - Raw(dml.SelectPendingAndRunningBuilds, after). - Scan(b) - - // variable we want to return - builds := []*library.BuildQueue{} - - // iterate through all query results - for _, build := range *b { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := build - - // convert query result to library type - builds = append(builds, tmp.ToLibrary()) - } - - return builds, result.Error -} - -// CreateBuild creates a new build in the database. -func (c *client) CreateBuild(b *library.Build) error { - c.Logger.WithFields(logrus.Fields{ - "build": b.GetNumber(), - }).Tracef("creating build %d in the database", b.GetNumber()) - - // cast to database type - build := database.BuildFromLibrary(b) - - // validate the necessary fields are populated - err := build.Validate() - if err != nil { - return err - } - - // send query to the database - return c.Sqlite. - Table(constants.TableBuild). - Create(build.Crop()).Error -} - -// UpdateBuild updates a build in the database. -func (c *client) UpdateBuild(b *library.Build) error { - c.Logger.WithFields(logrus.Fields{ - "build": b.GetNumber(), - }).Tracef("updating build %d in the database", b.GetNumber()) - - // cast to database type - build := database.BuildFromLibrary(b) - - // validate the necessary fields are populated - err := build.Validate() - if err != nil { - return err - } - - // send query to the database - return c.Sqlite. - Table(constants.TableBuild). - Save(build.Crop()).Error -} - -// DeleteBuild deletes a build by unique ID from the database. -func (c *client) DeleteBuild(id int64) error { - c.Logger.Tracef("deleting build %d in the database", id) - - // send query to the database - return c.Sqlite. - Table(constants.TableBuild). - Exec(dml.DeleteBuild, id).Error -} diff --git a/database/sqlite/build_count.go b/database/sqlite/build_count.go deleted file mode 100644 index 2a363ddcf..000000000 --- a/database/sqlite/build_count.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "github.com/go-vela/server/database/sqlite/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" -) - -// GetBuildCount gets a count of all builds from the database. -func (c *client) GetBuildCount() (int64, error) { - c.Logger.Trace("getting count of builds from the database") - - // variable to store query results - var b int64 - - // send query to the database and store result in variable - err := c.Sqlite. - Table(constants.TableBuild). - Raw(dml.SelectBuildsCount). - Pluck("count", &b).Error - - return b, err -} - -// GetBuildCountByStatus gets a count of all builds by status from the database. -func (c *client) GetBuildCountByStatus(status string) (int64, error) { - c.Logger.Tracef("getting count of builds by status %s from the database", status) - - // variable to store query results - var b int64 - - // send query to the database and store result in variable - err := c.Sqlite. - Table(constants.TableBuild). - Raw(dml.SelectBuildsCountByStatus, status). - Pluck("count", &b).Error - - return b, err -} - -// GetOrgBuildCount gets the count of all builds by repo ID from the database. -func (c *client) GetOrgBuildCount(org string, filters map[string]interface{}) (int64, error) { - c.Logger.WithFields(logrus.Fields{ - "org": org, - }).Tracef("getting count of builds for org %s from the database", org) - - // variable to store query results - var b int64 - - // send query to the database and store result in variable - err := c.Sqlite. - Table(constants.TableBuild). - Joins("JOIN repos ON builds.repo_id = repos.id and repos.org = ?", org). - Where(filters). - Count(&b).Error - - return b, err -} - -// GetRepoBuildCount gets the count of all builds by repo ID from the database. -func (c *client) GetRepoBuildCount(r *library.Repo, filters map[string]interface{}) (int64, error) { - c.Logger.WithFields(logrus.Fields{ - "org": r.GetOrg(), - "name": r.GetName(), - }).Tracef("getting count of builds for repo %s from the database", r.GetFullName()) - - // variable to store query results - var b int64 - - // send query to the database and store result in variable - err := c.Sqlite. - Table(constants.TableBuild). - Where("repo_id = ?", r.GetID()). - Where(filters). - Count(&b).Error - - return b, err -} diff --git a/database/sqlite/build_count_test.go b/database/sqlite/build_count_test.go deleted file mode 100644 index ec4c2120d..000000000 --- a/database/sqlite/build_count_test.go +++ /dev/null @@ -1,439 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "log" - "reflect" - "testing" - - "github.com/go-vela/server/database/sqlite/ddl" - "github.com/go-vela/types/constants" -) - -func init() { - // setup the test database client - _database, err := NewTest() - if err != nil { - log.Fatalf("unable to create new sqlite test database: %v", err) - } - - // create the build table - err = _database.Sqlite.Exec(ddl.CreateBuildTable).Error - if err != nil { - log.Fatalf("unable to create %s table: %v", constants.TableBuild, err) - } -} - -func TestSqlite_Client_GetBuildCount(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - // create the builds in the database - err := _database.CreateBuild(_buildOne) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - err = _database.CreateBuild(_buildTwo) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - got, err := _database.GetBuildCount() - - if test.failure { - if err == nil { - t.Errorf("GetBuildCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildCount is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetBuildCountByStatus(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetStatus("running") - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetStatus("running") - _buildTwo.SetDeployPayload(nil) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - // create the builds in the database - err := _database.CreateBuild(_buildOne) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - err = _database.CreateBuild(_buildTwo) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - got, err := _database.GetBuildCountByStatus("running") - - if test.failure { - if err == nil { - t.Errorf("GetBuildCountByStatus should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildCountByStatus returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildCountByStatus is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetOrgBuildCount(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - - filters := map[string]interface{}{} - - // run tests - for _, test := range tests { - // defer cleanup of the repos table - defer _database.Sqlite.Exec("delete from repos;") - - // create the repo in the database - err := _database.CreateRepo(_repo) - if err != nil { - t.Errorf("unable to create test repo: %v", err) - } - - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - // create the builds in the database - err = _database.CreateBuild(_buildOne) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - err = _database.CreateBuild(_buildTwo) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - got, err := _database.GetOrgBuildCount("foo", filters) - - if test.failure { - if err == nil { - t.Errorf("GetOrgBuildCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetOrgBuildCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetOrgBuildCount is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetOrgBuildCountByEvent(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetEvent("push") - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetEvent("push") - _buildTwo.SetDeployPayload(nil) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - - filters := map[string]interface{}{ - "event": "push", - } - - // run tests - for _, test := range tests { - // defer cleanup of the repos table - defer _database.Sqlite.Exec("delete from repos;") - - // create the repo in the database - err := _database.CreateRepo(_repo) - if err != nil { - t.Errorf("unable to create test repo: %v", err) - } - - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - // create the builds in the database - err = _database.CreateBuild(_buildOne) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - err = _database.CreateBuild(_buildTwo) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - got, err := _database.GetOrgBuildCount("foo", filters) - - if test.failure { - if err == nil { - t.Errorf("GetOrgBuildCountByEvent should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetOrgBuildCountByEvent returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetOrgBuildCountByEvent is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetRepoBuildCount(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - - filters := map[string]interface{}{} - - // run tests - for _, test := range tests { - // defer cleanup of the repos table - defer _database.Sqlite.Exec("delete from repos;") - - // create the repo in the database - err := _database.CreateRepo(_repo) - if err != nil { - t.Errorf("unable to create test repo: %v", err) - } - - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - // create the builds in the database - err = _database.CreateBuild(_buildOne) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - err = _database.CreateBuild(_buildTwo) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - got, err := _database.GetRepoBuildCount(_repo, filters) - - if test.failure { - if err == nil { - t.Errorf("GetRepoBuildCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetRepoBuildCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetRepoBuildCount is %v, want %v", got, test.want) - } - } -} diff --git a/database/sqlite/build_list.go b/database/sqlite/build_list.go deleted file mode 100644 index 300a7c9a8..000000000 --- a/database/sqlite/build_list.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "github.com/go-vela/server/database/sqlite/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" - "github.com/sirupsen/logrus" -) - -// GetBuildList gets a list of all builds from the database. -func (c *client) GetBuildList() ([]*library.Build, error) { - c.Logger.Trace("listing builds from the database") - - // variable to store query results - b := new([]database.Build) - - // send query to the database and store result in variable - err := c.Sqlite. - Table(constants.TableBuild). - Raw(dml.ListBuilds). - Scan(b).Error - - // variable we want to return - builds := []*library.Build{} - // iterate through all query results - for _, build := range *b { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := build - - // convert query result to library type - builds = append(builds, tmp.ToLibrary()) - } - - return builds, err -} - -// GetDeploymentBuildList gets a list of all builds from the database. -func (c *client) GetDeploymentBuildList(deployment string) ([]*library.Build, error) { - c.Logger.WithFields(logrus.Fields{ - "deployment": deployment, - }).Tracef("listing builds for deployment %s from the database", deployment) - - // variable to store query results - b := new([]database.Build) - filters := map[string]string{} - - if len(deployment) > 0 { - filters["source"] = deployment - } - // send query to the database and store result in variable - err := c.Sqlite. - Table(constants.TableBuild). - Select("*"). - Where(filters). - Limit(3). - Order("number DESC"). - Scan(b).Error - - // variable we want to return - builds := []*library.Build{} - // iterate through all query results - for _, build := range *b { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := build - - // convert query result to library type - builds = append(builds, tmp.ToLibrary()) - } - - return builds, err -} - -// GetOrgBuildList gets a list of all builds by org name from the database. -func (c *client) GetOrgBuildList(org string, filters map[string]interface{}, page int, perPage int) ([]*library.Build, int64, error) { - c.Logger.WithFields(logrus.Fields{ - "org": org, - }).Tracef("listing builds for org %s from the database", org) - - // variable to store query results - b := new([]database.Build) - builds := []*library.Build{} - count := int64(0) - - // // count the results - count, err := c.GetOrgBuildCount(org, filters) - - if err != nil { - return builds, 0, err - } - - // short-circuit if there are no results - if count == 0 { - return builds, 0, nil - } - - // calculate offset for pagination through results - offset := perPage * (page - 1) - - // send query to the database and store result in variable - err = c.Sqlite. - Table(constants.TableBuild). - Select("builds.*"). - Joins("JOIN repos ON builds.repo_id = repos.id AND repos.org = ?", org). - Where(filters). - Order("created DESC"). - Order("id"). - Limit(perPage). - Offset(offset). - Scan(b).Error - - // iterate through all query results - for _, build := range *b { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := build - - // convert query result to library type - builds = append(builds, tmp.ToLibrary()) - } - - return builds, count, err -} - -// GetRepoBuildList gets a list of all builds by repo ID from the database. -func (c *client) GetRepoBuildList(r *library.Repo, filters map[string]interface{}, before, after int64, page, perPage int) ([]*library.Build, int64, error) { - c.Logger.WithFields(logrus.Fields{ - "org": r.GetOrg(), - "repo": r.GetName(), - }).Tracef("listing builds for repo %s from the database", r.GetFullName()) - - // variable to store query results - b := new([]database.Build) - builds := []*library.Build{} - count := int64(0) - - // count the results - count, err := c.GetRepoBuildCount(r, filters) - if err != nil { - return builds, 0, err - } - - // short-circuit if there are no results - if count == 0 { - return builds, 0, nil - } - - // calculate offset for pagination through results - offset := perPage * (page - 1) - - // send query to the database and store result in variable - err = c.Sqlite. - Table(constants.TableBuild). - Where("repo_id = ?", r.GetID()). - Where("created < ?", before). - Where("created > ?", after). - Where(filters). - Order("number DESC"). - Limit(perPage). - Offset(offset). - Scan(b).Error - - // iterate through all query results - for _, build := range *b { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := build - - // convert query result to library type - builds = append(builds, tmp.ToLibrary()) - } - - return builds, count, err -} diff --git a/database/sqlite/build_list_test.go b/database/sqlite/build_list_test.go deleted file mode 100644 index 908c501bc..000000000 --- a/database/sqlite/build_list_test.go +++ /dev/null @@ -1,532 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "log" - "reflect" - "testing" - "time" - - "github.com/go-vela/server/database/sqlite/ddl" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" -) - -func init() { - // setup the test database client - _database, err := NewTest() - if err != nil { - log.Fatalf("unable to create new sqlite test database: %v", err) - } - - // create the build table - err = _database.Sqlite.Exec(ddl.CreateBuildTable).Error - if err != nil { - log.Fatalf("unable to create %s table: %v", constants.TableBuild, err) - } -} - -func TestSqlite_Client_GetBuildList(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want []*library.Build - }{ - { - failure: false, - want: []*library.Build{_buildOne, _buildTwo}, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - for _, build := range test.want { - // create the build in the database - err := _database.CreateBuild(build) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - } - - got, err := _database.GetBuildList() - - if test.failure { - if err == nil { - t.Errorf("GetBuildList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildList is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetDeploymentBuildList(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetEvent("deployment") - _buildOne.SetDeployPayload(nil) - _buildOne.SetSource("https://github.com/github/octocat/deployments/1") - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildOne.SetEvent("deployment") - _buildTwo.SetDeployPayload(nil) - _buildTwo.SetSource("https://github.com/github/octocat/deployments/1") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want []*library.Build - }{ - { - failure: false, - want: []*library.Build{_buildTwo, _buildOne}, - }, - } - // run tests - for _, test := range tests { - // defer cleanup of the repos table - defer _database.Sqlite.Exec("delete from repos;") - - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - for _, build := range test.want { - // create the build in the database - err := _database.CreateBuild(build) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - } - - got, err := _database.GetDeploymentBuildList("https://github.com/github/octocat/deployments/1") - - if test.failure { - if err == nil { - t.Errorf("GetDeploymentBuildList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetDeploymentBuildList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetDeploymentBuildList is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetOrgBuildList(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetEvent("push") - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildOne.SetEvent("deployment") - _buildTwo.SetDeployPayload(nil) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want []*library.Build - }{ - { - failure: false, - want: []*library.Build{_buildOne, _buildTwo}, - }, - } - - filters := map[string]interface{}{} - - // run tests - for _, test := range tests { - // defer cleanup of the repos table - defer _database.Sqlite.Exec("delete from repos;") - - // create the repo in the database - err := _database.CreateRepo(_repo) - if err != nil { - t.Errorf("unable to create test repo: %v", err) - } - - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - for _, build := range test.want { - // create the build in the database - err := _database.CreateBuild(build) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - } - - got, _, err := _database.GetOrgBuildList("foo", filters, 1, 10) - - if test.failure { - if err == nil { - t.Errorf("GetOrgBuildList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetOrgBuildList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetOrgBuildList is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetOrgBuildList_NonAdmin(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(2) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - - _repoOne := testRepo() - _repoOne.SetID(1) - _repoOne.SetUserID(1) - _repoOne.SetHash("baz") - _repoOne.SetOrg("foo") - _repoOne.SetName("bar") - _repoOne.SetFullName("foo/bar") - _repoOne.SetVisibility("public") - - _repoTwo := testRepo() - _repoTwo.SetID(2) - _repoTwo.SetUserID(1) - _repoTwo.SetHash("baz") - _repoTwo.SetOrg("bar") - _repoTwo.SetName("foo") - _repoTwo.SetFullName("bar/foo") - _repoTwo.SetVisibility("private") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want []*library.Build - }{ - { - failure: false, - want: []*library.Build{_buildOne}, - }, - } - - filters := map[string]interface{}{} - - repos := []*library.Repo{_repoOne, _repoTwo} - // run tests - for _, test := range tests { - // defer cleanup of the repos table - defer _database.Sqlite.Exec("delete from repos;") - - for _, repo := range repos { - // create the repo in the database - err := _database.CreateRepo(repo) - if err != nil { - t.Errorf("unable to create test repo: %v", err) - } - } - - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - for _, build := range test.want { - // create the build in the database - err := _database.CreateBuild(build) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - } - - got, _, err := _database.GetOrgBuildList("foo", filters, 1, 10) - - if test.failure { - if err == nil { - t.Errorf("GetOrgBuildList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetOrgBuildList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetOrgBuildList is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetOrgBuildListByEvent(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetEvent("push") - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetEvent("deployment") - _buildTwo.SetDeployPayload(nil) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want []*library.Build - }{ - { - failure: false, - want: []*library.Build{_buildOne}, - }, - } - - filters := map[string]interface{}{ - "event": "push", - } - - // run tests - for _, test := range tests { - // defer cleanup of the repos table - defer _database.Sqlite.Exec("delete from repos;") - - // create the repo in the database - err := _database.CreateRepo(_repo) - if err != nil { - t.Errorf("unable to create test repo: %v", err) - } - - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - for _, build := range []*library.Build{_buildTwo, _buildOne} { - // create the build in the database - err := _database.CreateBuild(build) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - } - - got, _, err := _database.GetOrgBuildList("foo", filters, 1, 10) - - if test.failure { - if err == nil { - t.Errorf("GetOrgBuildListByEvent should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetOrgBuildListByEvent returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetOrgBuildListByEvent is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetRepoBuildList(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetDeployPayload(nil) - _buildOne.SetCreated(1) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetDeployPayload(nil) - _buildTwo.SetCreated(2) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want []*library.Build - }{ - { - failure: false, - want: []*library.Build{_buildTwo, _buildOne}, - }, - } - - filters := map[string]interface{}{} - - // run tests - for _, test := range tests { - // defer cleanup of the repos table - defer _database.Sqlite.Exec("delete from repos;") - - // create the repo in the database - err := _database.CreateRepo(_repo) - if err != nil { - t.Errorf("unable to create test repo: %v", err) - } - - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - for _, build := range test.want { - // create the build in the database - err := _database.CreateBuild(build) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - } - - got, _, err := _database.GetRepoBuildList(_repo, filters, time.Now().UTC().Unix(), 0, 1, 10) - - if test.failure { - if err == nil { - t.Errorf("GetRepoBuildList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetRepoBuildList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetRepoBuildList is %v, want %v", got, test.want) - } - } -} diff --git a/database/sqlite/build_test.go b/database/sqlite/build_test.go deleted file mode 100644 index a0e8eb8c1..000000000 --- a/database/sqlite/build_test.go +++ /dev/null @@ -1,611 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "reflect" - "testing" - - "github.com/go-vela/types/library" -) - -func TestSqlite_Client_GetBuild(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - _build.SetDeployPayload(nil) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want *library.Build - }{ - { - failure: false, - want: _build, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - if test.want != nil { - // create the build in the database - err := _database.CreateBuild(test.want) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - } - - got, err := _database.GetBuild(1, _repo) - - // cleanup the builds table - _ = _database.Sqlite.Exec("DELETE FROM builds;") - - if test.failure { - if err == nil { - t.Errorf("GetBuild should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuild returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuild is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetBuildByID(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - _build.SetDeployPayload(nil) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want *library.Build - }{ - { - failure: false, - want: _build, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - if test.want != nil { - // create the build in the database - err := _database.CreateBuild(test.want) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - } - - got, err := _database.GetBuildByID(1) - - // cleanup the builds table - _ = _database.Sqlite.Exec("DELETE FROM builds;") - - if test.failure { - if err == nil { - t.Errorf("GetBuildByID should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildByID returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildByID is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetLastBuild(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - _build.SetDeployPayload(nil) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want *library.Build - }{ - { - failure: false, - want: _build, - }, - { - failure: false, - want: nil, - }, - } - - // run tests - for _, test := range tests { - if test.want != nil { - // create the build in the database - err := _database.CreateBuild(test.want) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - } - - got, err := _database.GetLastBuild(_repo) - - // cleanup the builds table - _ = _database.Sqlite.Exec("DELETE FROM builds;") - - if test.failure { - if err == nil { - t.Errorf("GetLastBuild should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetLastBuild returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetLastBuild is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetLastBuildByBranch(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - _build.SetBranch("master") - _build.SetDeployPayload(nil) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want *library.Build - }{ - { - failure: false, - want: _build, - }, - { - failure: false, - want: nil, - }, - } - - // run tests - for _, test := range tests { - if test.want != nil { - // create the build in the database - err := _database.CreateBuild(test.want) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - } - - got, err := _database.GetLastBuildByBranch(_repo, "master") - - // cleanup the builds table - _ = _database.Sqlite.Exec("DELETE FROM builds;") - - if test.failure { - if err == nil { - t.Errorf("GetLastBuildByBranch should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetLastBuildByBranch returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetLastBuildByBranch is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetPendingAndRunningBuilds(t *testing.T) { - // setup types - _buildOne := testBuild() - _buildOne.SetID(1) - _buildOne.SetRepoID(1) - _buildOne.SetNumber(1) - _buildOne.SetStatus("running") - _buildOne.SetCreated(1) - _buildOne.SetDeployPayload(nil) - - _buildTwo := testBuild() - _buildTwo.SetID(2) - _buildTwo.SetRepoID(1) - _buildTwo.SetNumber(2) - _buildTwo.SetStatus("pending") - _buildTwo.SetCreated(1) - _buildTwo.SetDeployPayload(nil) - - _queueOne := new(library.BuildQueue) - _queueOne.SetCreated(1) - _queueOne.SetFullName("foo/bar") - _queueOne.SetNumber(1) - _queueOne.SetStatus("running") - - _queueTwo := new(library.BuildQueue) - _queueTwo.SetCreated(1) - _queueTwo.SetFullName("foo/bar") - _queueTwo.SetNumber(2) - _queueTwo.SetStatus("pending") - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want []*library.BuildQueue - }{ - { - failure: false, - want: []*library.BuildQueue{_queueOne, _queueTwo}, - }, - { - failure: false, - want: []*library.BuildQueue{}, - }, - } - - // run tests - for _, test := range tests { - // create the repo in the database - err := _database.CreateRepo(_repo) - if err != nil { - t.Errorf("unable to create test repo: %v", err) - } - - if len(test.want) > 0 { - // create the builds in the database - err = _database.CreateBuild(_buildOne) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - err = _database.CreateBuild(_buildTwo) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - } - - got, err := _database.GetPendingAndRunningBuilds("0") - - // cleanup the repos table - _ = _database.Sqlite.Exec("DELETE FROM repos;") - // cleanup the builds table - _ = _database.Sqlite.Exec("DELETE FROM builds;") - - if test.failure { - if err == nil { - t.Errorf("GetPendingAndRunningBuilds should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetPendingAndRunningBuilds returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetPendingAndRunningBuilds is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_CreateBuild(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - err := _database.CreateBuild(_build) - - if test.failure { - if err == nil { - t.Errorf("CreateBuild should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("CreateBuild returned err: %v", err) - } - } -} - -func TestSqlite_Client_UpdateBuild(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - - _repo := testRepo() - _repo.SetID(1) - _repo.SetUserID(1) - _repo.SetHash("baz") - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - _repo.SetVisibility("public") - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - // create the build in the database - err = _database.CreateBuild(_build) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - err := _database.UpdateBuild(_build) - - if test.failure { - if err == nil { - t.Errorf("UpdateBuild should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("UpdateBuild returned err: %v", err) - } - } -} - -func TestSqlite_Client_DeleteBuild(t *testing.T) { - // setup types - _build := testBuild() - _build.SetID(1) - _build.SetRepoID(1) - _build.SetNumber(1) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the builds table - defer _database.Sqlite.Exec("delete from builds;") - - // create the build in the database - err = _database.CreateBuild(_build) - if err != nil { - t.Errorf("unable to create test build: %v", err) - } - - err = _database.DeleteBuild(1) - - if test.failure { - if err == nil { - t.Errorf("DeleteBuild should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("DeleteBuild returned err: %v", err) - } - } -} - -// testBuild is a test helper function to create a -// library Build type with all fields set to their -// zero values. -func testBuild() *library.Build { - i64 := int64(0) - i := 0 - str := "" - - return &library.Build{ - ID: &i64, - RepoID: &i64, - PipelineID: &i64, - Number: &i, - Parent: &i, - Event: &str, - EventAction: &str, - Status: &str, - Error: &str, - Enqueued: &i64, - Created: &i64, - Started: &i64, - Finished: &i64, - Deploy: &str, - Clone: &str, - Source: &str, - Title: &str, - Message: &str, - Commit: &str, - Sender: &str, - Author: &str, - Email: &str, - Link: &str, - Branch: &str, - Ref: &str, - BaseRef: &str, - HeadRef: &str, - Host: &str, - Runtime: &str, - Distribution: &str, - } -} diff --git a/database/sqlite/ddl/build.go b/database/sqlite/ddl/build.go deleted file mode 100644 index e4a713fdd..000000000 --- a/database/sqlite/ddl/build.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package ddl - -const ( - // CreateBuildTable represents a query to - // create the builds table for Vela. - CreateBuildTable = ` -CREATE TABLE -IF NOT EXISTS -builds ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - repo_id INTEGER, - pipeline_id INTEGER, - number INTEGER, - parent INTEGER, - event TEXT, - event_action TEXT, - status TEXT, - error TEXT, - enqueued INTEGER, - created INTEGER, - started INTEGER, - finished INTEGER, - deploy TEXT, - deploy_payload TEXT, - clone TEXT, - source TEXT, - title TEXT, - message TEXT, - 'commit' TEXT, - sender TEXT, - author TEXT, - email TEXT, - link TEXT, - branch TEXT, - ref TEXT, - base_ref TEXT, - head_ref TEXT, - host TEXT, - runtime TEXT, - distribution TEXT, - timestamp INTEGER, - UNIQUE(repo_id, number) -); -` - - // CreateBuildRepoIDIndex represents a query to create an - // index on the builds table for the repo_id column. - CreateBuildRepoIDIndex = ` -CREATE INDEX -IF NOT EXISTS -builds_repo_id -ON builds (repo_id); -` - - // CreateBuildStatusIndex represents a query to create an - // index on the builds table for the status column. - CreateBuildStatusIndex = ` -CREATE INDEX -IF NOT EXISTS -builds_status -ON builds (status); -` - - // CreateBuildCreatedIndex represents a query to create an - // index on the builds table for the created column. - CreateBuildCreatedIndex = ` -CREATE INDEX -IF NOT EXISTS -builds_created -ON builds (created); -` - - // CreateBuildSourceIndex represents a query to create an - // index on the builds table for the source column. - CreateBuildSourceIndex = ` -CREATE INDEX -IF NOT EXISTS -builds_source -ON builds (source); -` -) diff --git a/database/sqlite/ddl/doc.go b/database/sqlite/ddl/doc.go deleted file mode 100644 index 3e2bd5d2b..000000000 --- a/database/sqlite/ddl/doc.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -// Package ddl provides the Sqlite data definition language (DDL) for Vela. -// -// https://en.wikipedia.org/wiki/Data_definition_language -// -// Usage: -// -// import "github.com/go-vela/server/database/sqlite/ddl" -package ddl diff --git a/database/sqlite/dml/build.go b/database/sqlite/dml/build.go deleted file mode 100644 index fa9f94453..000000000 --- a/database/sqlite/dml/build.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package dml - -const ( - // ListBuilds represents a query to - // list all builds in the database. - ListBuilds = ` -SELECT * -FROM builds; -` - - // SelectBuildByID represents a query to select - // a build for its id in the database. - SelectBuildByID = ` -SELECT * -FROM builds -WHERE id = ? -LIMIT 1; -` - - // SelectRepoBuild represents a query to select - // a build for a repo_id in the database. - SelectRepoBuild = ` -SELECT * -FROM builds -WHERE repo_id = ? -AND number = ? -LIMIT 1; -` - - // SelectLastRepoBuild represents a query to select - // the last build for a repo_id in the database. - SelectLastRepoBuild = ` -SELECT * -FROM builds -WHERE repo_id = ? -ORDER BY number DESC -LIMIT 1; -` - // SelectLastRepoBuildByBranch represents a query to - // select the last build for a repo_id and branch name - // in the database. - SelectLastRepoBuildByBranch = ` -SELECT * -FROM builds -WHERE repo_id = ? -AND branch = ? -ORDER BY number DESC -LIMIT 1; -` - - // SelectBuildsCount represents a query to select - // the count of builds in the database. - SelectBuildsCount = ` -SELECT count(*) as count -FROM builds; -` - - // SelectBuildsCountByStatus represents a query to select - // the count of builds for a status in the database. - SelectBuildsCountByStatus = ` -SELECT count(*) as count -FROM builds -WHERE status = ?; -` - - // DeleteBuild represents a query to - // remove a build from the database. - DeleteBuild = ` -DELETE -FROM builds -WHERE id = ?; -` - - // SelectPendingAndRunningBuilds represents a joined query - // between the builds & repos table to select - // the created builds that are in pending or running builds status - // since the specified timeframe. - SelectPendingAndRunningBuilds = ` -SELECT builds.created, builds.number, builds.status, repos.full_name -FROM builds INNER JOIN repos -ON builds.repo_id = repos.id -WHERE builds.created > ? -AND (builds.status = 'running' OR builds.status = 'pending'); -` -) diff --git a/database/sqlite/dml/doc.go b/database/sqlite/dml/doc.go deleted file mode 100644 index 151d8a9a5..000000000 --- a/database/sqlite/dml/doc.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -// Package dml provides the Sqlite data manipulation language (DML) for Vela. -// -// https://en.wikipedia.org/wiki/Data_manipulation_language -// -// Usage: -// -// import "github.com/go-vela/server/database/sqlite/dml" -package dml diff --git a/database/sqlite/sqlite.go b/database/sqlite/sqlite.go index 3faa555ba..29f526643 100644 --- a/database/sqlite/sqlite.go +++ b/database/sqlite/sqlite.go @@ -5,9 +5,9 @@ package sqlite import ( - "fmt" "time" + "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/hook" "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" @@ -15,11 +15,9 @@ import ( "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" - "github.com/go-vela/server/database/sqlite/ddl" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" - "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" "gorm.io/driver/sqlite" @@ -50,6 +48,8 @@ type ( Sqlite *gorm.DB // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry + // https://pkg.go.dev/github.com/go-vela/server/database/build#BuildInterface + build.BuildInterface // https://pkg.go.dev/github.com/go-vela/server/database/hook#HookInterface hook.HookInterface // https://pkg.go.dev/github.com/go-vela/server/database/log#LogInterface @@ -177,12 +177,6 @@ func NewTest() (*client, error) { return nil, err } - // create the tables in the database - err = createTables(c) - if err != nil { - return nil, err - } - return c, nil } @@ -225,64 +219,6 @@ func setupDatabase(c *client) error { return nil } - // create the tables in the database - err = createTables(c) - if err != nil { - return err - } - - // create the indexes in the database - err = createIndexes(c) - if err != nil { - return err - } - - return nil -} - -// createTables is a helper function to setup -// the database with the necessary tables. -func createTables(c *client) error { - c.Logger.Trace("creating data tables in the sqlite database") - - // create the builds table - err := c.Sqlite.Exec(ddl.CreateBuildTable).Error - if err != nil { - return fmt.Errorf("unable to create %s table: %w", constants.TableBuild, err) - } - - return nil -} - -// createIndexes is a helper function to setup -// the database with the necessary indexes. -func createIndexes(c *client) error { - c.Logger.Trace("creating data indexes in the sqlite database") - - // create the builds_repo_id index for the builds table - err := c.Sqlite.Exec(ddl.CreateBuildRepoIDIndex).Error - if err != nil { - return fmt.Errorf("unable to create builds_repo_id index for the %s table: %w", constants.TableBuild, err) - } - - // create the builds_status index for the builds table - err = c.Sqlite.Exec(ddl.CreateBuildStatusIndex).Error - if err != nil { - return fmt.Errorf("unable to create builds_status index for the %s table: %w", constants.TableBuild, err) - } - - // create the builds_created index for the builds table - err = c.Sqlite.Exec(ddl.CreateBuildCreatedIndex).Error - if err != nil { - return fmt.Errorf("unable to create builds_created index for the %s table: %w", constants.TableBuild, err) - } - - // create the builds_source index for the builds table - err = c.Sqlite.Exec(ddl.CreateBuildSourceIndex).Error - if err != nil { - return fmt.Errorf("unable to create builds_source index for the %s table: %w", constants.TableBuild, err) - } - return nil } @@ -290,6 +226,18 @@ func createIndexes(c *client) error { func createServices(c *client) error { var err error + // create the database agnostic engine for builds + // + // https://pkg.go.dev/github.com/go-vela/server/database/build#New + c.BuildInterface, err = build.New( + build.WithClient(c.Sqlite), + build.WithLogger(c.Logger), + build.WithSkipCreation(c.config.SkipCreation), + ) + if err != nil { + return err + } + // create the database agnostic engine for hooks // // https://pkg.go.dev/github.com/go-vela/server/database/hook#New diff --git a/database/sqlite/sqlite_test.go b/database/sqlite/sqlite_test.go index 48ff34902..55c6940d0 100644 --- a/database/sqlite/sqlite_test.go +++ b/database/sqlite/sqlite_test.go @@ -7,8 +7,6 @@ package sqlite import ( "testing" "time" - - "github.com/go-vela/types/library" ) func TestSqlite_New(t *testing.T) { @@ -111,78 +109,6 @@ func TestSqlite_setupDatabase(t *testing.T) { } } -func TestSqlite_createTables(t *testing.T) { - // setup types - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := createTables(_database) - - if test.failure { - if err == nil { - t.Errorf("createTables should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("createTables returned err: %v", err) - } - } -} - -func TestSqlite_createIndexes(t *testing.T) { - // setup types - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := createIndexes(_database) - - if test.failure { - if err == nil { - t.Errorf("createIndexes should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("createIndexes returned err: %v", err) - } - } -} - func TestSqlite_createServices(t *testing.T) { // setup types // setup the test database client @@ -218,38 +144,3 @@ func TestSqlite_createServices(t *testing.T) { } } } - -// testRepo is a test helper function to create a -// library Repo type with all fields set to their -// zero values. -func testRepo() *library.Repo { - i64 := int64(0) - i := 0 - str := "" - b := false - - return &library.Repo{ - ID: &i64, - UserID: &i64, - Hash: &str, - Org: &str, - Name: &str, - FullName: &str, - Link: &str, - Clone: &str, - Branch: &str, - BuildLimit: &i64, - Timeout: &i64, - Counter: &i, - Visibility: &str, - Private: &b, - Trusted: &b, - Active: &b, - AllowPull: &b, - AllowPush: &b, - AllowDeploy: &b, - AllowTag: &b, - AllowComment: &b, - PreviousName: &str, - } -} diff --git a/router/middleware/build/build.go b/router/middleware/build/build.go index 87b25185b..6016adf3c 100644 --- a/router/middleware/build/build.go +++ b/router/middleware/build/build.go @@ -64,7 +64,7 @@ func Establish() gin.HandlerFunc { "user": u.GetName(), }).Debugf("reading build %s/%d", r.GetFullName(), number) - b, err := database.FromContext(c).GetBuild(number, r) + b, err := database.FromContext(c).GetBuildForRepo(r, number) if err != nil { retErr := fmt.Errorf("unable to read build %s/%d: %w", r.GetFullName(), number, err) util.HandleError(c, http.StatusNotFound, retErr) From f7c3c6b593305b1f0b1023c55a1519cd136e6d3d Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Fri, 26 May 2023 13:39:32 -0600 Subject: [PATCH 20/53] refactor(db): return hook created or updated (#861) Co-authored-by: Jordan Brockopp --- api/admin/hook.go | 4 ++-- api/hook/create.go | 5 +---- api/hook/update.go | 5 +---- api/repo/create.go | 2 +- api/repo/repair.go | 2 +- api/webhook.go | 9 +++------ database/hook/count_repo_test.go | 4 ++-- database/hook/count_test.go | 4 ++-- database/hook/create.go | 11 +++++------ database/hook/create_test.go | 2 +- database/hook/delete_test.go | 2 +- database/hook/get_repo_test.go | 2 +- database/hook/get_test.go | 2 +- database/hook/interface.go | 4 ++-- database/hook/last_repo_test.go | 2 +- database/hook/list_repo_test.go | 4 ++-- database/hook/list_test.go | 4 ++-- database/hook/update.go | 11 +++++------ database/hook/update_test.go | 4 ++-- 19 files changed, 36 insertions(+), 47 deletions(-) diff --git a/api/admin/hook.go b/api/admin/hook.go index 322142d27..789a74a25 100644 --- a/api/admin/hook.go +++ b/api/admin/hook.go @@ -64,7 +64,7 @@ func UpdateHook(c *gin.Context) { } // send API call to update the hook - err = database.FromContext(c).UpdateHook(input) + h, err := database.FromContext(c).UpdateHook(input) if err != nil { retErr := fmt.Errorf("unable to update hook %d: %w", input.GetID(), err) @@ -73,5 +73,5 @@ func UpdateHook(c *gin.Context) { return } - c.JSON(http.StatusOK, input) + c.JSON(http.StatusOK, h) } diff --git a/api/hook/create.go b/api/hook/create.go index 70f524524..b33d9fdfb 100644 --- a/api/hook/create.go +++ b/api/hook/create.go @@ -113,7 +113,7 @@ func CreateHook(c *gin.Context) { } // send API call to create the webhook - err = database.FromContext(c).CreateHook(input) + h, err := database.FromContext(c).CreateHook(input) if err != nil { retErr := fmt.Errorf("unable to create hook for repo %s: %w", r.GetFullName(), err) @@ -122,8 +122,5 @@ func CreateHook(c *gin.Context) { return } - // send API call to capture the created webhook - h, _ := database.FromContext(c).GetHookForRepo(r, input.GetNumber()) - c.JSON(http.StatusCreated, h) } diff --git a/api/hook/update.go b/api/hook/update.go index f8beeb1ec..2c5cdcfb7 100644 --- a/api/hook/update.go +++ b/api/hook/update.go @@ -157,7 +157,7 @@ func UpdateHook(c *gin.Context) { } // send API call to update the webhook - err = database.FromContext(c).UpdateHook(h) + h, err = database.FromContext(c).UpdateHook(h) if err != nil { retErr := fmt.Errorf("unable to update hook %s: %w", entry, err) @@ -166,8 +166,5 @@ func UpdateHook(c *gin.Context) { return } - // send API call to capture the updated user - h, _ = database.FromContext(c).GetHookForRepo(r, h.GetNumber()) - c.JSON(http.StatusOK, h) } diff --git a/api/repo/create.go b/api/repo/create.go index 91d42064e..e52cd23f4 100644 --- a/api/repo/create.go +++ b/api/repo/create.go @@ -316,7 +316,7 @@ func CreateRepo(c *gin.Context) { // update initialization hook h.SetRepoID(r.GetID()) // create first hook for repo in the database - err = database.FromContext(c).CreateHook(h) + _, err = database.FromContext(c).CreateHook(h) if err != nil { retErr := fmt.Errorf("unable to create initialization webhook for %s: %w", r.GetFullName(), err) diff --git a/api/repo/repair.go b/api/repo/repair.go index 424a7c0c1..192983d1c 100644 --- a/api/repo/repair.go +++ b/api/repo/repair.go @@ -107,7 +107,7 @@ func RepairRepo(c *gin.Context) { hook.SetRepoID(r.GetID()) - err = database.FromContext(c).CreateHook(hook) + _, err = database.FromContext(c).CreateHook(hook) if err != nil { retErr := fmt.Errorf("unable to create initialization webhook for %s: %w", r.GetFullName(), err) diff --git a/api/webhook.go b/api/webhook.go index 6881a03b7..fd958291c 100644 --- a/api/webhook.go +++ b/api/webhook.go @@ -177,7 +177,7 @@ func PostWebhook(c *gin.Context) { defer func() { // send API call to update the webhook - err = database.FromContext(c).UpdateHook(h) + _, err = database.FromContext(c).UpdateHook(h) if err != nil { logrus.Errorf("unable to update webhook %s/%d: %v", r.GetFullName(), h.GetNumber(), err) } @@ -219,7 +219,7 @@ func PostWebhook(c *gin.Context) { } // send API call to create the webhook - err = database.FromContext(c).CreateHook(h) + h, err = database.FromContext(c).CreateHook(h) if err != nil { retErr := fmt.Errorf("unable to create webhook %s/%d: %w", repo.GetFullName(), h.GetNumber(), err) util.HandleError(c, http.StatusInternalServerError, retErr) @@ -230,9 +230,6 @@ func PostWebhook(c *gin.Context) { return } - // send API call to capture the created webhook - h, _ = database.FromContext(c).GetHookForRepo(repo, h.GetNumber()) - // verify the webhook from the source control provider if c.Value("webhookvalidation").(bool) { err = scm.FromContext(c).VerifyWebhook(dupRequest, repo) @@ -765,7 +762,7 @@ func handleRepositoryEvent(c *gin.Context, m *types.Metadata, h *library.Hook, r defer func() { // send API call to update the webhook - err := database.FromContext(c).CreateHook(h) + _, err := database.FromContext(c).CreateHook(h) if err != nil { logrus.Errorf("unable to create webhook %s/%d: %v", r.GetFullName(), h.GetNumber(), err) } diff --git a/database/hook/count_repo_test.go b/database/hook/count_repo_test.go index 83fe35dab..5814764e6 100644 --- a/database/hook/count_repo_test.go +++ b/database/hook/count_repo_test.go @@ -48,12 +48,12 @@ func TestHook_Engine_CountHooksForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateHook(_hookOne) + _, err := _sqlite.CreateHook(_hookOne) if err != nil { t.Errorf("unable to create test repo for sqlite: %v", err) } - err = _sqlite.CreateHook(_hookTwo) + _, err = _sqlite.CreateHook(_hookTwo) if err != nil { t.Errorf("unable to create test hook for sqlite: %v", err) } diff --git a/database/hook/count_test.go b/database/hook/count_test.go index 6e4df9df4..dce97587f 100644 --- a/database/hook/count_test.go +++ b/database/hook/count_test.go @@ -41,12 +41,12 @@ func TestHook_Engine_CountHooks(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateHook(_hookOne) + _, err := _sqlite.CreateHook(_hookOne) if err != nil { t.Errorf("unable to create test hook for sqlite: %v", err) } - err = _sqlite.CreateHook(_hookTwo) + _, err = _sqlite.CreateHook(_hookTwo) if err != nil { t.Errorf("unable to create test hook for sqlite: %v", err) } diff --git a/database/hook/create.go b/database/hook/create.go index ce3745da3..e2bf5489d 100644 --- a/database/hook/create.go +++ b/database/hook/create.go @@ -12,7 +12,7 @@ import ( ) // CreateHook creates a new hook in the database. -func (e *engine) CreateHook(h *library.Hook) error { +func (e *engine) CreateHook(h *library.Hook) (*library.Hook, error) { e.logger.WithFields(logrus.Fields{ "hook": h.GetNumber(), }).Tracef("creating hook %d in the database", h.GetNumber()) @@ -27,12 +27,11 @@ func (e *engine) CreateHook(h *library.Hook) error { // https://pkg.go.dev/github.com/go-vela/types/database#Hook.Validate err := hook.Validate() if err != nil { - return err + return nil, err } + result := e.client.Table(constants.TableHook).Create(hook) + // send query to the database - return e.client. - Table(constants.TableHook). - Create(hook). - Error + return hook.ToLibrary(), result.Error } diff --git a/database/hook/create_test.go b/database/hook/create_test.go index 1fd03f2b9..f05bd8a9d 100644 --- a/database/hook/create_test.go +++ b/database/hook/create_test.go @@ -57,7 +57,7 @@ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) RETURNING "id"`). // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err := test.database.CreateHook(_hook) + _, err := test.database.CreateHook(_hook) if test.failure { if err == nil { diff --git a/database/hook/delete_test.go b/database/hook/delete_test.go index e5deab7ad..2fb91d567 100644 --- a/database/hook/delete_test.go +++ b/database/hook/delete_test.go @@ -31,7 +31,7 @@ func TestHook_Engine_DeleteHook(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateHook(_hook) + _, err := _sqlite.CreateHook(_hook) if err != nil { t.Errorf("unable to create test hook for sqlite: %v", err) } diff --git a/database/hook/get_repo_test.go b/database/hook/get_repo_test.go index eb1103cad..32fb9a813 100644 --- a/database/hook/get_repo_test.go +++ b/database/hook/get_repo_test.go @@ -43,7 +43,7 @@ func TestHook_Engine_GetHookForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateHook(_hook) + _, err := _sqlite.CreateHook(_hook) if err != nil { t.Errorf("unable to create test hook for sqlite: %v", err) } diff --git a/database/hook/get_test.go b/database/hook/get_test.go index 79e8a9856..b84fe7b13 100644 --- a/database/hook/get_test.go +++ b/database/hook/get_test.go @@ -36,7 +36,7 @@ func TestHook_Engine_GetHook(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateHook(_hook) + _, err := _sqlite.CreateHook(_hook) if err != nil { t.Errorf("unable to create test hook for sqlite: %v", err) } diff --git a/database/hook/interface.go b/database/hook/interface.go index a1e5c5fe0..4ed44fd84 100644 --- a/database/hook/interface.go +++ b/database/hook/interface.go @@ -31,7 +31,7 @@ type HookInterface interface { // CountHooksForRepo defines a function that gets the count of hooks by repo ID. CountHooksForRepo(*library.Repo) (int64, error) // CreateHook defines a function that creates a new hook. - CreateHook(*library.Hook) error + CreateHook(*library.Hook) (*library.Hook, error) // DeleteHook defines a function that deletes an existing hook. DeleteHook(*library.Hook) error // GetHook defines a function that gets a hook by ID. @@ -45,5 +45,5 @@ type HookInterface interface { // ListHooksForRepo defines a function that gets a list of hooks by repo ID. ListHooksForRepo(*library.Repo, int, int) ([]*library.Hook, int64, error) // UpdateHook defines a function that updates an existing hook. - UpdateHook(*library.Hook) error + UpdateHook(*library.Hook) (*library.Hook, error) } diff --git a/database/hook/last_repo_test.go b/database/hook/last_repo_test.go index c7efeab75..ecd4a3eea 100644 --- a/database/hook/last_repo_test.go +++ b/database/hook/last_repo_test.go @@ -43,7 +43,7 @@ func TestHook_Engine_LastHookForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateHook(_hook) + _, err := _sqlite.CreateHook(_hook) if err != nil { t.Errorf("unable to create test hook for sqlite: %v", err) } diff --git a/database/hook/list_repo_test.go b/database/hook/list_repo_test.go index 709de156e..c229f5674 100644 --- a/database/hook/list_repo_test.go +++ b/database/hook/list_repo_test.go @@ -58,12 +58,12 @@ func TestHook_Engine_ListHooksForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateHook(_hookOne) + _, err := _sqlite.CreateHook(_hookOne) if err != nil { t.Errorf("unable to create test hook for sqlite: %v", err) } - err = _sqlite.CreateHook(_hookTwo) + _, err = _sqlite.CreateHook(_hookTwo) if err != nil { t.Errorf("unable to create test hook for sqlite: %v", err) } diff --git a/database/hook/list_test.go b/database/hook/list_test.go index ba0784213..e2fb33772 100644 --- a/database/hook/list_test.go +++ b/database/hook/list_test.go @@ -51,12 +51,12 @@ func TestHook_Engine_ListHooks(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateHook(_hookOne) + _, err := _sqlite.CreateHook(_hookOne) if err != nil { t.Errorf("unable to create test hook for sqlite: %v", err) } - err = _sqlite.CreateHook(_hookTwo) + _, err = _sqlite.CreateHook(_hookTwo) if err != nil { t.Errorf("unable to create test hook for sqlite: %v", err) } diff --git a/database/hook/update.go b/database/hook/update.go index cfc746303..d7741f249 100644 --- a/database/hook/update.go +++ b/database/hook/update.go @@ -12,7 +12,7 @@ import ( ) // UpdateHook updates an existing hook in the database. -func (e *engine) UpdateHook(h *library.Hook) error { +func (e *engine) UpdateHook(h *library.Hook) (*library.Hook, error) { e.logger.WithFields(logrus.Fields{ "hook": h.GetNumber(), }).Tracef("updating hook %d in the database", h.GetNumber()) @@ -27,12 +27,11 @@ func (e *engine) UpdateHook(h *library.Hook) error { // https://pkg.go.dev/github.com/go-vela/types/database#Hook.Validate err := hook.Validate() if err != nil { - return err + return nil, err } + result := e.client.Table(constants.TableHook).Save(hook) + // send query to the database - return e.client. - Table(constants.TableHook). - Save(hook). - Error + return hook.ToLibrary(), result.Error } diff --git a/database/hook/update_test.go b/database/hook/update_test.go index a1f0d8de2..cbb309897 100644 --- a/database/hook/update_test.go +++ b/database/hook/update_test.go @@ -33,7 +33,7 @@ WHERE "id" = $14`). _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateHook(_hook) + _, err := _sqlite.CreateHook(_hook) if err != nil { t.Errorf("unable to create test hook for sqlite: %v", err) } @@ -59,7 +59,7 @@ WHERE "id" = $14`). // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err = test.database.UpdateHook(_hook) + _, err = test.database.UpdateHook(_hook) if test.failure { if err == nil { From 2f1b70ed763dd1b44dccefac14affc4da6f9f043 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Tue, 30 May 2023 13:23:12 -0600 Subject: [PATCH 21/53] feat(templates): add ability to call templates within a template (#794) * yeet * initial work * remove unnecessary validation check * adding some comments * remove edits to inline testing * fix typo * render inline init work * support for render_inline * address feedback * validate template depth * added tests for compile inline * add test for error with called render_inline template in step * drop vs code nonsense * add github compiler flags to env example --------- Co-authored-by: Jacob Floyd Co-authored-by: David May <49894298+wass3rw3rk@users.noreply.github.com> --- .env.example | 14 +- cmd/vela-server/main.go | 6 + cmd/vela-server/validate.go | 4 + compiler/engine.go | 4 +- compiler/native/compile.go | 31 +- compiler/native/compile_test.go | 192 ++++++++++ compiler/native/expand.go | 24 +- compiler/native/expand_test.go | 349 +++++++++++++++++- compiler/native/native.go | 4 + compiler/native/testdata/circular.yml | 16 + .../testdata/inline_circular_template.yml | 19 + .../testdata/inline_nested_template.yml | 19 + compiler/native/testdata/nested.yml | 23 ++ .../testdata/template-calls-itself.json | 18 + .../testdata/template-calls-template.json | 18 + compiler/template/native/render.go | 2 +- 16 files changed, 723 insertions(+), 20 deletions(-) create mode 100644 compiler/native/testdata/circular.yml create mode 100644 compiler/native/testdata/inline_circular_template.yml create mode 100644 compiler/native/testdata/inline_nested_template.yml create mode 100644 compiler/native/testdata/nested.yml create mode 100644 compiler/native/testdata/template-calls-itself.json create mode 100644 compiler/native/testdata/template-calls-template.json diff --git a/.env.example b/.env.example index c5bf492e9..fdffdf154 100644 --- a/.env.example +++ b/.env.example @@ -58,4 +58,16 @@ VELA_API=http://localhost:8080 # VELA_SCM_CLIENT= # github client secret from oauth application -# VELA_SCM_SECRET= \ No newline at end of file +# VELA_SCM_SECRET= + +# COMPILER FLAGS +# +# compiler github is whether or not the compiler uses github to pull templates +# +# default: false +# VELA_COMPILER_GITHUB= + +# compiler github url is the url used by the compiler to fetch templates +# +# default: https://github.com +# VELA_COMPILER_GITHUB_URL \ No newline at end of file diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 48b67af59..f9b160a8f 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -197,6 +197,12 @@ func main() { Usage: "modification retries, used by compiler, number of http requires that the modification http request will fail after", Value: 5, }, + &cli.IntFlag{ + EnvVars: []string{"VELA_MAX_TEMPLATE_DEPTH", "MAX_TEMPLATE_DEPTH"}, + Name: "max-template-depth", + Usage: "max template depth, used by compiler, maximum number of templates that can be called in a template chain", + Value: 3, + }, &cli.DurationFlag{ EnvVars: []string{"VELA_WORKER_ACTIVE_INTERVAL", "WORKER_ACTIVE_INTERVAL"}, Name: "worker-active-interval", diff --git a/cmd/vela-server/validate.go b/cmd/vela-server/validate.go index bce8b12fb..cac3b0424 100644 --- a/cmd/vela-server/validate.go +++ b/cmd/vela-server/validate.go @@ -117,5 +117,9 @@ func validateCompiler(c *cli.Context) error { } } + if c.Int("max-template-depth") < 1 { + return fmt.Errorf("max-template-depth (VELA_MAX_TEMPLATE_DEPTH) or (MAX_TEMPLATE_DEPTH) flag must be greater than 0") + } + return nil } diff --git a/compiler/engine.go b/compiler/engine.go index dfe0716c4..5d1d32993 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -73,8 +73,8 @@ type Engine interface { // for each templated step in every stage in a yaml configuration. ExpandStages(*yaml.Build, map[string]*yaml.Template, *pipeline.RuleData) (*yaml.Build, error) // ExpandSteps defines a function that injects the template - // for each templated step in a yaml configuration. - ExpandSteps(*yaml.Build, map[string]*yaml.Template, *pipeline.RuleData) (*yaml.Build, error) + // for each templated step in a yaml configuration with the provided template depth. + ExpandSteps(*yaml.Build, map[string]*yaml.Template, *pipeline.RuleData, int) (*yaml.Build, error) // Init Compiler Interface Functions diff --git a/compiler/native/compile.go b/compiler/native/compile.go index a36ddc113..f74e19bab 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -82,7 +82,7 @@ func (c *client) Compile(v interface{}) (*pipeline.Build, *library.Pipeline, err switch { case p.Metadata.RenderInline: - newPipeline, err := c.compileInline(p, nil) + newPipeline, err := c.compileInline(p, nil, c.TemplateDepth) if err != nil { return nil, _pipeline, err } @@ -117,7 +117,7 @@ func (c *client) CompileLite(v interface{}, template, substitute bool, localTemp _pipeline.SetType(c.repo.GetPipelineType()) if p.Metadata.RenderInline { - newPipeline, err := c.compileInline(p, localTemplates) + newPipeline, err := c.compileInline(p, localTemplates, c.TemplateDepth) if err != nil { return nil, _pipeline, err } @@ -169,7 +169,7 @@ func (c *client) CompileLite(v interface{}, template, substitute bool, localTemp } case len(p.Steps) > 0: // inject the templates into the steps - p, err = c.ExpandSteps(p, templates, nil) + p, err = c.ExpandSteps(p, templates, nil, c.TemplateDepth) if err != nil { return nil, _pipeline, err } @@ -194,10 +194,17 @@ func (c *client) CompileLite(v interface{}, template, substitute bool, localTemp } // compileInline parses and expands out inline pipelines. -func (c *client) compileInline(p *yaml.Build, localTemplates []string) (*yaml.Build, error) { +func (c *client) compileInline(p *yaml.Build, localTemplates []string, depth int) (*yaml.Build, error) { newPipeline := *p newPipeline.Templates = yaml.TemplateSlice{} + // return if max template depth has been reached + if depth == 0 { + retErr := fmt.Errorf("max template depth of %d exceeded", c.TemplateDepth) + + return nil, retErr + } + for _, template := range p.Templates { if c.local { for _, file := range localTemplates { @@ -231,6 +238,14 @@ func (c *client) compileInline(p *yaml.Build, localTemplates []string) (*yaml.Bu return nil, err } + // if template parsed contains a template reference, recurse with decremented depth + if len(parsed.Templates) > 0 && parsed.Metadata.RenderInline { + parsed, err = c.compileInline(parsed, localTemplates, depth-1) + if err != nil { + return nil, err + } + } + switch { case len(parsed.Environment) > 0: for key, value := range parsed.Environment { @@ -276,12 +291,6 @@ func (c *client) compileInline(p *yaml.Build, localTemplates []string) (*yaml.Bu } } - // validate the yaml configuration - err := c.Validate(&newPipeline) - if err != nil { - return nil, err - } - return &newPipeline, nil } @@ -307,7 +316,7 @@ func (c *client) compileSteps(p *yaml.Build, _pipeline *library.Pipeline, tmpls } // inject the templates into the steps - p, err = c.ExpandSteps(p, tmpls, r) + p, err = c.ExpandSteps(p, tmpls, r, c.TemplateDepth) if err != nil { return nil, _pipeline, err } diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index 5ea927b33..67c89f3a2 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -38,6 +38,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -417,6 +418,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -619,6 +621,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -883,6 +886,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -1112,6 +1116,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName(t *testi set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -1231,6 +1236,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName_Inline(t set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -1346,6 +1352,7 @@ func TestNative_Compile_InvalidType(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -1402,6 +1409,7 @@ func TestNative_Compile_Clone(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -1592,6 +1600,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -2161,6 +2170,7 @@ func Test_Compile_Inline(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -2360,6 +2370,179 @@ func Test_Compile_Inline(t *testing.T) { }, wantErr: false, }, + { + name: "nested templates", + args: args{ + file: "testdata/inline_nested_template.yml", + }, + want: &pipeline.Build{ + Version: "1", + ID: "__0", + Metadata: pipeline.Metadata{ + Clone: true, + Environment: []string{"steps", "services", "secrets"}, + }, + Stages: []*pipeline.Stage{ + { + Name: "init", + Environment: initEnv, + Steps: pipeline.ContainerSlice{ + &pipeline.Container{ + ID: "__0_init_init", + Directory: "/vela/src/foo//", + Environment: initEnv, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + }, + }, + { + Name: "clone", + Environment: initEnv, + Steps: pipeline.ContainerSlice{ + &pipeline.Container{ + ID: "__0_clone_clone", + Directory: "/vela/src/foo//", + Environment: initEnv, + Image: defaultCloneImage, + Name: "clone", + Number: 2, + Pull: "not_present", + }, + }, + }, + { + Name: "test", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_test_test", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo from inline", m, ""), + Image: "alpine", + Name: "test", + Pull: "not_present", + Number: 3, + }, + }, + }, + { + Name: "nested_test", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_nested_test_nested_test", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo from inline", m, ""), + Image: "alpine", + Name: "nested_test", + Pull: "not_present", + Number: 4, + }, + }, + }, + { + Name: "nested_golang_foo", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_nested_golang_foo_nested_golang_foo", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo hello from foo", m, ""), + Image: "golang:latest", + Name: "nested_golang_foo", + Pull: "not_present", + Number: 5, + }, + }, + }, + { + Name: "nested_golang_bar", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_nested_golang_bar_nested_golang_bar", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo hello from bar", m, ""), + Image: "golang:latest", + Name: "nested_golang_bar", + Pull: "not_present", + Number: 6, + }, + }, + }, + { + Name: "nested_golang_star", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_nested_golang_star_nested_golang_star", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo hello from star", m, ""), + Image: "golang:latest", + Name: "nested_golang_star", + Pull: "not_present", + Number: 7, + }, + }, + }, + { + Name: "nested_starlark_foo", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_nested_starlark_foo_nested_starlark_build_foo", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo hello from foo", m, ""), + Image: "alpine", + Name: "nested_starlark_build_foo", + Pull: "not_present", + Number: 8, + }, + }, + }, + { + Name: "nested_starlark_bar", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_nested_starlark_bar_nested_starlark_build_bar", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo hello from bar", m, ""), + Image: "alpine", + Name: "nested_starlark_build_bar", + Pull: "not_present", + Number: 9, + }, + }, + }, + }, + }, + wantErr: false, + }, { name: "root steps", args: args{ @@ -2468,6 +2651,14 @@ func Test_Compile_Inline(t *testing.T) { want: nil, wantErr: true, }, + { + name: "circular template call", + args: args{ + file: "testdata/inline_circular_template.yml", + }, + want: nil, + wantErr: true, + }, { name: "secrets", args: args{ @@ -2944,6 +3135,7 @@ func Test_CompileLite(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ diff --git a/compiler/native/expand.go b/compiler/native/expand.go index 102ae9774..5314cf25e 100644 --- a/compiler/native/expand.go +++ b/compiler/native/expand.go @@ -31,7 +31,7 @@ func (c *client) ExpandStages(s *yaml.Build, tmpls map[string]*yaml.Template, r // iterate through all stages for _, stage := range s.Stages { // inject the templates into the steps for the stage - p, err := c.ExpandSteps(&yaml.Build{Steps: stage.Steps, Secrets: s.Secrets, Services: s.Services, Environment: s.Environment}, tmpls, r) + p, err := c.ExpandSteps(&yaml.Build{Steps: stage.Steps, Secrets: s.Secrets, Services: s.Services, Environment: s.Environment}, tmpls, r, c.TemplateDepth) if err != nil { return nil, err } @@ -47,11 +47,18 @@ func (c *client) ExpandStages(s *yaml.Build, tmpls map[string]*yaml.Template, r // ExpandSteps injects the template for each // templated step in a yaml configuration. -func (c *client) ExpandSteps(s *yaml.Build, tmpls map[string]*yaml.Template, r *pipeline.RuleData) (*yaml.Build, error) { +func (c *client) ExpandSteps(s *yaml.Build, tmpls map[string]*yaml.Template, r *pipeline.RuleData, depth int) (*yaml.Build, error) { if len(tmpls) == 0 { return s, nil } + // return if max template depth has been reached + if depth == 0 { + retErr := fmt.Errorf("max template depth of %d exceeded", c.TemplateDepth) + + return s, retErr + } + steps := yaml.StepSlice{} secrets := s.Secrets services := s.Services @@ -117,6 +124,19 @@ func (c *client) ExpandSteps(s *yaml.Build, tmpls map[string]*yaml.Template, r * return s, err } + // if template references other templates, expand again + if len(tmplBuild.Templates) != 0 { + // if the tmplBuild has render_inline but the parent build does not, abort + if tmplBuild.Metadata.RenderInline && !s.Metadata.RenderInline { + return s, fmt.Errorf("cannot use render_inline inside a called template (%s)", step.Template.Name) + } + + tmplBuild, err = c.ExpandSteps(tmplBuild, mapFromTemplates(tmplBuild.Templates), r, depth-1) + if err != nil { + return s, err + } + } + // loop over secrets within template for _, secret := range tmplBuild.Secrets { found := false diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index c89d23d50..a32a2286d 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -43,6 +43,7 @@ func TestNative_ExpandStages(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) tmpls := map[string]*yaml.Template{ @@ -190,6 +191,7 @@ func TestNative_ExpandSteps(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) testRepo := new(library.Repo) @@ -317,7 +319,7 @@ func TestNative_ExpandSteps(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData)) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) if err != nil { t.Errorf("ExpandSteps_Type%s returned err: %v", test.name, err) } @@ -368,6 +370,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) tmpls := map[string]*yaml.Template{ @@ -582,7 +585,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { ruledata := new(pipeline.RuleData) ruledata.Branch = "main" - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, ruledata) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, ruledata, compiler.TemplateDepth) if err != nil { t.Errorf("ExpandSteps returned err: %v", err) } @@ -626,6 +629,7 @@ func TestNative_ExpandStepsStarlark(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) tmpls := map[string]*yaml.Template{ @@ -669,7 +673,7 @@ func TestNative_ExpandStepsStarlark(t *testing.T) { t.Errorf("Creating new compiler returned err: %v", err) } - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Secrets: yaml.SecretSlice{}, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, new(pipeline.RuleData)) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Secrets: yaml.SecretSlice{}, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, new(pipeline.RuleData), compiler.TemplateDepth) if err != nil { t.Errorf("ExpandSteps returned err: %v", err) } @@ -691,6 +695,345 @@ func TestNative_ExpandStepsStarlark(t *testing.T) { } } +func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.GET("/api/v3/repos/foo/bar/contents/:path", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/template.json") + }) + + engine.GET("/api/v3/repos/faz/baz/contents/:path", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/template-calls-template.json") + }) + + s := httptest.NewServer(engine) + defer s.Close() + + // setup types + set := flag.NewFlagSet("test", 0) + set.Bool("github-driver", true, "doc") + set.String("github-url", s.URL, "doc") + set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") + c := cli.NewContext(nil, set, nil) + + testBuild := new(library.Build) + + testBuild.SetID(1) + testBuild.SetCommit("123abc456def") + + testRepo := new(library.Repo) + + testRepo.SetID(1) + testRepo.SetOrg("foo") + testRepo.SetName("bar") + + tests := []struct { + name string + tmpls map[string]*yaml.Template + }{ + { + name: "Test 1", + tmpls: map[string]*yaml.Template{ + "chain": { + Name: "chain", + Source: "github.example.com/faz/baz/template.yml", + Type: "github", + }, + }, + }, + } + + steps := yaml.StepSlice{ + &yaml.Step{ + Name: "sample", + Template: yaml.StepTemplate{ + Name: "chain", + }, + }, + } + + globalEnvironment := raw.StringSliceMap{ + "foo": "test1", + "bar": "test2", + } + + wantSteps := yaml.StepSlice{ + &yaml.Step{ + Commands: []string{"./gradlew downloadDependencies"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "sample_call template_install", + Pull: "always", + }, + &yaml.Step{ + Commands: []string{"./gradlew check"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "sample_call template_test", + Pull: "always", + }, + &yaml.Step{ + Commands: []string{"./gradlew build"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "sample_call template_build", + Pull: "always", + }, + } + + wantSecrets := yaml.SecretSlice{ + &yaml.Secret{ + Name: "docker_username", + Key: "org/repo/foo/bar", + Engine: "native", + Type: "repo", + Origin: yaml.Origin{}, + }, + &yaml.Secret{ + Name: "foo_password", + Key: "org/repo/foo/password", + Engine: "vault", + Type: "repo", + Origin: yaml.Origin{}, + }, + } + + wantServices := yaml.ServiceSlice{ + &yaml.Service{ + Image: "postgres:12", + Name: "postgres", + Pull: "not_present", + }, + } + + wantEnvironment := raw.StringSliceMap{ + "foo": "test1", + "bar": "test2", + "star": "test3", + } + + // run test + compiler, err := New(c) + if err != nil { + t.Errorf("Creating new compiler returned err: %v", err) + } + + compiler.WithBuild(testBuild).WithRepo(testRepo) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + if err != nil { + t.Errorf("ExpandSteps_Type%s returned err: %v", test.name, err) + } + + if diff := cmp.Diff(build.Steps, wantSteps); diff != "" { + t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff) + } + + if diff := cmp.Diff(build.Secrets, wantSecrets); diff != "" { + t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff) + } + + if diff := cmp.Diff(build.Services, wantServices); diff != "" { + t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff) + } + + if diff := cmp.Diff(build.Environment, wantEnvironment); diff != "" { + t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff) + } + }) + } +} + +func TestNative_ExpandSteps_TemplateCallTemplate_CircularFail(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + engine.GET("/api/v3/repos/bad/design/contents/:path", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/template-calls-itself.json") + }) + + s := httptest.NewServer(engine) + defer s.Close() + + // setup types + set := flag.NewFlagSet("test", 0) + set.Bool("github-driver", true, "doc") + set.String("github-url", s.URL, "doc") + set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") + c := cli.NewContext(nil, set, nil) + + testBuild := new(library.Build) + + testBuild.SetID(1) + testBuild.SetCommit("123abc456def") + + testRepo := new(library.Repo) + + testRepo.SetID(1) + testRepo.SetOrg("foo") + testRepo.SetName("bar") + + tests := []struct { + name string + tmpls map[string]*yaml.Template + }{ + { + name: "Test 1", + tmpls: map[string]*yaml.Template{ + "circle": { + Name: "circle", + Source: "github.example.com/bad/design/template.yml", + Type: "github", + }, + }, + }, + } + + steps := yaml.StepSlice{ + &yaml.Step{ + Name: "sample", + Template: yaml.StepTemplate{ + Name: "circle", + }, + }, + } + + globalEnvironment := raw.StringSliceMap{ + "foo": "test1", + "bar": "test2", + } + + // run test + compiler, err := New(c) + if err != nil { + t.Errorf("Creating new compiler returned err: %v", err) + } + + compiler.WithBuild(testBuild).WithRepo(testRepo) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + if err == nil { + t.Errorf("ExpandSteps_Type%s should have returned an error", test.name) + } + }) + } +} + +func TestNative_ExpandSteps_CallTemplateWithRenderInline(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) { + body, err := convertFileToGithubResponse(c.Param("path")) + if err != nil { + t.Error(err) + } + c.JSON(http.StatusOK, body) + }) + + s := httptest.NewServer(engine) + defer s.Close() + + // setup types + set := flag.NewFlagSet("test", 0) + set.Bool("github-driver", true, "doc") + set.String("github-url", s.URL, "doc") + set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") + c := cli.NewContext(nil, set, nil) + + testBuild := new(library.Build) + + testBuild.SetID(1) + testBuild.SetCommit("123abc456def") + + testRepo := new(library.Repo) + + testRepo.SetID(1) + testRepo.SetOrg("foo") + testRepo.SetName("bar") + + tests := []struct { + name string + tmpls map[string]*yaml.Template + }{ + { + name: "Test 1", + tmpls: map[string]*yaml.Template{ + "render_inline": { + Name: "render_inline", + Source: "github.example.com/github/octocat/nested.yml", + Type: "github", + }, + }, + }, + } + + steps := yaml.StepSlice{ + &yaml.Step{ + Name: "sample", + Template: yaml.StepTemplate{ + Name: "render_inline", + }, + }, + } + + globalEnvironment := raw.StringSliceMap{ + "foo": "test1", + "bar": "test2", + } + + // run test + compiler, err := New(c) + if err != nil { + t.Errorf("Creating new compiler returned err: %v", err) + } + + compiler.WithBuild(testBuild).WithRepo(testRepo) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + if err == nil { + t.Errorf("ExpandSteps_Type%s should have returned an error", test.name) + } + }) + } +} + func TestNative_mapFromTemplates(t *testing.T) { // setup types str := "foo" diff --git a/compiler/native/native.go b/compiler/native/native.go index 61b411838..733a32407 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -32,6 +32,7 @@ type client struct { UsePrivateGithub bool ModificationService ModificationConfig CloneImage string + TemplateDepth int build *library.Build comment string @@ -71,6 +72,8 @@ func New(ctx *cli.Context) (*client, error) { // set the clone image to use for the injected clone step c.CloneImage = ctx.String("clone-image") + c.TemplateDepth = ctx.Int("max-template-depth") + if ctx.Bool("github-driver") { logrus.Tracef("setting up Private GitHub Client for %s", ctx.String("github-url")) // setup private github service @@ -110,6 +113,7 @@ func (c *client) Duplicate() compiler.Engine { cc.UsePrivateGithub = c.UsePrivateGithub cc.ModificationService = c.ModificationService cc.CloneImage = c.CloneImage + cc.TemplateDepth = c.TemplateDepth return cc } diff --git a/compiler/native/testdata/circular.yml b/compiler/native/testdata/circular.yml new file mode 100644 index 000000000..bd8327df3 --- /dev/null +++ b/compiler/native/testdata/circular.yml @@ -0,0 +1,16 @@ +metadata: + render_inline: true + template: true + +templates: + - name: bad + source: github.example.com/github/octocat/inline_circular_template.yml + type: github + +stages: + test: + steps: + - name: test + image: alpine + commands: + - echo from inline \ No newline at end of file diff --git a/compiler/native/testdata/inline_circular_template.yml b/compiler/native/testdata/inline_circular_template.yml new file mode 100644 index 000000000..4b0469c58 --- /dev/null +++ b/compiler/native/testdata/inline_circular_template.yml @@ -0,0 +1,19 @@ +version: "1" + +metadata: + render_inline: true + +templates: + - name: nested + source: github.example.com/github/octocat/circular.yml + type: github + vars: + image: golang:latest + +stages: + test: + steps: + - name: test + image: alpine + commands: + - echo from inline \ No newline at end of file diff --git a/compiler/native/testdata/inline_nested_template.yml b/compiler/native/testdata/inline_nested_template.yml new file mode 100644 index 000000000..0faaeb0c8 --- /dev/null +++ b/compiler/native/testdata/inline_nested_template.yml @@ -0,0 +1,19 @@ +version: "1" + +metadata: + render_inline: true + +templates: + - name: nested + source: github.example.com/github/octocat/nested.yml + type: github + vars: + image: golang:latest + +stages: + test: + steps: + - name: test + image: alpine + commands: + - echo from inline \ No newline at end of file diff --git a/compiler/native/testdata/nested.yml b/compiler/native/testdata/nested.yml new file mode 100644 index 000000000..2d44729d7 --- /dev/null +++ b/compiler/native/testdata/nested.yml @@ -0,0 +1,23 @@ +metadata: + render_inline: true + template: true + +templates: + - name: golang + source: github.example.com/github/octocat/golang_inline_stages.yml + format: golang + type: github + vars: + image: golang:latest + - name: starlark + source: github.example.com/github/octocat/starlark_inline_stages.star + format: starlark + type: github + +stages: + test: + steps: + - name: test + image: alpine + commands: + - echo from inline \ No newline at end of file diff --git a/compiler/native/testdata/template-calls-itself.json b/compiler/native/testdata/template-calls-itself.json new file mode 100644 index 000000000..a930d6780 --- /dev/null +++ b/compiler/native/testdata/template-calls-itself.json @@ -0,0 +1,18 @@ +{ + "type": "file", + "encoding": "base64", + "size": 5362, + "name": "template.yml", + "path": "template.yml", + "content": "dmVyc2lvbjogIjEiCgp0ZW1wbGF0ZXM6CiAgLSBuYW1lOiB0ZXN0CiAgICBzb3VyY2U6IGdpdGh1Yi5leGFtcGxlLmNvbS9iYWQvZGVzaWduL3RlbXBsYXRlLnltbAogICAgdHlwZTogZ2l0aHViCgpzdGVwczoKICAtIG5hbWU6IGNhbGwgdGVtcGxhdGUKICAgIHRlbXBsYXRlOgogICAgICBuYW1lOiB0ZXN0Cg==", + "sha": "3d21ec53a331a6f037a91c368710b99387d012c1", + "url": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", + "git_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", + "html_url": "https://github.com/octokit/octokit.rb/blob/master/template.yml", + "download_url": "https://raw.githubusercontent.com/octokit/octokit.rb/master/template.yml", + "_links": { + "git": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", + "self": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", + "html": "https://github.com/octokit/octokit.rb/blob/master/template.yml" + } +} \ No newline at end of file diff --git a/compiler/native/testdata/template-calls-template.json b/compiler/native/testdata/template-calls-template.json new file mode 100644 index 000000000..6fb0be303 --- /dev/null +++ b/compiler/native/testdata/template-calls-template.json @@ -0,0 +1,18 @@ +{ + "type": "file", + "encoding": "base64", + "size": 5362, + "name": "template.yml", + "path": "template.yml", + "content": "dmVyc2lvbjogIjEiCgp0ZW1wbGF0ZXM6CiAgLSBuYW1lOiB0ZXN0CiAgICBzb3VyY2U6IGdpdGh1Yi5leGFtcGxlLmNvbS9mb28vYmFyL3RlbXBsYXRlLnltbAogICAgdHlwZTogZ2l0aHViCgpzdGVwczoKICAtIG5hbWU6IGNhbGwgdGVtcGxhdGUKICAgIHRlbXBsYXRlOgogICAgICBuYW1lOiB0ZXN0CiAgICAgIHZhcnM6CiAgICAgICAgaW1hZ2U6IG9wZW5qZGs6bGF0ZXN0CiAgICAgICAgZW52aXJvbm1lbnQ6ICJ7IEdSQURMRV9VU0VSX0hPTUU6IC5ncmFkbGUsIEdSQURMRV9PUFRTOiAtRG9yZy5ncmFkbGUuZGFlbW9uPWZhbHNlIC1Eb3JnLmdyYWRsZS53b3JrZXJzLm1heD0xIC1Eb3JnLmdyYWRsZS5wYXJhbGxlbD1mYWxzZSB9IgogICAgICAgIHB1bGxfcG9saWN5OiAicHVsbDogdHJ1ZSIK", + "sha": "3d21ec53a331a6f037a91c368710b99387d012c1", + "url": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", + "git_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", + "html_url": "https://github.com/octokit/octokit.rb/blob/master/template.yml", + "download_url": "https://raw.githubusercontent.com/octokit/octokit.rb/master/template.yml", + "_links": { + "git": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", + "self": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", + "html": "https://github.com/octokit/octokit.rb/blob/master/template.yml" + } +} \ No newline at end of file diff --git a/compiler/template/native/render.go b/compiler/template/native/render.go index 95389fd1d..869e4588a 100644 --- a/compiler/template/native/render.go +++ b/compiler/template/native/render.go @@ -61,7 +61,7 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM config.Steps[index].Name = fmt.Sprintf("%s_%s", name, newStep.Name) } - return &types.Build{Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment}, nil + return &types.Build{Metadata: config.Metadata, Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment, Templates: config.Templates}, nil } // RenderBuild renders the templated build. From ca68a09f317b1e9e477b5e79f9cb5d0aa499fcde Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Tue, 30 May 2023 13:28:16 -0600 Subject: [PATCH 22/53] feat(api/admin): add endpoint to clean pending / running builds (#863) * feat(api/admin): add endpoint to clean pending / running builds * db tests * add mock * actually use the mock * info logrus logging for each table clean * add msg and finished to resources update --- api/admin/clean.go | 135 +++++++++++++++++++++++++++++++++ database/build/clean.go | 35 +++++++++ database/build/clean_test.go | 120 +++++++++++++++++++++++++++++ database/build/interface.go | 2 + database/service/clean.go | 35 +++++++++ database/service/clean_test.go | 131 ++++++++++++++++++++++++++++++++ database/service/interface.go | 2 + database/step/clean.go | 35 +++++++++ database/step/clean_test.go | 131 ++++++++++++++++++++++++++++++++ database/step/interface.go | 2 + mock/server/build.go | 22 ++++++ mock/server/server.go | 1 + router/admin.go | 4 + 13 files changed, 655 insertions(+) create mode 100644 api/admin/clean.go create mode 100644 database/build/clean.go create mode 100644 database/build/clean_test.go create mode 100644 database/service/clean.go create mode 100644 database/service/clean_test.go create mode 100644 database/step/clean.go create mode 100644 database/step/clean_test.go diff --git a/api/admin/clean.go b/api/admin/clean.go new file mode 100644 index 000000000..4895c3e55 --- /dev/null +++ b/api/admin/clean.go @@ -0,0 +1,135 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package admin + +import ( + "fmt" + "net/http" + "strconv" + "time" + + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types" + "github.com/go-vela/types/constants" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" +) + +// swagger:operation PUT /api/v1/admin/clean admin AdminCleanResources +// +// Update pending build resources to error status before a given time +// +// --- +// produces: +// - application/json +// parameters: +// - in: query +// name: before +// description: filter pending resources created before a certain time +// required: true +// type: integer +// - in: body +// name: body +// description: Payload containing error message +// required: true +// schema: +// "$ref": "#/definitions/Error" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully updated pending resources with error message +// schema: +// type: string +// '400': +// description: Unable to update resources — bad request +// schema: +// "$ref": "#/definitions/Error" +// '401': +// description: Unable to update resources — unauthorized +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to update resources +// schema: +// "$ref": "#/definitions/Error" + +// CleanResources represents the API handler to +// update any user stored in the database. +func CleanResources(c *gin.Context) { + u := user.Retrieve(c) + logrus.Infof("platform admin %s: updating pending resources in database", u.GetName()) + + // default error message + msg := "build cleaned by platform admin" + + // capture body from API request + input := new(types.Error) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for error message: %w", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // if a message is provided, set the error message to that + if input.Message != nil { + msg = *input.Message + } + + // capture before query parameter, default to max build timeout + before, err := strconv.ParseInt(c.DefaultQuery("before", fmt.Sprint((time.Now().Add(-(time.Minute * (constants.BuildTimeoutMax + 5)))).Unix())), 10, 64) + if err != nil { + retErr := fmt.Errorf("unable to convert before query parameter %s to int64: %w", c.Query("before"), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to clean builds + builds, err := database.FromContext(c).CleanBuilds(msg, before) + if err != nil { + retErr := fmt.Errorf("unable to update builds: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + logrus.Infof("platform admin %s: cleaned %d builds in database", u.GetName(), builds) + + // clean services + services, err := database.FromContext(c).CleanServices(msg, before) + if err != nil { + retErr := fmt.Errorf("%d builds cleaned. unable to update services: %w", builds, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + logrus.Infof("platform admin %s: cleaned %d services in database", u.GetName(), services) + + // clean steps + steps, err := database.FromContext(c).CleanSteps(msg, before) + if err != nil { + retErr := fmt.Errorf("%d builds cleaned. %d services cleaned. unable to update steps: %w", builds, services, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + logrus.Infof("platform admin %s: cleaned %d steps in database", u.GetName(), steps) + + c.JSON(http.StatusOK, fmt.Sprintf("%d builds cleaned. %d services cleaned. %d steps cleaned.", builds, services, steps)) +} diff --git a/database/build/clean.go b/database/build/clean.go new file mode 100644 index 000000000..5e0541b35 --- /dev/null +++ b/database/build/clean.go @@ -0,0 +1,35 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "time" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// CleanBuilds updates builds to an error with a provided message with a created timestamp prior to a defined moment. +func (e *engine) CleanBuilds(msg string, before int64) (int64, error) { + logrus.Tracef("cleaning pending or running builds in the database created prior to %d", before) + + b := new(library.Build) + b.SetStatus(constants.StatusError) + b.SetError(msg) + b.SetFinished(time.Now().UTC().Unix()) + + build := database.BuildFromLibrary(b) + + // send query to the database + result := e.client. + Table(constants.TableBuild). + Where("created < ?", before). + Where("status = 'running' OR status = 'pending'"). + Updates(build) + + return result.RowsAffected, result.Error +} diff --git a/database/build/clean_test.go b/database/build/clean_test.go new file mode 100644 index 000000000..ab91da075 --- /dev/null +++ b/database/build/clean_test.go @@ -0,0 +1,120 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "reflect" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestBuild_Engine_CleanBuilds(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetCreated(1) + _buildOne.SetStatus("pending") + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(1) + _buildTwo.SetNumber(2) + _buildTwo.SetCreated(2) + _buildTwo.SetStatus("running") + + // setup types + _buildThree := testBuild() + _buildThree.SetID(3) + _buildThree.SetRepoID(1) + _buildThree.SetNumber(3) + _buildThree.SetCreated(1) + _buildThree.SetStatus("success") + + _buildFour := testBuild() + _buildFour.SetID(4) + _buildFour.SetRepoID(1) + _buildFour.SetNumber(4) + _buildFour.SetCreated(5) + _buildFour.SetStatus("running") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the name query + _mock.ExpectExec(`UPDATE "builds" SET "status"=$1,"error"=$2,"finished"=$3,"deploy_payload"=$4 WHERE created < $5 AND (status = 'running' OR status = 'pending')`). + WithArgs("error", "msg", time.Now().UTC().Unix(), AnyArgument{}, 3). + WillReturnResult(sqlmock.NewResult(1, 2)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuild(_buildOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildThree) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuild(_buildFour) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 2, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 2, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CleanBuilds("msg", 3) + + if test.failure { + if err == nil { + t.Errorf("CleanBuilds for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CleanBuilds for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CleanBuilds for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/build/interface.go b/database/build/interface.go index e4264f6a8..6a14ad4f8 100644 --- a/database/build/interface.go +++ b/database/build/interface.go @@ -26,6 +26,8 @@ type BuildInterface interface { // // https://en.wikipedia.org/wiki/Data_manipulation_language + // CleanBuilds defines a function that sets pending or running builds to error created before a given time. + CleanBuilds(string, int64) (int64, error) // CountBuilds defines a function that gets the count of all builds. CountBuilds() (int64, error) // CountBuildsForDeployment defines a function that gets the count of builds by deployment url. diff --git a/database/service/clean.go b/database/service/clean.go new file mode 100644 index 000000000..e8bcac16d --- /dev/null +++ b/database/service/clean.go @@ -0,0 +1,35 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "time" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// CleanServices updates services to an error with a created timestamp prior to a defined moment. +func (e *engine) CleanServices(msg string, before int64) (int64, error) { + logrus.Tracef("cleaning pending or running steps in the database created prior to %d", before) + + s := new(library.Service) + s.SetStatus(constants.StatusError) + s.SetError(msg) + s.SetFinished(time.Now().UTC().Unix()) + + service := database.ServiceFromLibrary(s) + + // send query to the database + result := e.client. + Table(constants.TableService). + Where("created < ?", before). + Where("status = 'running' OR status = 'pending'"). + Updates(service) + + return result.RowsAffected, result.Error +} diff --git a/database/service/clean_test.go b/database/service/clean_test.go new file mode 100644 index 000000000..f30939880 --- /dev/null +++ b/database/service/clean_test.go @@ -0,0 +1,131 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "reflect" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestService_Engine_CleanService(t *testing.T) { + // setup types + _serviceOne := testService() + _serviceOne.SetID(1) + _serviceOne.SetRepoID(1) + _serviceOne.SetBuildID(1) + _serviceOne.SetNumber(1) + _serviceOne.SetName("foo") + _serviceOne.SetImage("bar") + _serviceOne.SetCreated(1) + _serviceOne.SetStatus("running") + + _serviceTwo := testService() + _serviceTwo.SetID(2) + _serviceTwo.SetRepoID(1) + _serviceTwo.SetBuildID(1) + _serviceTwo.SetNumber(2) + _serviceTwo.SetName("foo") + _serviceTwo.SetImage("bar") + _serviceTwo.SetCreated(1) + _serviceTwo.SetStatus("pending") + + _serviceThree := testService() + _serviceThree.SetID(3) + _serviceThree.SetRepoID(1) + _serviceThree.SetBuildID(1) + _serviceThree.SetNumber(3) + _serviceThree.SetName("foo") + _serviceThree.SetImage("bar") + _serviceThree.SetCreated(1) + _serviceThree.SetStatus("success") + + _serviceFour := testService() + _serviceFour.SetID(4) + _serviceFour.SetRepoID(1) + _serviceFour.SetBuildID(1) + _serviceFour.SetNumber(4) + _serviceFour.SetName("foo") + _serviceFour.SetImage("bar") + _serviceFour.SetCreated(5) + _serviceFour.SetStatus("pending") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the name query + _mock.ExpectExec(`UPDATE "services" SET "status"=$1,"error"=$2,"finished"=$3 WHERE created < $4 AND (status = 'running' OR status = 'pending')`). + WithArgs("error", "msg", time.Now().UTC().Unix(), 3). + WillReturnResult(sqlmock.NewResult(1, 2)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateService(_serviceOne) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + err = _sqlite.CreateService(_serviceTwo) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + err = _sqlite.CreateService(_serviceThree) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + err = _sqlite.CreateService(_serviceFour) + if err != nil { + t.Errorf("unable to create test service for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 2, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 2, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CleanServices("msg", 3) + + if test.failure { + if err == nil { + t.Errorf("CleanServices for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CleanServices for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CleanServices for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/service/interface.go b/database/service/interface.go index 29cb0f457..d22595859 100644 --- a/database/service/interface.go +++ b/database/service/interface.go @@ -24,6 +24,8 @@ type ServiceInterface interface { // // https://en.wikipedia.org/wiki/Data_manipulation_language + // CleanServices defines a function that sets running or pending services to error status before a given created time. + CleanServices(string, int64) (int64, error) // CountServices defines a function that gets the count of all services. CountServices() (int64, error) // CountServicesForBuild defines a function that gets the count of services by build ID. diff --git a/database/step/clean.go b/database/step/clean.go new file mode 100644 index 000000000..54e686e86 --- /dev/null +++ b/database/step/clean.go @@ -0,0 +1,35 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package step + +import ( + "time" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// CleanSteps updates steps to an error with a created timestamp prior to a defined moment. +func (e *engine) CleanSteps(msg string, before int64) (int64, error) { + logrus.Tracef("cleaning pending or running steps in the database created prior to %d", before) + + s := new(library.Step) + s.SetStatus(constants.StatusError) + s.SetError(msg) + s.SetFinished(time.Now().UTC().Unix()) + + step := database.StepFromLibrary(s) + + // send query to the database + result := e.client. + Table(constants.TableStep). + Where("created < ?", before). + Where("status = 'running' OR status = 'pending'"). + Updates(step) + + return result.RowsAffected, result.Error +} diff --git a/database/step/clean_test.go b/database/step/clean_test.go new file mode 100644 index 000000000..22709749a --- /dev/null +++ b/database/step/clean_test.go @@ -0,0 +1,131 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package step + +import ( + "reflect" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestStep_Engine_CleanStep(t *testing.T) { + // setup types + _stepOne := testStep() + _stepOne.SetID(1) + _stepOne.SetRepoID(1) + _stepOne.SetBuildID(1) + _stepOne.SetNumber(1) + _stepOne.SetName("foo") + _stepOne.SetImage("bar") + _stepOne.SetCreated(1) + _stepOne.SetStatus("running") + + _stepTwo := testStep() + _stepTwo.SetID(2) + _stepTwo.SetRepoID(1) + _stepTwo.SetBuildID(1) + _stepTwo.SetNumber(2) + _stepTwo.SetName("foo") + _stepTwo.SetImage("bar") + _stepTwo.SetCreated(1) + _stepTwo.SetStatus("pending") + + _stepThree := testStep() + _stepThree.SetID(3) + _stepThree.SetRepoID(1) + _stepThree.SetBuildID(1) + _stepThree.SetNumber(3) + _stepThree.SetName("foo") + _stepThree.SetImage("bar") + _stepThree.SetCreated(1) + _stepThree.SetStatus("success") + + _stepFour := testStep() + _stepFour.SetID(4) + _stepFour.SetRepoID(1) + _stepFour.SetBuildID(1) + _stepFour.SetNumber(4) + _stepFour.SetName("foo") + _stepFour.SetImage("bar") + _stepFour.SetCreated(5) + _stepFour.SetStatus("pending") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the name query + _mock.ExpectExec(`UPDATE "steps" SET "status"=$1,"error"=$2,"finished"=$3 WHERE created < $4 AND (status = 'running' OR status = 'pending')`). + WithArgs("error", "msg", time.Now().UTC().Unix(), 3). + WillReturnResult(sqlmock.NewResult(1, 2)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateStep(_stepOne) + if err != nil { + t.Errorf("unable to create test step for sqlite: %v", err) + } + + err = _sqlite.CreateStep(_stepTwo) + if err != nil { + t.Errorf("unable to create test step for sqlite: %v", err) + } + + err = _sqlite.CreateStep(_stepThree) + if err != nil { + t.Errorf("unable to create test step for sqlite: %v", err) + } + + err = _sqlite.CreateStep(_stepFour) + if err != nil { + t.Errorf("unable to create test step for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 2, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 2, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CleanSteps("msg", 3) + + if test.failure { + if err == nil { + t.Errorf("CleanSteps for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CleanSteps for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CleanSteps for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/step/interface.go b/database/step/interface.go index a410f2600..68423c8c9 100644 --- a/database/step/interface.go +++ b/database/step/interface.go @@ -24,6 +24,8 @@ type StepInterface interface { // // https://en.wikipedia.org/wiki/Data_manipulation_language + // CleanSteps defines a function that sets running or pending steps to error status before a given created time. + CleanSteps(string, int64) (int64, error) // CountSteps defines a function that gets the count of all steps. CountSteps() (int64, error) // CountStepsForBuild defines a function that gets the count of steps by build ID. diff --git a/mock/server/build.go b/mock/server/build.go index 37809bae4..db526cb29 100644 --- a/mock/server/build.go +++ b/mock/server/build.go @@ -149,6 +149,8 @@ const ( BuildTokenResp = `{ "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJidWlsZF9pZCI6MSwicmVwbyI6ImZvby9iYXIiLCJzdWIiOiJPY3RvY2F0IiwiaWF0IjoxNTE2MjM5MDIyfQ.hD7gXpaf9acnLBdOBa4GOEa5KZxdzd0ZvK6fGwaN4bc" }` + + CleanResourcesResp = "42 builds cleaned. 42 services cleaned. 42 steps cleaned." ) // getBuilds returns mock JSON for a http GET. @@ -337,3 +339,23 @@ func buildToken(c *gin.Context) { c.JSON(http.StatusOK, body) } + +// cleanResources has a query param :before returns mock JSON for a http PUT +// +// Pass "0" to :before to test receiving a http 500 response. Pass "1" to :before +// to test receiving a http 401 response. +func cleanResoures(c *gin.Context) { + before := c.Query("before") + + if strings.EqualFold(before, "0") { + c.AbortWithStatusJSON(http.StatusInternalServerError, "") + + return + } + + if strings.EqualFold(before, "1") { + c.AbortWithStatusJSON(http.StatusUnauthorized, "") + } + + c.JSON(http.StatusOK, CleanResourcesResp) +} diff --git a/mock/server/server.go b/mock/server/server.go index 8fb6b1061..d2f6ccbab 100644 --- a/mock/server/server.go +++ b/mock/server/server.go @@ -30,6 +30,7 @@ func FakeHandler() http.Handler { e.PUT("/api/v1/admin/step", updateStep) e.PUT("/api/v1/admin/user", updateUser) e.POST("/api/v1/admin/workers/:worker/register-token", registerToken) + e.PUT("api/v1/admin/clean", cleanResoures) // mock endpoints for build calls e.GET("/api/v1/repos/:org/:repo/builds/:build", getBuild) diff --git a/router/admin.go b/router/admin.go index f7fddf95f..96764617f 100644 --- a/router/admin.go +++ b/router/admin.go @@ -16,6 +16,7 @@ import ( // GET /api/v1/admin/builds/queue // GET /api/v1/admin/build/:id // PUT /api/v1/admin/build +// PUT /api/v1/admin/clean // PUT /api/v1/admin/deployment // PUT /api/v1/admin/hook // PUT /api/v1/admin/repo @@ -33,6 +34,9 @@ func AdminHandlers(base *gin.RouterGroup) { // Admin build endpoint _admin.PUT("/build", admin.UpdateBuild) + // Admin clean endpoint + _admin.PUT("/clean", admin.CleanResources) + // Admin deployment endpoint _admin.PUT("/deployment", admin.UpdateDeployment) From a5fc7c68a372938e5c6ed3be182a7052ea2a9f24 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Tue, 30 May 2023 13:33:58 -0600 Subject: [PATCH 23/53] refactor(api): move build logic to separate package (#864) * init commit * remove cleanBuild helper in favor of CleanBuild * planStep helper and address feedback and upgrade types * update path in swagger for org list builds * flaky linter overlord --------- Co-authored-by: dave vader <48764154+plyr4@users.noreply.github.com> --- api/build.go | 2185 --------------------- api/build/cancel.go | 289 +++ api/build/clean.go | 56 + api/build/create.go | 392 ++++ api/build/delete.go | 92 + api/build/doc.go | 10 + api/build/get.go | 70 + api/build/get_id.go | 118 ++ api/build/list_org.go | 195 ++ api/build/list_repo.go | 259 +++ api/build/plan.go | 71 + api/build/publish.go | 74 + api/build/restart.go | 362 ++++ api/build/skip.go | 41 + api/{build_test.go => build/skip_test.go} | 2 +- api/build/token.go | 119 ++ api/build/update.go | 186 ++ api/service/plan.go | 71 + api/step/plan.go | 97 + api/webhook/doc.go | 10 + api/{webhook.go => webhook/post.go} | 68 +- cmd/vela-server/schedule.go | 8 +- go.mod | 2 +- go.sum | 4 +- router/build.go | 28 +- router/repo.go | 4 +- router/router.go | 3 +- router/search.go | 6 +- 28 files changed, 2547 insertions(+), 2275 deletions(-) delete mode 100644 api/build.go create mode 100644 api/build/cancel.go create mode 100644 api/build/clean.go create mode 100644 api/build/create.go create mode 100644 api/build/delete.go create mode 100644 api/build/doc.go create mode 100644 api/build/get.go create mode 100644 api/build/get_id.go create mode 100644 api/build/list_org.go create mode 100644 api/build/list_repo.go create mode 100644 api/build/plan.go create mode 100644 api/build/publish.go create mode 100644 api/build/restart.go create mode 100644 api/build/skip.go rename api/{build_test.go => build/skip_test.go} (99%) create mode 100644 api/build/token.go create mode 100644 api/build/update.go create mode 100644 api/service/plan.go create mode 100644 api/step/plan.go create mode 100644 api/webhook/doc.go rename api/{webhook.go => webhook/post.go} (93%) diff --git a/api/build.go b/api/build.go deleted file mode 100644 index cd6c8b86f..000000000 --- a/api/build.go +++ /dev/null @@ -1,2185 +0,0 @@ -// Copyright (c) 2023 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package api - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strconv" - "strings" - "time" - - "github.com/gin-gonic/gin" - "github.com/go-vela/server/compiler" - "github.com/go-vela/server/database" - "github.com/go-vela/server/internal/token" - "github.com/go-vela/server/queue" - "github.com/go-vela/server/router/middleware/build" - "github.com/go-vela/server/router/middleware/claims" - "github.com/go-vela/server/router/middleware/executors" - "github.com/go-vela/server/router/middleware/org" - "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/router/middleware/user" - "github.com/go-vela/server/scm" - "github.com/go-vela/server/util" - "github.com/go-vela/types" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" - "github.com/go-vela/types/pipeline" - "github.com/sirupsen/logrus" -) - -// swagger:operation POST /api/v1/repos/{org}/{repo}/builds builds CreateBuild -// -// Create a build in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: body -// name: body -// description: Payload containing the build to update -// required: true -// schema: -// "$ref": "#/definitions/Build" -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Request processed but build was skipped -// schema: -// type: string -// '201': -// description: Successfully created the build -// type: json -// schema: -// "$ref": "#/definitions/Build" -// '400': -// description: Unable to create the build -// schema: -// "$ref": "#/definitions/Error" -// '404': -// description: Unable to create the build -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to create the build -// schema: -// "$ref": "#/definitions/Error" - -// CreateBuild represents the API handler to create a build in the configured backend. -// -//nolint:funlen,gocyclo // ignore function length and cyclomatic complexity -func CreateBuild(c *gin.Context) { - // capture middleware values - m := c.MustGet("metadata").(*types.Metadata) - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logger := logrus.WithFields(logrus.Fields{ - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }) - - logger.Infof("creating new build for repo %s", r.GetFullName()) - - // capture body from API request - input := new(library.Build) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for new build for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // verify the build has a valid event and the repo allows that event type - if (input.GetEvent() == constants.EventPush && !r.GetAllowPush()) || - (input.GetEvent() == constants.EventPull && !r.GetAllowPull()) || - (input.GetEvent() == constants.EventTag && !r.GetAllowTag()) || - (input.GetEvent() == constants.EventDeploy && !r.GetAllowDeploy()) { - retErr := fmt.Errorf("unable to create new build: %s does not have %s events enabled", r.GetFullName(), input.GetEvent()) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // send API call to capture the repo owner - u, err = database.FromContext(c).GetUser(r.GetUserID()) - if err != nil { - retErr := fmt.Errorf("unable to get owner for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // create SQL filters for querying pending and running builds for repo - filters := map[string]interface{}{ - "status": []string{constants.StatusPending, constants.StatusRunning}, - } - - // send API call to capture the number of pending or running builds for the repo - builds, err := database.FromContext(c).CountBuildsForRepo(r, filters) - if err != nil { - retErr := fmt.Errorf("unable to create new build: unable to get count of builds for repo %s", r.GetFullName()) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // check if the number of pending and running builds exceeds the limit for the repo - if builds >= r.GetBuildLimit() { - retErr := fmt.Errorf("unable to create new build: repo %s has exceeded the concurrent build limit of %d", r.GetFullName(), r.GetBuildLimit()) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update fields in build object - input.SetRepoID(r.GetID()) - input.SetStatus(constants.StatusPending) - input.SetCreated(time.Now().UTC().Unix()) - - // set the parent equal to the current repo counter - input.SetParent(r.GetCounter()) - // check if the parent is set to 0 - if input.GetParent() == 0 { - // parent should be "1" if it's the first build ran - input.SetParent(1) - } - - // update the build numbers based off repo counter - inc := r.GetCounter() + 1 - r.SetCounter(inc) - input.SetNumber(inc) - - // populate the build link if a web address is provided - if len(m.Vela.WebAddress) > 0 { - input.SetLink( - fmt.Sprintf("%s/%s/%d", m.Vela.WebAddress, r.GetFullName(), input.GetNumber()), - ) - } - - // variable to store changeset files - var files []string - // check if the build event is not issue_comment or pull_request - if !strings.EqualFold(input.GetEvent(), constants.EventComment) && - !strings.EqualFold(input.GetEvent(), constants.EventPull) { - // send API call to capture list of files changed for the commit - files, err = scm.FromContext(c).Changeset(u, r, input.GetCommit()) - if err != nil { - retErr := fmt.Errorf("unable to create new build: failed to get changeset for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - } - - // check if the build event is a pull_request - if strings.EqualFold(input.GetEvent(), constants.EventPull) { - // capture number from build - number, err := getPRNumberFromBuild(input) - if err != nil { - retErr := fmt.Errorf("unable to create new build: failed to get pull_request number for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture list of files changed for the pull request - files, err = scm.FromContext(c).ChangesetPR(u, r, number) - if err != nil { - retErr := fmt.Errorf("unable to create new build: failed to get changeset for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - } - - var ( - // variable to store the raw pipeline configuration - config []byte - // variable to store executable pipeline - p *pipeline.Build - // variable to store pipeline configuration - pipeline *library.Pipeline - // variable to store the pipeline type for the repository - pipelineType = r.GetPipelineType() - ) - - // send API call to attempt to capture the pipeline - pipeline, err = database.FromContext(c).GetPipelineForRepo(input.GetCommit(), r) - if err != nil { // assume the pipeline doesn't exist in the database yet - // send API call to capture the pipeline configuration file - config, err = scm.FromContext(c).ConfigBackoff(u, r, input.GetCommit()) - if err != nil { - retErr := fmt.Errorf("unable to create new build: failed to get pipeline configuration for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - } else { - config = pipeline.GetData() - } - - // ensure we use the expected pipeline type when compiling - // - // The pipeline type for a repo can change at any time which can break compiling - // existing pipelines in the system for that repo. To account for this, we update - // the repo pipeline type to match what was defined for the existing pipeline - // before compiling. After we're done compiling, we reset the pipeline type. - if len(pipeline.GetType()) > 0 { - r.SetPipelineType(pipeline.GetType()) - } - - var compiled *library.Pipeline - // parse and compile the pipeline configuration file - p, compiled, err = compiler.FromContext(c). - Duplicate(). - WithBuild(input). - WithCommit(input.GetCommit()). - WithFiles(files). - WithMetadata(m). - WithRepo(r). - WithUser(u). - Compile(config) - if err != nil { - retErr := fmt.Errorf("unable to compile pipeline configuration for %s/%d: %w", r.GetFullName(), input.GetNumber(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - // reset the pipeline type for the repo - // - // The pipeline type for a repo can change at any time which can break compiling - // existing pipelines in the system for that repo. To account for this, we update - // the repo pipeline type to match what was defined for the existing pipeline - // before compiling. After we're done compiling, we reset the pipeline type. - r.SetPipelineType(pipelineType) - - // skip the build if only the init or clone steps are found - skip := SkipEmptyBuild(p) - if skip != "" { - // set build to successful status - input.SetStatus(constants.StatusSuccess) - - // send API call to set the status on the commit - err = scm.FromContext(c).Status(u, input, r.GetOrg(), r.GetName()) - if err != nil { - logger.Errorf("unable to set commit status for %s/%d: %v", r.GetFullName(), input.GetNumber(), err) - } - - c.JSON(http.StatusOK, skip) - - return - } - - // check if the pipeline did not already exist in the database - // - //nolint:dupl // ignore duplicate code - if pipeline == nil { - pipeline = compiled - pipeline.SetRepoID(r.GetID()) - pipeline.SetCommit(input.GetCommit()) - pipeline.SetRef(input.GetRef()) - - // send API call to create the pipeline - err = database.FromContext(c).CreatePipeline(pipeline) - if err != nil { - retErr := fmt.Errorf("unable to create new build: failed to create pipeline for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // send API call to capture the created pipeline - pipeline, err = database.FromContext(c).GetPipelineForRepo(pipeline.GetCommit(), r) - if err != nil { - //nolint:lll // ignore long line length due to error message - retErr := fmt.Errorf("unable to create new build: failed to get new pipeline %s/%s: %w", r.GetFullName(), pipeline.GetCommit(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - } - - input.SetPipelineID(pipeline.GetID()) - - // create the objects from the pipeline in the database - err = PlanBuild(database.FromContext(c), p, input, r) - if err != nil { - util.HandleError(c, http.StatusInternalServerError, err) - - return - } - - // send API call to update repo for ensuring counter is incremented - err = database.FromContext(c).UpdateRepo(r) - if err != nil { - retErr := fmt.Errorf("unable to create new build: failed to update repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // send API call to capture the created build - input, _ = database.FromContext(c).GetBuildForRepo(r, input.GetNumber()) - - c.JSON(http.StatusCreated, input) - - // send API call to set the status on the commit - err = scm.FromContext(c).Status(u, input, r.GetOrg(), r.GetName()) - if err != nil { - logger.Errorf("unable to set commit status for build %s/%d: %v", r.GetFullName(), input.GetNumber(), err) - } - - // publish the build to the queue - go PublishToQueue( - queue.FromGinContext(c), - database.FromContext(c), - p, - input, - r, - u, - ) -} - -// SkipEmptyBuild checks if the build should be skipped due to it -// not containing any steps besides init or clone. -// -//nolint:goconst // ignore init and clone constants -func SkipEmptyBuild(p *pipeline.Build) string { - if len(p.Stages) == 1 { - if p.Stages[0].Name == "init" { - return "skipping build since only init stage found" - } - } - - if len(p.Stages) == 2 { - if p.Stages[0].Name == "init" && p.Stages[1].Name == "clone" { - return "skipping build since only init and clone stages found" - } - } - - if len(p.Steps) == 1 { - if p.Steps[0].Name == "init" { - return "skipping build since only init step found" - } - } - - if len(p.Steps) == 2 { - if p.Steps[0].Name == "init" && p.Steps[1].Name == "clone" { - return "skipping build since only init and clone steps found" - } - } - - return "" -} - -// swagger:operation GET /api/v1/search/builds/{id} builds GetBuildByID -// -// Get a single build by its id in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: id -// description: build id -// required: true -// type: number -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved build -// schema: -// "$ref": "#/definitions/Build" -// '400': -// description: Unable to retrieve the build -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to retrieve the build -// schema: -// "$ref": "#/definitions/Error" - -// GetBuildByID represents the API handler to capture a -// build by its id from the configured backend. -func GetBuildByID(c *gin.Context) { - // Variables that will hold the library types of the build and repo - var ( - b *library.Build - r *library.Repo - ) - - // Capture user from middleware - u := user.Retrieve(c) - - // Parse build ID from path - id, err := strconv.ParseInt(c.Param("id"), 10, 64) - - if err != nil { - retErr := fmt.Errorf("unable to parse build id: %w", err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": id, - "user": u.GetName(), - }).Infof("reading build %d", id) - - // Get build from database - b, err = database.FromContext(c).GetBuild(id) - if err != nil { - retErr := fmt.Errorf("unable to get build: %w", err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // Get repo from database using repo ID field from build - r, err = database.FromContext(c).GetRepo(b.GetRepoID()) - if err != nil { - retErr := fmt.Errorf("unable to get repo: %w", err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // Capture user access from SCM. We do this in order to ensure user has access and is not - // just retrieving any build using a random id number. - perm, err := scm.FromContext(c).RepoAccess(u, u.GetToken(), r.GetOrg(), r.GetName()) - if err != nil { - logrus.Errorf("unable to get user %s access level for repo %s", u.GetName(), r.GetFullName()) - } - - // Ensure that user has at least read access to repo to return the build - if perm == "none" && !u.GetAdmin() { - retErr := fmt.Errorf("unable to retrieve build %d: user does not have read access to repo %s", id, r.GetFullName()) - - util.HandleError(c, http.StatusUnauthorized, retErr) - - return - } - - c.JSON(http.StatusOK, b) -} - -// swagger:operation GET /api/v1/repos/{org}/{repo}/builds builds GetBuilds -// -// Get builds from the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: query -// name: event -// description: Filter by build event -// type: string -// enum: -// - push -// - pull_request -// - tag -// - deployment -// - comment -// - in: query -// name: commit -// description: Filter builds based on the commit hash -// type: string -// - in: query -// name: branch -// description: Filter builds by branch -// type: string -// - in: query -// name: status -// description: Filter by build status -// type: string -// enum: -// - canceled -// - error -// - failure -// - killed -// - pending -// - running -// - success -// - in: query -// name: page -// description: The page of results to retrieve -// type: integer -// default: 1 -// - in: query -// name: per_page -// description: How many results per page to return -// type: integer -// maximum: 100 -// default: 10 -// - in: query -// name: before -// description: filter builds created before a certain time -// type: integer -// default: 1 -// - in: query -// name: after -// description: filter builds created after a certain time -// type: integer -// default: 0 -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the builds -// schema: -// type: array -// items: -// "$ref": "#/definitions/Build" -// headers: -// X-Total-Count: -// description: Total number of results -// type: integer -// Link: -// description: see https://tools.ietf.org/html/rfc5988 -// type: string -// '400': -// description: Unable to retrieve the list of builds -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to retrieve the list of builds -// schema: -// "$ref": "#/definitions/Error" - -// GetBuilds represents the API handler to capture a -// list of builds for a repo from the configured backend. -func GetBuilds(c *gin.Context) { - // variables that will hold the build list, build list filters and total count - var ( - filters = map[string]interface{}{} - b []*library.Build - t int64 - ) - - // capture middleware values - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("reading builds for repo %s", r.GetFullName()) - - // capture the branch name parameter - branch := c.Query("branch") - // capture the event type parameter - event := c.Query("event") - // capture the status type parameter - status := c.Query("status") - // capture the commit hash parameter - commit := c.Query("commit") - - // check if branch filter was provided - if len(branch) > 0 { - // add branch to filters map - filters["branch"] = branch - } - // check if event filter was provided - if len(event) > 0 { - // verify the event provided is a valid event type - if event != constants.EventComment && event != constants.EventDeploy && - event != constants.EventPush && event != constants.EventPull && - event != constants.EventTag { - retErr := fmt.Errorf("unable to process event %s: invalid event type provided", event) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // add event to filters map - filters["event"] = event - } - // check if status filter was provided - if len(status) > 0 { - // verify the status provided is a valid status type - if status != constants.StatusCanceled && status != constants.StatusError && - status != constants.StatusFailure && status != constants.StatusKilled && - status != constants.StatusPending && status != constants.StatusRunning && - status != constants.StatusSuccess { - retErr := fmt.Errorf("unable to process status %s: invalid status type provided", status) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // add status to filters map - filters["status"] = status - } - - // check if commit hash filter was provided - if len(commit) > 0 { - // add commit to filters map - filters["commit"] = commit - } - - // capture page query parameter if present - page, err := strconv.Atoi(c.DefaultQuery("page", "1")) - if err != nil { - retErr := fmt.Errorf("unable to convert page query parameter for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // capture per_page query parameter if present - perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) - if err != nil { - retErr := fmt.Errorf("unable to convert per_page query parameter for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // ensure per_page isn't above or below allowed values - perPage = util.MaxInt(1, util.MinInt(100, perPage)) - - // capture before query parameter if present, default to now - before, err := strconv.ParseInt(c.DefaultQuery("before", strconv.FormatInt(time.Now().UTC().Unix(), 10)), 10, 64) - if err != nil { - retErr := fmt.Errorf("unable to convert before query parameter for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // capture after query parameter if present, default to 0 - after, err := strconv.ParseInt(c.DefaultQuery("after", "0"), 10, 64) - if err != nil { - retErr := fmt.Errorf("unable to convert after query parameter for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - b, t, err = database.FromContext(c).ListBuildsForRepo(r, filters, before, after, page, perPage) - if err != nil { - retErr := fmt.Errorf("unable to get builds for repo %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // create pagination object - pagination := Pagination{ - Page: page, - PerPage: perPage, - Total: t, - } - // set pagination headers - pagination.SetHeaderLink(c) - - c.JSON(http.StatusOK, b) -} - -// swagger:operation GET /api/v1/repos/{org} builds GetOrgBuilds -// -// Get a list of builds by org in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: query -// name: page -// description: The page of results to retrieve -// type: integer -// default: 1 -// - in: query -// name: per_page -// description: How many results per page to return -// type: integer -// maximum: 100 -// default: 10 -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved build list -// schema: -// type: array -// items: -// "$ref": "#/definitions/Build" -// headers: -// X-Total-Count: -// description: Total number of results -// type: integer -// Link: -// description: see https://tools.ietf.org/html/rfc5988 -// type: string -// '400': -// description: Unable to retrieve the list of builds -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to retrieve the list of builds -// schema: -// "$ref": "#/definitions/Error" - -// GetOrgBuilds represents the API handler to capture a -// list of builds associated with an org from the configured backend. -func GetOrgBuilds(c *gin.Context) { - // variables that will hold the build list, build list filters and total count - var ( - filters = map[string]interface{}{} - b []*library.Build - t int64 - ) - - // capture middleware values - o := org.Retrieve(c) - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "org": o, - "user": u.GetName(), - }).Infof("reading builds for org %s", o) - - // capture the branch name parameter - branch := c.Query("branch") - // capture the event type parameter - event := c.Query("event") - // capture the status type parameter - status := c.Query("status") - - // check if branch filter was provided - if len(branch) > 0 { - // add branch to filters map - filters["branch"] = branch - } - // check if event filter was provided - if len(event) > 0 { - // verify the event provided is a valid event type - if event != constants.EventComment && event != constants.EventDeploy && - event != constants.EventPush && event != constants.EventPull && - event != constants.EventTag { - retErr := fmt.Errorf("unable to process event %s: invalid event type provided", event) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // add event to filters map - filters["event"] = event - } - // check if status filter was provided - if len(status) > 0 { - // verify the status provided is a valid status type - if status != constants.StatusCanceled && status != constants.StatusError && - status != constants.StatusFailure && status != constants.StatusKilled && - status != constants.StatusPending && status != constants.StatusRunning && - status != constants.StatusSuccess { - retErr := fmt.Errorf("unable to process status %s: invalid status type provided", status) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // add status to filters map - filters["status"] = status - } - - // capture page query parameter if present - page, err := strconv.Atoi(c.DefaultQuery("page", "1")) - if err != nil { - retErr := fmt.Errorf("unable to convert page query parameter for org %s: %w", o, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // capture per_page query parameter if present - perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) - if err != nil { - retErr := fmt.Errorf("unable to convert per_page query parameter for Org %s: %w", o, err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // ensure per_page isn't above or below allowed values - perPage = util.MaxInt(1, util.MinInt(100, perPage)) - - // See if the user is an org admin to bypass individual permission checks - perm, err := scm.FromContext(c).OrgAccess(u, o) - if err != nil { - logrus.Errorf("unable to get user %s access level for org %s", u.GetName(), o) - } - // Only show public repos to non-admins - //nolint:goconst // ignore need for constant - if perm != "admin" { - filters["visibility"] = constants.VisibilityPublic - } - - // send API call to capture the list of builds for the org (and event type if passed in) - b, t, err = database.FromContext(c).ListBuildsForOrg(o, filters, page, perPage) - - if err != nil { - retErr := fmt.Errorf("unable to get builds for org %s: %w", o, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // create pagination object - pagination := Pagination{ - Page: page, - PerPage: perPage, - Total: t, - } - // set pagination headers - pagination.SetHeaderLink(c) - - c.JSON(http.StatusOK, b) -} - -// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build} builds GetBuild -// -// Get a build in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number to retrieve -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the build -// type: json -// schema: -// "$ref": "#/definitions/Build" - -// GetBuild represents the API handler to capture -// a build for a repo from the configured backend. -func GetBuild(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("reading build %s/%d", r.GetFullName(), b.GetNumber()) - - c.JSON(http.StatusOK, b) -} - -// swagger:operation POST /api/v1/repos/{org}/{repo}/builds/{build} builds RestartBuild -// -// Restart a build in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number to restart -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Request processed but build was skipped -// schema: -// type: string -// '201': -// description: Successfully restarted the build -// schema: -// "$ref": "#/definitions/Build" -// '400': -// description: Unable to restart the build -// schema: -// "$ref": "#/definitions/Error" -// '404': -// description: Unable to restart the build -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to restart the build -// schema: -// "$ref": "#/definitions/Error" - -// RestartBuild represents the API handler to restart an existing build in the configured backend. -// -//nolint:funlen // ignore statement count -func RestartBuild(c *gin.Context) { - // capture middleware values - m := c.MustGet("metadata").(*types.Metadata) - cl := claims.Retrieve(c) - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logger := logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }) - - logger.Infof("restarting build %s", entry) - - // send API call to capture the repo owner - u, err := database.FromContext(c).GetUser(r.GetUserID()) - if err != nil { - retErr := fmt.Errorf("unable to get owner for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // create SQL filters for querying pending and running builds for repo - filters := map[string]interface{}{ - "status": []string{constants.StatusPending, constants.StatusRunning}, - } - - // send API call to capture the number of pending or running builds for the repo - builds, err := database.FromContext(c).CountBuildsForRepo(r, filters) - if err != nil { - retErr := fmt.Errorf("unable to restart build: unable to get count of builds for repo %s", r.GetFullName()) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // check if the number of pending and running builds exceeds the limit for the repo - if builds >= r.GetBuildLimit() { - retErr := fmt.Errorf("unable to restart build: repo %s has exceeded the concurrent build limit of %d", r.GetFullName(), r.GetBuildLimit()) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // update fields in build object - b.SetID(0) - b.SetCreated(time.Now().UTC().Unix()) - b.SetEnqueued(0) - b.SetStarted(0) - b.SetFinished(0) - b.SetStatus(constants.StatusPending) - b.SetHost("") - b.SetRuntime("") - b.SetDistribution("") - b.SetSender(cl.Subject) - - // update the PR event action if action was never set - // for backwards compatibility with pre-0.14 releases. - if b.GetEvent() == constants.EventPull && b.GetEventAction() == "" { - // technically, the action could have been opened or synchronize. - // will not affect behavior of the pipeline since we did not - // support actions for builds where this would be the case. - b.SetEventAction(constants.ActionOpened) - } - - // set the parent equal to the restarted build number - b.SetParent(b.GetNumber()) - // update the build numbers based off repo counter - inc := r.GetCounter() + 1 - r.SetCounter(inc) - b.SetNumber(inc) - - // populate the build link if a web address is provided - if len(m.Vela.WebAddress) > 0 { - b.SetLink( - fmt.Sprintf("%s/%s/%d", m.Vela.WebAddress, r.GetFullName(), b.GetNumber()), - ) - } - - // variable to store changeset files - var files []string - // check if the build event is not issue_comment or pull_request - if !strings.EqualFold(b.GetEvent(), constants.EventComment) && - !strings.EqualFold(b.GetEvent(), constants.EventPull) { - // send API call to capture list of files changed for the commit - files, err = scm.FromContext(c).Changeset(u, r, b.GetCommit()) - if err != nil { - retErr := fmt.Errorf("unable to restart build: failed to get changeset for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - } - - // check if the build event is a pull_request - if strings.EqualFold(b.GetEvent(), constants.EventPull) { - // capture number from build - number, err := getPRNumberFromBuild(b) - if err != nil { - retErr := fmt.Errorf("unable to restart build: failed to get pull_request number for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture list of files changed for the pull request - files, err = scm.FromContext(c).ChangesetPR(u, r, number) - if err != nil { - retErr := fmt.Errorf("unable to restart build: failed to get changeset for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - } - - // variables to store pipeline configuration - var ( - // variable to store the raw pipeline configuration - config []byte - // variable to store executable pipeline - p *pipeline.Build - // variable to store pipeline configuration - pipeline *library.Pipeline - // variable to store the pipeline type for the repository - pipelineType = r.GetPipelineType() - ) - - // send API call to attempt to capture the pipeline - pipeline, err = database.FromContext(c).GetPipelineForRepo(b.GetCommit(), r) - if err != nil { // assume the pipeline doesn't exist in the database yet (before pipeline support was added) - // send API call to capture the pipeline configuration file - config, err = scm.FromContext(c).ConfigBackoff(u, r, b.GetCommit()) - if err != nil { - retErr := fmt.Errorf("unable to get pipeline configuration for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - } else { - config = pipeline.GetData() - } - - // ensure we use the expected pipeline type when compiling - // - // The pipeline type for a repo can change at any time which can break compiling - // existing pipelines in the system for that repo. To account for this, we update - // the repo pipeline type to match what was defined for the existing pipeline - // before compiling. After we're done compiling, we reset the pipeline type. - if len(pipeline.GetType()) > 0 { - r.SetPipelineType(pipeline.GetType()) - } - - var compiled *library.Pipeline - // parse and compile the pipeline configuration file - p, compiled, err = compiler.FromContext(c). - Duplicate(). - WithBuild(b). - WithCommit(b.GetCommit()). - WithFiles(files). - WithMetadata(m). - WithRepo(r). - WithUser(u). - Compile(config) - if err != nil { - retErr := fmt.Errorf("unable to compile pipeline configuration for %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - // reset the pipeline type for the repo - // - // The pipeline type for a repo can change at any time which can break compiling - // existing pipelines in the system for that repo. To account for this, we update - // the repo pipeline type to match what was defined for the existing pipeline - // before compiling. After we're done compiling, we reset the pipeline type. - r.SetPipelineType(pipelineType) - - // skip the build if only the init or clone steps are found - skip := SkipEmptyBuild(p) - if skip != "" { - // set build to successful status - b.SetStatus(constants.StatusSkipped) - - // send API call to set the status on the commit - err = scm.FromContext(c).Status(u, b, r.GetOrg(), r.GetName()) - if err != nil { - logrus.Errorf("unable to set commit status for %s/%d: %v", r.GetFullName(), b.GetNumber(), err) - } - - c.JSON(http.StatusOK, skip) - - return - } - - // check if the pipeline did not already exist in the database - // - //nolint:dupl // ignore duplicate code - if pipeline == nil { - pipeline = compiled - pipeline.SetRepoID(r.GetID()) - pipeline.SetCommit(b.GetCommit()) - pipeline.SetRef(b.GetRef()) - - // send API call to create the pipeline - err = database.FromContext(c).CreatePipeline(pipeline) - if err != nil { - retErr := fmt.Errorf("unable to create pipeline for %s: %w", r.GetFullName(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // send API call to capture the created pipeline - pipeline, err = database.FromContext(c).GetPipelineForRepo(pipeline.GetCommit(), r) - if err != nil { - //nolint:lll // ignore long line length due to error message - retErr := fmt.Errorf("unable to get new pipeline %s/%s: %w", r.GetFullName(), pipeline.GetCommit(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - } - - b.SetPipelineID(pipeline.GetID()) - - // create the objects from the pipeline in the database - err = PlanBuild(database.FromContext(c), p, b, r) - if err != nil { - util.HandleError(c, http.StatusInternalServerError, err) - - return - } - - // send API call to update repo for ensuring counter is incremented - err = database.FromContext(c).UpdateRepo(r) - if err != nil { - retErr := fmt.Errorf("unable to restart build: failed to update repo %s: %w", r.GetFullName(), err) - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // send API call to capture the restarted build - b, _ = database.FromContext(c).GetBuildForRepo(r, b.GetNumber()) - - c.JSON(http.StatusCreated, b) - - // send API call to set the status on the commit - err = scm.FromContext(c).Status(u, b, r.GetOrg(), r.GetName()) - if err != nil { - logger.Errorf("unable to set commit status for build %s: %v", entry, err) - } - - // publish the build to the queue - go PublishToQueue( - queue.FromGinContext(c), - database.FromContext(c), - p, - b, - r, - u, - ) -} - -// swagger:operation PUT /api/v1/repos/{org}/{repo}/builds/{build} builds UpdateBuild -// -// Updates a build in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number to update -// required: true -// type: integer -// - in: body -// name: body -// description: Payload containing the build to update -// required: true -// schema: -// "$ref": "#/definitions/Build" -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully updated the build -// schema: -// "$ref": "#/definitions/Build" -// '404': -// description: Unable to update the build -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to update the build -// schema: -// "$ref": "#/definitions/Error" - -// UpdateBuild represents the API handler to update -// a build for a repo in the configured backend. -func UpdateBuild(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("updating build %s", entry) - - // capture body from API request - input := new(library.Build) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for build %s: %w", entry, err) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - // update build fields if provided - if len(input.GetStatus()) > 0 { - // update status if set - b.SetStatus(input.GetStatus()) - } - - if len(input.GetError()) > 0 { - // update error if set - b.SetError(input.GetError()) - } - - if input.GetEnqueued() > 0 { - // update enqueued if set - b.SetEnqueued(input.GetEnqueued()) - } - - if input.GetStarted() > 0 { - // update started if set - b.SetStarted(input.GetStarted()) - } - - if input.GetFinished() > 0 { - // update finished if set - b.SetFinished(input.GetFinished()) - } - - if len(input.GetTitle()) > 0 { - // update title if set - b.SetTitle(input.GetTitle()) - } - - if len(input.GetMessage()) > 0 { - // update message if set - b.SetMessage(input.GetMessage()) - } - - if len(input.GetHost()) > 0 { - // update host if set - b.SetHost(input.GetHost()) - } - - if len(input.GetRuntime()) > 0 { - // update runtime if set - b.SetRuntime(input.GetRuntime()) - } - - if len(input.GetDistribution()) > 0 { - // update distribution if set - b.SetDistribution(input.GetDistribution()) - } - - // send API call to update the build - err = database.FromContext(c).UpdateBuild(b) - if err != nil { - retErr := fmt.Errorf("unable to update build %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the updated build - b, _ = database.FromContext(c).GetBuildForRepo(r, b.GetNumber()) - - c.JSON(http.StatusOK, b) - - // check if the build is in a "final" state - if b.GetStatus() == constants.StatusSuccess || - b.GetStatus() == constants.StatusFailure || - b.GetStatus() == constants.StatusCanceled || - b.GetStatus() == constants.StatusKilled || - b.GetStatus() == constants.StatusError { - // send API call to capture the repo owner - u, err := database.FromContext(c).GetUser(r.GetUserID()) - if err != nil { - logrus.Errorf("unable to get owner for build %s: %v", entry, err) - } - - // send API call to set the status on the commit - err = scm.FromContext(c).Status(u, b, r.GetOrg(), r.GetName()) - if err != nil { - logrus.Errorf("unable to set commit status for build %s: %v", entry, err) - } - } -} - -// swagger:operation DELETE /api/v1/repos/{org}/{repo}/builds/{build} builds DeleteBuild -// -// Delete a build in the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: build -// description: Build number to delete -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully deleted the build -// schema: -// type: string -// '400': -// description: Unable to delete the build -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to delete the build -// schema: -// "$ref": "#/definitions/Error" - -// DeleteBuild represents the API handler to remove -// a build for a repo from the configured backend. -func DeleteBuild(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("deleting build %s", entry) - - // send API call to remove the build - err := database.FromContext(c).DeleteBuild(b) - if err != nil { - retErr := fmt.Errorf("unable to delete build %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, fmt.Sprintf("build %s deleted", entry)) -} - -// getPRNumberFromBuild is a helper function to -// extract the pull request number from a Build. -func getPRNumberFromBuild(b *library.Build) (int, error) { - // parse out pull request number from base ref - // - // pattern: refs/pull/1/head - var parts []string - if strings.HasPrefix(b.GetRef(), "refs/pull/") { - parts = strings.Split(b.GetRef(), "/") - } - - // just being safe to avoid out of range index errors - if len(parts) < 3 { - return 0, fmt.Errorf("invalid ref: %s", b.GetRef()) - } - - // return the results of converting number to string - return strconv.Atoi(parts[2]) -} - -// PlanBuild is a helper function to plan the build for -// execution. This creates all resources, like steps -// and services, for the build in the configured backend. -// TODO: -// - return build and error. -func PlanBuild(database database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo) error { - // update fields in build object - b.SetCreated(time.Now().UTC().Unix()) - - // send API call to create the build - // TODO: return created build and error instead of just error - err := database.CreateBuild(b) - if err != nil { - // clean up the objects from the pipeline in the database - // TODO: - // - return build in CreateBuild - // - even if it was created, we need to get the new build id - // otherwise it will be 0, which attempts to INSERT instead - // of UPDATE-ing the existing build - which results in - // a constraint error (repo_id, number) - // - do we want to update the build or just delete it? - cleanBuild(database, b, nil, nil, err) - - return fmt.Errorf("unable to create new build for %s: %w", r.GetFullName(), err) - } - - // send API call to capture the created build - // TODO: this can be dropped once we return - // the created build above - b, err = database.GetBuildForRepo(r, b.GetNumber()) - if err != nil { - return fmt.Errorf("unable to get new build for %s: %w", r.GetFullName(), err) - } - - // plan all services for the build - services, err := planServices(database, p, b) - if err != nil { - // clean up the objects from the pipeline in the database - cleanBuild(database, b, services, nil, err) - - return err - } - - // plan all steps for the build - steps, err := planSteps(database, p, b) - if err != nil { - // clean up the objects from the pipeline in the database - cleanBuild(database, b, services, steps, err) - - return err - } - - return nil -} - -// planServices is a helper function to plan all services -// in the build for execution. This creates the services -// for the build in the configured backend. -func planServices(database database.Interface, p *pipeline.Build, b *library.Build) ([]*library.Service, error) { - // variable to store planned services - services := []*library.Service{} - - // iterate through all pipeline services - for _, service := range p.Services { - // create the service object - s := new(library.Service) - s.SetBuildID(b.GetID()) - s.SetRepoID(b.GetRepoID()) - s.SetName(service.Name) - s.SetImage(service.Image) - s.SetNumber(service.Number) - s.SetStatus(constants.StatusPending) - s.SetCreated(time.Now().UTC().Unix()) - - // send API call to create the service - err := database.CreateService(s) - if err != nil { - return services, fmt.Errorf("unable to create service %s: %w", s.GetName(), err) - } - - // send API call to capture the created service - s, err = database.GetServiceForBuild(b, s.GetNumber()) - if err != nil { - return services, fmt.Errorf("unable to get service %s: %w", s.GetName(), err) - } - - // populate environment variables from service library - // - // https://pkg.go.dev/github.com/go-vela/types/library#Service.Environment - err = service.MergeEnv(s.Environment()) - if err != nil { - return services, err - } - - // create the log object - l := new(library.Log) - l.SetServiceID(s.GetID()) - l.SetBuildID(b.GetID()) - l.SetRepoID(b.GetRepoID()) - l.SetData([]byte{}) - - // send API call to create the service logs - err = database.CreateLog(l) - if err != nil { - return services, fmt.Errorf("unable to create service logs for service %s: %w", s.GetName(), err) - } - } - - return services, nil -} - -// planSteps is a helper function to plan all steps -// in the build for execution. This creates the steps -// for the build in the configured backend. -func planSteps(database database.Interface, p *pipeline.Build, b *library.Build) ([]*library.Step, error) { - // variable to store planned steps - steps := []*library.Step{} - - // iterate through all pipeline stages - for _, stage := range p.Stages { - // iterate through all steps for each pipeline stage - for _, step := range stage.Steps { - // create the step object - s := new(library.Step) - s.SetBuildID(b.GetID()) - s.SetRepoID(b.GetRepoID()) - s.SetNumber(step.Number) - s.SetName(step.Name) - s.SetImage(step.Image) - s.SetStage(stage.Name) - s.SetStatus(constants.StatusPending) - s.SetCreated(time.Now().UTC().Unix()) - - // send API call to create the step - err := database.CreateStep(s) - if err != nil { - return steps, fmt.Errorf("unable to create step %s: %w", s.GetName(), err) - } - - // send API call to capture the created step - s, err = database.GetStepForBuild(b, s.GetNumber()) - if err != nil { - return steps, fmt.Errorf("unable to get step %s: %w", s.GetName(), err) - } - - // populate environment variables from step library - // - // https://pkg.go.dev/github.com/go-vela/types/library#step.Environment - err = step.MergeEnv(s.Environment()) - if err != nil { - return steps, err - } - - // create the log object - l := new(library.Log) - l.SetStepID(s.GetID()) - l.SetBuildID(b.GetID()) - l.SetRepoID(b.GetRepoID()) - l.SetData([]byte{}) - - // send API call to create the step logs - err = database.CreateLog(l) - if err != nil { - return nil, fmt.Errorf("unable to create logs for step %s: %w", s.GetName(), err) - } - - steps = append(steps, s) - } - } - - // iterate through all pipeline steps - for _, step := range p.Steps { - // create the step object - s := new(library.Step) - s.SetBuildID(b.GetID()) - s.SetRepoID(b.GetRepoID()) - s.SetNumber(step.Number) - s.SetName(step.Name) - s.SetImage(step.Image) - s.SetStatus(constants.StatusPending) - s.SetCreated(time.Now().UTC().Unix()) - - // send API call to create the step - err := database.CreateStep(s) - if err != nil { - return steps, fmt.Errorf("unable to create step %s: %w", s.GetName(), err) - } - - // send API call to capture the created step - s, err = database.GetStepForBuild(b, s.GetNumber()) - if err != nil { - return steps, fmt.Errorf("unable to get step %s: %w", s.GetName(), err) - } - - // populate environment variables from step library - // - // https://pkg.go.dev/github.com/go-vela/types/library#step.Environment - err = step.MergeEnv(s.Environment()) - if err != nil { - return steps, err - } - - // create the log object - l := new(library.Log) - l.SetStepID(s.GetID()) - l.SetBuildID(b.GetID()) - l.SetRepoID(b.GetRepoID()) - l.SetData([]byte{}) - - // send API call to create the step logs - err = database.CreateLog(l) - if err != nil { - return steps, fmt.Errorf("unable to create logs for step %s: %w", s.GetName(), err) - } - - steps = append(steps, s) - } - - return steps, nil -} - -// cleanBuild is a helper function to kill the build -// without execution. This will kill all resources, -// like steps and services, for the build in the -// configured backend. -func cleanBuild(database database.Interface, b *library.Build, services []*library.Service, steps []*library.Step, e error) { - // update fields in build object - b.SetError(fmt.Sprintf("unable to publish to queue: %s", e.Error())) - b.SetStatus(constants.StatusError) - b.SetFinished(time.Now().UTC().Unix()) - - // send API call to update the build - err := database.UpdateBuild(b) - if err != nil { - logrus.Errorf("unable to kill build %d: %v", b.GetNumber(), err) - } - - for _, s := range services { - // update fields in service object - s.SetStatus(constants.StatusKilled) - s.SetFinished(time.Now().UTC().Unix()) - - // send API call to update the service - err := database.UpdateService(s) - if err != nil { - logrus.Errorf("unable to kill service %s for build %d: %v", s.GetName(), b.GetNumber(), err) - } - } - - for _, s := range steps { - // update fields in step object - s.SetStatus(constants.StatusKilled) - s.SetFinished(time.Now().UTC().Unix()) - - // send API call to update the step - err := database.UpdateStep(s) - if err != nil { - logrus.Errorf("unable to kill step %s for build %d: %v", s.GetName(), b.GetNumber(), err) - } - } -} - -// swagger:operation DELETE /api/v1/repos/{org}/{repo}/builds/{build}/cancel builds CancelBuild -// -// Cancel a running build -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: build -// description: Build number to cancel -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully canceled the build -// schema: -// type: string -// '400': -// description: Unable to cancel build -// schema: -// "$ref": "#/definitions/Error" -// '404': -// description: Unable to cancel build -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to cancel build -// schema: -// "$ref": "#/definitions/Error" - -// CancelBuild represents the API handler to cancel a running build. -// -//nolint:funlen // ignore statement count -func CancelBuild(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - e := executors.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - u := user.Retrieve(c) - - entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "user": u.GetName(), - }).Infof("canceling build %s", entry) - - switch b.GetStatus() { - case constants.StatusRunning: - // retrieve the worker info - w, err := database.FromContext(c).GetWorkerForHostname(b.GetHost()) - if err != nil { - retErr := fmt.Errorf("unable to get worker for build %s: %w", entry, err) - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - for _, executor := range e { - // check each executor on the worker running the build to see if it's running the build we want to cancel - if strings.EqualFold(executor.Repo.GetFullName(), r.GetFullName()) && *executor.GetBuild().Number == b.GetNumber() { - // prepare the request to the worker - client := http.DefaultClient - client.Timeout = 30 * time.Second - - // set the API endpoint path we send the request to - u := fmt.Sprintf("%s/api/v1/executors/%d/build/cancel", w.GetAddress(), executor.GetID()) - - req, err := http.NewRequestWithContext(context.Background(), "DELETE", u, nil) - if err != nil { - retErr := fmt.Errorf("unable to form a request to %s: %w", u, err) - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - tm := c.MustGet("token-manager").(*token.Manager) - - // set mint token options - mto := &token.MintTokenOpts{ - Hostname: "vela-server", - TokenType: constants.WorkerAuthTokenType, - TokenDuration: time.Minute * 1, - } - - // mint token - tkn, err := tm.MintToken(mto) - if err != nil { - retErr := fmt.Errorf("unable to generate auth token: %w", err) - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // add the token to authenticate to the worker - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tkn)) - - // perform the request to the worker - resp, err := client.Do(req) - if err != nil { - retErr := fmt.Errorf("unable to connect to %s: %w", u, err) - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - defer resp.Body.Close() - - // Read Response Body - respBody, err := io.ReadAll(resp.Body) - if err != nil { - retErr := fmt.Errorf("unable to read response from %s: %w", u, err) - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - err = json.Unmarshal(respBody, b) - if err != nil { - retErr := fmt.Errorf("unable to parse response from %s: %w", u, err) - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - c.JSON(resp.StatusCode, b) - - return - } - } - case constants.StatusPending: - break - - default: - retErr := fmt.Errorf("found build %s but its status was %s", entry, b.GetStatus()) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // build has been abandoned - // update the status in the build table - b.SetStatus(constants.StatusCanceled) - - err := database.FromContext(c).UpdateBuild(b) - if err != nil { - retErr := fmt.Errorf("unable to update status for build %s: %w", entry, err) - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // retrieve the steps for the build from the step table - steps := []*library.Step{} - page := 1 - perPage := 100 - - for page > 0 { - // retrieve build steps (per page) from the database - stepsPart, _, err := database.FromContext(c).ListStepsForBuild(b, map[string]interface{}{}, page, perPage) - if err != nil { - retErr := fmt.Errorf("unable to retrieve steps for build %s: %w", entry, err) - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - // add page of steps to list steps - steps = append(steps, stepsPart...) - - // assume no more pages exist if under 100 results are returned - if len(stepsPart) < 100 { - page = 0 - } else { - page++ - } - } - - // iterate over each step for the build - // setting anything running or pending to canceled - for _, step := range steps { - if step.GetStatus() == constants.StatusRunning || step.GetStatus() == constants.StatusPending { - step.SetStatus(constants.StatusCanceled) - - err = database.FromContext(c).UpdateStep(step) - if err != nil { - retErr := fmt.Errorf("unable to update step %s for build %s: %w", step.GetName(), entry, err) - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - } - } - - // retrieve the services for the build from the service table - services := []*library.Service{} - page = 1 - - for page > 0 { - // retrieve build services (per page) from the database - servicesPart, _, err := database.FromContext(c).ListServicesForBuild(b, map[string]interface{}{}, page, perPage) - if err != nil { - retErr := fmt.Errorf("unable to retrieve services for build %s: %w", entry, err) - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - // add page of services to the list of services - services = append(services, servicesPart...) - - // assume no more pages exist if under 100 results are returned - if len(servicesPart) < 100 { - page = 0 - } else { - page++ - } - } - - // iterate over each service for the build - // setting anything running or pending to canceled - for _, service := range services { - if service.GetStatus() == constants.StatusRunning || service.GetStatus() == constants.StatusPending { - service.SetStatus(constants.StatusCanceled) - - err = database.FromContext(c).UpdateService(service) - if err != nil { - retErr := fmt.Errorf("unable to update service %s for build %s: %w", - service.GetName(), - entry, - err, - ) - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - } - } - - c.JSON(http.StatusOK, b) -} - -// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/token builds GetBuildToken -// -// Get a build token -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: repo -// description: Name of the repo -// required: true -// type: string -// - in: path -// name: org -// description: Name of the org -// required: true -// type: string -// - in: path -// name: build -// description: Build number -// required: true -// type: integer -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved build token -// schema: -// "$ref": "#/definitions/Token" -// '400': -// description: Bad request -// schema: -// "$ref": "#/definitions/Error" -// '409': -// description: Conflict (requested build token for build not in pending state) -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to generate build token -// schema: -// "$ref": "#/definitions/Error" - -// GetBuildToken represents the API handler to generate a build token. -func GetBuildToken(c *gin.Context) { - // capture middleware values - b := build.Retrieve(c) - o := org.Retrieve(c) - r := repo.Retrieve(c) - cl := claims.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "build": b.GetNumber(), - "org": o, - "repo": r.GetName(), - "user": cl.Subject, - }).Infof("generating build token for build %s/%d", r.GetFullName(), b.GetNumber()) - - // if build is not in a pending state, then a build token should not be needed - conflict - if !strings.EqualFold(b.GetStatus(), constants.StatusPending) { - retErr := fmt.Errorf("unable to mint build token: build is not in pending state") - util.HandleError(c, http.StatusConflict, retErr) - - return - } - - // retrieve token manager from context - tm := c.MustGet("token-manager").(*token.Manager) - - // set expiration to repo timeout plus configurable buffer - exp := (time.Duration(r.GetTimeout()) * time.Minute) + tm.BuildTokenBufferDuration - - // set mint token options - bmto := &token.MintTokenOpts{ - Hostname: cl.Subject, - BuildID: b.GetID(), - Repo: r.GetFullName(), - TokenType: constants.WorkerBuildTokenType, - TokenDuration: exp, - } - - // mint token - bt, err := tm.MintToken(bmto) - if err != nil { - retErr := fmt.Errorf("unable to generate build token: %w", err) - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, library.Token{Token: &bt}) -} diff --git a/api/build/cancel.go b/api/build/cancel.go new file mode 100644 index 000000000..ff63a698d --- /dev/null +++ b/api/build/cancel.go @@ -0,0 +1,289 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/internal/token" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/executors" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation DELETE /api/v1/repos/{org}/{repo}/builds/{build}/cancel builds CancelBuild +// +// Cancel a running build +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: build +// description: Build number to cancel +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully canceled the build +// schema: +// type: string +// '400': +// description: Unable to cancel build +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: Unable to cancel build +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to cancel build +// schema: +// "$ref": "#/definitions/Error" + +// CancelBuild represents the API handler to cancel a running build. +// +//nolint:funlen // ignore statement count +func CancelBuild(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + e := executors.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("canceling build %s", entry) + + switch b.GetStatus() { + case constants.StatusRunning: + // retrieve the worker info + w, err := database.FromContext(c).GetWorkerForHostname(b.GetHost()) + if err != nil { + retErr := fmt.Errorf("unable to get worker for build %s: %w", entry, err) + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + for _, executor := range e { + // check each executor on the worker running the build to see if it's running the build we want to cancel + if strings.EqualFold(executor.Repo.GetFullName(), r.GetFullName()) && *executor.GetBuild().Number == b.GetNumber() { + // prepare the request to the worker + client := http.DefaultClient + client.Timeout = 30 * time.Second + + // set the API endpoint path we send the request to + u := fmt.Sprintf("%s/api/v1/executors/%d/build/cancel", w.GetAddress(), executor.GetID()) + + req, err := http.NewRequestWithContext(context.Background(), "DELETE", u, nil) + if err != nil { + retErr := fmt.Errorf("unable to form a request to %s: %w", u, err) + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + tm := c.MustGet("token-manager").(*token.Manager) + + // set mint token options + mto := &token.MintTokenOpts{ + Hostname: "vela-server", + TokenType: constants.WorkerAuthTokenType, + TokenDuration: time.Minute * 1, + } + + // mint token + tkn, err := tm.MintToken(mto) + if err != nil { + retErr := fmt.Errorf("unable to generate auth token: %w", err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // add the token to authenticate to the worker + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tkn)) + + // perform the request to the worker + resp, err := client.Do(req) + if err != nil { + retErr := fmt.Errorf("unable to connect to %s: %w", u, err) + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + defer resp.Body.Close() + + // Read Response Body + respBody, err := io.ReadAll(resp.Body) + if err != nil { + retErr := fmt.Errorf("unable to read response from %s: %w", u, err) + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + err = json.Unmarshal(respBody, b) + if err != nil { + retErr := fmt.Errorf("unable to parse response from %s: %w", u, err) + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + c.JSON(resp.StatusCode, b) + + return + } + } + case constants.StatusPending: + break + + default: + retErr := fmt.Errorf("found build %s but its status was %s", entry, b.GetStatus()) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // build has been abandoned + // update the status in the build table + b.SetStatus(constants.StatusCanceled) + + err := database.FromContext(c).UpdateBuild(b) + if err != nil { + retErr := fmt.Errorf("unable to update status for build %s: %w", entry, err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // retrieve the steps for the build from the step table + steps := []*library.Step{} + page := 1 + perPage := 100 + + for page > 0 { + // retrieve build steps (per page) from the database + stepsPart, _, err := database.FromContext(c).ListStepsForBuild(b, map[string]interface{}{}, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to retrieve steps for build %s: %w", entry, err) + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // add page of steps to list steps + steps = append(steps, stepsPart...) + + // assume no more pages exist if under 100 results are returned + if len(stepsPart) < 100 { + page = 0 + } else { + page++ + } + } + + // iterate over each step for the build + // setting anything running or pending to canceled + for _, step := range steps { + if step.GetStatus() == constants.StatusRunning || step.GetStatus() == constants.StatusPending { + step.SetStatus(constants.StatusCanceled) + + err = database.FromContext(c).UpdateStep(step) + if err != nil { + retErr := fmt.Errorf("unable to update step %s for build %s: %w", step.GetName(), entry, err) + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + } + } + + // retrieve the services for the build from the service table + services := []*library.Service{} + page = 1 + + for page > 0 { + // retrieve build services (per page) from the database + servicesPart, _, err := database.FromContext(c).ListServicesForBuild(b, map[string]interface{}{}, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to retrieve services for build %s: %w", entry, err) + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // add page of services to the list of services + services = append(services, servicesPart...) + + // assume no more pages exist if under 100 results are returned + if len(servicesPart) < 100 { + page = 0 + } else { + page++ + } + } + + // iterate over each service for the build + // setting anything running or pending to canceled + for _, service := range services { + if service.GetStatus() == constants.StatusRunning || service.GetStatus() == constants.StatusPending { + service.SetStatus(constants.StatusCanceled) + + err = database.FromContext(c).UpdateService(service) + if err != nil { + retErr := fmt.Errorf("unable to update service %s for build %s: %w", + service.GetName(), + entry, + err, + ) + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + } + } + + c.JSON(http.StatusOK, b) +} diff --git a/api/build/clean.go b/api/build/clean.go new file mode 100644 index 000000000..02c37550c --- /dev/null +++ b/api/build/clean.go @@ -0,0 +1,56 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "fmt" + "time" + + "github.com/go-vela/server/database" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// cleanBuild is a helper function to kill the build +// without execution. This will kill all resources, +// like steps and services, for the build in the +// configured backend. +func CleanBuild(database database.Interface, b *library.Build, services []*library.Service, steps []*library.Step, e error) { + // update fields in build object + b.SetError(fmt.Sprintf("unable to publish to queue: %s", e.Error())) + b.SetStatus(constants.StatusError) + b.SetFinished(time.Now().UTC().Unix()) + + // send API call to update the build + err := database.UpdateBuild(b) + if err != nil { + logrus.Errorf("unable to kill build %d: %v", b.GetNumber(), err) + } + + for _, s := range services { + // update fields in service object + s.SetStatus(constants.StatusKilled) + s.SetFinished(time.Now().UTC().Unix()) + + // send API call to update the service + err := database.UpdateService(s) + if err != nil { + logrus.Errorf("unable to kill service %s for build %d: %v", s.GetName(), b.GetNumber(), err) + } + } + + for _, s := range steps { + // update fields in step object + s.SetStatus(constants.StatusKilled) + s.SetFinished(time.Now().UTC().Unix()) + + // send API call to update the step + err := database.UpdateStep(s) + if err != nil { + logrus.Errorf("unable to kill step %s for build %d: %v", s.GetName(), b.GetNumber(), err) + } + } +} diff --git a/api/build/create.go b/api/build/create.go new file mode 100644 index 000000000..3f695d791 --- /dev/null +++ b/api/build/create.go @@ -0,0 +1,392 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/compiler" + "github.com/go-vela/server/database" + "github.com/go-vela/server/queue" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/go-vela/types" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + "github.com/sirupsen/logrus" +) + +// swagger:operation POST /api/v1/repos/{org}/{repo}/builds builds CreateBuild +// +// Create a build in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: body +// name: body +// description: Payload containing the build to create +// required: true +// schema: +// "$ref": "#/definitions/Build" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Request processed but build was skipped +// schema: +// type: string +// '201': +// description: Successfully created the build +// type: json +// schema: +// "$ref": "#/definitions/Build" +// '400': +// description: Unable to create the build +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: Unable to create the build +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to create the build +// schema: +// "$ref": "#/definitions/Error" + +// CreateBuild represents the API handler to create a build in the configured backend. +// +//nolint:funlen,gocyclo // ignore function length and cyclomatic complexity +func CreateBuild(c *gin.Context) { + // capture middleware values + m := c.MustGet("metadata").(*types.Metadata) + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logger := logrus.WithFields(logrus.Fields{ + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }) + + logger.Infof("creating new build for repo %s", r.GetFullName()) + + // capture body from API request + input := new(library.Build) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for new build for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // verify the build has a valid event and the repo allows that event type + if (input.GetEvent() == constants.EventPush && !r.GetAllowPush()) || + (input.GetEvent() == constants.EventPull && !r.GetAllowPull()) || + (input.GetEvent() == constants.EventTag && !r.GetAllowTag()) || + (input.GetEvent() == constants.EventDeploy && !r.GetAllowDeploy()) { + retErr := fmt.Errorf("unable to create new build: %s does not have %s events enabled", r.GetFullName(), input.GetEvent()) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to capture the repo owner + u, err = database.FromContext(c).GetUser(r.GetUserID()) + if err != nil { + retErr := fmt.Errorf("unable to get owner for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // create SQL filters for querying pending and running builds for repo + filters := map[string]interface{}{ + "status": []string{constants.StatusPending, constants.StatusRunning}, + } + + // send API call to capture the number of pending or running builds for the repo + builds, err := database.FromContext(c).CountBuildsForRepo(r, filters) + if err != nil { + retErr := fmt.Errorf("unable to create new build: unable to get count of builds for repo %s", r.GetFullName()) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // check if the number of pending and running builds exceeds the limit for the repo + if builds >= r.GetBuildLimit() { + retErr := fmt.Errorf("unable to create new build: repo %s has exceeded the concurrent build limit of %d", r.GetFullName(), r.GetBuildLimit()) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update fields in build object + input.SetRepoID(r.GetID()) + input.SetStatus(constants.StatusPending) + input.SetCreated(time.Now().UTC().Unix()) + + // set the parent equal to the current repo counter + input.SetParent(r.GetCounter()) + // check if the parent is set to 0 + if input.GetParent() == 0 { + // parent should be "1" if it's the first build ran + input.SetParent(1) + } + + // update the build numbers based off repo counter + inc := r.GetCounter() + 1 + r.SetCounter(inc) + input.SetNumber(inc) + + // populate the build link if a web address is provided + if len(m.Vela.WebAddress) > 0 { + input.SetLink( + fmt.Sprintf("%s/%s/%d", m.Vela.WebAddress, r.GetFullName(), input.GetNumber()), + ) + } + + // variable to store changeset files + var files []string + // check if the build event is not issue_comment or pull_request + if !strings.EqualFold(input.GetEvent(), constants.EventComment) && + !strings.EqualFold(input.GetEvent(), constants.EventPull) { + // send API call to capture list of files changed for the commit + files, err = scm.FromContext(c).Changeset(u, r, input.GetCommit()) + if err != nil { + retErr := fmt.Errorf("unable to create new build: failed to get changeset for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + } + + // check if the build event is a pull_request + if strings.EqualFold(input.GetEvent(), constants.EventPull) { + // capture number from build + number, err := getPRNumberFromBuild(input) + if err != nil { + retErr := fmt.Errorf("unable to create new build: failed to get pull_request number for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture list of files changed for the pull request + files, err = scm.FromContext(c).ChangesetPR(u, r, number) + if err != nil { + retErr := fmt.Errorf("unable to create new build: failed to get changeset for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + } + + var ( + // variable to store the raw pipeline configuration + config []byte + // variable to store executable pipeline + p *pipeline.Build + // variable to store pipeline configuration + pipeline *library.Pipeline + // variable to store the pipeline type for the repository + pipelineType = r.GetPipelineType() + ) + + // send API call to attempt to capture the pipeline + pipeline, err = database.FromContext(c).GetPipelineForRepo(input.GetCommit(), r) + if err != nil { // assume the pipeline doesn't exist in the database yet + // send API call to capture the pipeline configuration file + config, err = scm.FromContext(c).ConfigBackoff(u, r, input.GetCommit()) + if err != nil { + retErr := fmt.Errorf("unable to create new build: failed to get pipeline configuration for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + } else { + config = pipeline.GetData() + } + + // ensure we use the expected pipeline type when compiling + // + // The pipeline type for a repo can change at any time which can break compiling + // existing pipelines in the system for that repo. To account for this, we update + // the repo pipeline type to match what was defined for the existing pipeline + // before compiling. After we're done compiling, we reset the pipeline type. + if len(pipeline.GetType()) > 0 { + r.SetPipelineType(pipeline.GetType()) + } + + var compiled *library.Pipeline + // parse and compile the pipeline configuration file + p, compiled, err = compiler.FromContext(c). + Duplicate(). + WithBuild(input). + WithFiles(files). + WithMetadata(m). + WithRepo(r). + WithUser(u). + Compile(config) + if err != nil { + retErr := fmt.Errorf("unable to compile pipeline configuration for %s/%d: %w", r.GetFullName(), input.GetNumber(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + // reset the pipeline type for the repo + // + // The pipeline type for a repo can change at any time which can break compiling + // existing pipelines in the system for that repo. To account for this, we update + // the repo pipeline type to match what was defined for the existing pipeline + // before compiling. After we're done compiling, we reset the pipeline type. + r.SetPipelineType(pipelineType) + + // skip the build if only the init or clone steps are found + skip := SkipEmptyBuild(p) + if skip != "" { + // set build to successful status + input.SetStatus(constants.StatusSuccess) + + // send API call to set the status on the commit + err = scm.FromContext(c).Status(u, input, r.GetOrg(), r.GetName()) + if err != nil { + logger.Errorf("unable to set commit status for %s/%d: %v", r.GetFullName(), input.GetNumber(), err) + } + + c.JSON(http.StatusOK, skip) + + return + } + + // check if the pipeline did not already exist in the database + // + //nolint:dupl // ignore duplicate code + if pipeline == nil { + pipeline = compiled + pipeline.SetRepoID(r.GetID()) + pipeline.SetCommit(input.GetCommit()) + pipeline.SetRef(input.GetRef()) + + // send API call to create the pipeline + err = database.FromContext(c).CreatePipeline(pipeline) + if err != nil { + retErr := fmt.Errorf("unable to create new build: failed to create pipeline for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to capture the created pipeline + pipeline, err = database.FromContext(c).GetPipelineForRepo(pipeline.GetCommit(), r) + if err != nil { + //nolint:lll // ignore long line length due to error message + retErr := fmt.Errorf("unable to create new build: failed to get new pipeline %s/%s: %w", r.GetFullName(), pipeline.GetCommit(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + } + + input.SetPipelineID(pipeline.GetID()) + + // create the objects from the pipeline in the database + err = PlanBuild(database.FromContext(c), p, input, r) + if err != nil { + util.HandleError(c, http.StatusInternalServerError, err) + + return + } + + // send API call to update repo for ensuring counter is incremented + err = database.FromContext(c).UpdateRepo(r) + if err != nil { + retErr := fmt.Errorf("unable to create new build: failed to update repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to capture the created build + input, _ = database.FromContext(c).GetBuildForRepo(r, input.GetNumber()) + + c.JSON(http.StatusCreated, input) + + // send API call to set the status on the commit + err = scm.FromContext(c).Status(u, input, r.GetOrg(), r.GetName()) + if err != nil { + logger.Errorf("unable to set commit status for build %s/%d: %v", r.GetFullName(), input.GetNumber(), err) + } + + // publish the build to the queue + go PublishToQueue( + queue.FromGinContext(c), + database.FromContext(c), + p, + input, + r, + u, + ) +} + +// getPRNumberFromBuild is a helper function to +// extract the pull request number from a Build. +func getPRNumberFromBuild(b *library.Build) (int, error) { + // parse out pull request number from base ref + // + // pattern: refs/pull/1/head + var parts []string + if strings.HasPrefix(b.GetRef(), "refs/pull/") { + parts = strings.Split(b.GetRef(), "/") + } + + // just being safe to avoid out of range index errors + if len(parts) < 3 { + return 0, fmt.Errorf("invalid ref: %s", b.GetRef()) + } + + // return the results of converting number to string + return strconv.Atoi(parts[2]) +} diff --git a/api/build/delete.go b/api/build/delete.go new file mode 100644 index 000000000..19a1fd17c --- /dev/null +++ b/api/build/delete.go @@ -0,0 +1,92 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/sirupsen/logrus" +) + +// swagger:operation DELETE /api/v1/repos/{org}/{repo}/builds/{build} builds DeleteBuild +// +// Delete a build in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number to delete +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully deleted the build +// schema: +// type: string +// '400': +// description: Unable to delete the build +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to delete the build +// schema: +// "$ref": "#/definitions/Error" + +// DeleteBuild represents the API handler to remove +// a build for a repo from the configured backend. +func DeleteBuild(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("deleting build %s", entry) + + // send API call to remove the build + err := database.FromContext(c).DeleteBuild(b) + if err != nil { + retErr := fmt.Errorf("unable to delete build %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, fmt.Sprintf("build %s deleted", entry)) +} diff --git a/api/build/doc.go b/api/build/doc.go new file mode 100644 index 000000000..94b6571e3 --- /dev/null +++ b/api/build/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package build provides the build handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/build" +package build diff --git a/api/build/get.go b/api/build/get.go new file mode 100644 index 000000000..7e476441e --- /dev/null +++ b/api/build/get.go @@ -0,0 +1,70 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build} builds GetBuild +// +// Get a build in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number to retrieve +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the build +// type: json +// schema: +// "$ref": "#/definitions/Build" + +// GetBuild represents the API handler to capture +// a build for a repo from the configured backend. +func GetBuild(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("reading build %s/%d", r.GetFullName(), b.GetNumber()) + + c.JSON(http.StatusOK, b) +} diff --git a/api/build/get_id.go b/api/build/get_id.go new file mode 100644 index 000000000..47fb18874 --- /dev/null +++ b/api/build/get_id.go @@ -0,0 +1,118 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/search/builds/{id} builds GetBuildByID +// +// Get a single build by its id in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: id +// description: build id +// required: true +// type: number +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved build +// schema: +// "$ref": "#/definitions/Build" +// '400': +// description: Unable to retrieve the build +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to retrieve the build +// schema: +// "$ref": "#/definitions/Error" + +// GetBuildByID represents the API handler to capture a +// build by its id from the configured backend. +func GetBuildByID(c *gin.Context) { + // Variables that will hold the library types of the build and repo + var ( + b *library.Build + r *library.Repo + ) + + // Capture user from middleware + u := user.Retrieve(c) + + // Parse build ID from path + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + + if err != nil { + retErr := fmt.Errorf("unable to parse build id: %w", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": id, + "user": u.GetName(), + }).Infof("reading build %d", id) + + // Get build from database + b, err = database.FromContext(c).GetBuild(id) + if err != nil { + retErr := fmt.Errorf("unable to get build: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // Get repo from database using repo ID field from build + r, err = database.FromContext(c).GetRepo(b.GetRepoID()) + if err != nil { + retErr := fmt.Errorf("unable to get repo: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // Capture user access from SCM. We do this in order to ensure user has access and is not + // just retrieving any build using a random id number. + perm, err := scm.FromContext(c).RepoAccess(u, u.GetToken(), r.GetOrg(), r.GetName()) + if err != nil { + logrus.Errorf("unable to get user %s access level for repo %s", u.GetName(), r.GetFullName()) + } + + // Ensure that user has at least read access to repo to return the build + if perm == "none" && !u.GetAdmin() { + retErr := fmt.Errorf("unable to retrieve build %d: user does not have read access to repo", id) + + util.HandleError(c, http.StatusUnauthorized, retErr) + + return + } + + c.JSON(http.StatusOK, b) +} diff --git a/api/build/list_org.go b/api/build/list_org.go new file mode 100644 index 000000000..1ccd8b45b --- /dev/null +++ b/api/build/list_org.go @@ -0,0 +1,195 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/api" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/repos/{org}/builds builds ListBuildsForOrg +// +// Get a list of builds by org in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: query +// name: page +// description: The page of results to retrieve +// type: integer +// default: 1 +// - in: query +// name: per_page +// description: How many results per page to return +// type: integer +// maximum: 100 +// default: 10 +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved build list +// schema: +// type: array +// items: +// "$ref": "#/definitions/Build" +// headers: +// X-Total-Count: +// description: Total number of results +// type: integer +// Link: +// description: see https://tools.ietf.org/html/rfc5988 +// type: string +// '400': +// description: Unable to retrieve the list of builds +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to retrieve the list of builds +// schema: +// "$ref": "#/definitions/Error" + +// ListBuildsForOrg represents the API handler to capture a +// list of builds associated with an org from the configured backend. +func ListBuildsForOrg(c *gin.Context) { + // variables that will hold the build list, build list filters and total count + var ( + filters = map[string]interface{}{} + b []*library.Build + t int64 + ) + + // capture middleware values + o := org.Retrieve(c) + u := user.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "user": u.GetName(), + }).Infof("listing builds for org %s", o) + + // capture the branch name parameter + branch := c.Query("branch") + // capture the event type parameter + event := c.Query("event") + // capture the status type parameter + status := c.Query("status") + + // check if branch filter was provided + if len(branch) > 0 { + // add branch to filters map + filters["branch"] = branch + } + // check if event filter was provided + if len(event) > 0 { + // verify the event provided is a valid event type + if event != constants.EventComment && event != constants.EventDeploy && + event != constants.EventPush && event != constants.EventPull && + event != constants.EventTag { + retErr := fmt.Errorf("unable to process event %s: invalid event type provided", event) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // add event to filters map + filters["event"] = event + } + // check if status filter was provided + if len(status) > 0 { + // verify the status provided is a valid status type + if status != constants.StatusCanceled && status != constants.StatusError && + status != constants.StatusFailure && status != constants.StatusKilled && + status != constants.StatusPending && status != constants.StatusRunning && + status != constants.StatusSuccess { + retErr := fmt.Errorf("unable to process status %s: invalid status type provided", status) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // add status to filters map + filters["status"] = status + } + + // capture page query parameter if present + page, err := strconv.Atoi(c.DefaultQuery("page", "1")) + if err != nil { + retErr := fmt.Errorf("unable to convert page query parameter for org %s: %w", o, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // capture per_page query parameter if present + perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) + if err != nil { + retErr := fmt.Errorf("unable to convert per_page query parameter for Org %s: %w", o, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // ensure per_page isn't above or below allowed values + perPage = util.MaxInt(1, util.MinInt(100, perPage)) + + // See if the user is an org admin to bypass individual permission checks + perm, err := scm.FromContext(c).OrgAccess(u, o) + if err != nil { + logrus.Errorf("unable to get user %s access level for org %s", u.GetName(), o) + } + // Only show public repos to non-admins + if perm != "admin" { + filters["visibility"] = constants.VisibilityPublic + } + + // send API call to capture the list of builds for the org (and event type if passed in) + b, t, err = database.FromContext(c).ListBuildsForOrg(o, filters, page, perPage) + + if err != nil { + retErr := fmt.Errorf("unable to list builds for org %s: %w", o, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // create pagination object + pagination := api.Pagination{ + Page: page, + PerPage: perPage, + Total: t, + } + // set pagination headers + pagination.SetHeaderLink(c) + + c.JSON(http.StatusOK, b) +} diff --git a/api/build/list_repo.go b/api/build/list_repo.go new file mode 100644 index 000000000..2dd71fc1a --- /dev/null +++ b/api/build/list_repo.go @@ -0,0 +1,259 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "fmt" + "net/http" + "strconv" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/api" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds builds ListBuildsForRepo +// +// Get builds from the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: query +// name: event +// description: Filter by build event +// type: string +// enum: +// - push +// - pull_request +// - tag +// - deployment +// - comment +// - in: query +// name: commit +// description: Filter builds based on the commit hash +// type: string +// - in: query +// name: branch +// description: Filter builds by branch +// type: string +// - in: query +// name: status +// description: Filter by build status +// type: string +// enum: +// - canceled +// - error +// - failure +// - killed +// - pending +// - running +// - success +// - in: query +// name: page +// description: The page of results to retrieve +// type: integer +// default: 1 +// - in: query +// name: per_page +// description: How many results per page to return +// type: integer +// maximum: 100 +// default: 10 +// - in: query +// name: before +// description: filter builds created before a certain time +// type: integer +// default: 1 +// - in: query +// name: after +// description: filter builds created after a certain time +// type: integer +// default: 0 +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved the builds +// schema: +// type: array +// items: +// "$ref": "#/definitions/Build" +// headers: +// X-Total-Count: +// description: Total number of results +// type: integer +// Link: +// description: see https://tools.ietf.org/html/rfc5988 +// type: string +// '400': +// description: Unable to retrieve the list of builds +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to retrieve the list of builds +// schema: +// "$ref": "#/definitions/Error" + +// ListBuildsForRepo represents the API handler to capture a +// list of builds for a repo from the configured backend. +func ListBuildsForRepo(c *gin.Context) { + // variables that will hold the build list, build list filters and total count + var ( + filters = map[string]interface{}{} + b []*library.Build + t int64 + ) + + // capture middleware values + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("listing builds for repo %s", r.GetFullName()) + + // capture the branch name parameter + branch := c.Query("branch") + // capture the event type parameter + event := c.Query("event") + // capture the status type parameter + status := c.Query("status") + // capture the commit hash parameter + commit := c.Query("commit") + + // check if branch filter was provided + if len(branch) > 0 { + // add branch to filters map + filters["branch"] = branch + } + // check if event filter was provided + if len(event) > 0 { + // verify the event provided is a valid event type + if event != constants.EventComment && event != constants.EventDeploy && + event != constants.EventPush && event != constants.EventPull && + event != constants.EventTag { + retErr := fmt.Errorf("unable to process event %s: invalid event type provided", event) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // add event to filters map + filters["event"] = event + } + // check if status filter was provided + if len(status) > 0 { + // verify the status provided is a valid status type + if status != constants.StatusCanceled && status != constants.StatusError && + status != constants.StatusFailure && status != constants.StatusKilled && + status != constants.StatusPending && status != constants.StatusRunning && + status != constants.StatusSuccess { + retErr := fmt.Errorf("unable to process status %s: invalid status type provided", status) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // add status to filters map + filters["status"] = status + } + + // check if commit hash filter was provided + if len(commit) > 0 { + // add commit to filters map + filters["commit"] = commit + } + + // capture page query parameter if present + page, err := strconv.Atoi(c.DefaultQuery("page", "1")) + if err != nil { + retErr := fmt.Errorf("unable to convert page query parameter for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // capture per_page query parameter if present + perPage, err := strconv.Atoi(c.DefaultQuery("per_page", "10")) + if err != nil { + retErr := fmt.Errorf("unable to convert per_page query parameter for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // ensure per_page isn't above or below allowed values + perPage = util.MaxInt(1, util.MinInt(100, perPage)) + + // capture before query parameter if present, default to now + before, err := strconv.ParseInt(c.DefaultQuery("before", strconv.FormatInt(time.Now().UTC().Unix(), 10)), 10, 64) + if err != nil { + retErr := fmt.Errorf("unable to convert before query parameter for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // capture after query parameter if present, default to 0 + after, err := strconv.ParseInt(c.DefaultQuery("after", "0"), 10, 64) + if err != nil { + retErr := fmt.Errorf("unable to convert after query parameter for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + b, t, err = database.FromContext(c).ListBuildsForRepo(r, filters, before, after, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to list builds for repo %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // create pagination object + pagination := api.Pagination{ + Page: page, + PerPage: perPage, + Total: t, + } + // set pagination headers + pagination.SetHeaderLink(c) + + c.JSON(http.StatusOK, b) +} diff --git a/api/build/plan.go b/api/build/plan.go new file mode 100644 index 000000000..5790ce4a5 --- /dev/null +++ b/api/build/plan.go @@ -0,0 +1,71 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "fmt" + "time" + + "github.com/go-vela/server/api/service" + "github.com/go-vela/server/api/step" + "github.com/go-vela/server/database" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +// PlanBuild is a helper function to plan the build for +// execution. This creates all resources, like steps +// and services, for the build in the configured backend. +// TODO: +// - return build and error. +func PlanBuild(database database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo) error { + // update fields in build object + b.SetCreated(time.Now().UTC().Unix()) + + // send API call to create the build + // TODO: return created build and error instead of just error + err := database.CreateBuild(b) + if err != nil { + // clean up the objects from the pipeline in the database + // TODO: + // - return build in CreateBuild + // - even if it was created, we need to get the new build id + // otherwise it will be 0, which attempts to INSERT instead + // of UPDATE-ing the existing build - which results in + // a constraint error (repo_id, number) + // - do we want to update the build or just delete it? + CleanBuild(database, b, nil, nil, err) + + return fmt.Errorf("unable to create new build for %s: %w", r.GetFullName(), err) + } + + // send API call to capture the created build + // TODO: this can be dropped once we return + // the created build above + b, err = database.GetBuildForRepo(r, b.GetNumber()) + if err != nil { + return fmt.Errorf("unable to get new build for %s: %w", r.GetFullName(), err) + } + + // plan all services for the build + services, err := service.PlanServices(database, p, b) + if err != nil { + // clean up the objects from the pipeline in the database + CleanBuild(database, b, services, nil, err) + + return err + } + + // plan all steps for the build + steps, err := step.PlanSteps(database, p, b) + if err != nil { + // clean up the objects from the pipeline in the database + CleanBuild(database, b, services, steps, err) + + return err + } + + return nil +} diff --git a/api/build/publish.go b/api/build/publish.go new file mode 100644 index 000000000..f126b8471 --- /dev/null +++ b/api/build/publish.go @@ -0,0 +1,74 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "context" + "encoding/json" + "time" + + "github.com/go-vela/server/database" + "github.com/go-vela/server/queue" + "github.com/go-vela/types" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + "github.com/sirupsen/logrus" +) + +// PublishToQueue is a helper function that creates +// a build item and publishes it to the queue. +func PublishToQueue(queue queue.Service, db database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo, u *library.User) { + item := types.ToItem(p, b, r, u) + + logrus.Infof("Converting queue item to json for build %d for %s", b.GetNumber(), r.GetFullName()) + + byteItem, err := json.Marshal(item) + if err != nil { + logrus.Errorf("Failed to convert item to json for build %d for %s: %v", b.GetNumber(), r.GetFullName(), err) + + // error out the build + CleanBuild(db, b, nil, nil, err) + + return + } + + logrus.Infof("Establishing route for build %d for %s", b.GetNumber(), r.GetFullName()) + + route, err := queue.Route(&p.Worker) + if err != nil { + logrus.Errorf("unable to set route for build %d for %s: %v", b.GetNumber(), r.GetFullName(), err) + + // error out the build + CleanBuild(db, b, nil, nil, err) + + return + } + + logrus.Infof("Publishing item for build %d for %s to queue %s", b.GetNumber(), r.GetFullName(), route) + + err = queue.Push(context.Background(), route, byteItem) + if err != nil { + logrus.Errorf("Retrying; Failed to publish build %d for %s: %v", b.GetNumber(), r.GetFullName(), err) + + err = queue.Push(context.Background(), route, byteItem) + if err != nil { + logrus.Errorf("Failed to publish build %d for %s: %v", b.GetNumber(), r.GetFullName(), err) + + // error out the build + CleanBuild(db, b, nil, nil, err) + + return + } + } + + // update fields in build object + b.SetEnqueued(time.Now().UTC().Unix()) + + // update the build in the db to reflect the time it was enqueued + err = db.UpdateBuild(b) + if err != nil { + logrus.Errorf("Failed to update build %d during publish to queue for %s: %v", b.GetNumber(), r.GetFullName(), err) + } +} diff --git a/api/build/restart.go b/api/build/restart.go new file mode 100644 index 000000000..302090a9c --- /dev/null +++ b/api/build/restart.go @@ -0,0 +1,362 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/compiler" + "github.com/go-vela/server/database" + "github.com/go-vela/server/queue" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/claims" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/go-vela/types" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + "github.com/sirupsen/logrus" +) + +// swagger:operation POST /api/v1/repos/{org}/{repo}/builds/{build} builds RestartBuild +// +// Restart a build in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number to restart +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Request processed but build was skipped +// schema: +// type: string +// '201': +// description: Successfully restarted the build +// schema: +// "$ref": "#/definitions/Build" +// '400': +// description: Unable to restart the build +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: Unable to restart the build +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to restart the build +// schema: +// "$ref": "#/definitions/Error" + +// RestartBuild represents the API handler to restart an existing build in the configured backend. +// +//nolint:funlen // ignore statement count +func RestartBuild(c *gin.Context) { + // capture middleware values + m := c.MustGet("metadata").(*types.Metadata) + cl := claims.Retrieve(c) + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + + entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logger := logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }) + + logger.Infof("restarting build %s", entry) + + // send API call to capture the repo owner + u, err := database.FromContext(c).GetUser(r.GetUserID()) + if err != nil { + retErr := fmt.Errorf("unable to get owner for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // create SQL filters for querying pending and running builds for repo + filters := map[string]interface{}{ + "status": []string{constants.StatusPending, constants.StatusRunning}, + } + + // send API call to capture the number of pending or running builds for the repo + builds, err := database.FromContext(c).CountBuildsForRepo(r, filters) + if err != nil { + retErr := fmt.Errorf("unable to restart build: unable to get count of builds for repo %s", r.GetFullName()) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // check if the number of pending and running builds exceeds the limit for the repo + if builds >= r.GetBuildLimit() { + retErr := fmt.Errorf("unable to restart build: repo %s has exceeded the concurrent build limit of %d", r.GetFullName(), r.GetBuildLimit()) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // update fields in build object + b.SetID(0) + b.SetCreated(time.Now().UTC().Unix()) + b.SetEnqueued(0) + b.SetStarted(0) + b.SetFinished(0) + b.SetStatus(constants.StatusPending) + b.SetHost("") + b.SetRuntime("") + b.SetDistribution("") + b.SetSender(cl.Subject) + + // update the PR event action if action was never set + // for backwards compatibility with pre-0.14 releases. + if b.GetEvent() == constants.EventPull && b.GetEventAction() == "" { + // technically, the action could have been opened or synchronize. + // will not affect behavior of the pipeline since we did not + // support actions for builds where this would be the case. + b.SetEventAction(constants.ActionOpened) + } + + // set the parent equal to the restarted build number + b.SetParent(b.GetNumber()) + // update the build numbers based off repo counter + inc := r.GetCounter() + 1 + r.SetCounter(inc) + b.SetNumber(inc) + + // populate the build link if a web address is provided + if len(m.Vela.WebAddress) > 0 { + b.SetLink( + fmt.Sprintf("%s/%s/%d", m.Vela.WebAddress, r.GetFullName(), b.GetNumber()), + ) + } + + // variable to store changeset files + var files []string + // check if the build event is not issue_comment or pull_request + if !strings.EqualFold(b.GetEvent(), constants.EventComment) && + !strings.EqualFold(b.GetEvent(), constants.EventPull) { + // send API call to capture list of files changed for the commit + files, err = scm.FromContext(c).Changeset(u, r, b.GetCommit()) + if err != nil { + retErr := fmt.Errorf("unable to restart build: failed to get changeset for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + } + + // check if the build event is a pull_request + if strings.EqualFold(b.GetEvent(), constants.EventPull) { + // capture number from build + number, err := getPRNumberFromBuild(b) + if err != nil { + retErr := fmt.Errorf("unable to restart build: failed to get pull_request number for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture list of files changed for the pull request + files, err = scm.FromContext(c).ChangesetPR(u, r, number) + if err != nil { + retErr := fmt.Errorf("unable to restart build: failed to get changeset for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + } + + // variables to store pipeline configuration + var ( + // variable to store the raw pipeline configuration + config []byte + // variable to store executable pipeline + p *pipeline.Build + // variable to store pipeline configuration + pipeline *library.Pipeline + // variable to store the pipeline type for the repository + pipelineType = r.GetPipelineType() + ) + + // send API call to attempt to capture the pipeline + pipeline, err = database.FromContext(c).GetPipelineForRepo(b.GetCommit(), r) + if err != nil { // assume the pipeline doesn't exist in the database yet (before pipeline support was added) + // send API call to capture the pipeline configuration file + config, err = scm.FromContext(c).ConfigBackoff(u, r, b.GetCommit()) + if err != nil { + retErr := fmt.Errorf("unable to get pipeline configuration for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + } else { + config = pipeline.GetData() + } + + // ensure we use the expected pipeline type when compiling + // + // The pipeline type for a repo can change at any time which can break compiling + // existing pipelines in the system for that repo. To account for this, we update + // the repo pipeline type to match what was defined for the existing pipeline + // before compiling. After we're done compiling, we reset the pipeline type. + if len(pipeline.GetType()) > 0 { + r.SetPipelineType(pipeline.GetType()) + } + + var compiled *library.Pipeline + // parse and compile the pipeline configuration file + p, compiled, err = compiler.FromContext(c). + Duplicate(). + WithBuild(b). + WithFiles(files). + WithMetadata(m). + WithRepo(r). + WithUser(u). + Compile(config) + if err != nil { + retErr := fmt.Errorf("unable to compile pipeline configuration for %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + // reset the pipeline type for the repo + // + // The pipeline type for a repo can change at any time which can break compiling + // existing pipelines in the system for that repo. To account for this, we update + // the repo pipeline type to match what was defined for the existing pipeline + // before compiling. After we're done compiling, we reset the pipeline type. + r.SetPipelineType(pipelineType) + + // skip the build if only the init or clone steps are found + skip := SkipEmptyBuild(p) + if skip != "" { + // set build to successful status + b.SetStatus(constants.StatusSkipped) + + // send API call to set the status on the commit + err = scm.FromContext(c).Status(u, b, r.GetOrg(), r.GetName()) + if err != nil { + logrus.Errorf("unable to set commit status for %s/%d: %v", r.GetFullName(), b.GetNumber(), err) + } + + c.JSON(http.StatusOK, skip) + + return + } + + // check if the pipeline did not already exist in the database + // + //nolint:dupl // ignore duplicate code + if pipeline == nil { + pipeline = compiled + pipeline.SetRepoID(r.GetID()) + pipeline.SetCommit(b.GetCommit()) + pipeline.SetRef(b.GetRef()) + + // send API call to create the pipeline + err = database.FromContext(c).CreatePipeline(pipeline) + if err != nil { + retErr := fmt.Errorf("unable to create pipeline for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to capture the created pipeline + pipeline, err = database.FromContext(c).GetPipelineForRepo(pipeline.GetCommit(), r) + if err != nil { + //nolint:lll // ignore long line length due to error message + retErr := fmt.Errorf("unable to get new pipeline %s/%s: %w", r.GetFullName(), pipeline.GetCommit(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + } + + b.SetPipelineID(pipeline.GetID()) + + // create the objects from the pipeline in the database + err = PlanBuild(database.FromContext(c), p, b, r) + if err != nil { + util.HandleError(c, http.StatusInternalServerError, err) + + return + } + + // send API call to update repo for ensuring counter is incremented + err = database.FromContext(c).UpdateRepo(r) + if err != nil { + retErr := fmt.Errorf("unable to restart build: failed to update repo %s: %w", r.GetFullName(), err) + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + // send API call to capture the restarted build + b, _ = database.FromContext(c).GetBuildForRepo(r, b.GetNumber()) + + c.JSON(http.StatusCreated, b) + + // send API call to set the status on the commit + err = scm.FromContext(c).Status(u, b, r.GetOrg(), r.GetName()) + if err != nil { + logger.Errorf("unable to set commit status for build %s: %v", entry, err) + } + + // publish the build to the queue + go PublishToQueue( + queue.FromGinContext(c), + database.FromContext(c), + p, + b, + r, + u, + ) +} diff --git a/api/build/skip.go b/api/build/skip.go new file mode 100644 index 000000000..4d5cf8af9 --- /dev/null +++ b/api/build/skip.go @@ -0,0 +1,41 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "github.com/go-vela/types/pipeline" +) + +// SkipEmptyBuild checks if the build should be skipped due to it +// not containing any steps besides init or clone. +// +//nolint:goconst // ignore init and clone constants +func SkipEmptyBuild(p *pipeline.Build) string { + if len(p.Stages) == 1 { + if p.Stages[0].Name == "init" { + return "skipping build since only init stage found" + } + } + + if len(p.Stages) == 2 { + if p.Stages[0].Name == "init" && p.Stages[1].Name == "clone" { + return "skipping build since only init and clone stages found" + } + } + + if len(p.Steps) == 1 { + if p.Steps[0].Name == "init" { + return "skipping build since only init step found" + } + } + + if len(p.Steps) == 2 { + if p.Steps[0].Name == "init" && p.Steps[1].Name == "clone" { + return "skipping build since only init and clone steps found" + } + } + + return "" +} diff --git a/api/build_test.go b/api/build/skip_test.go similarity index 99% rename from api/build_test.go rename to api/build/skip_test.go index 1fb395d7d..ca805a84e 100644 --- a/api/build_test.go +++ b/api/build/skip_test.go @@ -2,7 +2,7 @@ // // Use of this source code is governed by the LICENSE file in this repository. -package api +package build import ( "testing" diff --git a/api/build/token.go b/api/build/token.go new file mode 100644 index 000000000..da18a0322 --- /dev/null +++ b/api/build/token.go @@ -0,0 +1,119 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/internal/token" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/claims" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/token builds GetBuildToken +// +// Get a build token +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved build token +// schema: +// "$ref": "#/definitions/Token" +// '400': +// description: Bad request +// schema: +// "$ref": "#/definitions/Error" +// '409': +// description: Conflict (requested build token for build not in pending state) +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to generate build token +// schema: +// "$ref": "#/definitions/Error" + +// GetBuildToken represents the API handler to generate a build token. +func GetBuildToken(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + cl := claims.Retrieve(c) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "user": cl.Subject, + }).Infof("generating build token for build %s/%d", r.GetFullName(), b.GetNumber()) + + // if build is not in a pending state, then a build token should not be needed - conflict + if !strings.EqualFold(b.GetStatus(), constants.StatusPending) { + retErr := fmt.Errorf("unable to mint build token: build is not in pending state") + util.HandleError(c, http.StatusConflict, retErr) + + return + } + + // retrieve token manager from context + tm := c.MustGet("token-manager").(*token.Manager) + + // set expiration to repo timeout plus configurable buffer + exp := (time.Duration(r.GetTimeout()) * time.Minute) + tm.BuildTokenBufferDuration + + // set mint token options + bmto := &token.MintTokenOpts{ + Hostname: cl.Subject, + BuildID: b.GetID(), + Repo: r.GetFullName(), + TokenType: constants.WorkerBuildTokenType, + TokenDuration: exp, + } + + // mint token + bt, err := tm.MintToken(bmto) + if err != nil { + retErr := fmt.Errorf("unable to generate build token: %w", err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, library.Token{Token: &bt}) +} diff --git a/api/build/update.go b/api/build/update.go new file mode 100644 index 000000000..f9d3341fa --- /dev/null +++ b/api/build/update.go @@ -0,0 +1,186 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package build + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/claims" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// swagger:operation PUT /api/v1/repos/{org}/{repo}/builds/{build} builds UpdateBuild +// +// Updates a build in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number to update +// required: true +// type: integer +// - in: body +// name: body +// description: Payload containing the build to update +// required: true +// schema: +// "$ref": "#/definitions/Build" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully updated the build +// schema: +// "$ref": "#/definitions/Build" +// '404': +// description: Unable to update the build +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to update the build +// schema: +// "$ref": "#/definitions/Error" + +// UpdateBuild represents the API handler to update +// a build for a repo in the configured backend. +func UpdateBuild(c *gin.Context) { + // capture middleware values + cl := claims.Retrieve(c) + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + + entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "user": cl.Subject, + }).Infof("updating build %s", entry) + + // capture body from API request + input := new(library.Build) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for build %s: %w", entry, err) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // update build fields if provided + if len(input.GetStatus()) > 0 { + // update status if set + b.SetStatus(input.GetStatus()) + } + + if len(input.GetError()) > 0 { + // update error if set + b.SetError(input.GetError()) + } + + if input.GetEnqueued() > 0 { + // update enqueued if set + b.SetEnqueued(input.GetEnqueued()) + } + + if input.GetStarted() > 0 { + // update started if set + b.SetStarted(input.GetStarted()) + } + + if input.GetFinished() > 0 { + // update finished if set + b.SetFinished(input.GetFinished()) + } + + if len(input.GetTitle()) > 0 { + // update title if set + b.SetTitle(input.GetTitle()) + } + + if len(input.GetMessage()) > 0 { + // update message if set + b.SetMessage(input.GetMessage()) + } + + if len(input.GetHost()) > 0 { + // update host if set + b.SetHost(input.GetHost()) + } + + if len(input.GetRuntime()) > 0 { + // update runtime if set + b.SetRuntime(input.GetRuntime()) + } + + if len(input.GetDistribution()) > 0 { + // update distribution if set + b.SetDistribution(input.GetDistribution()) + } + + // send API call to update the build + err = database.FromContext(c).UpdateBuild(b) + if err != nil { + retErr := fmt.Errorf("unable to update build %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // send API call to capture the updated build + b, _ = database.FromContext(c).GetBuildForRepo(r, b.GetNumber()) + + c.JSON(http.StatusOK, b) + + // check if the build is in a "final" state + if b.GetStatus() == constants.StatusSuccess || + b.GetStatus() == constants.StatusFailure || + b.GetStatus() == constants.StatusCanceled || + b.GetStatus() == constants.StatusKilled || + b.GetStatus() == constants.StatusError { + // send API call to capture the repo owner + u, err := database.FromContext(c).GetUser(r.GetUserID()) + if err != nil { + logrus.Errorf("unable to get owner for build %s: %v", entry, err) + } + + // send API call to set the status on the commit + err = scm.FromContext(c).Status(u, b, r.GetOrg(), r.GetName()) + if err != nil { + logrus.Errorf("unable to set commit status for build %s: %v", entry, err) + } + } +} diff --git a/api/service/plan.go b/api/service/plan.go new file mode 100644 index 000000000..4912f38d1 --- /dev/null +++ b/api/service/plan.go @@ -0,0 +1,71 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package service + +import ( + "fmt" + "time" + + "github.com/go-vela/server/database" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +// PlanServices is a helper function to plan all services +// in the build for execution. This creates the services +// for the build in the configured backend. +func PlanServices(database database.Interface, p *pipeline.Build, b *library.Build) ([]*library.Service, error) { + // variable to store planned services + services := []*library.Service{} + + // iterate through all pipeline services + for _, service := range p.Services { + // create the service object + s := new(library.Service) + s.SetBuildID(b.GetID()) + s.SetRepoID(b.GetRepoID()) + s.SetName(service.Name) + s.SetImage(service.Image) + s.SetNumber(service.Number) + s.SetStatus(constants.StatusPending) + s.SetCreated(time.Now().UTC().Unix()) + + // send API call to create the service + err := database.CreateService(s) + if err != nil { + return services, fmt.Errorf("unable to create service %s: %w", s.GetName(), err) + } + + // send API call to capture the created service + s, err = database.GetServiceForBuild(b, s.GetNumber()) + if err != nil { + return services, fmt.Errorf("unable to get service %s: %w", s.GetName(), err) + } + + // populate environment variables from service library + // + // https://pkg.go.dev/github.com/go-vela/types/library#Service.Environment + err = service.MergeEnv(s.Environment()) + if err != nil { + return services, err + } + + // create the log object + l := new(library.Log) + l.SetServiceID(s.GetID()) + l.SetBuildID(b.GetID()) + l.SetRepoID(b.GetRepoID()) + l.SetData([]byte{}) + + // send API call to create the service logs + err = database.CreateLog(l) + if err != nil { + return services, fmt.Errorf("unable to create service logs for service %s: %w", s.GetName(), err) + } + } + + return services, nil +} diff --git a/api/step/plan.go b/api/step/plan.go new file mode 100644 index 000000000..b072cf4f7 --- /dev/null +++ b/api/step/plan.go @@ -0,0 +1,97 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package step + +import ( + "fmt" + "time" + + "github.com/go-vela/server/database" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +// PlanSteps is a helper function to plan all steps +// in the build for execution. This creates the steps +// for the build in the configured backend. +func PlanSteps(database database.Interface, p *pipeline.Build, b *library.Build) ([]*library.Step, error) { + // variable to store planned steps + steps := []*library.Step{} + + // iterate through all pipeline stages + for _, stage := range p.Stages { + // iterate through all steps for each pipeline stage + for _, step := range stage.Steps { + // create the step object + s, err := planStep(database, b, step, stage.Name) + if err != nil { + return steps, err + } + + steps = append(steps, s) + } + } + + // iterate through all pipeline steps + for _, step := range p.Steps { + s, err := planStep(database, b, step, "") + if err != nil { + return steps, err + } + + steps = append(steps, s) + } + + return steps, nil +} + +func planStep(database database.Interface, b *library.Build, c *pipeline.Container, stage string) (*library.Step, error) { + // create the step object + s := new(library.Step) + s.SetBuildID(b.GetID()) + s.SetRepoID(b.GetRepoID()) + s.SetNumber(c.Number) + s.SetName(c.Name) + s.SetImage(c.Image) + s.SetStage(stage) + s.SetStatus(constants.StatusPending) + s.SetCreated(time.Now().UTC().Unix()) + + // send API call to create the step + err := database.CreateStep(s) + if err != nil { + return nil, fmt.Errorf("unable to create step %s: %w", s.GetName(), err) + } + + // send API call to capture the created step + s, err = database.GetStepForBuild(b, s.GetNumber()) + if err != nil { + return nil, fmt.Errorf("unable to get step %s: %w", s.GetName(), err) + } + + // populate environment variables from step library + // + // https://pkg.go.dev/github.com/go-vela/types/library#step.Environment + err = c.MergeEnv(s.Environment()) + if err != nil { + return nil, err + } + + // create the log object + l := new(library.Log) + l.SetStepID(s.GetID()) + l.SetBuildID(b.GetID()) + l.SetRepoID(b.GetRepoID()) + l.SetData([]byte{}) + + // send API call to create the step logs + err = database.CreateLog(l) + if err != nil { + return nil, fmt.Errorf("unable to create logs for step %s: %w", s.GetName(), err) + } + + return s, nil +} diff --git a/api/webhook/doc.go b/api/webhook/doc.go new file mode 100644 index 000000000..0acc05476 --- /dev/null +++ b/api/webhook/doc.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +// Package webhook provides the webhook handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/webhook" +package webhook diff --git a/api/webhook.go b/api/webhook/post.go similarity index 93% rename from api/webhook.go rename to api/webhook/post.go index fd958291c..c25abb634 100644 --- a/api/webhook.go +++ b/api/webhook/post.go @@ -1,13 +1,12 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. -package api +package webhook import ( "bytes" "context" - "encoding/json" "fmt" "io" "net/http" @@ -16,6 +15,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/build" "github.com/go-vela/server/compiler" "github.com/go-vela/server/database" "github.com/go-vela/server/queue" @@ -537,7 +537,7 @@ func PostWebhook(c *gin.Context) { repo.SetPipelineType(pipelineType) // skip the build if only the init or clone steps are found - skip := SkipEmptyBuild(p) + skip := build.SkipEmptyBuild(p) if skip != "" { // set build to successful status b.SetStatus(constants.StatusSkipped) @@ -604,7 +604,7 @@ func PostWebhook(c *gin.Context) { // using the same Number and thus create a constraint // conflict; consider deleting the partially created // build object in the database - err = PlanBuild(database.FromContext(c), p, b, repo) + err = build.PlanBuild(database.FromContext(c), p, b, repo) if err != nil { retErr := fmt.Errorf("%s: %w", baseErr, err) @@ -691,7 +691,7 @@ func PostWebhook(c *gin.Context) { } // publish the build to the queue - go PublishToQueue( + go build.PublishToQueue( queue.FromGinContext(c), database.FromContext(c), p, @@ -701,62 +701,6 @@ func PostWebhook(c *gin.Context) { ) } -// PublishToQueue is a helper function that creates -// a build item and publishes it to the queue. -func PublishToQueue(queue queue.Service, db database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo, u *library.User) { - item := types.ToItem(p, b, r, u) - - logrus.Infof("Converting queue item to json for build %d for %s", b.GetNumber(), r.GetFullName()) - - byteItem, err := json.Marshal(item) - if err != nil { - logrus.Errorf("Failed to convert item to json for build %d for %s: %v", b.GetNumber(), r.GetFullName(), err) - - // error out the build - cleanBuild(db, b, nil, nil, err) - - return - } - - logrus.Infof("Establishing route for build %d for %s", b.GetNumber(), r.GetFullName()) - - route, err := queue.Route(&p.Worker) - if err != nil { - logrus.Errorf("unable to set route for build %d for %s: %v", b.GetNumber(), r.GetFullName(), err) - - // error out the build - cleanBuild(db, b, nil, nil, err) - - return - } - - logrus.Infof("Publishing item for build %d for %s to queue %s", b.GetNumber(), r.GetFullName(), route) - - err = queue.Push(context.Background(), route, byteItem) - if err != nil { - logrus.Errorf("Retrying; Failed to publish build %d for %s: %v", b.GetNumber(), r.GetFullName(), err) - - err = queue.Push(context.Background(), route, byteItem) - if err != nil { - logrus.Errorf("Failed to publish build %d for %s: %v", b.GetNumber(), r.GetFullName(), err) - - // error out the build - cleanBuild(db, b, nil, nil, err) - - return - } - } - - // update fields in build object - b.SetEnqueued(time.Now().UTC().Unix()) - - // update the build in the db to reflect the time it was enqueued - err = db.UpdateBuild(b) - if err != nil { - logrus.Errorf("Failed to update build %d during publish to queue for %s: %v", b.GetNumber(), r.GetFullName(), err) - } -} - func handleRepositoryEvent(c *gin.Context, m *types.Metadata, h *library.Hook, r *library.Repo) (*library.Repo, error) { logrus.Debugf("webhook is repository event, making necessary updates to repo %s", r.GetFullName()) diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index cb607c59b..a71688606 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -10,7 +10,7 @@ import ( "time" "github.com/adhocore/gronx" - "github.com/go-vela/server/api" + "github.com/go-vela/server/api/build" "github.com/go-vela/server/compiler" "github.com/go-vela/server/database" "github.com/go-vela/server/queue" @@ -281,7 +281,7 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat r.SetPipelineType(pipelineType) // skip the build if only the init or clone steps are found - skip := api.SkipEmptyBuild(p) + skip := build.SkipEmptyBuild(p) if skip != "" { return nil } @@ -325,7 +325,7 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat // using the same Number and thus create a constraint // conflict; consider deleting the partially created // build object in the database - err = api.PlanBuild(database, p, b, r) + err = build.PlanBuild(database, p, b, r) if err != nil { // check if the retry limit has been exceeded if i < retryLimit-1 { @@ -368,7 +368,7 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat } // publish the build to the queue - go api.PublishToQueue( + go build.PublishToQueue( queue, database, p, diff --git a/go.mod b/go.mod index 5ff15db7b..be974371f 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.0 github.com/go-playground/assert/v2 v2.2.0 - github.com/go-vela/types v0.19.3-0.20230519215217-0da8c8b5e90f + github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-cmp v0.5.9 github.com/google/go-github/v52 v52.0.0 diff --git a/go.sum b/go.sum index e988c342d..3b4b80685 100644 --- a/go.sum +++ b/go.sum @@ -138,8 +138,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= -github.com/go-vela/types v0.19.3-0.20230519215217-0da8c8b5e90f h1:13H381Djx9iFC3BSj2f/ac57HlaI3mQL0el9vM7a3+k= -github.com/go-vela/types v0.19.3-0.20230519215217-0da8c8b5e90f/go.mod h1:0lsuPfGyVyTWJSi2h3NS6uaEW6DgnFvIzaZu1sXYKrs= +github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c h1:eAApIK5e5MxFF8RzZAFsvTSdwq/AzdUrdhJHOGQ0ILc= +github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c/go.mod h1:0lsuPfGyVyTWJSi2h3NS6uaEW6DgnFvIzaZu1sXYKrs= github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= diff --git a/router/build.go b/router/build.go index 4aab6a9c7..22cd37aa4 100644 --- a/router/build.go +++ b/router/build.go @@ -6,10 +6,10 @@ package router import ( "github.com/gin-gonic/gin" - "github.com/go-vela/server/api" + "github.com/go-vela/server/api/build" "github.com/go-vela/server/api/log" "github.com/go-vela/server/router/middleware" - "github.com/go-vela/server/router/middleware/build" + bmiddleware "github.com/go-vela/server/router/middleware/build" "github.com/go-vela/server/router/middleware/executors" "github.com/go-vela/server/router/middleware/perm" ) @@ -48,27 +48,27 @@ func BuildHandlers(base *gin.RouterGroup) { // Builds endpoints builds := base.Group("/builds") { - builds.POST("", perm.MustAdmin(), middleware.Payload(), api.CreateBuild) - builds.GET("", perm.MustRead(), api.GetBuilds) + builds.POST("", perm.MustAdmin(), middleware.Payload(), build.CreateBuild) + builds.GET("", perm.MustRead(), build.ListBuildsForRepo) // Build endpoints - build := builds.Group("/:build", build.Establish()) + b := builds.Group("/:build", bmiddleware.Establish()) { - build.POST("", perm.MustWrite(), api.RestartBuild) - build.GET("", perm.MustRead(), api.GetBuild) - build.PUT("", perm.MustBuildAccess(), middleware.Payload(), api.UpdateBuild) - build.DELETE("", perm.MustPlatformAdmin(), api.DeleteBuild) - build.DELETE("/cancel", executors.Establish(), perm.MustWrite(), api.CancelBuild) - build.GET("/logs", perm.MustRead(), log.ListLogsForBuild) - build.GET("/token", perm.MustWorkerAuthToken(), api.GetBuildToken) + b.POST("", perm.MustWrite(), build.RestartBuild) + b.GET("", perm.MustRead(), build.GetBuild) + b.PUT("", perm.MustBuildAccess(), middleware.Payload(), build.UpdateBuild) + b.DELETE("", perm.MustPlatformAdmin(), build.DeleteBuild) + b.DELETE("/cancel", executors.Establish(), perm.MustWrite(), build.CancelBuild) + b.GET("/logs", perm.MustRead(), log.ListLogsForBuild) + b.GET("/token", perm.MustWorkerAuthToken(), build.GetBuildToken) // Service endpoints // * Log endpoints - ServiceHandlers(build) + ServiceHandlers(b) // Step endpoints // * Log endpoints - StepHandlers(build) + StepHandlers(b) } // end of build endpoints } // end of builds endpoints } diff --git a/router/repo.go b/router/repo.go index 6d0dbf240..7b7859f6a 100644 --- a/router/repo.go +++ b/router/repo.go @@ -6,7 +6,7 @@ package router import ( "github.com/gin-gonic/gin" - "github.com/go-vela/server/api" + "github.com/go-vela/server/api/build" "github.com/go-vela/server/api/repo" "github.com/go-vela/server/router/middleware" "github.com/go-vela/server/router/middleware/org" @@ -64,7 +64,7 @@ func RepoHandlers(base *gin.RouterGroup) { org := _repos.Group("/:org", org.Establish()) { org.GET("", repo.ListReposForOrg) - org.GET("/builds", api.GetOrgBuilds) + org.GET("/builds", build.ListBuildsForOrg) // Repo endpoints _repo := org.Group("/:repo", rmiddleware.Establish()) diff --git a/router/router.go b/router/router.go index 2ed598034..f95c6bf4b 100644 --- a/router/router.go +++ b/router/router.go @@ -35,6 +35,7 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/api" "github.com/go-vela/server/api/auth" + "github.com/go-vela/server/api/webhook" "github.com/go-vela/server/router/middleware" "github.com/go-vela/server/router/middleware/claims" "github.com/go-vela/server/router/middleware/org" @@ -84,7 +85,7 @@ func Load(options ...gin.HandlerFunc) *gin.Engine { r.GET("/version", api.Version) // Webhook endpoint - r.POST("/webhook", api.PostWebhook) + r.POST("/webhook", webhook.PostWebhook) // Authentication endpoints authenticate := r.Group("/authenticate") diff --git a/router/search.go b/router/search.go index 2aaaec699..62e3542db 100644 --- a/router/search.go +++ b/router/search.go @@ -6,7 +6,7 @@ package router import ( "github.com/gin-gonic/gin" - "github.com/go-vela/server/api" + "github.com/go-vela/server/api/build" ) // SearchHandlers is a function that extends the provided base router group @@ -18,9 +18,9 @@ func SearchHandlers(base *gin.RouterGroup) { search := base.Group("/search") { // Build endpoint - build := search.Group("/builds") + b := search.Group("/builds") { - build.GET("/:id", api.GetBuildByID) + b.GET("/:id", build.GetBuildByID) } } // end of search endpoints } From a6002745dc0634a8592d82185f980d615e89dbe2 Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Fri, 2 Jun 2023 09:36:26 -0500 Subject: [PATCH 24/53] feat(database): add agnostic engine (#868) --- database/close.go | 18 +++++ database/close_test.go | 83 +++++++++++++++++++ database/database.go | 58 +++++++++++++- database/database_test.go | 79 +++++++++++++++++- database/driver.go | 10 +++ database/driver_test.go | 47 +++++++++++ database/validate.go | 75 +++++++++++++++++ database/validate_test.go | 165 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 532 insertions(+), 3 deletions(-) create mode 100644 database/close.go create mode 100644 database/close_test.go create mode 100644 database/driver.go create mode 100644 database/driver_test.go create mode 100644 database/validate.go create mode 100644 database/validate_test.go diff --git a/database/close.go b/database/close.go new file mode 100644 index 000000000..0596b972e --- /dev/null +++ b/database/close.go @@ -0,0 +1,18 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +// Close stops and terminates the connection to the database. +func (e *engine) Close() error { + e.Logger.Tracef("closing connection to the %s database", e.Driver()) + + // capture database/sql database from gorm.io/gorm database + _sql, err := e.Database.DB() + if err != nil { + return err + } + + return _sql.Close() +} diff --git a/database/close_test.go b/database/close_test.go new file mode 100644 index 000000000..b555e78d5 --- /dev/null +++ b/database/close_test.go @@ -0,0 +1,83 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "testing" + + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +func TestDatabase_Engine_Close(t *testing.T) { + _postgres, _mock := testPostgres(t) + defer _postgres.Close() + // ensure the mock expects the close + _mock.ExpectClose() + + // create a test database without mocking the call + _unmocked, _ := testPostgres(t) + + _sqlite := testSqlite(t) + defer _sqlite.Close() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + name: "success with postgres", + failure: false, + database: _postgres, + }, + { + name: "success with sqlite3", + failure: false, + database: _sqlite, + }, + { + name: "failure without mocked call", + failure: true, + database: _unmocked, + }, + { + name: "failure with invalid gorm database", + failure: true, + database: &engine{ + Config: &Config{ + Driver: "invalid", + }, + Database: &gorm.DB{ + Config: &gorm.Config{ + ConnPool: nil, + }, + }, + Logger: logrus.NewEntry(logrus.StandardLogger()), + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.Close() + + if test.failure { + if err == nil { + t.Errorf("Close for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("Close for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/database.go b/database/database.go index ad26dd82a..bf0ca2724 100644 --- a/database/database.go +++ b/database/database.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. @@ -6,10 +6,64 @@ package database import ( "fmt" + "time" + "github.com/go-vela/server/database/build" + "github.com/go-vela/server/database/hook" + "github.com/go-vela/server/database/log" + "github.com/go-vela/server/database/pipeline" + "github.com/go-vela/server/database/repo" + "github.com/go-vela/server/database/schedule" + "github.com/go-vela/server/database/secret" + "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/step" + "github.com/go-vela/server/database/user" + "github.com/go-vela/server/database/worker" "github.com/go-vela/types/constants" - "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +type ( + // Config represents the settings required to create the engine that implements the Interface. + Config struct { + // specifies the address to use for the database engine + Address string + // specifies the level of compression to use for the database engine + CompressionLevel int + // specifies the maximum idle connections for the database engine + ConnectionIdle int + // specifies the connection duration to use for the database engine + ConnectionLife time.Duration + // specifies the maximum open connections for the database engine + ConnectionOpen int + // specifies the driver to use for the database engine + Driver string + // specifies the encryption key to use for the database engine + EncryptionKey string + // specifies to skip creating tables and indexes for the database engine + SkipCreation bool + } + + // engine represents the functionality that implements the Interface. + engine struct { + Config *Config + Database *gorm.DB + Logger *logrus.Entry + + build.BuildInterface + hook.HookInterface + log.LogInterface + pipeline.PipelineInterface + repo.RepoInterface + schedule.ScheduleInterface + secret.SecretInterface + service.ServiceInterface + step.StepInterface + user.UserInterface + worker.WorkerInterface + } ) // New creates and returns a Vela service capable of diff --git a/database/database_test.go b/database/database_test.go index 14081a382..77de76d5b 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. @@ -7,6 +7,13 @@ package database import ( "testing" "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/sirupsen/logrus" + + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" ) func TestDatabase_New(t *testing.T) { @@ -86,3 +93,73 @@ func TestDatabase_New(t *testing.T) { } } } + +// testPostgres is a helper function to create a Postgres engine for testing. +func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { + // create the engine with test configuration + _engine := &engine{ + Config: &Config{ + CompressionLevel: 3, + ConnectionLife: 30 * time.Minute, + ConnectionIdle: 2, + ConnectionOpen: 0, + Driver: "postgres", + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + Logger: logrus.NewEntry(logrus.StandardLogger()), + } + + // create the new mock sql database + _sql, _mock, err := sqlmock.New( + sqlmock.MonitorPingsOption(true), + sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual), + ) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + // ensure the mock expects the ping + _mock.ExpectPing() + + // create the new mock Postgres database client + _engine.Database, err = gorm.Open( + postgres.New(postgres.Config{Conn: _sql}), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new test postgres database: %v", err) + } + + return _engine, _mock +} + +// testSqlite is a helper function to create a Sqlite engine for testing. +func testSqlite(t *testing.T) *engine { + var err error + + // create the engine with test configuration + _engine := &engine{ + Config: &Config{ + Address: "file::memory:?cache=shared", + CompressionLevel: 3, + ConnectionLife: 30 * time.Minute, + ConnectionIdle: 2, + ConnectionOpen: 0, + Driver: "sqlite3", + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + Logger: logrus.NewEntry(logrus.StandardLogger()), + } + + // create the new mock Sqlite database client + _engine.Database, err = gorm.Open( + sqlite.Open(_engine.Config.Address), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new test sqlite database: %v", err) + } + + return _engine +} diff --git a/database/driver.go b/database/driver.go new file mode 100644 index 000000000..a81662846 --- /dev/null +++ b/database/driver.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +// Driver outputs the configured database driver. +func (e *engine) Driver() string { + return e.Config.Driver +} diff --git a/database/driver_test.go b/database/driver_test.go new file mode 100644 index 000000000..6d382d8cd --- /dev/null +++ b/database/driver_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "strings" + "testing" +) + +func TestDatabase_Engine_Driver(t *testing.T) { + _postgres, _ := testPostgres(t) + defer _postgres.Close() + + _sqlite := testSqlite(t) + defer _sqlite.Close() + + // setup tests + tests := []struct { + name string + database *engine + want string + }{ + { + name: "success with postgres", + database: _postgres, + want: "postgres", + }, + { + name: "success with sqlite3", + database: _sqlite, + want: "sqlite3", + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := test.database.Driver() + + if !strings.EqualFold(got, test.want) { + t.Errorf("Driver for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/validate.go b/database/validate.go new file mode 100644 index 000000000..c16c07cd2 --- /dev/null +++ b/database/validate.go @@ -0,0 +1,75 @@ +// Copyright (c) 2023 Target Brands, Ine. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "fmt" + "strings" + + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" +) + +// Validate verifies the required fields from the provided configuration are populated correctly. +func (c *Config) Validate() error { + logrus.Trace("validating database configuration for engine") + + // verify a database driver was provided + if len(c.Driver) == 0 { + return fmt.Errorf("no database driver provided") + } + + // verify a database address was provided + if len(c.Address) == 0 { + return fmt.Errorf("no database address provided") + } + + // check if the database address has a trailing slash + if strings.HasSuffix(c.Address, "/") { + return fmt.Errorf("invalid database address provided: address must not have trailing slash") + } + + // verify a database encryption key was provided + if len(c.EncryptionKey) == 0 { + return fmt.Errorf("no database encryption key provided") + } + + // check the database encryption key length - enforce AES-256 by forcing 32 characters in the key + if len(c.EncryptionKey) != 32 { + return fmt.Errorf("invalid database encryption key provided: key length (%d) must be 32 characters", len(c.EncryptionKey)) + } + + // verify the database compression level is valid + switch c.CompressionLevel { + case constants.CompressionNegOne: + fallthrough + case constants.CompressionZero: + fallthrough + case constants.CompressionOne: + fallthrough + case constants.CompressionTwo: + fallthrough + case constants.CompressionThree: + fallthrough + case constants.CompressionFour: + fallthrough + case constants.CompressionFive: + fallthrough + case constants.CompressionSix: + fallthrough + case constants.CompressionSeven: + fallthrough + case constants.CompressionEight: + fallthrough + case constants.CompressionNine: + break + default: + return fmt.Errorf("invalid database compression level provided: level (%d) must be between %d and %d", + c.CompressionLevel, constants.CompressionNegOne, constants.CompressionNine, + ) + } + + return nil +} diff --git a/database/validate_test.go b/database/validate_test.go new file mode 100644 index 000000000..356705fe2 --- /dev/null +++ b/database/validate_test.go @@ -0,0 +1,165 @@ +// Copyright (c) 2023 Target Brands, Ine. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "testing" + "time" +) + +func TestDatabase_Config_Validate(t *testing.T) { + // setup tests + tests := []struct { + failure bool + name string + config *Config + }{ + { + name: "success with postgres", + failure: false, + config: &Config{ + Driver: "postgres", + Address: "postgres://foo:bar@localhost:5432/vela", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "success with sqlite3", + failure: false, + config: &Config{ + Driver: "sqlite3", + Address: "file::memory:?cache=shared", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "success with negative compression level", + failure: false, + config: &Config{ + Driver: "postgres", + Address: "postgres://foo:bar@localhost:5432/vela", + CompressionLevel: -1, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "failure with empty driver", + failure: true, + config: &Config{ + Driver: "", + Address: "postgres://foo:bar@localhost:5432/vela", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "failure with empty address", + failure: true, + config: &Config{ + Driver: "postgres", + Address: "", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "failure with invalid address", + failure: true, + config: &Config{ + Driver: "postgres", + Address: "postgres://foo:bar@localhost:5432/vela/", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "failure with invalid compression level", + failure: true, + config: &Config{ + Driver: "postgres", + Address: "postgres://foo:bar@localhost:5432/vela", + CompressionLevel: 10, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "failure with empty encryption key", + failure: true, + config: &Config{ + Driver: "postgres", + Address: "postgres://foo:bar@localhost:5432/vela", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "", + SkipCreation: false, + }, + }, + { + name: "failure with invalid encryption key", + failure: true, + config: &Config{ + Driver: "postgres", + Address: "postgres://foo:bar@localhost:5432/vela", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0", + SkipCreation: false, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.config.Validate() + + if test.failure { + if err == nil { + t.Errorf("Validate for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("Validate for %s returned err: %v", test.name, err) + } + }) + } +} From e4bfa4e038167efa514c0b45aee419432aacc007 Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Mon, 5 Jun 2023 10:23:43 -0500 Subject: [PATCH 25/53] feat(database): add engine.Ping() (#869) --- database/ping.go | 42 ++++++++++++++++++++++ database/ping_test.go | 82 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 database/ping.go create mode 100644 database/ping_test.go diff --git a/database/ping.go b/database/ping.go new file mode 100644 index 000000000..7ac1a09c2 --- /dev/null +++ b/database/ping.go @@ -0,0 +1,42 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "fmt" + "time" +) + +// Ping sends a "ping" request with backoff to the database. +func (e *engine) Ping() error { + e.Logger.Tracef("sending ping request to the %s database", e.Driver()) + + // create a loop to attempt ping requests 5 times + for i := 0; i < 5; i++ { + // capture database/sql database from gorm.io/gorm database + _sql, err := e.Database.DB() + if err != nil { + return err + } + + // send ping request to database + err = _sql.Ping() + if err != nil { + // create the duration of time to sleep for before attempting to retry + duration := time.Duration(i+1) * time.Second + + e.Logger.Warnf("unable to ping %s database - retrying in %v", e.Driver(), duration) + + // sleep for loop iteration in seconds + time.Sleep(duration) + + continue + } + + return nil + } + + return fmt.Errorf("unable to successfully ping %s database", e.Driver()) +} diff --git a/database/ping_test.go b/database/ping_test.go new file mode 100644 index 000000000..7a34ad122 --- /dev/null +++ b/database/ping_test.go @@ -0,0 +1,82 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "testing" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +func TestDatabase_Engine_Ping(t *testing.T) { + _postgres, _mock := testPostgres(t) + defer _postgres.Close() + // ensure the mock expects the ping + _mock.ExpectPing() + + // create a test database without mocking the call + _unmocked, _ := testPostgres(t) + + _sqlite := testSqlite(t) + defer _sqlite.Close() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + name: "success with postgres", + failure: false, + database: _postgres, + }, + { + name: "success with sqlite", + failure: false, + database: _sqlite, + }, + { + name: "failure without mocked call", + failure: true, + database: _unmocked, + }, + { + name: "failure with invalid gorm database", + failure: true, + database: &engine{ + Config: &Config{ + Driver: "invalid", + }, + Database: &gorm.DB{ + Config: &gorm.Config{ + ConnPool: nil, + }, + }, + Logger: logrus.NewEntry(logrus.StandardLogger()), + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.Ping() + + if test.failure { + if err == nil { + t.Errorf("Ping for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("Ping for %s returned err: %v", test.name, err) + } + }) + } +} From e8bdb157111bb2da2a0cf867c297a2954a791b25 Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Wed, 7 Jun 2023 09:21:43 -0500 Subject: [PATCH 26/53] feat(database): add engine.NewResources() (#870) --- database/resource.go | 143 ++++++++++++++++++++++++++++++++++++++ database/resource_test.go | 112 +++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 database/resource.go create mode 100644 database/resource_test.go diff --git a/database/resource.go b/database/resource.go new file mode 100644 index 000000000..93122eadc --- /dev/null +++ b/database/resource.go @@ -0,0 +1,143 @@ +// Copyright (c) 2023 Target Brands, Ine. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "github.com/go-vela/server/database/build" + "github.com/go-vela/server/database/hook" + "github.com/go-vela/server/database/log" + "github.com/go-vela/server/database/pipeline" + "github.com/go-vela/server/database/repo" + "github.com/go-vela/server/database/schedule" + "github.com/go-vela/server/database/secret" + "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/step" + "github.com/go-vela/server/database/user" + "github.com/go-vela/server/database/worker" +) + +// NewResources creates and returns the database agnostic engines for resources. +func (e *engine) NewResources() error { + var err error + + // create the database agnostic engine for builds + e.BuildInterface, err = build.New( + build.WithClient(e.Database), + build.WithLogger(e.Logger), + build.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for hooks + e.HookInterface, err = hook.New( + hook.WithClient(e.Database), + hook.WithLogger(e.Logger), + hook.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for logs + e.LogInterface, err = log.New( + log.WithClient(e.Database), + log.WithCompressionLevel(e.Config.CompressionLevel), + log.WithLogger(e.Logger), + log.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for pipelines + e.PipelineInterface, err = pipeline.New( + pipeline.WithClient(e.Database), + pipeline.WithCompressionLevel(e.Config.CompressionLevel), + pipeline.WithLogger(e.Logger), + pipeline.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for repos + e.RepoInterface, err = repo.New( + repo.WithClient(e.Database), + repo.WithEncryptionKey(e.Config.EncryptionKey), + repo.WithLogger(e.Logger), + repo.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for schedules + e.ScheduleInterface, err = schedule.New( + schedule.WithClient(e.Database), + schedule.WithLogger(e.Logger), + schedule.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for secrets + // + // https://pkg.go.dev/github.com/go-vela/server/database/secret#New + e.SecretInterface, err = secret.New( + secret.WithClient(e.Database), + secret.WithEncryptionKey(e.Config.EncryptionKey), + secret.WithLogger(e.Logger), + secret.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for services + e.ServiceInterface, err = service.New( + service.WithClient(e.Database), + service.WithLogger(e.Logger), + service.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for steps + e.StepInterface, err = step.New( + step.WithClient(e.Database), + step.WithLogger(e.Logger), + step.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for users + e.UserInterface, err = user.New( + user.WithClient(e.Database), + user.WithEncryptionKey(e.Config.EncryptionKey), + user.WithLogger(e.Logger), + user.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for workers + e.WorkerInterface, err = worker.New( + worker.WithClient(e.Database), + worker.WithLogger(e.Logger), + worker.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + return nil +} diff --git a/database/resource_test.go b/database/resource_test.go new file mode 100644 index 000000000..36a4694e6 --- /dev/null +++ b/database/resource_test.go @@ -0,0 +1,112 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/server/database/build" + "github.com/go-vela/server/database/hook" + "github.com/go-vela/server/database/log" + "github.com/go-vela/server/database/pipeline" + "github.com/go-vela/server/database/repo" + "github.com/go-vela/server/database/schedule" + "github.com/go-vela/server/database/secret" + "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/step" + "github.com/go-vela/server/database/user" + "github.com/go-vela/server/database/worker" +) + +func TestDatabase_Engine_NewResources(t *testing.T) { + _postgres, _mock := testPostgres(t) + defer _postgres.Close() + + // ensure the mock expects the build queries + _mock.ExpectExec(build.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the hook queries + _mock.ExpectExec(hook.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(hook.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the log queries + _mock.ExpectExec(log.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(log.CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the pipeline queries + _mock.ExpectExec(pipeline.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(pipeline.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the repo queries + _mock.ExpectExec(repo.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(repo.CreateOrgNameIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the schedule queries + _mock.ExpectExec(schedule.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(schedule.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the secret queries + _mock.ExpectExec(secret.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(secret.CreateTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(secret.CreateTypeOrgTeam).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(secret.CreateTypeOrg).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the service queries + _mock.ExpectExec(service.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the step queries + _mock.ExpectExec(step.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the user queries + _mock.ExpectExec(user.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(user.CreateUserRefreshIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the worker queries + _mock.ExpectExec(worker.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(worker.CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + // create a test database without mocking the call + _unmocked, _ := testPostgres(t) + + _sqlite := testSqlite(t) + defer _sqlite.Close() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + name: "success with postgres", + failure: false, + database: _postgres, + }, + { + name: "success with sqlite3", + failure: false, + database: _sqlite, + }, + { + name: "failure without mocked call", + failure: true, + database: _unmocked, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.NewResources() + + if test.failure { + if err == nil { + t.Errorf("NewResources for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("NewResources for %s returned err: %v", test.name, err) + } + }) + } +} From 63d323f4af2039cc83bb1d8153f095ed552daa65 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Thu, 8 Jun 2023 08:39:23 -0600 Subject: [PATCH 27/53] fix(mock): correct query param mocks (#866) Co-authored-by: David May <49894298+wass3rw3rk@users.noreply.github.com> --- mock/server/build.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mock/server/build.go b/mock/server/build.go index db526cb29..6d989d8af 100644 --- a/mock/server/build.go +++ b/mock/server/build.go @@ -342,18 +342,18 @@ func buildToken(c *gin.Context) { // cleanResources has a query param :before returns mock JSON for a http PUT // -// Pass "0" to :before to test receiving a http 500 response. Pass "1" to :before +// Pass "1" to :before to test receiving a http 500 response. Pass "2" to :before // to test receiving a http 401 response. func cleanResoures(c *gin.Context) { before := c.Query("before") - if strings.EqualFold(before, "0") { + if strings.EqualFold(before, "1") { c.AbortWithStatusJSON(http.StatusInternalServerError, "") return } - if strings.EqualFold(before, "1") { + if strings.EqualFold(before, "2") { c.AbortWithStatusJSON(http.StatusUnauthorized, "") } From da734388cd3b8415bf714415aadd76a3f20cfdf7 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Thu, 8 Jun 2023 08:51:22 -0600 Subject: [PATCH 28/53] refactor(db/log): return library log on created and updated (#865) Co-authored-by: dave vader <48764154+plyr4@users.noreply.github.com> --- api/log/create_service.go | 5 +---- api/log/create_step.go | 5 +---- api/log/update_service.go | 5 +---- api/log/update_step.go | 5 +---- api/service/plan.go | 2 +- api/step/plan.go | 2 +- database/log/count_build_test.go | 4 ++-- database/log/count_test.go | 4 ++-- database/log/create.go | 15 +++++++-------- database/log/create_test.go | 2 +- database/log/delete_test.go | 2 +- database/log/get_service_test.go | 2 +- database/log/get_step_test.go | 2 +- database/log/get_test.go | 2 +- database/log/interface.go | 4 ++-- database/log/list_build_test.go | 4 ++-- database/log/list_test.go | 4 ++-- database/log/update.go | 15 +++++++-------- database/log/update_test.go | 6 +++--- 19 files changed, 38 insertions(+), 52 deletions(-) diff --git a/api/log/create_service.go b/api/log/create_service.go index dd559073c..c6014f09b 100644 --- a/api/log/create_service.go +++ b/api/log/create_service.go @@ -113,7 +113,7 @@ func CreateServiceLog(c *gin.Context) { input.SetRepoID(r.GetID()) // send API call to create the logs - err = database.FromContext(c).CreateLog(input) + l, err := database.FromContext(c).CreateLog(input) if err != nil { retErr := fmt.Errorf("unable to create logs for service %s: %w", entry, err) @@ -122,8 +122,5 @@ func CreateServiceLog(c *gin.Context) { return } - // send API call to capture the created log - l, _ := database.FromContext(c).GetLogForService(s) - c.JSON(http.StatusCreated, l) } diff --git a/api/log/create_step.go b/api/log/create_step.go index 0f8a1ca5a..5bc83c646 100644 --- a/api/log/create_step.go +++ b/api/log/create_step.go @@ -113,7 +113,7 @@ func CreateStepLog(c *gin.Context) { input.SetRepoID(r.GetID()) // send API call to create the logs - err = database.FromContext(c).CreateLog(input) + l, err := database.FromContext(c).CreateLog(input) if err != nil { retErr := fmt.Errorf("unable to create logs for step %s: %w", entry, err) @@ -122,8 +122,5 @@ func CreateStepLog(c *gin.Context) { return } - // send API call to capture the created log - l, _ := database.FromContext(c).GetLogForStep(s) - c.JSON(http.StatusCreated, l) } diff --git a/api/log/update_service.go b/api/log/update_service.go index 746fd63e5..82b56f13b 100644 --- a/api/log/update_service.go +++ b/api/log/update_service.go @@ -124,7 +124,7 @@ func UpdateServiceLog(c *gin.Context) { } // send API call to update the log - err = database.FromContext(c).UpdateLog(l) + l, err = database.FromContext(c).UpdateLog(l) if err != nil { retErr := fmt.Errorf("unable to update logs for service %s: %w", entry, err) @@ -133,8 +133,5 @@ func UpdateServiceLog(c *gin.Context) { return } - // send API call to capture the updated log - l, _ = database.FromContext(c).GetLogForService(s) - c.JSON(http.StatusOK, l) } diff --git a/api/log/update_step.go b/api/log/update_step.go index a86a74113..dcb4a99d1 100644 --- a/api/log/update_step.go +++ b/api/log/update_step.go @@ -124,7 +124,7 @@ func UpdateStepLog(c *gin.Context) { } // send API call to update the log - err = database.FromContext(c).UpdateLog(l) + l, err = database.FromContext(c).UpdateLog(l) if err != nil { retErr := fmt.Errorf("unable to update logs for step %s: %w", entry, err) @@ -133,8 +133,5 @@ func UpdateStepLog(c *gin.Context) { return } - // send API call to capture the updated log - l, _ = database.FromContext(c).GetLogForStep(s) - c.JSON(http.StatusOK, l) } diff --git a/api/service/plan.go b/api/service/plan.go index 4912f38d1..fecd03f24 100644 --- a/api/service/plan.go +++ b/api/service/plan.go @@ -61,7 +61,7 @@ func PlanServices(database database.Interface, p *pipeline.Build, b *library.Bui l.SetData([]byte{}) // send API call to create the service logs - err = database.CreateLog(l) + _, err = database.CreateLog(l) if err != nil { return services, fmt.Errorf("unable to create service logs for service %s: %w", s.GetName(), err) } diff --git a/api/step/plan.go b/api/step/plan.go index b072cf4f7..c74a52614 100644 --- a/api/step/plan.go +++ b/api/step/plan.go @@ -88,7 +88,7 @@ func planStep(database database.Interface, b *library.Build, c *pipeline.Contain l.SetData([]byte{}) // send API call to create the step logs - err = database.CreateLog(l) + _, err = database.CreateLog(l) if err != nil { return nil, fmt.Errorf("unable to create logs for step %s: %w", s.GetName(), err) } diff --git a/database/log/count_build_test.go b/database/log/count_build_test.go index d462eedf0..e168f80ec 100644 --- a/database/log/count_build_test.go +++ b/database/log/count_build_test.go @@ -43,12 +43,12 @@ func TestLog_Engine_CountLogsForBuild(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateLog(_service) + _, err := _sqlite.CreateLog(_service) if err != nil { t.Errorf("unable to create test service log for sqlite: %v", err) } - err = _sqlite.CreateLog(_step) + _, err = _sqlite.CreateLog(_step) if err != nil { t.Errorf("unable to create test step log for sqlite: %v", err) } diff --git a/database/log/count_test.go b/database/log/count_test.go index 99e75a767..2af820203 100644 --- a/database/log/count_test.go +++ b/database/log/count_test.go @@ -37,12 +37,12 @@ func TestLog_Engine_CountLogs(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateLog(_service) + _, err := _sqlite.CreateLog(_service) if err != nil { t.Errorf("unable to create test service log for sqlite: %v", err) } - err = _sqlite.CreateLog(_step) + _, err = _sqlite.CreateLog(_step) if err != nil { t.Errorf("unable to create test step log for sqlite: %v", err) } diff --git a/database/log/create.go b/database/log/create.go index 978da9a1f..b1767c01e 100644 --- a/database/log/create.go +++ b/database/log/create.go @@ -14,7 +14,7 @@ import ( ) // CreateLog creates a new log in the database. -func (e *engine) CreateLog(l *library.Log) error { +func (e *engine) CreateLog(l *library.Log) (*library.Log, error) { // check what the log entry is for switch { case l.GetServiceID() > 0: @@ -33,7 +33,7 @@ func (e *engine) CreateLog(l *library.Log) error { // https://pkg.go.dev/github.com/go-vela/types/database#Log.Validate err := log.Validate() if err != nil { - return err + return nil, err } // compress log data for the resource @@ -43,15 +43,14 @@ func (e *engine) CreateLog(l *library.Log) error { if err != nil { switch { case l.GetServiceID() > 0: - return fmt.Errorf("unable to compress log for service %d for build %d: %w", l.GetServiceID(), l.GetBuildID(), err) + return nil, fmt.Errorf("unable to compress log for service %d for build %d: %w", l.GetServiceID(), l.GetBuildID(), err) case l.GetStepID() > 0: - return fmt.Errorf("unable to compress log for step %d for build %d: %w", l.GetStepID(), l.GetBuildID(), err) + return nil, fmt.Errorf("unable to compress log for step %d for build %d: %w", l.GetStepID(), l.GetBuildID(), err) } } // send query to the database - return e.client. - Table(constants.TableLog). - Create(log). - Error + result := e.client.Table(constants.TableLog).Create(log) + + return log.ToLibrary(), result.Error } diff --git a/database/log/create_test.go b/database/log/create_test.go index 1574e88a8..68e0a2d40 100644 --- a/database/log/create_test.go +++ b/database/log/create_test.go @@ -73,7 +73,7 @@ VALUES ($1,$2,$3,$4,$5,$6) RETURNING "id"`). for _, test := range tests { t.Run(test.name, func(t *testing.T) { for _, log := range test.logs { - err := test.database.CreateLog(log) + _, err := test.database.CreateLog(log) if test.failure { if err == nil { diff --git a/database/log/delete_test.go b/database/log/delete_test.go index 15329a0af..dbfdbca00 100644 --- a/database/log/delete_test.go +++ b/database/log/delete_test.go @@ -29,7 +29,7 @@ func TestLog_Engine_DeleteLog(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateLog(_log) + _, err := _sqlite.CreateLog(_log) if err != nil { t.Errorf("unable to create test log for sqlite: %v", err) } diff --git a/database/log/get_service_test.go b/database/log/get_service_test.go index 26f42813c..3ff82d6c8 100644 --- a/database/log/get_service_test.go +++ b/database/log/get_service_test.go @@ -42,7 +42,7 @@ func TestLog_Engine_GetLogForService(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateLog(_log) + _, err := _sqlite.CreateLog(_log) if err != nil { t.Errorf("unable to create test log for sqlite: %v", err) } diff --git a/database/log/get_step_test.go b/database/log/get_step_test.go index 39f019039..d092e30a9 100644 --- a/database/log/get_step_test.go +++ b/database/log/get_step_test.go @@ -42,7 +42,7 @@ func TestLog_Engine_GetLogForStep(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateLog(_log) + _, err := _sqlite.CreateLog(_log) if err != nil { t.Errorf("unable to create test log for sqlite: %v", err) } diff --git a/database/log/get_test.go b/database/log/get_test.go index 31325e3ac..7127d3165 100644 --- a/database/log/get_test.go +++ b/database/log/get_test.go @@ -35,7 +35,7 @@ func TestLog_Engine_GetLog(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateLog(_log) + _, err := _sqlite.CreateLog(_log) if err != nil { t.Errorf("unable to create test log for sqlite: %v", err) } diff --git a/database/log/interface.go b/database/log/interface.go index 8c72a5098..5b71be967 100644 --- a/database/log/interface.go +++ b/database/log/interface.go @@ -31,7 +31,7 @@ type LogInterface interface { // CountLogsForBuild defines a function that gets the count of logs by build ID. CountLogsForBuild(*library.Build) (int64, error) // CreateLog defines a function that creates a new log. - CreateLog(*library.Log) error + CreateLog(*library.Log) (*library.Log, error) // DeleteLog defines a function that deletes an existing log. DeleteLog(*library.Log) error // GetLog defines a function that gets a log by ID. @@ -45,5 +45,5 @@ type LogInterface interface { // ListLogsForBuild defines a function that gets a list of logs by build ID. ListLogsForBuild(*library.Build, int, int) ([]*library.Log, int64, error) // UpdateLog defines a function that updates an existing log. - UpdateLog(*library.Log) error + UpdateLog(*library.Log) (*library.Log, error) } diff --git a/database/log/list_build_test.go b/database/log/list_build_test.go index fb08236e9..7ffcc0f46 100644 --- a/database/log/list_build_test.go +++ b/database/log/list_build_test.go @@ -54,12 +54,12 @@ func TestLog_Engine_ListLogsForBuild(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateLog(_service) + _, err := _sqlite.CreateLog(_service) if err != nil { t.Errorf("unable to create test service log for sqlite: %v", err) } - err = _sqlite.CreateLog(_step) + _, err = _sqlite.CreateLog(_step) if err != nil { t.Errorf("unable to create test step log for sqlite: %v", err) } diff --git a/database/log/list_test.go b/database/log/list_test.go index 0cf420255..343af9d5a 100644 --- a/database/log/list_test.go +++ b/database/log/list_test.go @@ -48,12 +48,12 @@ func TestLog_Engine_ListLogs(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateLog(_service) + _, err := _sqlite.CreateLog(_service) if err != nil { t.Errorf("unable to create test service log for sqlite: %v", err) } - err = _sqlite.CreateLog(_step) + _, err = _sqlite.CreateLog(_step) if err != nil { t.Errorf("unable to create test step log for sqlite: %v", err) } diff --git a/database/log/update.go b/database/log/update.go index fb7165004..62ba865bc 100644 --- a/database/log/update.go +++ b/database/log/update.go @@ -14,7 +14,7 @@ import ( ) // UpdateLog updates an existing log in the database. -func (e *engine) UpdateLog(l *library.Log) error { +func (e *engine) UpdateLog(l *library.Log) (*library.Log, error) { // check what the log entry is for switch { case l.GetServiceID() > 0: @@ -33,7 +33,7 @@ func (e *engine) UpdateLog(l *library.Log) error { // https://pkg.go.dev/github.com/go-vela/types/database#Log.Validate err := log.Validate() if err != nil { - return err + return nil, err } // compress log data for the resource @@ -43,15 +43,14 @@ func (e *engine) UpdateLog(l *library.Log) error { if err != nil { switch { case l.GetServiceID() > 0: - return fmt.Errorf("unable to compress log for service %d for build %d: %w", l.GetServiceID(), l.GetBuildID(), err) + return nil, fmt.Errorf("unable to compress log for service %d for build %d: %w", l.GetServiceID(), l.GetBuildID(), err) case l.GetStepID() > 0: - return fmt.Errorf("unable to compress log for step %d for build %d: %w", l.GetStepID(), l.GetBuildID(), err) + return nil, fmt.Errorf("unable to compress log for step %d for build %d: %w", l.GetStepID(), l.GetBuildID(), err) } } // send query to the database - return e.client. - Table(constants.TableLog). - Save(log). - Error + result := e.client.Table(constants.TableLog).Save(log) + + return log.ToLibrary(), result.Error } diff --git a/database/log/update_test.go b/database/log/update_test.go index 0b4e8e127..9659cfb65 100644 --- a/database/log/update_test.go +++ b/database/log/update_test.go @@ -47,12 +47,12 @@ WHERE "id" = $6`). _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateLog(_service) + _, err := _sqlite.CreateLog(_service) if err != nil { t.Errorf("unable to create test service log for sqlite: %v", err) } - err = _sqlite.CreateLog(_step) + _, err = _sqlite.CreateLog(_step) if err != nil { t.Errorf("unable to create test step log for sqlite: %v", err) } @@ -82,7 +82,7 @@ WHERE "id" = $6`). for _, test := range tests { t.Run(test.name, func(t *testing.T) { for _, log := range test.logs { - err = test.database.UpdateLog(log) + _, err = test.database.UpdateLog(log) if test.failure { if err == nil { From 470791d51178a9f96a33eb288c3fd102a5ae6dc7 Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Thu, 8 Jun 2023 11:45:26 -0500 Subject: [PATCH 29/53] refactor(database): use agnostic engine (#873) * feat(database): add agnostic engine * feat(database): add config.Validate() * feat(database): add engine.Close() * feat(database): add engine.Driver() * chore: minor fixes * feat(database): add engine.Ping() * feat(database): add engine.NewResources() * refactor(database): use agnostic engine * feat(database): add FromCLIContext() * chore: use database.FromCLIContext() * chore(database): remove unused code * chore: misc updates * chore: fix typos --------- Co-authored-by: Easton Crupper <65553218+ecrupper@users.noreply.github.com> --- cmd/vela-server/database.go | 35 ------ cmd/vela-server/server.go | 3 +- database/context.go | 21 +++- database/context_test.go | 177 +++++++++++++++++--------- database/database.go | 98 +++++++++++---- database/database_test.go | 45 ++++--- database/flags.go | 9 +- database/interface.go | 18 ++- database/setup.go | 140 --------------------- database/setup_test.go | 245 ------------------------------------ 10 files changed, 256 insertions(+), 535 deletions(-) delete mode 100644 cmd/vela-server/database.go delete mode 100644 database/setup.go delete mode 100644 database/setup_test.go diff --git a/cmd/vela-server/database.go b/cmd/vela-server/database.go deleted file mode 100644 index e4db992ec..000000000 --- a/cmd/vela-server/database.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package main - -import ( - "github.com/go-vela/server/database" - - "github.com/sirupsen/logrus" - - "github.com/urfave/cli/v2" -) - -// helper function to setup the database from the CLI arguments. -func setupDatabase(c *cli.Context) (database.Interface, error) { - logrus.Debug("Creating database client from CLI configuration") - - // database configuration - _setup := &database.Setup{ - Driver: c.String("database.driver"), - Address: c.String("database.addr"), - CompressionLevel: c.Int("database.compression.level"), - ConnectionLife: c.Duration("database.connection.life"), - ConnectionIdle: c.Int("database.connection.idle"), - ConnectionOpen: c.Int("database.connection.open"), - EncryptionKey: c.String("database.encryption.key"), - SkipCreation: c.Bool("database.skip_creation"), - } - - // setup the database - // - // https://pkg.go.dev/github.com/go-vela/server/database?tab=doc#New - return database.New(_setup) -} diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 80508e4b0..0d95e88e5 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -15,6 +15,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" "github.com/go-vela/server/router" "github.com/go-vela/server/router/middleware" "github.com/sirupsen/logrus" @@ -61,7 +62,7 @@ func server(c *cli.Context) error { return err } - database, err := setupDatabase(c) + database, err := database.FromCLIContext(c) if err != nil { return err } diff --git a/database/context.go b/database/context.go index f3a872602..d27d73550 100644 --- a/database/context.go +++ b/database/context.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. @@ -6,6 +6,9 @@ package database import ( "context" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" ) const key = "database" @@ -35,3 +38,19 @@ func FromContext(c context.Context) Interface { func ToContext(c Setter, d Interface) { c.Set(key, d) } + +// FromCLIContext creates and returns a database engine from the urfave/cli context. +func FromCLIContext(c *cli.Context) (Interface, error) { + logrus.Debug("creating database engine from CLI configuration") + + return New(&Config{ + Address: c.String("database.addr"), + CompressionLevel: c.Int("database.compression.level"), + ConnectionLife: c.Duration("database.connection.life"), + ConnectionIdle: c.Int("database.connection.idle"), + ConnectionOpen: c.Int("database.connection.open"), + Driver: c.String("database.driver"), + EncryptionKey: c.String("database.encryption.key"), + SkipCreation: c.Bool("database.skip_creation"), + }) +} diff --git a/database/context_test.go b/database/context_test.go index 382567a43..4037b26c0 100644 --- a/database/context_test.go +++ b/database/context_test.go @@ -1,91 +1,152 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. package database import ( + "flag" + "reflect" "testing" + "time" "github.com/gin-gonic/gin" - "github.com/go-vela/server/database/sqlite" + "github.com/urfave/cli/v2" ) func TestDatabase_FromContext(t *testing.T) { - // setup types - want, _ := sqlite.NewTest() + _postgres, _ := testPostgres(t) + defer _postgres.Close() - defer func() { _sql, _ := want.Sqlite.DB(); _sql.Close() }() - - // setup context gin.SetMode(gin.TestMode) - context, _ := gin.CreateTestContext(nil) - context.Set(key, want) - - // run test - got := FromContext(context) - - if got != want { - t.Errorf("FromContext is %v, want %v", got, want) + ctx, _ := gin.CreateTestContext(nil) + ctx.Set(key, _postgres) + + typeCtx, _ := gin.CreateTestContext(nil) + typeCtx.Set(key, nil) + + nilCtx, _ := gin.CreateTestContext(nil) + nilCtx.Set(key, nil) + + // setup tests + tests := []struct { + name string + context *gin.Context + want Interface + }{ + { + name: "success", + context: ctx, + want: _postgres, + }, + { + name: "failure with nil", + context: nilCtx, + want: nil, + }, + { + name: "failure with wrong type", + context: typeCtx, + want: nil, + }, } -} -func TestDatabase_FromContext_Bad(t *testing.T) { - // setup context - gin.SetMode(gin.TestMode) - context, _ := gin.CreateTestContext(nil) - context.Set(key, nil) - - // run test - got := FromContext(context) - - if got != nil { - t.Errorf("FromContext is %v, want nil", got) + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := FromContext(test.context) + if !reflect.DeepEqual(got, test.want) { + t.Errorf("FromContext for %s is %v, want %v", test.name, got, test.want) + } + }) } } -func TestDatabase_FromContext_WrongType(t *testing.T) { - // setup context - gin.SetMode(gin.TestMode) +func TestDatabase_ToContext(t *testing.T) { context, _ := gin.CreateTestContext(nil) - context.Set(key, 1) - - // run test - got := FromContext(context) - if got != nil { - t.Errorf("FromContext is %v, want nil", got) + _postgres, _ := testPostgres(t) + defer _postgres.Close() + + _sqlite := testSqlite(t) + defer _sqlite.Close() + + // setup tests + tests := []struct { + name string + database *engine + want *engine + }{ + { + name: "success with postgres", + database: _postgres, + want: _postgres, + }, + { + name: "success with sqlite3", + database: _sqlite, + want: _sqlite, + }, } -} - -func TestDatabase_FromContext_Empty(t *testing.T) { - // setup context - gin.SetMode(gin.TestMode) - context, _ := gin.CreateTestContext(nil) - // run test - got := FromContext(context) + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ToContext(context, test.want) - if got != nil { - t.Errorf("FromContext is %v, want nil", got) + got := context.Value(key) + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToContext for %s is %v, want %v", test.name, got, test.want) + } + }) } } -func TestDatabase_ToContext(t *testing.T) { - // setup types - want, _ := sqlite.NewTest() +func TestDatabase_FromCLIContext(t *testing.T) { + flags := flag.NewFlagSet("test", 0) + flags.String("database.driver", "sqlite3", "doc") + flags.String("database.addr", "file::memory:?cache=shared", "doc") + flags.Int("database.compression.level", 3, "doc") + flags.Duration("database.connection.life", 10*time.Second, "doc") + flags.Int("database.connection.idle", 5, "doc") + flags.Int("database.connection.open", 20, "doc") + flags.String("database.encryption.key", "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", "doc") + flags.Bool("database.skip_creation", true, "doc") + + // setup tests + tests := []struct { + name string + failure bool + context *cli.Context + }{ + { + name: "success", + failure: false, + context: cli.NewContext(&cli.App{Name: "vela"}, flags, nil), + }, + { + name: "failure", + failure: true, + context: cli.NewContext(&cli.App{Name: "vela"}, flag.NewFlagSet("test", 0), nil), + }, + } - defer func() { _sql, _ := want.Sqlite.DB(); _sql.Close() }() + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := FromCLIContext(test.context) - // setup context - gin.SetMode(gin.TestMode) - context, _ := gin.CreateTestContext(nil) - ToContext(context, want) + if test.failure { + if err == nil { + t.Errorf("FromCLIContext for %s should have returned err", test.name) + } - // run test - got := context.Value(key) + return + } - if got != want { - t.Errorf("ToContext is %v, want %v", got, want) + if err != nil { + t.Errorf("FromCLIContext for %s returned err: %v", test.name, err) + } + }) } } diff --git a/database/database.go b/database/database.go index bf0ca2724..5fe3de951 100644 --- a/database/database.go +++ b/database/database.go @@ -22,6 +22,8 @@ import ( "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" "gorm.io/gorm" ) @@ -32,10 +34,10 @@ type ( Address string // specifies the level of compression to use for the database engine CompressionLevel int - // specifies the maximum idle connections for the database engine - ConnectionIdle int // specifies the connection duration to use for the database engine ConnectionLife time.Duration + // specifies the maximum idle connections for the database engine + ConnectionIdle int // specifies the maximum open connections for the database engine ConnectionOpen int // specifies the driver to use for the database engine @@ -66,38 +68,86 @@ type ( } ) -// New creates and returns a Vela service capable of -// integrating with the configured database provider. +// New creates and returns an engine capable of integrating with the configured database provider. // -// Currently the following database providers are supported: +// Currently, the following database providers are supported: // -// * Postgres -// * Sqlite -// . -func New(s *Setup) (Interface, error) { - // validate the setup being provided - // - // https://pkg.go.dev/github.com/go-vela/server/database?tab=doc#Setup.Validate - err := s.Validate() +// * postgres +// * sqlite3 +func New(c *Config) (Interface, error) { + // validate the configuration being provided + err := c.Validate() if err != nil { return nil, err } - logrus.Debug("creating database service from setup") + // create new database engine + e := &engine{ + Config: c, + Database: new(gorm.DB), + Logger: logrus.NewEntry(logrus.StandardLogger()).WithField("database", c.Driver), + } + + e.Logger.Trace("creating database engine from configuration") // process the database driver being provided - switch s.Driver { + switch c.Driver { case constants.DriverPostgres: - // handle the Postgres database driver being provided - // - // https://pkg.go.dev/github.com/go-vela/server/database?tab=doc#Setup.Postgres - return s.Postgres() + // create the new Postgres database client + e.Database, err = gorm.Open(postgres.Open(e.Config.Address), &gorm.Config{}) + if err != nil { + return nil, err + } case constants.DriverSqlite: - // handle the Sqlite database driver being provided - // - // https://pkg.go.dev/github.com/go-vela/server/database?tab=doc#Setup.Sqlite - return s.Sqlite() + // create the new Sqlite database client + e.Database, err = gorm.Open(sqlite.Open(e.Config.Address), &gorm.Config{}) + if err != nil { + return nil, err + } default: // handle an invalid database driver being provided - return nil, fmt.Errorf("invalid database driver provided: %s", s.Driver) + return nil, fmt.Errorf("invalid database driver provided: %s", c.Driver) } + + // capture database/sql database from gorm.io/gorm database + db, err := e.Database.DB() + if err != nil { + return nil, err + } + + // set the maximum amount of time a connection may be reused + db.SetConnMaxLifetime(e.Config.ConnectionLife) + // set the maximum number of connections in the idle connection pool + db.SetMaxIdleConns(e.Config.ConnectionIdle) + // set the maximum number of open connections to the database + db.SetMaxOpenConns(e.Config.ConnectionOpen) + + // verify connection to the database + err = e.Ping() + if err != nil { + return nil, err + } + + // create database agnostic engines for resources + err = e.NewResources() + if err != nil { + return nil, err + } + + return e, nil +} + +// NewTest creates and returns an engine that integrates with an in-memory database provider. +// +// This function is ONLY intended to be used for testing purposes. +func NewTest() (Interface, error) { + return New(&Config{ + Address: "file::memory:?cache=shared", + CompressionLevel: 3, + ConnectionLife: 30 * time.Minute, + ConnectionIdle: 2, + ConnectionOpen: 0, + Driver: "sqlite3", + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }) } diff --git a/database/database_test.go b/database/database_test.go index 77de76d5b..6d18421bc 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -20,11 +20,13 @@ func TestDatabase_New(t *testing.T) { // setup tests tests := []struct { failure bool - setup *Setup + name string + config *Config }{ { + name: "failure with postgres", failure: true, - setup: &Setup{ + config: &Config{ Driver: "postgres", Address: "postgres://foo:bar@localhost:5432/vela", CompressionLevel: 3, @@ -36,8 +38,9 @@ func TestDatabase_New(t *testing.T) { }, }, { + name: "success with sqlite3", failure: false, - setup: &Setup{ + config: &Config{ Driver: "sqlite3", Address: "file::memory:?cache=shared", CompressionLevel: 3, @@ -49,10 +52,11 @@ func TestDatabase_New(t *testing.T) { }, }, { + name: "failure with invalid config", failure: true, - setup: &Setup{ - Driver: "mysql", - Address: "foo:bar@tcp(localhost:3306)/vela?charset=utf8mb4&parseTime=True&loc=Local", + config: &Config{ + Driver: "postgres", + Address: "", CompressionLevel: 3, ConnectionLife: 10 * time.Second, ConnectionIdle: 5, @@ -62,10 +66,11 @@ func TestDatabase_New(t *testing.T) { }, }, { + name: "failure with invalid driver", failure: true, - setup: &Setup{ - Driver: "postgres", - Address: "", + config: &Config{ + Driver: "mysql", + Address: "foo:bar@tcp(localhost:3306)/vela?charset=utf8mb4&parseTime=True&loc=Local", CompressionLevel: 3, ConnectionLife: 10 * time.Second, ConnectionIdle: 5, @@ -78,19 +83,21 @@ func TestDatabase_New(t *testing.T) { // run tests for _, test := range tests { - _, err := New(test.setup) + t.Run(test.name, func(t *testing.T) { + _, err := New(test.config) - if test.failure { - if err == nil { - t.Errorf("New should have returned err") - } + if test.failure { + if err == nil { + t.Errorf("New for %s should have returned err", test.name) + } - continue - } + return + } - if err != nil { - t.Errorf("New returned err: %v", err) - } + if err != nil { + t.Errorf("New for %s returned err: %v", test.name, err) + } + }) } } diff --git a/database/flags.go b/database/flags.go index 75d3c109b..f8d4fd36e 100644 --- a/database/flags.go +++ b/database/flags.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. @@ -11,13 +11,8 @@ import ( "github.com/urfave/cli/v2" ) -// Flags represents all supported command line -// interface (CLI) flags for the database. -// -// https://pkg.go.dev/github.com/urfave/cli?tab=doc#Flag +// Flags represents all supported command line interface (CLI) flags for the database. var Flags = []cli.Flag{ - // Database Flags - &cli.StringFlag{ EnvVars: []string{"VELA_DATABASE_DRIVER", "DATABASE_DRIVER"}, FilePath: "/vela/database/driver", diff --git a/database/interface.go b/database/interface.go index 975206c93..eed4b27c8 100644 --- a/database/interface.go +++ b/database/interface.go @@ -18,15 +18,23 @@ import ( "github.com/go-vela/server/database/worker" ) -// Interface represents the interface for Vela integrating -// with the different supported Database backends. +// Interface represents the interface for integrating with the supported database providers. type Interface interface { - // Database Interface Functions + // Generic Interface Functions - // Driver defines a function that outputs - // the configured database driver. + // TODO: Add this function to the interface once other code has been updated to use the agnostic engine. + // + // Close defines a function that stops and terminates the connection to the database. + // Close() error + + // Driver defines a function that outputs the configured database driver. Driver() string + // Ping defines a function that sends a "ping" request to the configured database. + Ping() error + + // Resource Interface Functions + // BuildInterface defines the interface for builds stored in the database. build.BuildInterface diff --git a/database/setup.go b/database/setup.go deleted file mode 100644 index a4b553d4d..000000000 --- a/database/setup.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package database - -import ( - "fmt" - "strings" - "time" - - "github.com/go-vela/server/database/postgres" - "github.com/go-vela/server/database/sqlite" - "github.com/go-vela/types/constants" - "github.com/sirupsen/logrus" -) - -// Setup represents the configuration necessary for -// creating a Vela service capable of integrating -// with a configured database system. -type Setup struct { - // Database Configuration - - // specifies the driver to use for the database client - Driver string - // specifies the address to use for the database client - Address string - // specifies the level of compression to use for the database client - CompressionLevel int - // specifies the connection duration to use for the database client - ConnectionLife time.Duration - // specifies the maximum idle connections for the database client - ConnectionIdle int - // specifies the maximum open connections for the database client - ConnectionOpen int - // specifies the encryption key to use for the database client - EncryptionKey string - // specifies to skip creating tables and indexes for the database client - SkipCreation bool -} - -// Postgres creates and returns a Vela service capable of -// integrating with a Postgres database system. -func (s *Setup) Postgres() (Interface, error) { - logrus.Trace("creating postgres database client from setup") - - // create new Postgres database service - // - // https://pkg.go.dev/github.com/go-vela/server/database/postgres?tab=doc#New - return postgres.New( - postgres.WithAddress(s.Address), - postgres.WithCompressionLevel(s.CompressionLevel), - postgres.WithConnectionLife(s.ConnectionLife), - postgres.WithConnectionIdle(s.ConnectionIdle), - postgres.WithConnectionOpen(s.ConnectionOpen), - postgres.WithEncryptionKey(s.EncryptionKey), - postgres.WithSkipCreation(s.SkipCreation), - ) -} - -// Sqlite creates and returns a Vela service capable of -// integrating with a Sqlite database system. -func (s *Setup) Sqlite() (Interface, error) { - logrus.Trace("creating sqlite database client from setup") - - // create new Sqlite database service - // - // https://pkg.go.dev/github.com/go-vela/server/database/sqlite?tab=doc#New - return sqlite.New( - sqlite.WithAddress(s.Address), - sqlite.WithCompressionLevel(s.CompressionLevel), - sqlite.WithConnectionLife(s.ConnectionLife), - sqlite.WithConnectionIdle(s.ConnectionIdle), - sqlite.WithConnectionOpen(s.ConnectionOpen), - sqlite.WithEncryptionKey(s.EncryptionKey), - sqlite.WithSkipCreation(s.SkipCreation), - ) -} - -// Validate verifies the necessary fields for the -// provided configuration are populated correctly. -func (s *Setup) Validate() error { - logrus.Trace("validating database setup for client") - - // verify a database driver was provided - if len(s.Driver) == 0 { - return fmt.Errorf("no database driver provided") - } - - // verify a database address was provided - if len(s.Address) == 0 { - return fmt.Errorf("no database address provided") - } - - // check if the database address has a trailing slash - if strings.HasSuffix(s.Address, "/") { - return fmt.Errorf("database address must not have trailing slash") - } - - // verify a database encryption key was provided - if len(s.EncryptionKey) == 0 { - return fmt.Errorf("no database encryption key provided") - } - - // verify the database compression level is valid - switch s.CompressionLevel { - case constants.CompressionNegOne: - fallthrough - case constants.CompressionZero: - fallthrough - case constants.CompressionOne: - fallthrough - case constants.CompressionTwo: - fallthrough - case constants.CompressionThree: - fallthrough - case constants.CompressionFour: - fallthrough - case constants.CompressionFive: - fallthrough - case constants.CompressionSix: - fallthrough - case constants.CompressionSeven: - fallthrough - case constants.CompressionEight: - fallthrough - case constants.CompressionNine: - break - default: - return fmt.Errorf("database compression level must be between %d and %d - provided level: %d", constants.CompressionNegOne, constants.CompressionNine, s.CompressionLevel) - } - - // enforce AES-256 for the encryption key - explicitly check for 32 characters in the key - if len(s.EncryptionKey) != 32 { - return fmt.Errorf("database encryption key must have 32 characters - provided length: %d", len(s.EncryptionKey)) - } - - // setup is valid - return nil -} diff --git a/database/setup_test.go b/database/setup_test.go deleted file mode 100644 index 4ec2c958c..000000000 --- a/database/setup_test.go +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package database - -import ( - "testing" - "time" -) - -func TestDatabase_Setup_Postgres(t *testing.T) { - // setup types - _setup := &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - } - - // setup tests - tests := []struct { - failure bool - setup *Setup - }{ - { - failure: true, - setup: _setup, - }, - { - failure: true, - setup: &Setup{Driver: "postgres"}, - }, - } - - // run tests - for _, test := range tests { - _, err := test.setup.Postgres() - - if test.failure { - if err == nil { - t.Errorf("Postgres should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("Postgres returned err: %v", err) - } - } -} - -func TestDatabase_Setup_Sqlite(t *testing.T) { - // setup types - _setup := &Setup{ - Driver: "sqlite3", - Address: "file::memory:?cache=shared", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - } - - // setup tests - tests := []struct { - failure bool - setup *Setup - }{ - { - failure: false, - setup: _setup, - }, - { - failure: true, - setup: &Setup{Driver: "sqlite3"}, - }, - } - - // run tests - for _, test := range tests { - _, err := test.setup.Sqlite() - - if test.failure { - if err == nil { - t.Errorf("Sqlite should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("Sqlite returned err: %v", err) - } - } -} - -func TestDatabase_Setup_Validate(t *testing.T) { - // setup tests - tests := []struct { - failure bool - setup *Setup - }{ - { - failure: false, - setup: &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - { - failure: false, - setup: &Setup{ - Driver: "sqlite3", - Address: "file::memory:?cache=shared", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - { - failure: false, - setup: &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: -1, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - { - failure: true, - setup: &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela/", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - { - failure: true, - setup: &Setup{ - Driver: "", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - { - failure: true, - setup: &Setup{ - Driver: "postgres", - Address: "", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - { - failure: true, - setup: &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "", - SkipCreation: false, - }, - }, - { - failure: true, - setup: &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0", - SkipCreation: false, - }, - }, - { - failure: true, - setup: &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: 10, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - } - - // run tests - for _, test := range tests { - err := test.setup.Validate() - - if test.failure { - if err == nil { - t.Errorf("Validate should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("Validate returned err: %v", err) - } - } -} From 3e887957406f4c68a87d3e4c837e9f9ba1b2c548 Mon Sep 17 00:00:00 2001 From: dave vader <48764154+plyr4@users.noreply.github.com> Date: Fri, 9 Jun 2023 08:47:21 -0500 Subject: [PATCH 30/53] fix(schedules): increment repo counter after read (#877) --- cmd/vela-server/schedule.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index a71688606..532dd65b7 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -233,7 +233,6 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat } // set the build numbers based off repo counter - r.SetCounter(r.GetCounter() + 1) b.SetNumber(r.GetCounter() + 1) // set the parent equal to the current repo counter b.SetParent(r.GetCounter()) @@ -242,6 +241,7 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat // parent should be "1" if it's the first build ran b.SetParent(1) } + r.SetCounter(r.GetCounter() + 1) // set the build link if a web address is provided if len(metadata.Vela.WebAddress) > 0 { From ad8d7d706204f65252685c1499466d981414ca61 Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Fri, 9 Jun 2023 09:29:34 -0500 Subject: [PATCH 31/53] refactor(testing): use database.NewTest() (#878) * refactor(internal): sqlite.NewTest() -> database.NewTest() * refactor(router): sqlite.NewTest() -> database.NewTest() * refactor(secret): sqlite.NewTest() -> database.NewTest() * chore(database): add Close() to Interface{} --------- Co-authored-by: dave vader <48764154+plyr4@users.noreply.github.com> --- database/interface.go | 4 +- internal/token/refresh_test.go | 24 +- router/middleware/build/build_test.go | 54 ++-- router/middleware/claims/claims_test.go | 17 +- router/middleware/database_test.go | 9 +- router/middleware/org/org_test.go | 19 +- router/middleware/perm/perm_test.go | 280 +++++++++++--------- router/middleware/pipeline/pipeline_test.go | 42 +-- router/middleware/repo/repo_test.go | 35 ++- router/middleware/secret_test.go | 12 +- router/middleware/service/service_test.go | 72 ++--- router/middleware/step/step_test.go | 72 ++--- router/middleware/user/user_test.go | 41 +-- router/middleware/worker/worker_test.go | 19 +- secret/context_test.go | 25 +- secret/native/count_test.go | 26 +- secret/native/create_test.go | 42 +-- secret/native/delete_test.go | 20 +- secret/native/driver_test.go | 7 +- secret/native/get_test.go | 21 +- secret/native/list_test.go | 31 ++- secret/native/native_test.go | 8 +- secret/native/opts_test.go | 8 +- secret/native/update_test.go | 20 +- secret/secret_test.go | 11 +- secret/setup_test.go | 20 +- 26 files changed, 533 insertions(+), 406 deletions(-) diff --git a/database/interface.go b/database/interface.go index eed4b27c8..3ed775707 100644 --- a/database/interface.go +++ b/database/interface.go @@ -22,10 +22,8 @@ import ( type Interface interface { // Generic Interface Functions - // TODO: Add this function to the interface once other code has been updated to use the agnostic engine. - // // Close defines a function that stops and terminates the connection to the database. - // Close() error + Close() error // Driver defines a function that outputs the configured database driver. Driver() string diff --git a/internal/token/refresh_test.go b/internal/token/refresh_test.go index a1873f01e..e306f9ed6 100644 --- a/internal/token/refresh_test.go +++ b/internal/token/refresh_test.go @@ -10,10 +10,10 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/go-vela/server/database/sqlite" + "github.com/go-vela/server/database" "github.com/go-vela/types/constants" "github.com/go-vela/types/library" - jwt "github.com/golang-jwt/jwt/v5" + "github.com/golang-jwt/jwt/v5" ) func TestTokenManager_Refresh(t *testing.T) { @@ -45,12 +45,14 @@ func TestTokenManager_Refresh(t *testing.T) { u.SetRefreshToken(rt) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteUser(u) + db.Close() }() _ = db.CreateUser(u) @@ -102,12 +104,14 @@ func TestTokenManager_Refresh_Expired(t *testing.T) { u.SetRefreshToken(rt) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteUser(u) + db.Close() }() _ = db.CreateUser(u) diff --git a/router/middleware/build/build_test.go b/router/middleware/build/build_test.go index 1822164b4..10af76ab7 100644 --- a/router/middleware/build/build_test.go +++ b/router/middleware/build/build_test.go @@ -10,11 +10,9 @@ import ( "reflect" "testing" - "github.com/go-vela/server/router/middleware/org" - "github.com/gin-gonic/gin" "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" + "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/types/library" ) @@ -85,13 +83,15 @@ func TestBuild_Establish(t *testing.T) { got := new(library.Build) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from builds;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(want) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -129,9 +129,11 @@ func TestBuild_Establish(t *testing.T) { func TestBuild_Establish_NoRepo(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // setup context gin.SetMode(gin.TestMode) @@ -164,12 +166,14 @@ func TestBuild_Establish_NoBuildParameter(t *testing.T) { r.SetVisibility("public") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -210,12 +214,14 @@ func TestBuild_Establish_InvalidBuildParameter(t *testing.T) { r.SetVisibility("public") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -256,12 +262,14 @@ func TestBuild_Establish_NoBuild(t *testing.T) { r.SetVisibility("public") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) diff --git a/router/middleware/claims/claims_test.go b/router/middleware/claims/claims_test.go index 8934a4afd..da28eb319 100644 --- a/router/middleware/claims/claims_test.go +++ b/router/middleware/claims/claims_test.go @@ -13,15 +13,12 @@ import ( "testing" "time" + "github.com/gin-gonic/gin" "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" "github.com/go-vela/server/internal/token" - "github.com/golang-jwt/jwt/v5" - "github.com/go-vela/types/constants" "github.com/go-vela/types/library" - - "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt/v5" ) func TestClaims_Retrieve(t *testing.T) { @@ -271,12 +268,14 @@ func TestClaims_Establish_BadToken(t *testing.T) { u.SetHash("abc") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteUser(u) + db.Close() }() _ = db.CreateUser(u) diff --git a/router/middleware/database_test.go b/router/middleware/database_test.go index a3480a9f5..b0ba77b04 100644 --- a/router/middleware/database_test.go +++ b/router/middleware/database_test.go @@ -12,16 +12,17 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" ) func TestMiddleware_Database(t *testing.T) { // setup types var got database.Interface - want, _ := sqlite.NewTest() - - defer func() { _sql, _ := want.Sqlite.DB(); _sql.Close() }() + want, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer want.Close() // setup context gin.SetMode(gin.TestMode) diff --git a/router/middleware/org/org_test.go b/router/middleware/org/org_test.go index f52e2aea3..aa64710cc 100644 --- a/router/middleware/org/org_test.go +++ b/router/middleware/org/org_test.go @@ -12,7 +12,6 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" "github.com/go-vela/types/library" ) @@ -60,12 +59,14 @@ func TestOrg_Establish(t *testing.T) { got := "" // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -100,9 +101,11 @@ func TestOrg_Establish(t *testing.T) { func TestOrg_Establish_NoOrgParameter(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // setup context gin.SetMode(gin.TestMode) diff --git a/router/middleware/perm/perm_test.go b/router/middleware/perm/perm_test.go index 755db4a12..34498d21d 100644 --- a/router/middleware/perm/perm_test.go +++ b/router/middleware/perm/perm_test.go @@ -11,21 +11,19 @@ import ( "testing" "time" + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/router/middleware/build" "github.com/go-vela/server/router/middleware/claims" "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/repo" - "github.com/golang-jwt/jwt/v5" - - "github.com/gin-gonic/gin" - "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/scm" "github.com/go-vela/server/scm/github" "github.com/go-vela/types/constants" "github.com/go-vela/types/library" + "github.com/golang-jwt/jwt/v5" ) func TestPerm_MustPlatformAdmin(t *testing.T) { @@ -55,12 +53,14 @@ func TestPerm_MustPlatformAdmin(t *testing.T) { tok, _ := tm.MintToken(mto) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteUser(u) + db.Close() }() _ = db.CreateUser(u) @@ -141,12 +141,14 @@ func TestPerm_MustPlatformAdmin_NotAdmin(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteUser(u) + db.Close() }() _ = db.CreateUser(u) @@ -266,12 +268,14 @@ func TestPerm_MustWorkerRegisterToken_PlatAdmin(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteUser(u) + db.Close() }() _ = db.CreateUser(u) @@ -432,13 +436,15 @@ func TestPerm_MustBuildAccess(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -519,14 +525,16 @@ func TestPerm_MustBuildAccess_PlatAdmin(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - db.Sqlite.Exec("delete from builds;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -603,13 +611,15 @@ func TestPerm_MustBuildToken_WrongBuild(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -685,13 +695,15 @@ func TestPerm_MustSecretAdmin_BuildToken_Repo(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -764,13 +776,15 @@ func TestPerm_MustSecretAdmin_BuildToken_Org(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -843,13 +857,15 @@ func TestPerm_MustSecretAdmin_BuildToken_Shared(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -922,13 +938,15 @@ func TestPerm_MustAdmin(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -1018,13 +1036,15 @@ func TestPerm_MustAdmin_PlatAdmin(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -1114,13 +1134,15 @@ func TestPerm_MustAdmin_NotAdmin(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -1210,13 +1232,15 @@ func TestPerm_MustWrite(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -1306,13 +1330,15 @@ func TestPerm_MustWrite_PlatAdmin(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -1402,13 +1428,15 @@ func TestPerm_MustWrite_RepoAdmin(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -1498,13 +1526,15 @@ func TestPerm_MustWrite_NotWrite(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -1594,13 +1624,15 @@ func TestPerm_MustRead(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -1690,13 +1722,15 @@ func TestPerm_MustRead_PlatAdmin(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -1786,13 +1820,15 @@ func TestPerm_MustRead_WorkerBuildToken(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from builds") - db.Sqlite.Exec("delete from repos;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateBuild(b) @@ -1868,13 +1904,15 @@ func TestPerm_MustRead_RepoAdmin(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -1964,13 +2002,15 @@ func TestPerm_MustRead_RepoWrite(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -2060,13 +2100,15 @@ func TestPerm_MustRead_RepoPublic(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) @@ -2156,13 +2198,15 @@ func TestPerm_MustRead_NotRead(t *testing.T) { context, engine := gin.CreateTestContext(resp) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) diff --git a/router/middleware/pipeline/pipeline_test.go b/router/middleware/pipeline/pipeline_test.go index ec4e809ed..4618524fd 100644 --- a/router/middleware/pipeline/pipeline_test.go +++ b/router/middleware/pipeline/pipeline_test.go @@ -17,7 +17,6 @@ import ( "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/router/middleware/claims" "github.com/go-vela/server/router/middleware/org" @@ -97,13 +96,15 @@ func TestPipeline_Establish(t *testing.T) { got := new(library.Pipeline) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from pipelines;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeletePipeline(want) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -141,8 +142,11 @@ func TestPipeline_Establish(t *testing.T) { func TestPipeline_Establish_NoRepo(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // setup context gin.SetMode(gin.TestMode) @@ -175,12 +179,14 @@ func TestPipeline_Establish_NoPipelineParameter(t *testing.T) { r.SetVisibility("public") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -276,13 +282,15 @@ func TestPipeline_Establish_NoPipeline(t *testing.T) { } // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.DeleteUser(u) + db.Close() }() _ = db.CreateRepo(r) diff --git a/router/middleware/repo/repo_test.go b/router/middleware/repo/repo_test.go index d812e1ba7..83051c09c 100644 --- a/router/middleware/repo/repo_test.go +++ b/router/middleware/repo/repo_test.go @@ -14,7 +14,6 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" "github.com/go-vela/types/library" ) @@ -67,12 +66,14 @@ func TestRepo_Establish(t *testing.T) { got := new(library.Repo) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(want) + db.Close() }() _ = db.CreateRepo(want) @@ -108,9 +109,11 @@ func TestRepo_Establish(t *testing.T) { func TestRepo_Establish_NoOrgParameter(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // setup context gin.SetMode(gin.TestMode) @@ -136,9 +139,11 @@ func TestRepo_Establish_NoOrgParameter(t *testing.T) { func TestRepo_Establish_NoRepoParameter(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // setup context gin.SetMode(gin.TestMode) @@ -164,9 +169,11 @@ func TestRepo_Establish_NoRepoParameter(t *testing.T) { func TestRepo_Establish_NoRepo(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // setup context gin.SetMode(gin.TestMode) diff --git a/router/middleware/secret_test.go b/router/middleware/secret_test.go index 55ee9edcf..9ca21fd24 100644 --- a/router/middleware/secret_test.go +++ b/router/middleware/secret_test.go @@ -10,7 +10,7 @@ import ( "reflect" "testing" - "github.com/go-vela/server/database/sqlite" + "github.com/go-vela/server/database" "github.com/go-vela/server/secret" "github.com/go-vela/server/secret/native" @@ -51,14 +51,16 @@ func TestMiddleware_Secret(t *testing.T) { func TestMiddleware_Secrets(t *testing.T) { // setup types - d, _ := sqlite.NewTest() - - defer func() { _sql, _ := d.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() var got secret.Service want, _ := native.New( - native.WithDatabase(d), + native.WithDatabase(db), ) s := map[string]secret.Service{"native": want} diff --git a/router/middleware/service/service_test.go b/router/middleware/service/service_test.go index bf38fc5ad..091a3e170 100644 --- a/router/middleware/service/service_test.go +++ b/router/middleware/service/service_test.go @@ -10,12 +10,10 @@ import ( "reflect" "testing" - "github.com/go-vela/server/router/middleware/org" - "github.com/gin-gonic/gin" "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/types/library" ) @@ -74,14 +72,16 @@ func TestService_Establish(t *testing.T) { got := new(library.Service) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from builds;") - db.Sqlite.Exec("delete from services;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.DeleteService(want) + db.Close() }() _ = db.CreateRepo(r) @@ -121,9 +121,11 @@ func TestService_Establish(t *testing.T) { func TestService_Establish_NoRepo(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // setup context gin.SetMode(gin.TestMode) @@ -159,12 +161,14 @@ func TestService_Establish_NoBuild(t *testing.T) { r.SetVisibility("public") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -210,13 +214,15 @@ func TestService_Establish_NoServiceParameter(t *testing.T) { b.SetNumber(1) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from builds;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -264,13 +270,15 @@ func TestService_Establish_InvalidServiceParameter(t *testing.T) { b.SetNumber(1) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from builds;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -318,13 +326,15 @@ func TestService_Establish_NoService(t *testing.T) { b.SetNumber(1) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from builds;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) diff --git a/router/middleware/step/step_test.go b/router/middleware/step/step_test.go index b9a0d8e9f..2d9e7ac12 100644 --- a/router/middleware/step/step_test.go +++ b/router/middleware/step/step_test.go @@ -10,12 +10,10 @@ import ( "reflect" "testing" - "github.com/go-vela/server/router/middleware/org" - "github.com/gin-gonic/gin" "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/types/library" ) @@ -76,14 +74,16 @@ func TestStep_Establish(t *testing.T) { got := new(library.Step) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from builds;") - db.Sqlite.Exec("delete from steps;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.DeleteStep(want) + db.Close() }() _ = db.CreateRepo(r) @@ -123,9 +123,11 @@ func TestStep_Establish(t *testing.T) { func TestStep_Establish_NoRepo(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // setup context gin.SetMode(gin.TestMode) @@ -161,12 +163,14 @@ func TestStep_Establish_NoBuild(t *testing.T) { r.SetVisibility("public") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -212,13 +216,15 @@ func TestStep_Establish_NoStepParameter(t *testing.T) { b.SetNumber(1) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from builds;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -266,13 +272,15 @@ func TestStep_Establish_InvalidStepParameter(t *testing.T) { b.SetNumber(1) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from builds;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) @@ -320,13 +328,15 @@ func TestStep_Establish_NoStep(t *testing.T) { b.SetNumber(1) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from repos;") - db.Sqlite.Exec("delete from builds;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteBuild(b) + db.DeleteRepo(r) + db.Close() }() _ = db.CreateRepo(r) diff --git a/router/middleware/user/user_test.go b/router/middleware/user/user_test.go index 7561f8a7d..0ea18dfcc 100644 --- a/router/middleware/user/user_test.go +++ b/router/middleware/user/user_test.go @@ -12,18 +12,15 @@ import ( "testing" "time" + "github.com/gin-gonic/gin" "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/router/middleware/claims" "github.com/go-vela/server/scm" "github.com/go-vela/server/scm/github" - "github.com/golang-jwt/jwt/v5" - "github.com/go-vela/types/constants" "github.com/go-vela/types/library" - - "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt/v5" ) func TestUser_Retrieve(t *testing.T) { @@ -89,12 +86,14 @@ func TestUser_Establish(t *testing.T) { }) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from users;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteUser(want) + db.Close() }() _ = db.CreateUser(want) @@ -151,9 +150,11 @@ func TestUser_Establish_NoToken(t *testing.T) { UserRefreshTokenDuration: time.Minute * 30, } // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // setup context gin.SetMode(gin.TestMode) @@ -238,9 +239,11 @@ func TestUser_Establish_NoAuthorizeUser(t *testing.T) { } // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // setup context gin.SetMode(gin.TestMode) @@ -285,9 +288,11 @@ func TestUser_Establish_NoUser(t *testing.T) { secret := "superSecret" // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // setup context gin.SetMode(gin.TestMode) diff --git a/router/middleware/worker/worker_test.go b/router/middleware/worker/worker_test.go index cad19f15e..1cfeadfc7 100644 --- a/router/middleware/worker/worker_test.go +++ b/router/middleware/worker/worker_test.go @@ -12,7 +12,6 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" "github.com/go-vela/types/library" ) @@ -48,12 +47,14 @@ func TestWorker_Establish(t *testing.T) { got := new(library.Worker) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from workers;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteWorker(want) + db.Close() }() _ = db.CreateWorker(want) @@ -88,9 +89,11 @@ func TestWorker_Establish(t *testing.T) { func TestWorker_Establish_NoWorkerParameter(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // setup context gin.SetMode(gin.TestMode) diff --git a/secret/context_test.go b/secret/context_test.go index e9f2de71c..87f6afe06 100644 --- a/secret/context_test.go +++ b/secret/context_test.go @@ -7,20 +7,21 @@ package secret import ( "testing" - "github.com/go-vela/server/database/sqlite" - "github.com/go-vela/server/secret/native" - "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/secret/native" ) func TestSecret_FromContext(t *testing.T) { // setup types - d, _ := sqlite.NewTest() - - defer func() { _sql, _ := d.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() want, err := native.New( - native.WithDatabase(d), + native.WithDatabase(db), ) if err != nil { t.Errorf("New returned err: %v", err) @@ -82,12 +83,14 @@ func TestSecret_FromContext_Empty(t *testing.T) { func TestSecret_ToContext(t *testing.T) { // setup types - d, _ := sqlite.NewTest() - - defer func() { _sql, _ := d.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() want, err := native.New( - native.WithDatabase(d), + native.WithDatabase(db), ) if err != nil { t.Errorf("New returned err: %v", err) diff --git a/secret/native/count_test.go b/secret/native/count_test.go index faed10a88..2d1080029 100644 --- a/secret/native/count_test.go +++ b/secret/native/count_test.go @@ -7,7 +7,7 @@ package native import ( "testing" - "github.com/go-vela/server/database/sqlite" + "github.com/go-vela/server/database" "github.com/go-vela/types/library" ) @@ -28,12 +28,14 @@ func TestNative_Count(t *testing.T) { want := 1 // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from secrets;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteSecret(sec) + db.Close() }() _ = db.CreateSecret(sec) @@ -56,11 +58,13 @@ func TestNative_Count(t *testing.T) { } } -func TestNative_Count_Invalid(t *testing.T) { +func TestNative_Count_Empty(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - _sql, _ := db.Sqlite.DB() - _sql.Close() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // run test s, err := New( @@ -71,8 +75,8 @@ func TestNative_Count_Invalid(t *testing.T) { } got, err := s.Count("repo", "foo", "bar", []string{}) - if err == nil { - t.Errorf("Count should have returned err") + if err != nil { + t.Errorf("Count returned err: %v", err) } if got != 0 { diff --git a/secret/native/create_test.go b/secret/native/create_test.go index 08fd4b18b..328b22ae0 100644 --- a/secret/native/create_test.go +++ b/secret/native/create_test.go @@ -8,7 +8,7 @@ import ( "reflect" "testing" - "github.com/go-vela/server/database/sqlite" + "github.com/go-vela/server/database" "github.com/go-vela/types/library" ) @@ -31,12 +31,14 @@ func TestNative_Create_Org(t *testing.T) { want.SetUpdatedBy("user2") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from secrets;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteSecret(want) + db.Close() }() // run test @@ -78,12 +80,14 @@ func TestNative_Create_Repo(t *testing.T) { want.SetUpdatedBy("user2") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from secrets;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteSecret(want) + db.Close() }() // run test @@ -125,12 +129,14 @@ func TestNative_Create_Shared(t *testing.T) { want.SetUpdatedBy("user2") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from secrets;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteSecret(want) + db.Close() }() // run test @@ -172,12 +178,14 @@ func TestNative_Create_Invalid(t *testing.T) { sec.SetUpdatedBy("user2") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from secrets;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteSecret(sec) + db.Close() }() // run test diff --git a/secret/native/delete_test.go b/secret/native/delete_test.go index 743288f62..e1c49c133 100644 --- a/secret/native/delete_test.go +++ b/secret/native/delete_test.go @@ -7,7 +7,7 @@ package native import ( "testing" - "github.com/go-vela/server/database/sqlite" + "github.com/go-vela/server/database" "github.com/go-vela/types/library" ) @@ -28,12 +28,14 @@ func TestNative_Delete(t *testing.T) { sec.SetUpdatedAt(1) // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from secrets;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteSecret(sec) + db.Close() }() _ = db.CreateSecret(sec) @@ -54,9 +56,11 @@ func TestNative_Delete(t *testing.T) { func TestNative_Delete_Invalid(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // run test s, err := New( diff --git a/secret/native/driver_test.go b/secret/native/driver_test.go index 81cddaeb4..ec0038df5 100644 --- a/secret/native/driver_test.go +++ b/secret/native/driver_test.go @@ -8,18 +8,17 @@ import ( "reflect" "testing" - "github.com/go-vela/server/database/sqlite" + "github.com/go-vela/server/database" "github.com/go-vela/types/constants" ) func TestNative_Driver(t *testing.T) { // setup types - db, err := sqlite.NewTest() + db, err := database.NewTest() if err != nil { t.Errorf("unable to create database service: %v", err) } - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + defer db.Close() want := constants.DriverNative diff --git a/secret/native/get_test.go b/secret/native/get_test.go index 33dc174ce..438ede6a7 100644 --- a/secret/native/get_test.go +++ b/secret/native/get_test.go @@ -8,7 +8,7 @@ import ( "reflect" "testing" - "github.com/go-vela/server/database/sqlite" + "github.com/go-vela/server/database" "github.com/go-vela/types/library" ) @@ -31,12 +31,15 @@ func TestNative_Get(t *testing.T) { want.SetUpdatedBy("user2") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() defer func() { - db.Sqlite.Exec("delete from secrets;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteSecret(want) + db.Close() }() // run test @@ -61,9 +64,11 @@ func TestNative_Get(t *testing.T) { func TestNative_Get_Invalid(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // run test s, err := New( diff --git a/secret/native/list_test.go b/secret/native/list_test.go index ac9fd1c67..ae0870cd6 100644 --- a/secret/native/list_test.go +++ b/secret/native/list_test.go @@ -8,7 +8,7 @@ import ( "reflect" "testing" - "github.com/go-vela/server/database/sqlite" + "github.com/go-vela/server/database" "github.com/go-vela/types/library" ) @@ -49,12 +49,15 @@ func TestNative_List(t *testing.T) { want := []*library.Secret{sTwo, sOne} // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from secrets;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteSecret(sOne) + db.DeleteSecret(sTwo) + db.Close() }() // run test @@ -79,11 +82,13 @@ func TestNative_List(t *testing.T) { } } -func TestNative_List_Invalid(t *testing.T) { +func TestNative_List_Empty(t *testing.T) { // setup database - db, _ := sqlite.NewTest() - _sql, _ := db.Sqlite.DB() - _sql.Close() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // run test s, err := New( @@ -94,11 +99,11 @@ func TestNative_List_Invalid(t *testing.T) { } got, err := s.List("repo", "foo", "bar", 1, 10, []string{}) - if err == nil { - t.Errorf("List should have returned err") + if err != nil { + t.Errorf("List returned err: %v", err) } - if got != nil { - t.Errorf("List is %v, want nil", got) + if len(got) > 0 { + t.Errorf("List is %v, want []", got) } } diff --git a/secret/native/native_test.go b/secret/native/native_test.go index 1b7daaa2f..0a5e3ee49 100644 --- a/secret/native/native_test.go +++ b/secret/native/native_test.go @@ -8,17 +8,15 @@ import ( "testing" "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" ) func TestNative_New(t *testing.T) { // setup types - db, err := sqlite.NewTest() + db, err := database.NewTest() if err != nil { - t.Errorf("unable to create database service: %v", err) + t.Errorf("unable to create test database engine: %v", err) } - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + defer db.Close() // setup tests tests := []struct { diff --git a/secret/native/opts_test.go b/secret/native/opts_test.go index 1cda40700..0d9d968df 100644 --- a/secret/native/opts_test.go +++ b/secret/native/opts_test.go @@ -9,17 +9,15 @@ import ( "testing" "github.com/go-vela/server/database" - "github.com/go-vela/server/database/sqlite" ) func TestNative_ClientOpt_WithDatabase(t *testing.T) { // setup types - db, err := sqlite.NewTest() + db, err := database.NewTest() if err != nil { - t.Errorf("unable to create database service: %v", err) + t.Errorf("unable to create test database engine: %v", err) } - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + defer db.Close() // setup tests tests := []struct { diff --git a/secret/native/update_test.go b/secret/native/update_test.go index 1538fd368..f8c7bf3cb 100644 --- a/secret/native/update_test.go +++ b/secret/native/update_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/go-vela/server/database/sqlite" + "github.com/go-vela/server/database" "github.com/go-vela/types/library" ) @@ -48,12 +48,14 @@ func TestNative_Update(t *testing.T) { want.SetUpdatedBy("user2") // setup database - db, _ := sqlite.NewTest() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } defer func() { - db.Sqlite.Exec("delete from secrets;") - _sql, _ := db.Sqlite.DB() - _sql.Close() + db.DeleteSecret(original) + db.Close() }() _ = db.CreateSecret(original) @@ -85,9 +87,11 @@ func TestNative_Update_Invalid(t *testing.T) { sec.SetValue("foob") // setup database - db, _ := sqlite.NewTest() - - defer func() { _sql, _ := db.Sqlite.DB(); _sql.Close() }() + db, err := database.NewTest() + if err != nil { + t.Errorf("unable to create test database engine: %v", err) + } + defer db.Close() // run test s, err := New( diff --git a/secret/secret_test.go b/secret/secret_test.go index 03cc6a432..21ab33ee4 100644 --- a/secret/secret_test.go +++ b/secret/secret_test.go @@ -7,17 +7,16 @@ package secret import ( "testing" - "github.com/go-vela/server/database/sqlite" + "github.com/go-vela/server/database" ) func TestSecret_New(t *testing.T) { // setup types - _database, err := sqlite.NewTest() + db, err := database.NewTest() if err != nil { - t.Errorf("unable to create database service: %v", err) + t.Errorf("unable to create test database engine: %v", err) } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() + defer db.Close() // setup tests tests := []struct { @@ -28,7 +27,7 @@ func TestSecret_New(t *testing.T) { failure: false, setup: &Setup{ Driver: "native", - Database: _database, + Database: db, }, }, { diff --git a/secret/setup_test.go b/secret/setup_test.go index da5c535c9..ef4492681 100644 --- a/secret/setup_test.go +++ b/secret/setup_test.go @@ -8,21 +8,20 @@ import ( "reflect" "testing" - "github.com/go-vela/server/database/sqlite" + "github.com/go-vela/server/database" ) func TestSecret_Setup_Native(t *testing.T) { // setup types - _database, err := sqlite.NewTest() + db, err := database.NewTest() if err != nil { - t.Errorf("unable to create database service: %v", err) + t.Errorf("unable to create test database engine: %v", err) } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() + defer db.Close() _setup := &Setup{ Driver: "native", - Database: _database, + Database: db, } _native, err := _setup.Native() @@ -126,12 +125,11 @@ func TestSecret_Setup_Vault(t *testing.T) { func TestSecret_Setup_Validate(t *testing.T) { // setup types - _database, err := sqlite.NewTest() + db, err := database.NewTest() if err != nil { - t.Errorf("unable to create database service: %v", err) + t.Errorf("unable to create test database engine: %v", err) } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() + defer db.Close() // setup tests tests := []struct { @@ -142,7 +140,7 @@ func TestSecret_Setup_Validate(t *testing.T) { failure: false, setup: &Setup{ Driver: "native", - Database: _database, + Database: db, }, }, { From f8c795f7aa038582de479fffb0a3b072b0565ed1 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Fri, 9 Jun 2023 08:50:07 -0600 Subject: [PATCH 32/53] enhance(log): do not return log object for POST and PUT requests (#879) --- api/log/create_service.go | 6 ++---- api/log/create_step.go | 6 ++---- api/log/update_service.go | 4 ++-- api/log/update_step.go | 4 ++-- api/service/plan.go | 2 +- api/step/plan.go | 2 +- database/log/count_build_test.go | 4 ++-- database/log/count_test.go | 4 ++-- database/log/create.go | 15 ++++++++------- database/log/create_test.go | 2 +- database/log/delete_test.go | 2 +- database/log/get_service_test.go | 2 +- database/log/get_step_test.go | 2 +- database/log/get_test.go | 2 +- database/log/interface.go | 4 ++-- database/log/list_build_test.go | 4 ++-- database/log/list_test.go | 4 ++-- database/log/update.go | 15 ++++++++------- database/log/update_test.go | 6 +++--- mock/server/log.go | 28 ++++------------------------ 20 files changed, 48 insertions(+), 70 deletions(-) diff --git a/api/log/create_service.go b/api/log/create_service.go index c6014f09b..bf8cd4ec1 100644 --- a/api/log/create_service.go +++ b/api/log/create_service.go @@ -61,8 +61,6 @@ import ( // responses: // '201': // description: Successfully created the service logs -// schema: -// "$ref": "#/definitions/Log" // '400': // description: Unable to create the service logs // schema: @@ -113,7 +111,7 @@ func CreateServiceLog(c *gin.Context) { input.SetRepoID(r.GetID()) // send API call to create the logs - l, err := database.FromContext(c).CreateLog(input) + err = database.FromContext(c).CreateLog(input) if err != nil { retErr := fmt.Errorf("unable to create logs for service %s: %w", entry, err) @@ -122,5 +120,5 @@ func CreateServiceLog(c *gin.Context) { return } - c.JSON(http.StatusCreated, l) + c.JSON(http.StatusCreated, nil) } diff --git a/api/log/create_step.go b/api/log/create_step.go index 5bc83c646..d2abed7a9 100644 --- a/api/log/create_step.go +++ b/api/log/create_step.go @@ -61,8 +61,6 @@ import ( // responses: // '201': // description: Successfully created the logs for step -// schema: -// "$ref": "#/definitions/Log" // '400': // description: Unable to create the logs for a step // schema: @@ -113,7 +111,7 @@ func CreateStepLog(c *gin.Context) { input.SetRepoID(r.GetID()) // send API call to create the logs - l, err := database.FromContext(c).CreateLog(input) + err = database.FromContext(c).CreateLog(input) if err != nil { retErr := fmt.Errorf("unable to create logs for step %s: %w", entry, err) @@ -122,5 +120,5 @@ func CreateStepLog(c *gin.Context) { return } - c.JSON(http.StatusCreated, l) + c.JSON(http.StatusCreated, nil) } diff --git a/api/log/update_service.go b/api/log/update_service.go index 82b56f13b..92d96e1f1 100644 --- a/api/log/update_service.go +++ b/api/log/update_service.go @@ -124,7 +124,7 @@ func UpdateServiceLog(c *gin.Context) { } // send API call to update the log - l, err = database.FromContext(c).UpdateLog(l) + err = database.FromContext(c).UpdateLog(l) if err != nil { retErr := fmt.Errorf("unable to update logs for service %s: %w", entry, err) @@ -133,5 +133,5 @@ func UpdateServiceLog(c *gin.Context) { return } - c.JSON(http.StatusOK, l) + c.JSON(http.StatusOK, nil) } diff --git a/api/log/update_step.go b/api/log/update_step.go index dcb4a99d1..14a642d57 100644 --- a/api/log/update_step.go +++ b/api/log/update_step.go @@ -124,7 +124,7 @@ func UpdateStepLog(c *gin.Context) { } // send API call to update the log - l, err = database.FromContext(c).UpdateLog(l) + err = database.FromContext(c).UpdateLog(l) if err != nil { retErr := fmt.Errorf("unable to update logs for step %s: %w", entry, err) @@ -133,5 +133,5 @@ func UpdateStepLog(c *gin.Context) { return } - c.JSON(http.StatusOK, l) + c.JSON(http.StatusOK, nil) } diff --git a/api/service/plan.go b/api/service/plan.go index fecd03f24..4912f38d1 100644 --- a/api/service/plan.go +++ b/api/service/plan.go @@ -61,7 +61,7 @@ func PlanServices(database database.Interface, p *pipeline.Build, b *library.Bui l.SetData([]byte{}) // send API call to create the service logs - _, err = database.CreateLog(l) + err = database.CreateLog(l) if err != nil { return services, fmt.Errorf("unable to create service logs for service %s: %w", s.GetName(), err) } diff --git a/api/step/plan.go b/api/step/plan.go index c74a52614..b072cf4f7 100644 --- a/api/step/plan.go +++ b/api/step/plan.go @@ -88,7 +88,7 @@ func planStep(database database.Interface, b *library.Build, c *pipeline.Contain l.SetData([]byte{}) // send API call to create the step logs - _, err = database.CreateLog(l) + err = database.CreateLog(l) if err != nil { return nil, fmt.Errorf("unable to create logs for step %s: %w", s.GetName(), err) } diff --git a/database/log/count_build_test.go b/database/log/count_build_test.go index e168f80ec..d462eedf0 100644 --- a/database/log/count_build_test.go +++ b/database/log/count_build_test.go @@ -43,12 +43,12 @@ func TestLog_Engine_CountLogsForBuild(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - _, err := _sqlite.CreateLog(_service) + err := _sqlite.CreateLog(_service) if err != nil { t.Errorf("unable to create test service log for sqlite: %v", err) } - _, err = _sqlite.CreateLog(_step) + err = _sqlite.CreateLog(_step) if err != nil { t.Errorf("unable to create test step log for sqlite: %v", err) } diff --git a/database/log/count_test.go b/database/log/count_test.go index 2af820203..99e75a767 100644 --- a/database/log/count_test.go +++ b/database/log/count_test.go @@ -37,12 +37,12 @@ func TestLog_Engine_CountLogs(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - _, err := _sqlite.CreateLog(_service) + err := _sqlite.CreateLog(_service) if err != nil { t.Errorf("unable to create test service log for sqlite: %v", err) } - _, err = _sqlite.CreateLog(_step) + err = _sqlite.CreateLog(_step) if err != nil { t.Errorf("unable to create test step log for sqlite: %v", err) } diff --git a/database/log/create.go b/database/log/create.go index b1767c01e..978da9a1f 100644 --- a/database/log/create.go +++ b/database/log/create.go @@ -14,7 +14,7 @@ import ( ) // CreateLog creates a new log in the database. -func (e *engine) CreateLog(l *library.Log) (*library.Log, error) { +func (e *engine) CreateLog(l *library.Log) error { // check what the log entry is for switch { case l.GetServiceID() > 0: @@ -33,7 +33,7 @@ func (e *engine) CreateLog(l *library.Log) (*library.Log, error) { // https://pkg.go.dev/github.com/go-vela/types/database#Log.Validate err := log.Validate() if err != nil { - return nil, err + return err } // compress log data for the resource @@ -43,14 +43,15 @@ func (e *engine) CreateLog(l *library.Log) (*library.Log, error) { if err != nil { switch { case l.GetServiceID() > 0: - return nil, fmt.Errorf("unable to compress log for service %d for build %d: %w", l.GetServiceID(), l.GetBuildID(), err) + return fmt.Errorf("unable to compress log for service %d for build %d: %w", l.GetServiceID(), l.GetBuildID(), err) case l.GetStepID() > 0: - return nil, fmt.Errorf("unable to compress log for step %d for build %d: %w", l.GetStepID(), l.GetBuildID(), err) + return fmt.Errorf("unable to compress log for step %d for build %d: %w", l.GetStepID(), l.GetBuildID(), err) } } // send query to the database - result := e.client.Table(constants.TableLog).Create(log) - - return log.ToLibrary(), result.Error + return e.client. + Table(constants.TableLog). + Create(log). + Error } diff --git a/database/log/create_test.go b/database/log/create_test.go index 68e0a2d40..1574e88a8 100644 --- a/database/log/create_test.go +++ b/database/log/create_test.go @@ -73,7 +73,7 @@ VALUES ($1,$2,$3,$4,$5,$6) RETURNING "id"`). for _, test := range tests { t.Run(test.name, func(t *testing.T) { for _, log := range test.logs { - _, err := test.database.CreateLog(log) + err := test.database.CreateLog(log) if test.failure { if err == nil { diff --git a/database/log/delete_test.go b/database/log/delete_test.go index dbfdbca00..15329a0af 100644 --- a/database/log/delete_test.go +++ b/database/log/delete_test.go @@ -29,7 +29,7 @@ func TestLog_Engine_DeleteLog(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - _, err := _sqlite.CreateLog(_log) + err := _sqlite.CreateLog(_log) if err != nil { t.Errorf("unable to create test log for sqlite: %v", err) } diff --git a/database/log/get_service_test.go b/database/log/get_service_test.go index 3ff82d6c8..26f42813c 100644 --- a/database/log/get_service_test.go +++ b/database/log/get_service_test.go @@ -42,7 +42,7 @@ func TestLog_Engine_GetLogForService(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - _, err := _sqlite.CreateLog(_log) + err := _sqlite.CreateLog(_log) if err != nil { t.Errorf("unable to create test log for sqlite: %v", err) } diff --git a/database/log/get_step_test.go b/database/log/get_step_test.go index d092e30a9..39f019039 100644 --- a/database/log/get_step_test.go +++ b/database/log/get_step_test.go @@ -42,7 +42,7 @@ func TestLog_Engine_GetLogForStep(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - _, err := _sqlite.CreateLog(_log) + err := _sqlite.CreateLog(_log) if err != nil { t.Errorf("unable to create test log for sqlite: %v", err) } diff --git a/database/log/get_test.go b/database/log/get_test.go index 7127d3165..31325e3ac 100644 --- a/database/log/get_test.go +++ b/database/log/get_test.go @@ -35,7 +35,7 @@ func TestLog_Engine_GetLog(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - _, err := _sqlite.CreateLog(_log) + err := _sqlite.CreateLog(_log) if err != nil { t.Errorf("unable to create test log for sqlite: %v", err) } diff --git a/database/log/interface.go b/database/log/interface.go index 5b71be967..8c72a5098 100644 --- a/database/log/interface.go +++ b/database/log/interface.go @@ -31,7 +31,7 @@ type LogInterface interface { // CountLogsForBuild defines a function that gets the count of logs by build ID. CountLogsForBuild(*library.Build) (int64, error) // CreateLog defines a function that creates a new log. - CreateLog(*library.Log) (*library.Log, error) + CreateLog(*library.Log) error // DeleteLog defines a function that deletes an existing log. DeleteLog(*library.Log) error // GetLog defines a function that gets a log by ID. @@ -45,5 +45,5 @@ type LogInterface interface { // ListLogsForBuild defines a function that gets a list of logs by build ID. ListLogsForBuild(*library.Build, int, int) ([]*library.Log, int64, error) // UpdateLog defines a function that updates an existing log. - UpdateLog(*library.Log) (*library.Log, error) + UpdateLog(*library.Log) error } diff --git a/database/log/list_build_test.go b/database/log/list_build_test.go index 7ffcc0f46..fb08236e9 100644 --- a/database/log/list_build_test.go +++ b/database/log/list_build_test.go @@ -54,12 +54,12 @@ func TestLog_Engine_ListLogsForBuild(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - _, err := _sqlite.CreateLog(_service) + err := _sqlite.CreateLog(_service) if err != nil { t.Errorf("unable to create test service log for sqlite: %v", err) } - _, err = _sqlite.CreateLog(_step) + err = _sqlite.CreateLog(_step) if err != nil { t.Errorf("unable to create test step log for sqlite: %v", err) } diff --git a/database/log/list_test.go b/database/log/list_test.go index 343af9d5a..0cf420255 100644 --- a/database/log/list_test.go +++ b/database/log/list_test.go @@ -48,12 +48,12 @@ func TestLog_Engine_ListLogs(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - _, err := _sqlite.CreateLog(_service) + err := _sqlite.CreateLog(_service) if err != nil { t.Errorf("unable to create test service log for sqlite: %v", err) } - _, err = _sqlite.CreateLog(_step) + err = _sqlite.CreateLog(_step) if err != nil { t.Errorf("unable to create test step log for sqlite: %v", err) } diff --git a/database/log/update.go b/database/log/update.go index 62ba865bc..fb7165004 100644 --- a/database/log/update.go +++ b/database/log/update.go @@ -14,7 +14,7 @@ import ( ) // UpdateLog updates an existing log in the database. -func (e *engine) UpdateLog(l *library.Log) (*library.Log, error) { +func (e *engine) UpdateLog(l *library.Log) error { // check what the log entry is for switch { case l.GetServiceID() > 0: @@ -33,7 +33,7 @@ func (e *engine) UpdateLog(l *library.Log) (*library.Log, error) { // https://pkg.go.dev/github.com/go-vela/types/database#Log.Validate err := log.Validate() if err != nil { - return nil, err + return err } // compress log data for the resource @@ -43,14 +43,15 @@ func (e *engine) UpdateLog(l *library.Log) (*library.Log, error) { if err != nil { switch { case l.GetServiceID() > 0: - return nil, fmt.Errorf("unable to compress log for service %d for build %d: %w", l.GetServiceID(), l.GetBuildID(), err) + return fmt.Errorf("unable to compress log for service %d for build %d: %w", l.GetServiceID(), l.GetBuildID(), err) case l.GetStepID() > 0: - return nil, fmt.Errorf("unable to compress log for step %d for build %d: %w", l.GetStepID(), l.GetBuildID(), err) + return fmt.Errorf("unable to compress log for step %d for build %d: %w", l.GetStepID(), l.GetBuildID(), err) } } // send query to the database - result := e.client.Table(constants.TableLog).Save(log) - - return log.ToLibrary(), result.Error + return e.client. + Table(constants.TableLog). + Save(log). + Error } diff --git a/database/log/update_test.go b/database/log/update_test.go index 9659cfb65..0b4e8e127 100644 --- a/database/log/update_test.go +++ b/database/log/update_test.go @@ -47,12 +47,12 @@ WHERE "id" = $6`). _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - _, err := _sqlite.CreateLog(_service) + err := _sqlite.CreateLog(_service) if err != nil { t.Errorf("unable to create test service log for sqlite: %v", err) } - _, err = _sqlite.CreateLog(_step) + err = _sqlite.CreateLog(_step) if err != nil { t.Errorf("unable to create test step log for sqlite: %v", err) } @@ -82,7 +82,7 @@ WHERE "id" = $6`). for _, test := range tests { t.Run(test.name, func(t *testing.T) { for _, log := range test.logs { - _, err = test.database.UpdateLog(log) + err = test.database.UpdateLog(log) if test.failure { if err == nil { diff --git a/mock/server/log.go b/mock/server/log.go index 8a101c4d2..3becfff38 100644 --- a/mock/server/log.go +++ b/mock/server/log.go @@ -51,12 +51,7 @@ func getServiceLog(c *gin.Context) { // addServiceLog returns mock JSON for a http GET. func addServiceLog(c *gin.Context) { - data := []byte(LogResp) - - var body library.Log - _ = json.Unmarshal(data, &body) - - c.JSON(http.StatusCreated, body) + c.JSON(http.StatusCreated, nil) } // updateServiceLog has a param :service returns mock JSON for a http PUT. @@ -73,12 +68,7 @@ func updateServiceLog(c *gin.Context) { return } - data := []byte(LogResp) - - var body library.Log - _ = json.Unmarshal(data, &body) - - c.JSON(http.StatusOK, body) + c.JSON(http.StatusOK, nil) } // removeServiceLog has a param :service returns mock JSON for a http DELETE. @@ -122,12 +112,7 @@ func getStepLog(c *gin.Context) { // addStepLog returns mock JSON for a http GET. func addStepLog(c *gin.Context) { - data := []byte(LogResp) - - var body library.Log - _ = json.Unmarshal(data, &body) - - c.JSON(http.StatusCreated, body) + c.JSON(http.StatusCreated, nil) } // updateStepLog has a param :step returns mock JSON for a http PUT. @@ -144,12 +129,7 @@ func updateStepLog(c *gin.Context) { return } - data := []byte(LogResp) - - var body library.Log - _ = json.Unmarshal(data, &body) - - c.JSON(http.StatusOK, body) + c.JSON(http.StatusOK, nil) } // removeStepLog has a param :step returns mock JSON for a http DELETE. From e2e86a745c36cdb2e516a75d5eb1b9929894c6ea Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Fri, 9 Jun 2023 10:16:34 -0500 Subject: [PATCH 33/53] enhance(clone): upgrade target/vela-git to v0.8.0 (#876) --- cmd/vela-server/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index f9b160a8f..19463fe0b 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -84,7 +84,7 @@ func main() { EnvVars: []string{"VELA_CLONE_IMAGE"}, Name: "clone-image", Usage: "the clone image to use for the injected clone step", - Value: "target/vela-git:v0.7.0@sha256:c2e8794556d6debceeaa2c82ff3cc9e8e6ed045b723419e3ff050409f25cc258", + Value: "target/vela-git:v0.8.0@sha256:02de004ae9dbf184c70039cb9ce431c31d6e7580eb9e6ec64a97ebf108aa65cb", }, &cli.StringSliceFlag{ EnvVars: []string{"VELA_REPO_ALLOWLIST"}, From 0410f86a295d5b8bc2d79dd14ed0947bafcb5dae Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Fri, 9 Jun 2023 16:20:11 -0500 Subject: [PATCH 34/53] chore(database): remove unused packages (#880) * refactor(internal): sqlite.NewTest() -> database.NewTest() * refactor(router): sqlite.NewTest() -> database.NewTest() * refactor(secret): sqlite.NewTest() -> database.NewTest() * chore(database): add Close() to Interface{} * chore(database): remove unused postgres package * chore(database): remove unused sqlite package --------- Co-authored-by: Easton Crupper <65553218+ecrupper@users.noreply.github.com> --- database/postgres/doc.go | 11 - database/postgres/driver.go | 12 - database/postgres/driver_test.go | 32 --- database/postgres/opts.go | 107 -------- database/postgres/opts_test.go | 287 -------------------- database/postgres/ping.go | 45 ---- database/postgres/ping_test.go | 65 ----- database/postgres/postgres.go | 411 ----------------------------- database/postgres/postgres_test.go | 240 ----------------- database/sqlite/doc.go | 11 - database/sqlite/driver.go | 12 - database/sqlite/driver_test.go | 32 --- database/sqlite/opts.go | 107 -------- database/sqlite/opts_test.go | 287 -------------------- database/sqlite/ping.go | 45 ---- database/sqlite/ping_test.go | 62 ----- database/sqlite/sqlite.go | 367 -------------------------- database/sqlite/sqlite_test.go | 146 ---------- 18 files changed, 2279 deletions(-) delete mode 100644 database/postgres/doc.go delete mode 100644 database/postgres/driver.go delete mode 100644 database/postgres/driver_test.go delete mode 100644 database/postgres/opts.go delete mode 100644 database/postgres/opts_test.go delete mode 100644 database/postgres/ping.go delete mode 100644 database/postgres/ping_test.go delete mode 100644 database/postgres/postgres.go delete mode 100644 database/postgres/postgres_test.go delete mode 100644 database/sqlite/doc.go delete mode 100644 database/sqlite/driver.go delete mode 100644 database/sqlite/driver_test.go delete mode 100644 database/sqlite/opts.go delete mode 100644 database/sqlite/opts_test.go delete mode 100644 database/sqlite/ping.go delete mode 100644 database/sqlite/ping_test.go delete mode 100644 database/sqlite/sqlite.go delete mode 100644 database/sqlite/sqlite_test.go diff --git a/database/postgres/doc.go b/database/postgres/doc.go deleted file mode 100644 index 0fbbbd274..000000000 --- a/database/postgres/doc.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -// Package postgres provides the ability for Vela to -// integrate with Postgres as a SQL backend. -// -// Usage: -// -// import "github.com/go-vela/server/database/postgres" -package postgres diff --git a/database/postgres/driver.go b/database/postgres/driver.go deleted file mode 100644 index 2a61cae42..000000000 --- a/database/postgres/driver.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import "github.com/go-vela/types/constants" - -// Driver outputs the configured database driver. -func (c *client) Driver() string { - return constants.DriverPostgres -} diff --git a/database/postgres/driver_test.go b/database/postgres/driver_test.go deleted file mode 100644 index e54d0b0f5..000000000 --- a/database/postgres/driver_test.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "reflect" - "testing" - - "github.com/go-vela/types/constants" -) - -func TestPostgres_Client_Driver(t *testing.T) { - // setup types - want := constants.DriverPostgres - - // setup the test database client - _database, _, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // run test - got := _database.Driver() - - if !reflect.DeepEqual(got, want) { - t.Errorf("Driver is %v, want %v", got, want) - } -} diff --git a/database/postgres/opts.go b/database/postgres/opts.go deleted file mode 100644 index 0564f4c0d..000000000 --- a/database/postgres/opts.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "fmt" - "time" -) - -// ClientOpt represents a configuration option to initialize the database client for Postgres. -type ClientOpt func(*client) error - -// WithAddress sets the address in the database client for Postgres. -func WithAddress(address string) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring address in postgres database client") - - // check if the Postgres address provided is empty - if len(address) == 0 { - return fmt.Errorf("no Postgres address provided") - } - - // set the address in the postgres client - c.config.Address = address - - return nil - } -} - -// WithCompressionLevel sets the compression level in the database client for Postgres. -func WithCompressionLevel(level int) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring compression level in postgres database client") - - // set the compression level in the postgres client - c.config.CompressionLevel = level - - return nil - } -} - -// WithConnectionLife sets the connection duration in the database client for Postgres. -func WithConnectionLife(duration time.Duration) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring connection duration in postgres database client") - - // set the connection duration in the postgres client - c.config.ConnectionLife = duration - - return nil - } -} - -// WithConnectionIdle sets the maximum idle connections in the database client for Postgres. -func WithConnectionIdle(idle int) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring maximum idle connections in postgres database client") - - // set the maximum idle connections in the postgres client - c.config.ConnectionIdle = idle - - return nil - } -} - -// WithConnectionOpen sets the maximum open connections in the database client for Postgres. -func WithConnectionOpen(open int) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring maximum open connections in postgres database client") - - // set the maximum open connections in the postgres client - c.config.ConnectionOpen = open - - return nil - } -} - -// WithEncryptionKey sets the encryption key in the database client for Postgres. -func WithEncryptionKey(key string) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring encryption key in postgres database client") - - // check if the Postgres encryption key provided is empty - if len(key) == 0 { - return fmt.Errorf("no Postgres encryption key provided") - } - - // set the encryption key in the postgres client - c.config.EncryptionKey = key - - return nil - } -} - -// WithSkipCreation sets the skip creation logic in the database client for Postgres. -func WithSkipCreation(skipCreation bool) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring skip creating objects in postgres database client") - - // set to skip creating tables and indexes in the postgres client - c.config.SkipCreation = skipCreation - - return nil - } -} diff --git a/database/postgres/opts_test.go b/database/postgres/opts_test.go deleted file mode 100644 index cb3e687da..000000000 --- a/database/postgres/opts_test.go +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "reflect" - "testing" - "time" - - "github.com/sirupsen/logrus" -) - -func TestPostgres_ClientOpt_WithAddress(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - failure bool - address string - want string - }{ - { - failure: false, - address: "postgres://foo:bar@localhost:5432/vela", - want: "postgres://foo:bar@localhost:5432/vela", - }, - { - failure: true, - address: "", - want: "", - }, - } - - // run tests - for _, test := range tests { - err := WithAddress(test.address)(c) - - if test.failure { - if err == nil { - t.Errorf("WithAddress should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("WithAddress returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.Address, test.want) { - t.Errorf("WithAddress is %v, want %v", c.config.Address, test.want) - } - } -} - -func TestPostgres_ClientOpt_WithCompressionLevel(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - level int - want int - }{ - { - level: 3, - want: 3, - }, - { - level: 0, - want: 0, - }, - } - - // run tests - for _, test := range tests { - err := WithCompressionLevel(test.level)(c) - - if err != nil { - t.Errorf("WithCompressionLevel returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.CompressionLevel, test.want) { - t.Errorf("WithCompressionLevel is %v, want %v", c.config.CompressionLevel, test.want) - } - } -} - -func TestPostgres_ClientOpt_WithConnectionLife(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - duration time.Duration - want time.Duration - }{ - { - duration: 10 * time.Second, - want: 10 * time.Second, - }, - { - duration: 0, - want: 0, - }, - } - - // run tests - for _, test := range tests { - err := WithConnectionLife(test.duration)(c) - - if err != nil { - t.Errorf("WithConnectionLife returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.ConnectionLife, test.want) { - t.Errorf("WithConnectionLife is %v, want %v", c.config.ConnectionLife, test.want) - } - } -} - -func TestPostgres_ClientOpt_WithConnectionIdle(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - idle int - want int - }{ - { - idle: 5, - want: 5, - }, - { - idle: 0, - want: 0, - }, - } - - // run tests - for _, test := range tests { - err := WithConnectionIdle(test.idle)(c) - - if err != nil { - t.Errorf("WithConnectionIdle returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.ConnectionIdle, test.want) { - t.Errorf("WithConnectionIdle is %v, want %v", c.config.ConnectionIdle, test.want) - } - } -} - -func TestPostgres_ClientOpt_WithConnectionOpen(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - open int - want int - }{ - { - open: 10, - want: 10, - }, - { - open: 0, - want: 0, - }, - } - - // run tests - for _, test := range tests { - err := WithConnectionOpen(test.open)(c) - - if err != nil { - t.Errorf("WithConnectionOpen returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.ConnectionOpen, test.want) { - t.Errorf("WithConnectionOpen is %v, want %v", c.config.ConnectionOpen, test.want) - } - } -} - -func TestPostgres_ClientOpt_WithEncryptionKey(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - failure bool - key string - want string - }{ - { - failure: false, - key: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - want: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - }, - { - failure: true, - key: "", - want: "", - }, - } - - // run tests - for _, test := range tests { - err := WithEncryptionKey(test.key)(c) - - if test.failure { - if err == nil { - t.Errorf("WithEncryptionKey should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("WithEncryptionKey returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.EncryptionKey, test.want) { - t.Errorf("WithEncryptionKey is %v, want %v", c.config.EncryptionKey, test.want) - } - } -} - -func TestPostgres_ClientOpt_WithSkipCreation(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - skipCreation bool - want bool - }{ - { - skipCreation: true, - want: true, - }, - { - skipCreation: false, - want: false, - }, - } - - // run tests - for _, test := range tests { - err := WithSkipCreation(test.skipCreation)(c) - - if err != nil { - t.Errorf("WithSkipCreation returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.SkipCreation, test.want) { - t.Errorf("WithSkipCreation is %v, want %v", c.config.SkipCreation, test.want) - } - } -} diff --git a/database/postgres/ping.go b/database/postgres/ping.go deleted file mode 100644 index acdc29782..000000000 --- a/database/postgres/ping.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "fmt" - "time" -) - -// Ping sends a "ping" request with backoff to the database. -func (c *client) Ping() error { - c.Logger.Trace("sending ping requests to the database") - - // create a loop to attempt ping requests 5 times - for i := 0; i < 5; i++ { - // capture database/sql database from gorm database - // - // https://pkg.go.dev/gorm.io/gorm#DB.DB - _sql, err := c.Postgres.DB() - if err != nil { - return err - } - - // send ping request to database - // - // https://pkg.go.dev/database/sql#DB.Ping - err = _sql.Ping() - if err != nil { - c.Logger.Debugf("unable to ping database - retrying in %v", time.Duration(i)*time.Second) - - // sleep for loop iteration in seconds - time.Sleep(time.Duration(i) * time.Second) - - // continue to next iteration of the loop - continue - } - - // able to ping database so return with no error - return nil - } - - return fmt.Errorf("unable to successfully ping database") -} diff --git a/database/postgres/ping_test.go b/database/postgres/ping_test.go deleted file mode 100644 index 563f41016..000000000 --- a/database/postgres/ping_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "testing" -) - -func TestPostgres_Client_Ping(t *testing.T) { - // setup types - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // ensure the mock expects the ping - _mock.ExpectPing() - - // setup the closed test database client - _closed, _, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - // capture the closed test sql database - _sql, _ := _closed.Postgres.DB() - // close the test sql database to simulate failures to ping - _sql.Close() - - // setup tests - tests := []struct { - failure bool - database *client - }{ - { - failure: false, - database: _database, - }, - { - failure: true, - database: _closed, - }, - } - - // run tests - for _, test := range tests { - err = test.database.Ping() - - if test.failure { - if err == nil { - t.Errorf("Ping should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("Ping returned err: %v", err) - } - } -} diff --git a/database/postgres/postgres.go b/database/postgres/postgres.go deleted file mode 100644 index fea05037d..000000000 --- a/database/postgres/postgres.go +++ /dev/null @@ -1,411 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "time" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/go-vela/server/database/build" - "github.com/go-vela/server/database/hook" - "github.com/go-vela/server/database/log" - "github.com/go-vela/server/database/pipeline" - "github.com/go-vela/server/database/repo" - "github.com/go-vela/server/database/schedule" - "github.com/go-vela/server/database/secret" - "github.com/go-vela/server/database/service" - "github.com/go-vela/server/database/step" - "github.com/go-vela/server/database/user" - "github.com/go-vela/server/database/worker" - "github.com/sirupsen/logrus" - - "gorm.io/driver/postgres" - "gorm.io/gorm" -) - -type ( - config struct { - // specifies the address to use for the Postgres client - Address string - // specifies the level of compression to use for the Postgres client - CompressionLevel int - // specifies the connection duration to use for the Postgres client - ConnectionLife time.Duration - // specifies the maximum idle connections for the Postgres client - ConnectionIdle int - // specifies the maximum open connections for the Postgres client - ConnectionOpen int - // specifies the encryption key to use for the Postgres client - EncryptionKey string - // specifies to skip creating tables and indexes for the Postgres client - SkipCreation bool - } - - client struct { - config *config - // https://pkg.go.dev/gorm.io/gorm#DB - Postgres *gorm.DB - // https://pkg.go.dev/github.com/sirupsen/logrus#Entry - Logger *logrus.Entry - // https://pkg.go.dev/github.com/go-vela/server/database/build#BuildInterface - build.BuildInterface - // https://pkg.go.dev/github.com/go-vela/server/database/hook#HookInterface - hook.HookInterface - // https://pkg.go.dev/github.com/go-vela/server/database/log#LogInterface - log.LogInterface - // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#PipelineInterface - pipeline.PipelineInterface - // https://pkg.go.dev/github.com/go-vela/server/database/repo#RepoInterface - repo.RepoInterface - // https://pkg.go.dev/github.com/go-vela/server/database/schedule#ScheduleInterface - schedule.ScheduleInterface - // https://pkg.go.dev/github.com/go-vela/server/database/secret#SecretInterface - secret.SecretInterface - // https://pkg.go.dev/github.com/go-vela/server/database/service#ServiceInterface - service.ServiceInterface - // https://pkg.go.dev/github.com/go-vela/server/database/step#StepInterface - step.StepInterface - // https://pkg.go.dev/github.com/go-vela/server/database/user#UserInterface - user.UserInterface - // https://pkg.go.dev/github.com/go-vela/server/database/worker#WorkerInterface - worker.WorkerInterface - } -) - -// New returns a Database implementation that integrates with a Postgres instance. -// -//nolint:revive // ignore returning unexported client -func New(opts ...ClientOpt) (*client, error) { - // create new Postgres client - c := new(client) - - // create new fields - c.config = new(config) - c.Postgres = new(gorm.DB) - - // create new logger for the client - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#StandardLogger - logger := logrus.StandardLogger() - - // create new logger for the client - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#NewEntry - c.Logger = logrus.NewEntry(logger).WithField("database", c.Driver()) - - // apply all provided configuration options - for _, opt := range opts { - err := opt(c) - if err != nil { - return nil, err - } - } - - // create the new Postgres database client - // - // https://pkg.go.dev/gorm.io/gorm#Open - _postgres, err := gorm.Open(postgres.Open(c.config.Address), &gorm.Config{}) - if err != nil { - return nil, err - } - - // set the Postgres database client in the Postgres client - c.Postgres = _postgres - - // setup database with proper configuration - err = setupDatabase(c) - if err != nil { - return nil, err - } - - // create the services for the database - err = createServices(c) - if err != nil { - return nil, err - } - - return c, nil -} - -// NewTest returns a Database implementation that integrates with a fake Postgres instance. -// -// This function is intended for running tests only. -// -//nolint:revive // ignore returning unexported client -func NewTest() (*client, sqlmock.Sqlmock, error) { - // create new Postgres client - c := new(client) - - // create new fields - c.config = &config{ - CompressionLevel: 3, - ConnectionLife: 30 * time.Minute, - ConnectionIdle: 2, - ConnectionOpen: 0, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - } - c.Postgres = new(gorm.DB) - - // create new logger for the client - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#StandardLogger - logger := logrus.StandardLogger() - - // create new logger for the client - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#NewEntry - c.Logger = logrus.NewEntry(logger).WithField("database", c.Driver()) - - // create the new mock sql database - // - // https://pkg.go.dev/github.com/DATA-DOG/go-sqlmock#New - _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - if err != nil { - return nil, nil, err - } - - // ensure the mock expects the build queries - _mock.ExpectExec(build.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(build.CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(build.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(build.CreateSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(build.CreateStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the hook queries - _mock.ExpectExec(hook.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(hook.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the log queries - _mock.ExpectExec(log.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(log.CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the pipeline queries - _mock.ExpectExec(pipeline.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(pipeline.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the repo queries - _mock.ExpectExec(repo.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(repo.CreateOrgNameIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the schedule queries - _mock.ExpectExec(schedule.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(schedule.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the secret queries - _mock.ExpectExec(secret.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(secret.CreateTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(secret.CreateTypeOrgTeam).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(secret.CreateTypeOrg).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the service queries - _mock.ExpectExec(service.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the step queries - _mock.ExpectExec(step.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the user queries - _mock.ExpectExec(user.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(user.CreateUserRefreshIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the worker queries - _mock.ExpectExec(worker.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(worker.CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - - // create the new mock Postgres database client - // - // https://pkg.go.dev/gorm.io/gorm#Open - c.Postgres, err = gorm.Open( - postgres.New(postgres.Config{Conn: _sql}), - &gorm.Config{SkipDefaultTransaction: true}, - ) - if err != nil { - return nil, nil, err - } - - // setup database with proper configuration - err = createServices(c) - if err != nil { - return nil, nil, err - } - - return c, _mock, nil -} - -// setupDatabase is a helper function to setup -// the database with the proper configuration. -func setupDatabase(c *client) error { - // capture database/sql database from gorm database - // - // https://pkg.go.dev/gorm.io/gorm#DB.DB - _sql, err := c.Postgres.DB() - if err != nil { - return err - } - - // set the maximum amount of time a connection may be reused - // - // https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime - _sql.SetConnMaxLifetime(c.config.ConnectionLife) - - // set the maximum number of connections in the idle connection pool - // - // https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns - _sql.SetMaxIdleConns(c.config.ConnectionIdle) - - // set the maximum number of open connections to the database - // - // https://golang.org/pkg/database/sql/#DB.SetMaxOpenConns - _sql.SetMaxOpenConns(c.config.ConnectionOpen) - - // verify connection to the database - err = c.Ping() - if err != nil { - return err - } - - // check if we should skip creating database objects - if c.config.SkipCreation { - c.Logger.Warning("skipping creation of data tables and indexes in the postgres database") - - return nil - } - - return nil -} - -// createServices is a helper function to create the database services. -func createServices(c *client) error { - var err error - - // create the database agnostic engine for builds - // - // https://pkg.go.dev/github.com/go-vela/server/database/build#New - c.BuildInterface, err = build.New( - build.WithClient(c.Postgres), - build.WithLogger(c.Logger), - build.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for hooks - // - // https://pkg.go.dev/github.com/go-vela/server/database/hook#New - c.HookInterface, err = hook.New( - hook.WithClient(c.Postgres), - hook.WithLogger(c.Logger), - hook.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for logs - // - // https://pkg.go.dev/github.com/go-vela/server/database/log#New - c.LogInterface, err = log.New( - log.WithClient(c.Postgres), - log.WithCompressionLevel(c.config.CompressionLevel), - log.WithLogger(c.Logger), - log.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for pipelines - // - // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#New - c.PipelineInterface, err = pipeline.New( - pipeline.WithClient(c.Postgres), - pipeline.WithCompressionLevel(c.config.CompressionLevel), - pipeline.WithLogger(c.Logger), - pipeline.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for repos - // - // https://pkg.go.dev/github.com/go-vela/server/database/repo#New - c.RepoInterface, err = repo.New( - repo.WithClient(c.Postgres), - repo.WithEncryptionKey(c.config.EncryptionKey), - repo.WithLogger(c.Logger), - repo.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for schedules - // - // https://pkg.go.dev/github.com/go-vela/server/database/schedule#New - c.ScheduleInterface, err = schedule.New( - schedule.WithClient(c.Postgres), - schedule.WithLogger(c.Logger), - schedule.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for secrets - // - // https://pkg.go.dev/github.com/go-vela/server/database/secret#New - c.SecretInterface, err = secret.New( - secret.WithClient(c.Postgres), - secret.WithEncryptionKey(c.config.EncryptionKey), - secret.WithLogger(c.Logger), - secret.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for services - // - // https://pkg.go.dev/github.com/go-vela/server/database/service#New - c.ServiceInterface, err = service.New( - service.WithClient(c.Postgres), - service.WithLogger(c.Logger), - service.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for steps - // - // https://pkg.go.dev/github.com/go-vela/server/database/step#New - c.StepInterface, err = step.New( - step.WithClient(c.Postgres), - step.WithLogger(c.Logger), - step.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for users - // - // https://pkg.go.dev/github.com/go-vela/server/database/user#New - c.UserInterface, err = user.New( - user.WithClient(c.Postgres), - user.WithEncryptionKey(c.config.EncryptionKey), - user.WithLogger(c.Logger), - user.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for workers - // - // https://pkg.go.dev/github.com/go-vela/server/database/worker#New - c.WorkerInterface, err = worker.New( - worker.WithClient(c.Postgres), - worker.WithLogger(c.Logger), - worker.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - return nil -} diff --git a/database/postgres/postgres_test.go b/database/postgres/postgres_test.go deleted file mode 100644 index a8f74d1ff..000000000 --- a/database/postgres/postgres_test.go +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "testing" - "time" - - "github.com/go-vela/server/database/schedule" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/go-vela/server/database/build" - "github.com/go-vela/server/database/hook" - "github.com/go-vela/server/database/log" - "github.com/go-vela/server/database/pipeline" - "github.com/go-vela/server/database/repo" - "github.com/go-vela/server/database/secret" - "github.com/go-vela/server/database/service" - "github.com/go-vela/server/database/step" - "github.com/go-vela/server/database/user" - "github.com/go-vela/server/database/worker" -) - -func TestPostgres_New(t *testing.T) { - // setup tests - tests := []struct { - failure bool - address string - want string - }{ - { - failure: true, - address: "postgres://foo:bar@localhost:5432/vela", - want: "postgres://foo:bar@localhost:5432/vela", - }, - { - failure: true, - address: "", - want: "", - }, - } - - // run tests - for _, test := range tests { - _, err := New( - WithAddress(test.address), - WithCompressionLevel(3), - WithConnectionLife(10*time.Second), - WithConnectionIdle(5), - WithConnectionOpen(20), - WithEncryptionKey("A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW"), - WithSkipCreation(false), - ) - - if test.failure { - if err == nil { - t.Errorf("New should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("New returned err: %v", err) - } - } -} - -func TestPostgres_setupDatabase(t *testing.T) { - // setup types - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // ensure the mock expects the ping - _mock.ExpectPing() - - // ensure the mock expects the build queries - _mock.ExpectExec(build.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(build.CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(build.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(build.CreateSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(build.CreateStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the hook queries - _mock.ExpectExec(hook.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(hook.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the log queries - _mock.ExpectExec(log.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(log.CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the pipeline queries - _mock.ExpectExec(pipeline.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(pipeline.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the repo queries - _mock.ExpectExec(repo.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(repo.CreateOrgNameIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the schedule queries - _mock.ExpectExec(schedule.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(schedule.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the secret queries - _mock.ExpectExec(secret.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(secret.CreateTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(secret.CreateTypeOrgTeam).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(secret.CreateTypeOrg).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the service queries - _mock.ExpectExec(service.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the step queries - _mock.ExpectExec(step.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the user queries - _mock.ExpectExec(user.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(user.CreateUserRefreshIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the worker queries - _mock.ExpectExec(worker.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(worker.CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - - // setup the skip test database client - _skipDatabase, _skipMock, err := NewTest() - if err != nil { - t.Errorf("unable to create new skip postgres test database: %v", err) - } - - defer func() { _sql, _ := _skipDatabase.Postgres.DB(); _sql.Close() }() - - err = WithSkipCreation(true)(_skipDatabase) - if err != nil { - t.Errorf("unable to set SkipCreation for postgres test database: %v", err) - } - - // ensure the mock expects the ping - _skipMock.ExpectPing() - - tests := []struct { - failure bool - database *client - }{ - { - failure: false, - database: _database, - }, - { - failure: false, - database: _skipDatabase, - }, - } - - // run tests - for _, test := range tests { - err := setupDatabase(test.database) - - if test.failure { - if err == nil { - t.Errorf("setupDatabase should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("setupDatabase returned err: %v", err) - } - } -} - -func TestPostgres_createServices(t *testing.T) { - // setup types - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // ensure the mock expects the build queries - _mock.ExpectExec(build.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(build.CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(build.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(build.CreateSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(build.CreateStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the hook queries - _mock.ExpectExec(hook.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(hook.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the log queries - _mock.ExpectExec(log.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(log.CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the pipeline queries - _mock.ExpectExec(pipeline.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(pipeline.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the repo queries - _mock.ExpectExec(repo.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(repo.CreateOrgNameIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the schedule queries - _mock.ExpectExec(schedule.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(schedule.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the secret queries - _mock.ExpectExec(secret.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(secret.CreateTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(secret.CreateTypeOrgTeam).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(secret.CreateTypeOrg).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the service queries - _mock.ExpectExec(service.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the step queries - _mock.ExpectExec(step.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the user queries - _mock.ExpectExec(user.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(user.CreateUserRefreshIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - // ensure the mock expects the worker queries - _mock.ExpectExec(worker.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(worker.CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := createServices(_database) - - if test.failure { - if err == nil { - t.Errorf("createServices should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("createServices returned err: %v", err) - } - } -} diff --git a/database/sqlite/doc.go b/database/sqlite/doc.go deleted file mode 100644 index a27ddb46f..000000000 --- a/database/sqlite/doc.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -// Package sqlite provides the ability for Vela to -// integrate with Sqlite as a SQL backend. -// -// Usage: -// -// import "github.com/go-vela/server/database/sqlite" -package sqlite diff --git a/database/sqlite/driver.go b/database/sqlite/driver.go deleted file mode 100644 index 208853dd5..000000000 --- a/database/sqlite/driver.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import "github.com/go-vela/types/constants" - -// Driver outputs the configured database driver. -func (c *client) Driver() string { - return constants.DriverSqlite -} diff --git a/database/sqlite/driver_test.go b/database/sqlite/driver_test.go deleted file mode 100644 index 83a2aae29..000000000 --- a/database/sqlite/driver_test.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "reflect" - "testing" - - "github.com/go-vela/types/constants" -) - -func TestSqlite_Client_Driver(t *testing.T) { - // setup types - want := constants.DriverSqlite - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // run test - got := _database.Driver() - - if !reflect.DeepEqual(got, want) { - t.Errorf("Driver is %v, want %v", got, want) - } -} diff --git a/database/sqlite/opts.go b/database/sqlite/opts.go deleted file mode 100644 index 406dddcc8..000000000 --- a/database/sqlite/opts.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "fmt" - "time" -) - -// ClientOpt represents a configuration option to initialize the database client for Sqlite. -type ClientOpt func(*client) error - -// WithAddress sets the Sqlite address in the database client for Sqlite. -func WithAddress(address string) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring address in sqlite database client") - - // check if the Sqlite address provided is empty - if len(address) == 0 { - return fmt.Errorf("no Sqlite address provided") - } - - // set the address in the sqlite client - c.config.Address = address - - return nil - } -} - -// WithCompressionLevel sets the compression level in the database client for Sqlite. -func WithCompressionLevel(level int) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring compression level in sqlite database client") - - // set the compression level in the sqlite client - c.config.CompressionLevel = level - - return nil - } -} - -// WithConnectionLife sets the connection duration in the database client for Sqlite. -func WithConnectionLife(duration time.Duration) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring connection duration in sqlite database client") - - // set the connection duration in the sqlite client - c.config.ConnectionLife = duration - - return nil - } -} - -// WithConnectionIdle sets the maximum idle connections in the database client for Sqlite. -func WithConnectionIdle(idle int) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring maximum idle connections in sqlite database client") - - // set the maximum idle connections in the sqlite client - c.config.ConnectionIdle = idle - - return nil - } -} - -// WithConnectionOpen sets the maximum open connections in the database client for Sqlite. -func WithConnectionOpen(open int) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring maximum open connections in sqlite database client") - - // set the maximum open connections in the sqlite client - c.config.ConnectionOpen = open - - return nil - } -} - -// WithEncryptionKey sets the encryption key in the database client for Sqlite. -func WithEncryptionKey(key string) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring encryption key in sqlite database client") - - // check if the Sqlite encryption key provided is empty - if len(key) == 0 { - return fmt.Errorf("no Sqlite encryption key provided") - } - - // set the encryption key in the sqlite client - c.config.EncryptionKey = key - - return nil - } -} - -// WithSkipCreation sets the skip creation logic in the database client for Sqlite. -func WithSkipCreation(skipCreation bool) ClientOpt { - return func(c *client) error { - c.Logger.Trace("configuring skip creating objects in sqlite database client") - - // set to skip creating tables and indexes in the sqlite client - c.config.SkipCreation = skipCreation - - return nil - } -} diff --git a/database/sqlite/opts_test.go b/database/sqlite/opts_test.go deleted file mode 100644 index 42f70c58e..000000000 --- a/database/sqlite/opts_test.go +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "reflect" - "testing" - "time" - - "github.com/sirupsen/logrus" -) - -func TestSqlite_ClientOpt_WithAddress(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - failure bool - address string - want string - }{ - { - failure: false, - address: "sqlite://foo:bar@localhost:5432/vela", - want: "sqlite://foo:bar@localhost:5432/vela", - }, - { - failure: true, - address: "", - want: "", - }, - } - - // run tests - for _, test := range tests { - err := WithAddress(test.address)(c) - - if test.failure { - if err == nil { - t.Errorf("WithAddress should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("WithAddress returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.Address, test.want) { - t.Errorf("WithAddress is %v, want %v", c.config.Address, test.want) - } - } -} - -func TestSqlite_ClientOpt_WithCompressionLevel(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - level int - want int - }{ - { - level: 3, - want: 3, - }, - { - level: 0, - want: 0, - }, - } - - // run tests - for _, test := range tests { - err := WithCompressionLevel(test.level)(c) - - if err != nil { - t.Errorf("WithCompressionLevel returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.CompressionLevel, test.want) { - t.Errorf("WithCompressionLevel is %v, want %v", c.config.CompressionLevel, test.want) - } - } -} - -func TestSqlite_ClientOpt_WithConnectionLife(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - duration time.Duration - want time.Duration - }{ - { - duration: 10 * time.Second, - want: 10 * time.Second, - }, - { - duration: 0, - want: 0, - }, - } - - // run tests - for _, test := range tests { - err := WithConnectionLife(test.duration)(c) - - if err != nil { - t.Errorf("WithConnectionLife returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.ConnectionLife, test.want) { - t.Errorf("WithConnectionLife is %v, want %v", c.config.ConnectionLife, test.want) - } - } -} - -func TestSqlite_ClientOpt_WithConnectionIdle(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - idle int - want int - }{ - { - idle: 5, - want: 5, - }, - { - idle: 0, - want: 0, - }, - } - - // run tests - for _, test := range tests { - err := WithConnectionIdle(test.idle)(c) - - if err != nil { - t.Errorf("WithConnectionIdle returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.ConnectionIdle, test.want) { - t.Errorf("WithConnectionIdle is %v, want %v", c.config.ConnectionIdle, test.want) - } - } -} - -func TestSqlite_ClientOpt_WithConnectionOpen(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - open int - want int - }{ - { - open: 10, - want: 10, - }, - { - open: 0, - want: 0, - }, - } - - // run tests - for _, test := range tests { - err := WithConnectionOpen(test.open)(c) - - if err != nil { - t.Errorf("WithConnectionOpen returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.ConnectionOpen, test.want) { - t.Errorf("WithConnectionOpen is %v, want %v", c.config.ConnectionOpen, test.want) - } - } -} - -func TestSqlite_ClientOpt_WithEncryptionKey(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - failure bool - key string - want string - }{ - { - failure: false, - key: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - want: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - }, - { - failure: true, - key: "", - want: "", - }, - } - - // run tests - for _, test := range tests { - err := WithEncryptionKey(test.key)(c) - - if test.failure { - if err == nil { - t.Errorf("WithEncryptionKey should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("WithEncryptionKey returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.EncryptionKey, test.want) { - t.Errorf("WithEncryptionKey is %v, want %v", c.config.EncryptionKey, test.want) - } - } -} - -func TestSqlite_ClientOpt_WithSkipCreation(t *testing.T) { - // setup types - c := new(client) - c.config = new(config) - logger := logrus.StandardLogger() - c.Logger = logrus.NewEntry(logger) - - // setup tests - tests := []struct { - skipCreation bool - want bool - }{ - { - skipCreation: true, - want: true, - }, - { - skipCreation: false, - want: false, - }, - } - - // run tests - for _, test := range tests { - err := WithSkipCreation(test.skipCreation)(c) - - if err != nil { - t.Errorf("WithSkipCreation returned err: %v", err) - } - - if !reflect.DeepEqual(c.config.SkipCreation, test.want) { - t.Errorf("WithSkipCreation is %v, want %v", c.config.SkipCreation, test.want) - } - } -} diff --git a/database/sqlite/ping.go b/database/sqlite/ping.go deleted file mode 100644 index a23b9f707..000000000 --- a/database/sqlite/ping.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "fmt" - "time" -) - -// Ping sends a "ping" request with backoff to the database. -func (c *client) Ping() error { - c.Logger.Trace("sending ping requests to the database") - - // create a loop to attempt ping requests 5 times - for i := 0; i < 5; i++ { - // capture database/sql database from gorm database - // - // https://pkg.go.dev/gorm.io/gorm#DB.DB - _sql, err := c.Sqlite.DB() - if err != nil { - return err - } - - // send ping request to database - // - // https://pkg.go.dev/database/sql#DB.Ping - err = _sql.Ping() - if err != nil { - c.Logger.Debugf("unable to ping database - retrying in %v", time.Duration(i)*time.Second) - - // sleep for loop iteration in seconds - time.Sleep(time.Duration(i) * time.Second) - - // continue to next iteration of the loop - continue - } - - // able to ping database so return with no error - return nil - } - - return fmt.Errorf("unable to successfully ping database") -} diff --git a/database/sqlite/ping_test.go b/database/sqlite/ping_test.go deleted file mode 100644 index 04c207bcd..000000000 --- a/database/sqlite/ping_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "testing" -) - -func TestSqlite_Client_Ping(t *testing.T) { - // setup types - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { - _sql, _ := _database.Sqlite.DB() - _sql.Close() - }() - - _bad, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - // close the bad database to simulate failures to ping - _sql, _ := _bad.Sqlite.DB() - _sql.Close() - - // setup tests - tests := []struct { - failure bool - database *client - }{ - { - failure: false, - database: _database, - }, - { - failure: true, - database: _bad, - }, - } - - // run tests - for _, test := range tests { - err = test.database.Ping() - - if test.failure { - if err == nil { - t.Errorf("Ping should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("Ping returned err: %v", err) - } - } -} diff --git a/database/sqlite/sqlite.go b/database/sqlite/sqlite.go deleted file mode 100644 index 29f526643..000000000 --- a/database/sqlite/sqlite.go +++ /dev/null @@ -1,367 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "time" - - "github.com/go-vela/server/database/build" - "github.com/go-vela/server/database/hook" - "github.com/go-vela/server/database/log" - "github.com/go-vela/server/database/pipeline" - "github.com/go-vela/server/database/repo" - "github.com/go-vela/server/database/schedule" - "github.com/go-vela/server/database/secret" - "github.com/go-vela/server/database/service" - "github.com/go-vela/server/database/step" - "github.com/go-vela/server/database/user" - "github.com/go-vela/server/database/worker" - "github.com/sirupsen/logrus" - - "gorm.io/driver/sqlite" - "gorm.io/gorm" -) - -type ( - config struct { - // specifies the address to use for the Sqlite client - Address string - // specifies the level of compression to use for the Sqlite client - CompressionLevel int - // specifies the connection duration to use for the Sqlite client - ConnectionLife time.Duration - // specifies the maximum idle connections for the Sqlite client - ConnectionIdle int - // specifies the maximum open connections for the Sqlite client - ConnectionOpen int - // specifies the encryption key to use for the Sqlite client - EncryptionKey string - // specifies to skip creating tables and indexes for the Sqlite client - SkipCreation bool - } - - client struct { - config *config - // https://pkg.go.dev/gorm.io/gorm#DB - Sqlite *gorm.DB - // https://pkg.go.dev/github.com/sirupsen/logrus#Entry - Logger *logrus.Entry - // https://pkg.go.dev/github.com/go-vela/server/database/build#BuildInterface - build.BuildInterface - // https://pkg.go.dev/github.com/go-vela/server/database/hook#HookInterface - hook.HookInterface - // https://pkg.go.dev/github.com/go-vela/server/database/log#LogInterface - log.LogInterface - // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#PipelineInterface - pipeline.PipelineInterface - // https://pkg.go.dev/github.com/go-vela/server/database/repo#RepoInterface - repo.RepoInterface - // https://pkg.go.dev/github.com/go-vela/server/database/schedule#ScheduleInterface - schedule.ScheduleInterface - // https://pkg.go.dev/github.com/go-vela/server/database/secret#SecretInterface - secret.SecretInterface - // https://pkg.go.dev/github.com/go-vela/server/database/service#ServiceInterface - service.ServiceInterface - // https://pkg.go.dev/github.com/go-vela/server/database/step#StepInterface - step.StepInterface - // https://pkg.go.dev/github.com/go-vela/server/database/user#UserInterface - user.UserInterface - // https://pkg.go.dev/github.com/go-vela/server/database/worker#WorkerInterface - worker.WorkerInterface - } -) - -// New returns a Database implementation that integrates with a Sqlite instance. -// -//nolint:revive // ignore returning unexported client -func New(opts ...ClientOpt) (*client, error) { - // create new Sqlite client - c := new(client) - - // create new fields - c.config = new(config) - c.Sqlite = new(gorm.DB) - - // create new logger for the client - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#StandardLogger - logger := logrus.StandardLogger() - - // create new logger for the client - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#NewEntry - c.Logger = logrus.NewEntry(logger).WithField("database", c.Driver()) - - // apply all provided configuration options - for _, opt := range opts { - err := opt(c) - if err != nil { - return nil, err - } - } - - // create the new Sqlite database client - // - // https://pkg.go.dev/gorm.io/gorm#Open - _sqlite, err := gorm.Open(sqlite.Open(c.config.Address), &gorm.Config{}) - if err != nil { - return nil, err - } - - // set the Sqlite database client in the Sqlite client - c.Sqlite = _sqlite - - // setup database with proper configuration - err = setupDatabase(c) - if err != nil { - return nil, err - } - - // create the services for the database - err = createServices(c) - if err != nil { - return nil, err - } - - return c, nil -} - -// NewTest returns a Database implementation that integrates with a fake Sqlite instance. -// -// This function is intended for running tests only. -// -//nolint:revive // ignore returning unexported client -func NewTest() (*client, error) { - // create new Sqlite client - c := new(client) - - // create new fields - c.config = &config{ - CompressionLevel: 3, - ConnectionLife: 30 * time.Minute, - ConnectionIdle: 2, - ConnectionOpen: 0, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - } - c.Sqlite = new(gorm.DB) - - // create new logger for the client - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#StandardLogger - logger := logrus.StandardLogger() - - // create new logger for the client - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#NewEntry - c.Logger = logrus.NewEntry(logger).WithField("database", c.Driver()) - - // create the new Sqlite database client - // - // https://pkg.go.dev/gorm.io/gorm#Open - _sqlite, err := gorm.Open( - sqlite.Open("file::memory:?cache=shared"), - &gorm.Config{SkipDefaultTransaction: true}, - ) - if err != nil { - return nil, err - } - - c.Sqlite = _sqlite - - // setup database with proper configuration - err = createServices(c) - if err != nil { - return nil, err - } - - return c, nil -} - -// setupDatabase is a helper function to setup -// the database with the proper configuration. -func setupDatabase(c *client) error { - // capture database/sql database from gorm database - // - // https://pkg.go.dev/gorm.io/gorm#DB.DB - _sql, err := c.Sqlite.DB() - if err != nil { - return err - } - - // set the maximum amount of time a connection may be reused - // - // https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime - _sql.SetConnMaxLifetime(c.config.ConnectionLife) - - // set the maximum number of connections in the idle connection pool - // - // https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns - _sql.SetMaxIdleConns(c.config.ConnectionIdle) - - // set the maximum number of open connections to the database - // - // https://golang.org/pkg/database/sql/#DB.SetMaxOpenConns - _sql.SetMaxOpenConns(c.config.ConnectionOpen) - - // verify connection to the database - err = c.Ping() - if err != nil { - return err - } - - // check if we should skip creating database objects - if c.config.SkipCreation { - c.Logger.Warning("skipping creation of data tables and indexes in the sqlite database") - - return nil - } - - return nil -} - -// createServices is a helper function to create the database services. -func createServices(c *client) error { - var err error - - // create the database agnostic engine for builds - // - // https://pkg.go.dev/github.com/go-vela/server/database/build#New - c.BuildInterface, err = build.New( - build.WithClient(c.Sqlite), - build.WithLogger(c.Logger), - build.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for hooks - // - // https://pkg.go.dev/github.com/go-vela/server/database/hook#New - c.HookInterface, err = hook.New( - hook.WithClient(c.Sqlite), - hook.WithLogger(c.Logger), - hook.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for logs - // - // https://pkg.go.dev/github.com/go-vela/server/database/log#New - c.LogInterface, err = log.New( - log.WithClient(c.Sqlite), - log.WithCompressionLevel(c.config.CompressionLevel), - log.WithLogger(c.Logger), - log.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for pipelines - // - // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#New - c.PipelineInterface, err = pipeline.New( - pipeline.WithClient(c.Sqlite), - pipeline.WithCompressionLevel(c.config.CompressionLevel), - pipeline.WithLogger(c.Logger), - pipeline.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for repos - // - // https://pkg.go.dev/github.com/go-vela/server/database/repo#New - c.RepoInterface, err = repo.New( - repo.WithClient(c.Sqlite), - repo.WithEncryptionKey(c.config.EncryptionKey), - repo.WithLogger(c.Logger), - repo.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for schedules - // - // https://pkg.go.dev/github.com/go-vela/server/database/schedule#New - c.ScheduleInterface, err = schedule.New( - schedule.WithClient(c.Sqlite), - schedule.WithLogger(c.Logger), - schedule.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for secrets - // - // https://pkg.go.dev/github.com/go-vela/server/database/secret#New - c.SecretInterface, err = secret.New( - secret.WithClient(c.Sqlite), - secret.WithEncryptionKey(c.config.EncryptionKey), - secret.WithLogger(c.Logger), - secret.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for services - // - // https://pkg.go.dev/github.com/go-vela/server/database/service#New - c.ServiceInterface, err = service.New( - service.WithClient(c.Sqlite), - service.WithLogger(c.Logger), - service.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for steps - // - // https://pkg.go.dev/github.com/go-vela/server/database/step#New - c.StepInterface, err = step.New( - step.WithClient(c.Sqlite), - step.WithLogger(c.Logger), - step.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for users - // - // https://pkg.go.dev/github.com/go-vela/server/database/user#New - c.UserInterface, err = user.New( - user.WithClient(c.Sqlite), - user.WithEncryptionKey(c.config.EncryptionKey), - user.WithLogger(c.Logger), - user.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - // create the database agnostic engine for workers - // - // https://pkg.go.dev/github.com/go-vela/server/database/worker#New - c.WorkerInterface, err = worker.New( - worker.WithClient(c.Sqlite), - worker.WithLogger(c.Logger), - worker.WithSkipCreation(c.config.SkipCreation), - ) - if err != nil { - return err - } - - return nil -} diff --git a/database/sqlite/sqlite_test.go b/database/sqlite/sqlite_test.go deleted file mode 100644 index 55c6940d0..000000000 --- a/database/sqlite/sqlite_test.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "testing" - "time" -) - -func TestSqlite_New(t *testing.T) { - // setup tests - tests := []struct { - failure bool - address string - want string - }{ - { - failure: false, - address: ":memory:", - want: ":memory:", - }, - { - failure: true, - address: "", - want: "", - }, - } - - // run tests - for _, test := range tests { - _, err := New( - WithAddress(test.address), - WithCompressionLevel(3), - WithConnectionLife(10*time.Second), - WithConnectionIdle(5), - WithConnectionOpen(20), - WithEncryptionKey("A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW"), - WithSkipCreation(false), - ) - - if test.failure { - if err == nil { - t.Errorf("New should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("New returned err: %v", err) - } - } -} - -func TestSqlite_setupDatabase(t *testing.T) { - // setup types - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup the skip test database client - _skipDatabase, err := NewTest() - if err != nil { - t.Errorf("unable to create new skip sqlite test database: %v", err) - } - - defer func() { _sql, _ := _skipDatabase.Sqlite.DB(); _sql.Close() }() - - err = WithSkipCreation(true)(_skipDatabase) - if err != nil { - t.Errorf("unable to set SkipCreation for sqlite test database: %v", err) - } - - tests := []struct { - failure bool - database *client - }{ - { - failure: false, - database: _database, - }, - { - failure: false, - database: _skipDatabase, - }, - } - - // run tests - for _, test := range tests { - err := setupDatabase(test.database) - - if test.failure { - if err == nil { - t.Errorf("setupDatabase should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("setupDatabase returned err: %v", err) - } - } -} - -func TestSqlite_createServices(t *testing.T) { - // setup types - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := createServices(_database) - - if test.failure { - if err == nil { - t.Errorf("createServices should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("createServices returned err: %v", err) - } - } -} From d51b1c33e4b0ca2e1b940a0c8a2dc5797d7ad8c9 Mon Sep 17 00:00:00 2001 From: Jordan Sussman Date: Mon, 12 Jun 2023 09:38:25 -0500 Subject: [PATCH 35/53] refactor(templates): use yaml for tests instead of base64 encoded json (#881) * refactor(templates): use yaml for tests instead of base64 encoded json * remove duplicative file --- compiler/native/compile_test.go | 40 +++++---- compiler/native/expand_test.go | 86 ++++++++++--------- compiler/native/testdata/gradle.yml | 70 +++++++++++++++ compiler/native/testdata/long_template.yml | 59 +++++++++++++ compiler/native/testdata/maven.yml | 70 +++++++++++++++ .../testdata/stages_pipeline_template.yml | 2 +- .../testdata/steps_pipeline_template.yml | 2 +- .../testdata/template-calls-itself.json | 18 ---- .../testdata/template-calls-template.json | 18 ---- compiler/native/testdata/template-gradle.json | 18 ---- compiler/native/testdata/template-maven.json | 18 ---- .../native/testdata/template-starlark.json | 18 ---- compiler/native/testdata/template.json | 18 ---- compiler/native/testdata/template.star | 6 +- .../native/testdata/template_calls_itself.yml | 11 +++ .../testdata/template_calls_template.yml | 15 ++++ 16 files changed, 300 insertions(+), 169 deletions(-) create mode 100644 compiler/native/testdata/gradle.yml create mode 100644 compiler/native/testdata/long_template.yml create mode 100644 compiler/native/testdata/maven.yml delete mode 100644 compiler/native/testdata/template-calls-itself.json delete mode 100644 compiler/native/testdata/template-calls-template.json delete mode 100644 compiler/native/testdata/template-gradle.json delete mode 100644 compiler/native/testdata/template-maven.json delete mode 100644 compiler/native/testdata/template-starlark.json delete mode 100644 compiler/native/testdata/template.json create mode 100644 compiler/native/testdata/template_calls_itself.yml create mode 100644 compiler/native/testdata/template_calls_template.yml diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index 67c89f3a2..07b420dde 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -606,10 +606,12 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { _, engine := gin.CreateTestContext(resp) // setup mock server - engine.GET("/api/v3/repos/:org/:name/contents/:path", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/template.json") + engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) { + body, err := convertFileToGithubResponse(c.Param("path")) + if err != nil { + t.Error(err) + } + c.JSON(http.StatusOK, body) }) s := httptest.NewServer(engine) @@ -871,10 +873,12 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { _, engine := gin.CreateTestContext(resp) // setup mock server - engine.GET("/api/v3/repos/foo/bar/contents/:path", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/template.json") + engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) { + body, err := convertFileToGithubResponse(c.Param("path")) + if err != nil { + t.Error(err) + } + c.JSON(http.StatusOK, body) }) s := httptest.NewServer(engine) @@ -1338,10 +1342,12 @@ func TestNative_Compile_InvalidType(t *testing.T) { _, engine := gin.CreateTestContext(resp) // setup mock server - engine.GET("/api/v3/repos/foo/bar/contents/:path", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/template.json") + engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) { + body, err := convertFileToGithubResponse(c.Param("path")) + if err != nil { + t.Error(err) + } + c.JSON(http.StatusOK, body) }) s := httptest.NewServer(engine) @@ -1886,10 +1892,12 @@ func Test_client_modifyConfig(t *testing.T) { _, engine := gin.CreateTestContext(resp) // setup mock server - engine.GET("/api/v3/repos/foo/bar/contents/:path", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/template.json") + engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) { + body, err := convertFileToGithubResponse(c.Param("path")) + if err != nil { + t.Error(err) + } + c.JSON(http.StatusOK, body) }) m := &types.Metadata{ diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index a32a2286d..d2a8a92b6 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -29,10 +29,12 @@ func TestNative_ExpandStages(t *testing.T) { _, engine := gin.CreateTestContext(resp) // setup mock server - engine.GET("/api/v3/repos/foo/bar/contents/:path", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/template.json") + engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) { + body, err := convertFileToGithubResponse(c.Param("path")) + if err != nil { + t.Error(err) + } + c.JSON(http.StatusOK, body) }) s := httptest.NewServer(engine) @@ -49,7 +51,7 @@ func TestNative_ExpandStages(t *testing.T) { tmpls := map[string]*yaml.Template{ "gradle": { Name: "gradle", - Source: "github.example.com/foo/bar/template.yml", + Source: "github.example.com/foo/bar/long_template.yml", Type: "github", }, } @@ -177,10 +179,12 @@ func TestNative_ExpandSteps(t *testing.T) { _, engine := gin.CreateTestContext(resp) // setup mock server - engine.GET("/api/v3/repos/foo/bar/contents/:path", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/template.json") + engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) { + body, err := convertFileToGithubResponse(c.Param("path")) + if err != nil { + t.Error(err) + } + c.JSON(http.StatusOK, body) }) s := httptest.NewServer(engine) @@ -209,7 +213,7 @@ func TestNative_ExpandSteps(t *testing.T) { tmpls: map[string]*yaml.Template{ "gradle": { Name: "gradle", - Source: "github.example.com/foo/bar/template.yml", + Source: "github.example.com/foo/bar/long_template.yml", Type: "github", }, }, @@ -219,7 +223,7 @@ func TestNative_ExpandSteps(t *testing.T) { tmpls: map[string]*yaml.Template{ "gradle": { Name: "gradle", - Source: "template.yml", + Source: "long_template.yml", Type: "file", }, }, @@ -351,15 +355,12 @@ func TestNative_ExpandStepsMulti(t *testing.T) { _, engine := gin.CreateTestContext(resp) // setup mock server - engine.GET("/api/v3/repos/foo/bar/contents/:path", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/template-gradle.json") - }) - engine.GET("/api/v3/repos/bar/foo/contents/:path", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/template-maven.json") + engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) { + body, err := convertFileToGithubResponse(c.Param("path")) + if err != nil { + t.Error(err) + } + c.JSON(http.StatusOK, body) }) s := httptest.NewServer(engine) @@ -538,7 +539,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { "auth_method": "token", "username": "octocat", "items": []interface{}{ - map[interface{}]interface{}{string("path"): string("docker"), string("source"): string("secret/docker")}, + map[interface{}]interface{}{"path": "docker", "source": "secret/docker"}, }, }, }, @@ -559,7 +560,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { "auth_method": "token", "username": "octocat", "items": []interface{}{ - map[interface{}]interface{}{string("path"): string("docker"), string("source"): string("secret/docker")}, + map[interface{}]interface{}{"path": "docker", "source": "secret/docker"}, }, }, }, @@ -615,10 +616,12 @@ func TestNative_ExpandStepsStarlark(t *testing.T) { _, engine := gin.CreateTestContext(resp) // setup mock server - engine.GET("/api/v3/repos/foo/bar/contents/:path", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/template-starlark.json") + engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) { + body, err := convertFileToGithubResponse(c.Param("path")) + if err != nil { + t.Error(err) + } + c.JSON(http.StatusOK, body) }) s := httptest.NewServer(engine) @@ -703,16 +706,12 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { _, engine := gin.CreateTestContext(resp) // setup mock server - engine.GET("/api/v3/repos/foo/bar/contents/:path", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/template.json") - }) - - engine.GET("/api/v3/repos/faz/baz/contents/:path", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/template-calls-template.json") + engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) { + body, err := convertFileToGithubResponse(c.Param("path")) + if err != nil { + t.Error(err) + } + c.JSON(http.StatusOK, body) }) s := httptest.NewServer(engine) @@ -746,7 +745,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { tmpls: map[string]*yaml.Template{ "chain": { Name: "chain", - Source: "github.example.com/faz/baz/template.yml", + Source: "github.example.com/faz/baz/template_calls_template.yml", Type: "github", }, }, @@ -872,10 +871,13 @@ func TestNative_ExpandSteps_TemplateCallTemplate_CircularFail(t *testing.T) { resp := httptest.NewRecorder() _, engine := gin.CreateTestContext(resp) - engine.GET("/api/v3/repos/bad/design/contents/:path", func(c *gin.Context) { - c.Header("Content-Type", "application/json") - c.Status(http.StatusOK) - c.File("testdata/template-calls-itself.json") + // setup mock server + engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) { + body, err := convertFileToGithubResponse(c.Param("path")) + if err != nil { + t.Error(err) + } + c.JSON(http.StatusOK, body) }) s := httptest.NewServer(engine) @@ -909,7 +911,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate_CircularFail(t *testing.T) { tmpls: map[string]*yaml.Template{ "circle": { Name: "circle", - Source: "github.example.com/bad/design/template.yml", + Source: "github.example.com/bad/design/template_calls_itself.yml", Type: "github", }, }, diff --git a/compiler/native/testdata/gradle.yml b/compiler/native/testdata/gradle.yml new file mode 100644 index 000000000..ed1e8e0d1 --- /dev/null +++ b/compiler/native/testdata/gradle.yml @@ -0,0 +1,70 @@ +metadata: + template: true + +steps: + - name: install + commands: + - ./gradlew downloadDependencies + environment: {{ .environment }} + image: {{ .image }} + {{ .pull_policy }} + + - name: test + commands: + - ./gradlew check + environment: {{ .environment }} + image: {{ .image }} + {{ .pull_policy }} + + - name: build + commands: + - ./gradlew build + environment: {{ .environment }} + image: {{ .image }} + {{ .pull_policy }} + +secrets: + - name: docker_username + key: org/repo/foo/bar + engine: native + type: repo + + - name: foo_password + key: org/repo/foo/password + engine: vault + type: repo + + - name: vault_token + + - origin: + name: private vault + image: target/secret-vault:latest + pull: always + secrets: [ vault_token ] + parameters: + addr: vault.example.com + auth_method: token + username: octocat + items: + - source: secret/docker + path: docker + + {{ if .secret }} + +- name: bar_password + key: org/repo/bar/password + engine: vault + type: repo + + {{ end }} + +services: + - name: postgres + image: postgres:12 + + {{ if .service }} + + - name: redis + key: redis:6 + + {{ end }} diff --git a/compiler/native/testdata/long_template.yml b/compiler/native/testdata/long_template.yml new file mode 100644 index 000000000..c9f92ee44 --- /dev/null +++ b/compiler/native/testdata/long_template.yml @@ -0,0 +1,59 @@ +environment: + star: "test3" + bar: "test4" + +metadata: + template: true + +steps: + - name: install + commands: + - ./gradlew downloadDependencies + environment: {{ .environment }} + image: {{ .image }} + {{ .pull_policy }} + + - name: test + commands: + - ./gradlew check + environment: {{ .environment }} + image: {{ .image }} + {{ .pull_policy }} + + - name: build + commands: + - ./gradlew build + environment: {{ .environment }} + image: {{ .image }} + {{ .pull_policy }} + +secrets: + - name: docker_username + key: org/repo/foo/bar + engine: native + type: repo + + - name: foo_password + key: org/repo/foo/password + engine: vault + type: repo + + {{ if .secret }} + + - name: bar_password + key: org/repo/bar/password + engine: vault + type: repo + + {{ end }} + +services: + - name: postgres + image: postgres:12 + + {{ if .service }} + + - name: redis + key: redis:6 + + {{ end }} diff --git a/compiler/native/testdata/maven.yml b/compiler/native/testdata/maven.yml new file mode 100644 index 000000000..fd6ff73b1 --- /dev/null +++ b/compiler/native/testdata/maven.yml @@ -0,0 +1,70 @@ +metadata: + template: true + +steps: + - name: install + commands: + - mvn downloadDependencies + environment: {{ .environment }} + image: {{ .image }} + {{ .pull_policy }} + + - name: test + commands: + - mvn check + environment: {{ .environment }} + image: {{ .image }} + {{ .pull_policy }} + + - name: build + commands: + - mvn build + environment: {{ .environment }} + image: {{ .image }} + {{ .pull_policy }} + +secrets: + - name: docker_username + key: org/repo/foo/bar + engine: native + type: repo + + - name: foo_password + key: org/repo/foo/password + engine: vault + type: repo + + - name: vault_token + + - origin: + name: private vault + image: target/secret-vault:latest + pull: always + secrets: [ vault_token ] + parameters: + addr: vault.example.com + auth_method: token + username: octocat + items: + - source: secret/docker + path: docker + + {{ if .secret }} + +- name: bar_password + key: org/repo/bar/password + engine: vault + type: repo + + {{ end }} + +services: + - name: postgres + image: postgres:12 + + {{ if .service }} + + - name: redis + key: redis:6 + + {{ end }} diff --git a/compiler/native/testdata/stages_pipeline_template.yml b/compiler/native/testdata/stages_pipeline_template.yml index 0354ec737..25189ed73 100644 --- a/compiler/native/testdata/stages_pipeline_template.yml +++ b/compiler/native/testdata/stages_pipeline_template.yml @@ -6,7 +6,7 @@ metadata: templates: - name: gradle - source: github.example.com/github/octocat/template.yml + source: github.example.com/github/octocat/long_template.yml type: github stages: diff --git a/compiler/native/testdata/steps_pipeline_template.yml b/compiler/native/testdata/steps_pipeline_template.yml index 9e8402e8e..7d8e8db28 100644 --- a/compiler/native/testdata/steps_pipeline_template.yml +++ b/compiler/native/testdata/steps_pipeline_template.yml @@ -5,7 +5,7 @@ metadata: templates: - name: gradle - source: github.example.com/foo/bar/template.yml + source: github.example.com/foo/bar/long_template.yml type: github steps: diff --git a/compiler/native/testdata/template-calls-itself.json b/compiler/native/testdata/template-calls-itself.json deleted file mode 100644 index a930d6780..000000000 --- a/compiler/native/testdata/template-calls-itself.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "file", - "encoding": "base64", - "size": 5362, - "name": "template.yml", - "path": "template.yml", - "content": "dmVyc2lvbjogIjEiCgp0ZW1wbGF0ZXM6CiAgLSBuYW1lOiB0ZXN0CiAgICBzb3VyY2U6IGdpdGh1Yi5leGFtcGxlLmNvbS9iYWQvZGVzaWduL3RlbXBsYXRlLnltbAogICAgdHlwZTogZ2l0aHViCgpzdGVwczoKICAtIG5hbWU6IGNhbGwgdGVtcGxhdGUKICAgIHRlbXBsYXRlOgogICAgICBuYW1lOiB0ZXN0Cg==", - "sha": "3d21ec53a331a6f037a91c368710b99387d012c1", - "url": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", - "git_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", - "html_url": "https://github.com/octokit/octokit.rb/blob/master/template.yml", - "download_url": "https://raw.githubusercontent.com/octokit/octokit.rb/master/template.yml", - "_links": { - "git": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", - "self": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", - "html": "https://github.com/octokit/octokit.rb/blob/master/template.yml" - } -} \ No newline at end of file diff --git a/compiler/native/testdata/template-calls-template.json b/compiler/native/testdata/template-calls-template.json deleted file mode 100644 index 6fb0be303..000000000 --- a/compiler/native/testdata/template-calls-template.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "file", - "encoding": "base64", - "size": 5362, - "name": "template.yml", - "path": "template.yml", - "content": "dmVyc2lvbjogIjEiCgp0ZW1wbGF0ZXM6CiAgLSBuYW1lOiB0ZXN0CiAgICBzb3VyY2U6IGdpdGh1Yi5leGFtcGxlLmNvbS9mb28vYmFyL3RlbXBsYXRlLnltbAogICAgdHlwZTogZ2l0aHViCgpzdGVwczoKICAtIG5hbWU6IGNhbGwgdGVtcGxhdGUKICAgIHRlbXBsYXRlOgogICAgICBuYW1lOiB0ZXN0CiAgICAgIHZhcnM6CiAgICAgICAgaW1hZ2U6IG9wZW5qZGs6bGF0ZXN0CiAgICAgICAgZW52aXJvbm1lbnQ6ICJ7IEdSQURMRV9VU0VSX0hPTUU6IC5ncmFkbGUsIEdSQURMRV9PUFRTOiAtRG9yZy5ncmFkbGUuZGFlbW9uPWZhbHNlIC1Eb3JnLmdyYWRsZS53b3JrZXJzLm1heD0xIC1Eb3JnLmdyYWRsZS5wYXJhbGxlbD1mYWxzZSB9IgogICAgICAgIHB1bGxfcG9saWN5OiAicHVsbDogdHJ1ZSIK", - "sha": "3d21ec53a331a6f037a91c368710b99387d012c1", - "url": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", - "git_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", - "html_url": "https://github.com/octokit/octokit.rb/blob/master/template.yml", - "download_url": "https://raw.githubusercontent.com/octokit/octokit.rb/master/template.yml", - "_links": { - "git": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", - "self": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", - "html": "https://github.com/octokit/octokit.rb/blob/master/template.yml" - } -} \ No newline at end of file diff --git a/compiler/native/testdata/template-gradle.json b/compiler/native/testdata/template-gradle.json deleted file mode 100644 index 8ad5f9091..000000000 --- a/compiler/native/testdata/template-gradle.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "file", - "encoding": "base64", - "size": 5362, - "name": "gradle.yml", - "path": "gradle.yml", - "content": "bWV0YWRhdGE6CiAgdGVtcGxhdGU6IHRydWUKCnN0ZXBzOgogIC0gbmFtZTogaW5zdGFsbAogICAgY29tbWFuZHM6CiAgICAgIC0gLi9ncmFkbGV3IGRvd25sb2FkRGVwZW5kZW5jaWVzCiAgICBlbnZpcm9ubWVudDoge3sgLmVudmlyb25tZW50IH19CiAgICBpbWFnZToge3sgLmltYWdlIH19CiAgICB7eyAucHVsbF9wb2xpY3kgfX0KCiAgLSBuYW1lOiB0ZXN0CiAgICBjb21tYW5kczoKICAgICAgLSAuL2dyYWRsZXcgY2hlY2sKICAgIGVudmlyb25tZW50OiB7eyAuZW52aXJvbm1lbnQgfX0KICAgIGltYWdlOiB7eyAuaW1hZ2UgfX0KICAgIHt7IC5wdWxsX3BvbGljeSB9fQoKICAtIG5hbWU6IGJ1aWxkCiAgICBjb21tYW5kczoKICAgICAgLSAuL2dyYWRsZXcgYnVpbGQKICAgIGVudmlyb25tZW50OiB7eyAuZW52aXJvbm1lbnQgfX0KICAgIGltYWdlOiB7eyAuaW1hZ2UgfX0KICAgIHt7IC5wdWxsX3BvbGljeSB9fQoKc2VjcmV0czoKICAtIG5hbWU6IGRvY2tlcl91c2VybmFtZQogICAga2V5OiBvcmcvcmVwby9mb28vYmFyCiAgICBlbmdpbmU6IG5hdGl2ZQogICAgdHlwZTogcmVwbwoKICAtIG5hbWU6IGZvb19wYXNzd29yZAogICAga2V5OiBvcmcvcmVwby9mb28vcGFzc3dvcmQKICAgIGVuZ2luZTogdmF1bHQKICAgIHR5cGU6IHJlcG8KCiAgLSBuYW1lOiB2YXVsdF90b2tlbgoKICAtIG9yaWdpbjoKICAgICAgbmFtZTogcHJpdmF0ZSB2YXVsdAogICAgICBpbWFnZTogdGFyZ2V0L3NlY3JldC12YXVsdDpsYXRlc3QKICAgICAgcHVsbDogYWx3YXlzCiAgICAgIHNlY3JldHM6IFsgdmF1bHRfdG9rZW4gXQogICAgICBwYXJhbWV0ZXJzOgogICAgICAgIGFkZHI6IHZhdWx0LmV4YW1wbGUuY29tCiAgICAgICAgYXV0aF9tZXRob2Q6IHRva2VuCiAgICAgICAgdXNlcm5hbWU6IG9jdG9jYXQKICAgICAgICBpdGVtczoKICAgICAgICAgIC0gc291cmNlOiBzZWNyZXQvZG9ja2VyCiAgICAgICAgICAgIHBhdGg6IGRvY2tlcgoKe3sgaWYgLnNlY3JldCB9fQoKICAtIG5hbWU6IGJhcl9wYXNzd29yZAogICAga2V5OiBvcmcvcmVwby9iYXIvcGFzc3dvcmQKICAgIGVuZ2luZTogdmF1bHQKICAgIHR5cGU6IHJlcG8KCnt7IGVuZCB9fQoKc2VydmljZXM6CiAgIC0gbmFtZTogcG9zdGdyZXMKICAgICBpbWFnZTogcG9zdGdyZXM6MTIKCiB7eyBpZiAuc2VydmljZSB9fQoKICAgLSBuYW1lOiByZWRpcwogICAgIGtleTogcmVkaXM6NgoKIHt7IGVuZCB9fQo=", - "sha": "3d21ec53a331a6f037a91c368710b99387d012c1", - "url": "https://api.github.com/repos/octokit/octokit.rb/contents/gradle.yml", - "git_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", - "html_url": "https://github.com/octokit/octokit.rb/blob/master/gradle.yml", - "download_url": "https://raw.githubusercontent.com/octokit/octokit.rb/master/gradle.yml", - "_links": { - "git": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", - "self": "https://api.github.com/repos/octokit/octokit.rb/contents/gradle.yml", - "html": "https://github.com/octokit/octokit.rb/blob/master/gradle.yml" - } -} diff --git a/compiler/native/testdata/template-maven.json b/compiler/native/testdata/template-maven.json deleted file mode 100644 index afc763414..000000000 --- a/compiler/native/testdata/template-maven.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "file", - "encoding": "base64", - "size": 5362, - "name": "maven.yml", - "path": "maven.yml", - "content": "bWV0YWRhdGE6CiAgdGVtcGxhdGU6IHRydWUKCnN0ZXBzOgogIC0gbmFtZTogaW5zdGFsbAogICAgY29tbWFuZHM6CiAgICAgIC0gbXZuIGRvd25sb2FkRGVwZW5kZW5jaWVzCiAgICBlbnZpcm9ubWVudDoge3sgLmVudmlyb25tZW50IH19CiAgICBpbWFnZToge3sgLmltYWdlIH19CiAgICB7eyAucHVsbF9wb2xpY3kgfX0KCiAgLSBuYW1lOiB0ZXN0CiAgICBjb21tYW5kczoKICAgICAgLSBtdm4gY2hlY2sKICAgIGVudmlyb25tZW50OiB7eyAuZW52aXJvbm1lbnQgfX0KICAgIGltYWdlOiB7eyAuaW1hZ2UgfX0KICAgIHt7IC5wdWxsX3BvbGljeSB9fQoKICAtIG5hbWU6IGJ1aWxkCiAgICBjb21tYW5kczoKICAgICAgLSBtdm4gYnVpbGQKICAgIGVudmlyb25tZW50OiB7eyAuZW52aXJvbm1lbnQgfX0KICAgIGltYWdlOiB7eyAuaW1hZ2UgfX0KICAgIHt7IC5wdWxsX3BvbGljeSB9fQoKc2VjcmV0czoKICAtIG5hbWU6IGRvY2tlcl91c2VybmFtZQogICAga2V5OiBvcmcvcmVwby9mb28vYmFyCiAgICBlbmdpbmU6IG5hdGl2ZQogICAgdHlwZTogcmVwbwoKICAtIG5hbWU6IGZvb19wYXNzd29yZAogICAga2V5OiBvcmcvcmVwby9mb28vcGFzc3dvcmQKICAgIGVuZ2luZTogdmF1bHQKICAgIHR5cGU6IHJlcG8KCiAgLSBuYW1lOiB2YXVsdF90b2tlbgoKICAtIG9yaWdpbjoKICAgICAgbmFtZTogcHJpdmF0ZSB2YXVsdAogICAgICBpbWFnZTogdGFyZ2V0L3NlY3JldC12YXVsdDpsYXRlc3QKICAgICAgcHVsbDogYWx3YXlzCiAgICAgIHNlY3JldHM6IFsgdmF1bHRfdG9rZW4gXQogICAgICBwYXJhbWV0ZXJzOgogICAgICAgIGFkZHI6IHZhdWx0LmV4YW1wbGUuY29tCiAgICAgICAgYXV0aF9tZXRob2Q6IHRva2VuCiAgICAgICAgdXNlcm5hbWU6IG9jdG9jYXQKICAgICAgICBpdGVtczoKICAgICAgICAgIC0gc291cmNlOiBzZWNyZXQvZG9ja2VyCiAgICAgICAgICAgIHBhdGg6IGRvY2tlcgoKe3sgaWYgLnNlY3JldCB9fQoKICAtIG5hbWU6IGJhcl9wYXNzd29yZAogICAga2V5OiBvcmcvcmVwby9iYXIvcGFzc3dvcmQKICAgIGVuZ2luZTogdmF1bHQKICAgIHR5cGU6IHJlcG8KCnt7IGVuZCB9fQoKc2VydmljZXM6CiAgIC0gbmFtZTogcG9zdGdyZXMKICAgICBpbWFnZTogcG9zdGdyZXM6MTIKCiB7eyBpZiAuc2VydmljZSB9fQoKICAgLSBuYW1lOiByZWRpcwogICAgIGtleTogcmVkaXM6NgoKIHt7IGVuZCB9fQo=", - "sha": "3d21ec53a331a6f037a91c368710b99387d012c1", - "url": "https://api.github.com/repos/octokit/octokit.rb/contents/maven.yml", - "git_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", - "html_url": "https://github.com/octokit/octokit.rb/blob/master/maven.yml", - "download_url": "https://raw.githubusercontent.com/octokit/octokit.rb/master/maven.yml", - "_links": { - "git": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", - "self": "https://api.github.com/repos/octokit/octokit.rb/contents/maven.yml", - "html": "https://github.com/octokit/octokit.rb/blob/master/maven.yml" - } -} diff --git a/compiler/native/testdata/template-starlark.json b/compiler/native/testdata/template-starlark.json deleted file mode 100644 index 5a29da9df..000000000 --- a/compiler/native/testdata/template-starlark.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "file", - "encoding": "base64", - "size": 5362, - "name": "template.star", - "path": "template.star", - "content": "ZGVmIG1haW4oY3R4KToKICByZXR1cm4gewogICAgJ3ZlcnNpb24nOiAnMScsCiAgICAnZW52aXJvbm1lbnQnOiB7CiAgICAgICdzdGFyJzogJ3Rlc3QzJywKICAgICAgJ2Jhcic6ICd0ZXN0NCcsCiAgICB9LAogICAgJ3N0ZXBzJzogWwogICAgICB7CiAgICAgICAgJ25hbWUnOiAnYnVpbGQnLAogICAgICAgICdpbWFnZSc6ICdnb2xhbmc6bGF0ZXN0JywKICAgICAgICAnY29tbWFuZHMnOiBbCiAgICAgICAgICAnZ28gYnVpbGQnLAogICAgICAgICAgJ2dvIHRlc3QnLAogICAgICAgIF0KICAgICAgfSwKICAgIF0sCn0K\n", - "sha": "3d21ec53a331a6f037a91c368710b99387d012c1", - "url": "https://api.github.com/repos/octokit/octokit.rb/contents/template.star", - "git_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", - "html_url": "https://github.com/octokit/octokit.rb/blob/master/template.star", - "download_url": "https://raw.githubusercontent.com/octokit/octokit.rb/master/template.star", - "_links": { - "git": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", - "self": "https://api.github.com/repos/octokit/octokit.rb/contents/template.star", - "html": "https://github.com/octokit/octokit.rb/blob/master/template.star" - } -} diff --git a/compiler/native/testdata/template.json b/compiler/native/testdata/template.json deleted file mode 100644 index f65b38008..000000000 --- a/compiler/native/testdata/template.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "file", - "encoding": "base64", - "size": 5362, - "name": "template.yml", - "path": "template.yml", - "content": "ZW52aXJvbm1lbnQ6CiAgc3RhcjogInRlc3QzIgogIGJhcjogInRlc3Q0IgoKbWV0YWRhdGE6CiAgdGVtcGxhdGU6IHRydWUKCnN0ZXBzOgogIC0gbmFtZTogaW5zdGFsbAogICAgY29tbWFuZHM6CiAgICAgIC0gLi9ncmFkbGV3IGRvd25sb2FkRGVwZW5kZW5jaWVzCiAgICBlbnZpcm9ubWVudDoge3sgLmVudmlyb25tZW50IH19CiAgICBpbWFnZToge3sgLmltYWdlIH19CiAgICB7eyAucHVsbF9wb2xpY3kgfX0KCiAgLSBuYW1lOiB0ZXN0CiAgICBjb21tYW5kczoKICAgICAgLSAuL2dyYWRsZXcgY2hlY2sKICAgIGVudmlyb25tZW50OiB7eyAuZW52aXJvbm1lbnQgfX0KICAgIGltYWdlOiB7eyAuaW1hZ2UgfX0KICAgIHt7IC5wdWxsX3BvbGljeSB9fQoKICAtIG5hbWU6IGJ1aWxkCiAgICBjb21tYW5kczoKICAgICAgLSAuL2dyYWRsZXcgYnVpbGQKICAgIGVudmlyb25tZW50OiB7eyAuZW52aXJvbm1lbnQgfX0KICAgIGltYWdlOiB7eyAuaW1hZ2UgfX0KICAgIHt7IC5wdWxsX3BvbGljeSB9fQoKc2VjcmV0czoKICAtIG5hbWU6IGRvY2tlcl91c2VybmFtZQogICAga2V5OiBvcmcvcmVwby9mb28vYmFyCiAgICBlbmdpbmU6IG5hdGl2ZQogICAgdHlwZTogcmVwbwoKICAtIG5hbWU6IGZvb19wYXNzd29yZAogICAga2V5OiBvcmcvcmVwby9mb28vcGFzc3dvcmQKICAgIGVuZ2luZTogdmF1bHQKICAgIHR5cGU6IHJlcG8KCnt7IGlmIC5zZWNyZXQgfX0KCiAgLSBuYW1lOiBiYXJfcGFzc3dvcmQKICAgIGtleTogb3JnL3JlcG8vYmFyL3Bhc3N3b3JkCiAgICBlbmdpbmU6IHZhdWx0CiAgICB0eXBlOiByZXBvCgp7eyBlbmQgfX0KCnNlcnZpY2VzOgogICAtIG5hbWU6IHBvc3RncmVzCiAgICAgaW1hZ2U6IHBvc3RncmVzOjEyCgoge3sgaWYgLnNlcnZpY2UgfX0KCiAgIC0gbmFtZTogcmVkaXMKICAgICBrZXk6IHJlZGlzOjYKCiB7eyBlbmQgfX0K", - "sha": "3d21ec53a331a6f037a91c368710b99387d012c1", - "url": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", - "git_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", - "html_url": "https://github.com/octokit/octokit.rb/blob/master/template.yml", - "download_url": "https://raw.githubusercontent.com/octokit/octokit.rb/master/template.yml", - "_links": { - "git": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", - "self": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", - "html": "https://github.com/octokit/octokit.rb/blob/master/template.yml" - } -} diff --git a/compiler/native/testdata/template.star b/compiler/native/testdata/template.star index 079d70943..9c2815791 100644 --- a/compiler/native/testdata/template.star +++ b/compiler/native/testdata/template.star @@ -1,6 +1,10 @@ def main(ctx): return { 'version': '1', + 'environment': { + 'star': 'test3', + 'bar': 'test4', + }, 'steps': [ { 'name': 'build', @@ -11,4 +15,4 @@ def main(ctx): ] }, ], -} \ No newline at end of file +} diff --git a/compiler/native/testdata/template_calls_itself.yml b/compiler/native/testdata/template_calls_itself.yml new file mode 100644 index 000000000..9b467a675 --- /dev/null +++ b/compiler/native/testdata/template_calls_itself.yml @@ -0,0 +1,11 @@ +version: "1" + +templates: + - name: test + source: github.example.com/bad/design/template_calls_itself.yml + type: github + +steps: + - name: call template + template: + name: test diff --git a/compiler/native/testdata/template_calls_template.yml b/compiler/native/testdata/template_calls_template.yml new file mode 100644 index 000000000..b6197169d --- /dev/null +++ b/compiler/native/testdata/template_calls_template.yml @@ -0,0 +1,15 @@ +version: "1" + +templates: + - name: test + source: github.example.com/foo/bar/long_template.yml + type: github + +steps: + - name: call template + template: + name: test + vars: + image: openjdk:latest + environment: "{ GRADLE_USER_HOME: .gradle, GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false }" + pull_policy: "pull: true" From ca3e1e3f8cf72017f4eb38f928de611bd20291d3 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Tue, 13 Jun 2023 10:39:49 -0600 Subject: [PATCH 36/53] refactor(database): return pipeline on created and updated (#882) * refactor(database): return pipeline on created and updated * fix weird error handling --- api/build/create.go | 13 +----------- api/build/restart.go | 13 +----------- api/pipeline/create.go | 12 +---------- api/pipeline/update.go | 12 +---------- api/webhook/post.go | 15 +------------- cmd/vela-server/schedule.go | 8 +------- database/pipeline/count_repo_test.go | 4 ++-- database/pipeline/count_test.go | 4 ++-- database/pipeline/create.go | 21 +++++++++++++------- database/pipeline/create_test.go | 8 +++++++- database/pipeline/delete_test.go | 2 +- database/pipeline/get_repo_test.go | 2 +- database/pipeline/get_test.go | 2 +- database/pipeline/interface.go | 4 ++-- database/pipeline/list_repo_test.go | 4 ++-- database/pipeline/list_test.go | 4 ++-- database/pipeline/update.go | 22 ++++++++++++++------- database/pipeline/update_test.go | 10 ++++++++-- router/middleware/pipeline/pipeline_test.go | 2 +- 19 files changed, 64 insertions(+), 98 deletions(-) diff --git a/api/build/create.go b/api/build/create.go index 3f695d791..38a8867c2 100644 --- a/api/build/create.go +++ b/api/build/create.go @@ -308,7 +308,7 @@ func CreateBuild(c *gin.Context) { pipeline.SetRef(input.GetRef()) // send API call to create the pipeline - err = database.FromContext(c).CreatePipeline(pipeline) + pipeline, err = database.FromContext(c).CreatePipeline(pipeline) if err != nil { retErr := fmt.Errorf("unable to create new build: failed to create pipeline for %s: %w", r.GetFullName(), err) @@ -316,17 +316,6 @@ func CreateBuild(c *gin.Context) { return } - - // send API call to capture the created pipeline - pipeline, err = database.FromContext(c).GetPipelineForRepo(pipeline.GetCommit(), r) - if err != nil { - //nolint:lll // ignore long line length due to error message - retErr := fmt.Errorf("unable to create new build: failed to get new pipeline %s/%s: %w", r.GetFullName(), pipeline.GetCommit(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } } input.SetPipelineID(pipeline.GetID()) diff --git a/api/build/restart.go b/api/build/restart.go index 302090a9c..8bb85e798 100644 --- a/api/build/restart.go +++ b/api/build/restart.go @@ -299,7 +299,7 @@ func RestartBuild(c *gin.Context) { pipeline.SetRef(b.GetRef()) // send API call to create the pipeline - err = database.FromContext(c).CreatePipeline(pipeline) + pipeline, err = database.FromContext(c).CreatePipeline(pipeline) if err != nil { retErr := fmt.Errorf("unable to create pipeline for %s: %w", r.GetFullName(), err) @@ -307,17 +307,6 @@ func RestartBuild(c *gin.Context) { return } - - // send API call to capture the created pipeline - pipeline, err = database.FromContext(c).GetPipelineForRepo(pipeline.GetCommit(), r) - if err != nil { - //nolint:lll // ignore long line length due to error message - retErr := fmt.Errorf("unable to get new pipeline %s/%s: %w", r.GetFullName(), pipeline.GetCommit(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } } b.SetPipelineID(pipeline.GetID()) diff --git a/api/pipeline/create.go b/api/pipeline/create.go index 2b8bbfaf0..f6a4938bc 100644 --- a/api/pipeline/create.go +++ b/api/pipeline/create.go @@ -98,7 +98,7 @@ func CreatePipeline(c *gin.Context) { input.SetRepoID(r.GetID()) // send API call to create the pipeline - err = database.FromContext(c).CreatePipeline(input) + p, err := database.FromContext(c).CreatePipeline(input) if err != nil { retErr := fmt.Errorf("unable to create pipeline %s/%s: %w", r.GetFullName(), input.GetCommit(), err) @@ -107,15 +107,5 @@ func CreatePipeline(c *gin.Context) { return } - // send API call to capture the created pipeline - p, err := database.FromContext(c).GetPipelineForRepo(input.GetCommit(), r) - if err != nil { - retErr := fmt.Errorf("unable to capture pipeline %s/%s: %w", r.GetFullName(), input.GetCommit(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - c.JSON(http.StatusCreated, p) } diff --git a/api/pipeline/update.go b/api/pipeline/update.go index c769cfb7b..9c10d40b6 100644 --- a/api/pipeline/update.go +++ b/api/pipeline/update.go @@ -170,7 +170,7 @@ func UpdatePipeline(c *gin.Context) { } // send API call to update the pipeline - err = database.FromContext(c).UpdatePipeline(p) + p, err = database.FromContext(c).UpdatePipeline(p) if err != nil { retErr := fmt.Errorf("unable to update pipeline %s: %w", entry, err) @@ -179,15 +179,5 @@ func UpdatePipeline(c *gin.Context) { return } - // send API call to capture the updated pipeline - p, err = database.FromContext(c).GetPipelineForRepo(p.GetCommit(), r) - if err != nil { - retErr := fmt.Errorf("unable to capture pipeline %s: %w", entry, err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - c.JSON(http.StatusOK, p) } diff --git a/api/webhook/post.go b/api/webhook/post.go index c25abb634..2832941dc 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -561,7 +561,7 @@ func PostWebhook(c *gin.Context) { pipeline.SetRef(b.GetRef()) // send API call to create the pipeline - err = database.FromContext(c).CreatePipeline(pipeline) + pipeline, err = database.FromContext(c).CreatePipeline(pipeline) if err != nil { retErr := fmt.Errorf("%s: failed to create pipeline for %s: %w", baseErr, repo.GetFullName(), err) @@ -580,19 +580,6 @@ func PostWebhook(c *gin.Context) { return } - - // send API call to capture the created pipeline - pipeline, err = database.FromContext(c).GetPipelineForRepo(pipeline.GetCommit(), repo) - if err != nil { - //nolint:lll // ignore long line length due to error message - retErr := fmt.Errorf("%s: failed to get new pipeline %s/%s: %w", baseErr, repo.GetFullName(), pipeline.GetCommit(), err) - util.HandleError(c, http.StatusInternalServerError, retErr) - - h.SetStatus(constants.StatusFailure) - h.SetError(retErr.Error()) - - return - } } b.SetPipelineID(pipeline.GetID()) diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index 532dd65b7..ab0a56b90 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -294,7 +294,7 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat pipeline.SetRef(b.GetRef()) // send API call to create the pipeline - err = database.CreatePipeline(pipeline) + pipeline, err = database.CreatePipeline(pipeline) if err != nil { err = fmt.Errorf("failed to create pipeline for %s: %w", r.GetFullName(), err) @@ -308,12 +308,6 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat return err } - - // send API call to capture the created pipeline - pipeline, err = database.GetPipelineForRepo(pipeline.GetCommit(), r) - if err != nil { - return fmt.Errorf("unable to get new pipeline %s/%s: %w", r.GetFullName(), pipeline.GetCommit(), err) - } } b.SetPipelineID(pipeline.GetID()) diff --git a/database/pipeline/count_repo_test.go b/database/pipeline/count_repo_test.go index 9cb91b72b..cc3650d12 100644 --- a/database/pipeline/count_repo_test.go +++ b/database/pipeline/count_repo_test.go @@ -42,12 +42,12 @@ func TestPipeline_Engine_CountPipelinesForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreatePipeline(_pipelineOne) + _, err := _sqlite.CreatePipeline(_pipelineOne) if err != nil { t.Errorf("unable to create test pipeline for sqlite: %v", err) } - err = _sqlite.CreatePipeline(_pipelineTwo) + _, err = _sqlite.CreatePipeline(_pipelineTwo) if err != nil { t.Errorf("unable to create test pipeline for sqlite: %v", err) } diff --git a/database/pipeline/count_test.go b/database/pipeline/count_test.go index a811b01df..bbc654fb5 100644 --- a/database/pipeline/count_test.go +++ b/database/pipeline/count_test.go @@ -41,12 +41,12 @@ func TestPipeline_Engine_CountPipelines(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreatePipeline(_pipelineOne) + _, err := _sqlite.CreatePipeline(_pipelineOne) if err != nil { t.Errorf("unable to create test pipeline for sqlite: %v", err) } - err = _sqlite.CreatePipeline(_pipelineTwo) + _, err = _sqlite.CreatePipeline(_pipelineTwo) if err != nil { t.Errorf("unable to create test pipeline for sqlite: %v", err) } diff --git a/database/pipeline/create.go b/database/pipeline/create.go index 5d6e4771f..424c24c23 100644 --- a/database/pipeline/create.go +++ b/database/pipeline/create.go @@ -12,7 +12,7 @@ import ( ) // CreatePipeline creates a new pipeline in the database. -func (e *engine) CreatePipeline(p *library.Pipeline) error { +func (e *engine) CreatePipeline(p *library.Pipeline) (*library.Pipeline, error) { e.logger.WithFields(logrus.Fields{ "pipeline": p.GetCommit(), }).Tracef("creating pipeline %s in the database", p.GetCommit()) @@ -27,7 +27,7 @@ func (e *engine) CreatePipeline(p *library.Pipeline) error { // https://pkg.go.dev/github.com/go-vela/types/database#Pipeline.Validate err := pipeline.Validate() if err != nil { - return err + return nil, err } // compress data for the pipeline @@ -35,12 +35,19 @@ func (e *engine) CreatePipeline(p *library.Pipeline) error { // https://pkg.go.dev/github.com/go-vela/types/database#Pipeline.Compress err = pipeline.Compress(e.config.CompressionLevel) if err != nil { - return err + return nil, err } // send query to the database - return e.client. - Table(constants.TablePipeline). - Create(pipeline). - Error + err = e.client.Table(constants.TablePipeline).Create(pipeline).Error + if err != nil { + return nil, err + } + + err = pipeline.Decompress() + if err != nil { + return nil, err + } + + return pipeline.ToLibrary(), nil } diff --git a/database/pipeline/create_test.go b/database/pipeline/create_test.go index be29ad8fd..5a19c7e84 100644 --- a/database/pipeline/create_test.go +++ b/database/pipeline/create_test.go @@ -5,6 +5,7 @@ package pipeline import ( + "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -19,6 +20,7 @@ func TestPipeline_Engine_CreatePipeline(t *testing.T) { _pipeline.SetRef("refs/heads/master") _pipeline.SetType("yaml") _pipeline.SetVersion("1") + _pipeline.SetData([]byte{}) _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -57,7 +59,7 @@ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err := test.database.CreatePipeline(_pipeline) + got, err := test.database.CreatePipeline(_pipeline) if test.failure { if err == nil { @@ -70,6 +72,10 @@ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). if err != nil { t.Errorf("CreatePipeline for %s returned err: %v", test.name, err) } + + if !reflect.DeepEqual(got, _pipeline) { + t.Errorf("CreatePipeline for %s returned %s, want %s", test.name, got, _pipeline) + } }) } } diff --git a/database/pipeline/delete_test.go b/database/pipeline/delete_test.go index ff3e7f892..6bef174c7 100644 --- a/database/pipeline/delete_test.go +++ b/database/pipeline/delete_test.go @@ -31,7 +31,7 @@ func TestPipeline_Engine_DeletePipeline(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreatePipeline(_pipeline) + _, err := _sqlite.CreatePipeline(_pipeline) if err != nil { t.Errorf("unable to create test pipeline for sqlite: %v", err) } diff --git a/database/pipeline/get_repo_test.go b/database/pipeline/get_repo_test.go index bb956d469..ab810ba5f 100644 --- a/database/pipeline/get_repo_test.go +++ b/database/pipeline/get_repo_test.go @@ -37,7 +37,7 @@ func TestPipeline_Engine_GetPipelineForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreatePipeline(_pipeline) + _, err := _sqlite.CreatePipeline(_pipeline) if err != nil { t.Errorf("unable to create test pipeline for sqlite: %v", err) } diff --git a/database/pipeline/get_test.go b/database/pipeline/get_test.go index 050a06c3d..f7c3565d3 100644 --- a/database/pipeline/get_test.go +++ b/database/pipeline/get_test.go @@ -37,7 +37,7 @@ func TestPipeline_Engine_GetPipeline(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreatePipeline(_pipeline) + _, err := _sqlite.CreatePipeline(_pipeline) if err != nil { t.Errorf("unable to create test pipeline for sqlite: %v", err) } diff --git a/database/pipeline/interface.go b/database/pipeline/interface.go index eaafa59fe..c28c30da8 100644 --- a/database/pipeline/interface.go +++ b/database/pipeline/interface.go @@ -31,7 +31,7 @@ type PipelineInterface interface { // CountPipelinesForRepo defines a function that gets the count of pipelines by repo ID. CountPipelinesForRepo(*library.Repo) (int64, error) // CreatePipeline defines a function that creates a new pipeline. - CreatePipeline(*library.Pipeline) error + CreatePipeline(*library.Pipeline) (*library.Pipeline, error) // DeletePipeline defines a function that deletes an existing pipeline. DeletePipeline(*library.Pipeline) error // GetPipeline defines a function that gets a pipeline by ID. @@ -43,5 +43,5 @@ type PipelineInterface interface { // ListPipelinesForRepo defines a function that gets a list of pipelines by repo ID. ListPipelinesForRepo(*library.Repo, int, int) ([]*library.Pipeline, int64, error) // UpdatePipeline defines a function that updates an existing pipeline. - UpdatePipeline(*library.Pipeline) error + UpdatePipeline(*library.Pipeline) (*library.Pipeline, error) } diff --git a/database/pipeline/list_repo_test.go b/database/pipeline/list_repo_test.go index c1eaef193..dc64e8a0f 100644 --- a/database/pipeline/list_repo_test.go +++ b/database/pipeline/list_repo_test.go @@ -53,12 +53,12 @@ func TestPipeline_Engine_ListPipelinesForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreatePipeline(_pipelineOne) + _, err := _sqlite.CreatePipeline(_pipelineOne) if err != nil { t.Errorf("unable to create test pipeline for sqlite: %v", err) } - err = _sqlite.CreatePipeline(_pipelineTwo) + _, err = _sqlite.CreatePipeline(_pipelineTwo) if err != nil { t.Errorf("unable to create test pipeline for sqlite: %v", err) } diff --git a/database/pipeline/list_test.go b/database/pipeline/list_test.go index bcd5dcef5..36f65199d 100644 --- a/database/pipeline/list_test.go +++ b/database/pipeline/list_test.go @@ -53,12 +53,12 @@ func TestPipeline_Engine_ListPipelines(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreatePipeline(_pipelineOne) + _, err := _sqlite.CreatePipeline(_pipelineOne) if err != nil { t.Errorf("unable to create test pipeline for sqlite: %v", err) } - err = _sqlite.CreatePipeline(_pipelineTwo) + _, err = _sqlite.CreatePipeline(_pipelineTwo) if err != nil { t.Errorf("unable to create test pipeline for sqlite: %v", err) } diff --git a/database/pipeline/update.go b/database/pipeline/update.go index 85add5411..59b552164 100644 --- a/database/pipeline/update.go +++ b/database/pipeline/update.go @@ -12,7 +12,7 @@ import ( ) // UpdatePipeline updates an existing pipeline in the database. -func (e *engine) UpdatePipeline(p *library.Pipeline) error { +func (e *engine) UpdatePipeline(p *library.Pipeline) (*library.Pipeline, error) { e.logger.WithFields(logrus.Fields{ "pipeline": p.GetCommit(), }).Tracef("updating pipeline %s in the database", p.GetCommit()) @@ -27,7 +27,7 @@ func (e *engine) UpdatePipeline(p *library.Pipeline) error { // https://pkg.go.dev/github.com/go-vela/types/database#Pipeline.Validate err := pipeline.Validate() if err != nil { - return err + return nil, err } // compress data for the pipeline @@ -35,12 +35,20 @@ func (e *engine) UpdatePipeline(p *library.Pipeline) error { // https://pkg.go.dev/github.com/go-vela/types/database#Pipeline.Compress err = pipeline.Compress(e.config.CompressionLevel) if err != nil { - return err + return nil, err } // send query to the database - return e.client. - Table(constants.TablePipeline). - Save(pipeline). - Error + err = e.client.Table(constants.TablePipeline).Save(pipeline).Error + if err != nil { + return nil, err + } + + // decompress pipeline to return + err = pipeline.Decompress() + if err != nil { + return nil, err + } + + return pipeline.ToLibrary(), nil } diff --git a/database/pipeline/update_test.go b/database/pipeline/update_test.go index 33e18d738..d321d63b5 100644 --- a/database/pipeline/update_test.go +++ b/database/pipeline/update_test.go @@ -5,6 +5,7 @@ package pipeline import ( + "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -19,6 +20,7 @@ func TestPipeline_Engine_UpdatePipeline(t *testing.T) { _pipeline.SetRef("refs/heads/master") _pipeline.SetType("yaml") _pipeline.SetVersion("1") + _pipeline.SetData([]byte{}) _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -33,7 +35,7 @@ WHERE "id" = $15`). _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreatePipeline(_pipeline) + _, err := _sqlite.CreatePipeline(_pipeline) if err != nil { t.Errorf("unable to create test pipeline for sqlite: %v", err) } @@ -59,7 +61,7 @@ WHERE "id" = $15`). // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err = test.database.UpdatePipeline(_pipeline) + got, err := test.database.UpdatePipeline(_pipeline) if test.failure { if err == nil { @@ -72,6 +74,10 @@ WHERE "id" = $15`). if err != nil { t.Errorf("UpdatePipeline for %s returned err: %v", test.name, err) } + + if !reflect.DeepEqual(got, _pipeline) { + t.Errorf("UpdatePipeline for %s returned %s, want %s", test.name, got, _pipeline) + } }) } } diff --git a/router/middleware/pipeline/pipeline_test.go b/router/middleware/pipeline/pipeline_test.go index 4618524fd..a7c76c892 100644 --- a/router/middleware/pipeline/pipeline_test.go +++ b/router/middleware/pipeline/pipeline_test.go @@ -108,7 +108,7 @@ func TestPipeline_Establish(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreatePipeline(want) + _, _ = db.CreatePipeline(want) // setup context gin.SetMode(gin.TestMode) From e06d557dd12cb1c6477b7403d7815f091cf25e51 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 13 Jun 2023 13:56:55 -0500 Subject: [PATCH 37/53] fix(deps): update deps (patch) (#835) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 29 +++++++++++------------ go.sum | 72 +++++++++++++++++++++++++--------------------------------- 2 files changed, 46 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index be974371f..445f5c13c 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,9 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 - github.com/adhocore/gronx v1.6.2 - github.com/alicebob/miniredis/v2 v2.30.2 - github.com/aws/aws-sdk-go v1.44.248 + github.com/adhocore/gronx v1.6.3 + github.com/alicebob/miniredis/v2 v2.30.3 + github.com/aws/aws-sdk-go v1.44.281 github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.0 @@ -22,23 +22,23 @@ require ( github.com/goware/urlx v0.3.2 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-multierror v1.1.1 - github.com/hashicorp/go-retryablehttp v0.7.2 - github.com/hashicorp/vault/api v1.9.1 + github.com/hashicorp/go-retryablehttp v0.7.4 + github.com/hashicorp/vault/api v1.9.2 github.com/joho/godotenv v1.5.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.15.0 - github.com/redis/go-redis/v9 v9.0.3 - github.com/sirupsen/logrus v1.9.0 + github.com/prometheus/client_golang v1.15.1 + github.com/redis/go-redis/v9 v9.0.5 + github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.9.5 - github.com/urfave/cli/v2 v2.25.1 + github.com/urfave/cli/v2 v2.25.6 go.starlark.net v0.0.0-20230302034142-4b1e35fe2254 golang.org/x/oauth2 v0.7.0 golang.org/x/sync v0.1.0 gopkg.in/square/go-jose.v2 v2.6.0 - gorm.io/driver/postgres v1.5.0 + gorm.io/driver/postgres v1.5.2 gorm.io/driver/sqlite v1.4.4 - gorm.io/gorm v1.25.0 - k8s.io/apimachinery v0.27.1 + gorm.io/gorm v1.25.1 + k8s.io/apimachinery v0.27.2 ) require ( @@ -59,6 +59,7 @@ require ( github.com/fatih/color v1.10.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-jose/go-jose/v3 v3.0.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -80,7 +81,7 @@ require ( github.com/imdario/mergo v0.3.11 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.3.0 // indirect + github.com/jackc/pgx/v5 v5.3.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -113,7 +114,7 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yuin/gopher-lua v1.1.0 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect - golang.org/x/crypto v0.7.0 // indirect + golang.org/x/crypto v0.8.0 // indirect golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect diff --git a/go.sum b/go.sum index 3b4b80685..e705f7c03 100644 --- a/go.sum +++ b/go.sum @@ -57,17 +57,17 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/adhocore/gronx v1.6.2 h1:/Pg6cuHFJmUGRIYWhRFjb6iL9fdzNmoMPj+/r6L01KU= -github.com/adhocore/gronx v1.6.2/go.mod h1:7oUY1WAU8rEJWmAxXR2DN0JaO4gi9khSgKjiRypqteg= +github.com/adhocore/gronx v1.6.3 h1:bnm5vieTrY3QQPpsfB0hrAaeaHDpuZTUC2LLCVMLe9c= +github.com/adhocore/gronx v1.6.3/go.mod h1:7oUY1WAU8rEJWmAxXR2DN0JaO4gi9khSgKjiRypqteg= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis/v2 v2.11.1/go.mod h1:UA48pmi7aSazcGAvcdKcBB49z521IC9VjTTRz2nIaJE= -github.com/alicebob/miniredis/v2 v2.30.2 h1:lc1UAUT9ZA7h4srlfBmBt2aorm5Yftk9nBjxz7EyY9I= -github.com/alicebob/miniredis/v2 v2.30.2/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg= +github.com/alicebob/miniredis/v2 v2.30.3 h1:hrqDB4cHFSHQf4gO3xu6YKQg8PqJpNjLYsQAFYHstqw= +github.com/alicebob/miniredis/v2 v2.30.3/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.44.248 h1:GvkxpgsxqNc03LmhXiaxKpzbyxndnex7V+OThLx4g5M= -github.com/aws/aws-sdk-go v1.44.248/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.281 h1:z/ptheJvINaIAsKXthxONM+toTKw2pxyk700Hfm6yUw= +github.com/aws/aws-sdk-go v1.44.281/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -100,7 +100,6 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -126,6 +125,8 @@ github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH8 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -233,8 +234,8 @@ github.com/hashicorp/go-hclog v0.16.2 h1:K4ev2ib4LdQETX5cSZBG0DVLk1jwGqSPXBjdah3 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= -github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 h1:om4Al8Oy7kCm/B86rLCLah4Dt5Aa0Fr5rYBG60OzwHQ= @@ -248,8 +249,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/vault/api v1.9.1 h1:LtY/I16+5jVGU8rufyyAkwopgq/HpUnxFBg+QLOAV38= -github.com/hashicorp/vault/api v1.9.1/go.mod h1:78kktNcQYbBGSrOjQfHjXN32OhhxXnbYl3zxpd2uPUs= +github.com/hashicorp/vault/api v1.9.2 h1:YjkZLJ7K3inKgMZ0wzCU9OHqc+UqMQyXsPXnf3Cl2as= +github.com/hashicorp/vault/api v1.9.2/go.mod h1:jo5Y/ET+hNyz+JnKDt8XLAdKs+AM0G5W0Vp1IrFI8N8= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -260,9 +261,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.3.0 h1:/NQi8KHMpKWHInxXesC8yD4DhkXPrVhmnwYkjp9AmBA= -github.com/jackc/pgx/v5 v5.3.0/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= -github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgx/v5 v5.3.1 h1:Fcr8QJ1ZeLi5zsPZqQeUZhNhxfkkKBOgJuYkJHoBOtU= +github.com/jackc/pgx/v5 v5.3.1/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -284,13 +284,10 @@ github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBF github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.10.8 h1:3fdt97i/cwSU83+E0hZTC/Xpc9mTZxc6UWSCRcSbxiE= @@ -336,8 +333,8 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= @@ -345,10 +342,9 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k= -github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= +github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= +github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -358,8 +354,8 @@ github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkB github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= @@ -381,8 +377,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU= github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/urfave/cli/v2 v2.25.1 h1:zw8dSP7ghX0Gmm8vugrs6q9Ku0wzweqPyshy+syu9Gw= -github.com/urfave/cli/v2 v2.25.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.6 h1:yuSkgDSZfH3L1CjF2/5fNNg2KbM47pY2EvjBq4ESQnU= +github.com/urfave/cli/v2 v2.25.6/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -407,15 +403,15 @@ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUu golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -485,7 +481,6 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -557,7 +552,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -565,7 +559,6 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -575,7 +568,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -733,7 +725,6 @@ google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -745,14 +736,13 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U= -gorm.io/driver/postgres v1.5.0/go.mod h1:FUZXzO+5Uqg5zzwzv4KK49R8lvGIyscBOqYrtI1Ce9A= +gorm.io/driver/postgres v1.5.2 h1:ytTDxxEv+MplXOfFe3Lzm7SjG09fcdb3Z/c056DTBx0= +gorm.io/driver/postgres v1.5.2/go.mod h1:fmpX0m2I1PKuR7mKZiEluwrP3hbs+ps7JIGMUBpCgl8= gorm.io/driver/sqlite v1.4.4 h1:gIufGoR0dQzjkyqDyYSCvsYR6fba1Gw5YKDqKeChxFc= gorm.io/driver/sqlite v1.4.4/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI= gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= -gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= -gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU= -gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/gorm v1.25.1 h1:nsSALe5Pr+cM3V1qwwQ7rOkw+6UeLrX5O4v3llhHa64= +gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -760,8 +750,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/apimachinery v0.27.1 h1:EGuZiLI95UQQcClhanryclaQE6xjg1Bts6/L3cD7zyc= -k8s.io/apimachinery v0.27.1/go.mod h1:5ikh59fK3AJ287GUvpUsryoMFtH9zj/ARfWCo3AyXTM= +k8s.io/apimachinery v0.27.2 h1:vBjGaKKieaIreI+oQwELalVG4d8f3YAMNpWLzDXkxeg= +k8s.io/apimachinery v0.27.2/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= From e17c9710cc42276759564c40f894c318738fa460 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Thu, 15 Jun 2023 10:07:37 -0600 Subject: [PATCH 38/53] refactor(database): return build object on created and updated (#884) * refactor(database): return build object on created and updated * address feedback --- api/admin/build.go | 4 ++-- api/build/cancel.go | 2 +- api/build/clean.go | 2 +- api/build/plan.go | 11 +---------- api/build/publish.go | 2 +- api/build/update.go | 5 +---- api/webhook/post.go | 2 +- database/build/clean_test.go | 8 ++++---- database/build/count_deployment_test.go | 4 ++-- database/build/count_org_test.go | 4 ++-- database/build/count_repo_test.go | 4 ++-- database/build/count_status_test.go | 4 ++-- database/build/count_test.go | 4 ++-- database/build/create.go | 14 ++++++++------ database/build/create_test.go | 7 ++++++- database/build/delete_test.go | 2 +- database/build/get_repo_test.go | 2 +- database/build/get_test.go | 2 +- database/build/interface.go | 4 ++-- database/build/last_repo_test.go | 2 +- database/build/list_deployment_test.go | 4 ++-- database/build/list_org_test.go | 4 ++-- database/build/list_pending_running_test.go | 4 ++-- database/build/list_repo_test.go | 4 ++-- database/build/list_test.go | 4 ++-- database/build/update.go | 14 ++++++++------ database/build/update_test.go | 9 +++++++-- router/middleware/build/build_test.go | 2 +- router/middleware/perm/perm_test.go | 14 +++++++------- router/middleware/service/service_test.go | 8 ++++---- router/middleware/step/step_test.go | 8 ++++---- 31 files changed, 83 insertions(+), 81 deletions(-) diff --git a/api/admin/build.go b/api/admin/build.go index 71a343e88..0666c2fad 100644 --- a/api/admin/build.go +++ b/api/admin/build.go @@ -116,7 +116,7 @@ func UpdateBuild(c *gin.Context) { } // send API call to update the build - err = database.FromContext(c).UpdateBuild(input) + b, err := database.FromContext(c).UpdateBuild(input) if err != nil { retErr := fmt.Errorf("unable to update build %d: %w", input.GetID(), err) @@ -125,5 +125,5 @@ func UpdateBuild(c *gin.Context) { return } - c.JSON(http.StatusOK, input) + c.JSON(http.StatusOK, b) } diff --git a/api/build/cancel.go b/api/build/cancel.go index ff63a698d..350573911 100644 --- a/api/build/cancel.go +++ b/api/build/cancel.go @@ -190,7 +190,7 @@ func CancelBuild(c *gin.Context) { // update the status in the build table b.SetStatus(constants.StatusCanceled) - err := database.FromContext(c).UpdateBuild(b) + b, err := database.FromContext(c).UpdateBuild(b) if err != nil { retErr := fmt.Errorf("unable to update status for build %s: %w", entry, err) util.HandleError(c, http.StatusInternalServerError, retErr) diff --git a/api/build/clean.go b/api/build/clean.go index 02c37550c..6fc0ba10b 100644 --- a/api/build/clean.go +++ b/api/build/clean.go @@ -25,7 +25,7 @@ func CleanBuild(database database.Interface, b *library.Build, services []*libra b.SetFinished(time.Now().UTC().Unix()) // send API call to update the build - err := database.UpdateBuild(b) + b, err := database.UpdateBuild(b) if err != nil { logrus.Errorf("unable to kill build %d: %v", b.GetNumber(), err) } diff --git a/api/build/plan.go b/api/build/plan.go index 5790ce4a5..eff7aa630 100644 --- a/api/build/plan.go +++ b/api/build/plan.go @@ -26,11 +26,10 @@ func PlanBuild(database database.Interface, p *pipeline.Build, b *library.Build, // send API call to create the build // TODO: return created build and error instead of just error - err := database.CreateBuild(b) + b, err := database.CreateBuild(b) if err != nil { // clean up the objects from the pipeline in the database // TODO: - // - return build in CreateBuild // - even if it was created, we need to get the new build id // otherwise it will be 0, which attempts to INSERT instead // of UPDATE-ing the existing build - which results in @@ -41,14 +40,6 @@ func PlanBuild(database database.Interface, p *pipeline.Build, b *library.Build, return fmt.Errorf("unable to create new build for %s: %w", r.GetFullName(), err) } - // send API call to capture the created build - // TODO: this can be dropped once we return - // the created build above - b, err = database.GetBuildForRepo(r, b.GetNumber()) - if err != nil { - return fmt.Errorf("unable to get new build for %s: %w", r.GetFullName(), err) - } - // plan all services for the build services, err := service.PlanServices(database, p, b) if err != nil { diff --git a/api/build/publish.go b/api/build/publish.go index f126b8471..a74bc265c 100644 --- a/api/build/publish.go +++ b/api/build/publish.go @@ -67,7 +67,7 @@ func PublishToQueue(queue queue.Service, db database.Interface, p *pipeline.Buil b.SetEnqueued(time.Now().UTC().Unix()) // update the build in the db to reflect the time it was enqueued - err = db.UpdateBuild(b) + _, err = db.UpdateBuild(b) if err != nil { logrus.Errorf("Failed to update build %d during publish to queue for %s: %v", b.GetNumber(), r.GetFullName(), err) } diff --git a/api/build/update.go b/api/build/update.go index f9d3341fa..8dde2cd77 100644 --- a/api/build/update.go +++ b/api/build/update.go @@ -151,7 +151,7 @@ func UpdateBuild(c *gin.Context) { } // send API call to update the build - err = database.FromContext(c).UpdateBuild(b) + b, err = database.FromContext(c).UpdateBuild(b) if err != nil { retErr := fmt.Errorf("unable to update build %s: %w", entry, err) @@ -160,9 +160,6 @@ func UpdateBuild(c *gin.Context) { return } - // send API call to capture the updated build - b, _ = database.FromContext(c).GetBuildForRepo(r, b.GetNumber()) - c.JSON(http.StatusOK, b) // check if the build is in a "final" state diff --git a/api/webhook/post.go b/api/webhook/post.go index 2832941dc..c4670d8ea 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -893,7 +893,7 @@ func renameRepository(h *library.Hook, r *library.Repo, c *gin.Context, m *types fmt.Sprintf("%s/%s/%d", m.Vela.WebAddress, dbR.GetFullName(), build.GetNumber()), ) - err = database.FromContext(c).UpdateBuild(build) + _, err = database.FromContext(c).UpdateBuild(build) if err != nil { return nil, fmt.Errorf("unable to update build for repo %s: %w", dbR.GetFullName(), err) } diff --git a/database/build/clean_test.go b/database/build/clean_test.go index ab91da075..05b606b3f 100644 --- a/database/build/clean_test.go +++ b/database/build/clean_test.go @@ -54,22 +54,22 @@ func TestBuild_Engine_CleanBuilds(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_buildOne) + _, err := _sqlite.CreateBuild(_buildOne) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildTwo) + _, err = _sqlite.CreateBuild(_buildTwo) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildThree) + _, err = _sqlite.CreateBuild(_buildThree) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildFour) + _, err = _sqlite.CreateBuild(_buildFour) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/count_deployment_test.go b/database/build/count_deployment_test.go index 53c0c1df6..611ca6247 100644 --- a/database/build/count_deployment_test.go +++ b/database/build/count_deployment_test.go @@ -44,12 +44,12 @@ func TestBuild_Engine_CountBuildsForDeployment(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_buildOne) + _, err := _sqlite.CreateBuild(_buildOne) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildTwo) + _, err = _sqlite.CreateBuild(_buildTwo) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/count_org_test.go b/database/build/count_org_test.go index 429149e54..d2c2b2c4d 100644 --- a/database/build/count_org_test.go +++ b/database/build/count_org_test.go @@ -67,12 +67,12 @@ func TestBuild_Engine_CountBuildsForOrg(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_buildOne) + _, err := _sqlite.CreateBuild(_buildOne) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildTwo) + _, err = _sqlite.CreateBuild(_buildTwo) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/count_repo_test.go b/database/build/count_repo_test.go index 673887183..313dcba70 100644 --- a/database/build/count_repo_test.go +++ b/database/build/count_repo_test.go @@ -46,12 +46,12 @@ func TestBuild_Engine_CountBuildsForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_buildOne) + _, err := _sqlite.CreateBuild(_buildOne) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildTwo) + _, err = _sqlite.CreateBuild(_buildTwo) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/count_status_test.go b/database/build/count_status_test.go index f88759d1c..34c7ce525 100644 --- a/database/build/count_status_test.go +++ b/database/build/count_status_test.go @@ -39,12 +39,12 @@ func TestBuild_Engine_CountBuildsForStatus(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_buildOne) + _, err := _sqlite.CreateBuild(_buildOne) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildTwo) + _, err = _sqlite.CreateBuild(_buildTwo) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/count_test.go b/database/build/count_test.go index 6254648ee..f63e1430b 100644 --- a/database/build/count_test.go +++ b/database/build/count_test.go @@ -37,12 +37,12 @@ func TestBuild_Engine_CountBuilds(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_buildOne) + _, err := _sqlite.CreateBuild(_buildOne) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildTwo) + _, err = _sqlite.CreateBuild(_buildTwo) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/create.go b/database/build/create.go index 911c6f148..52c425e9f 100644 --- a/database/build/create.go +++ b/database/build/create.go @@ -13,7 +13,7 @@ import ( ) // CreateBuild creates a new build in the database. -func (e *engine) CreateBuild(b *library.Build) error { +func (e *engine) CreateBuild(b *library.Build) (*library.Build, error) { e.logger.WithFields(logrus.Fields{ "build": b.GetNumber(), }).Tracef("creating build %d in the database", b.GetNumber()) @@ -28,12 +28,14 @@ func (e *engine) CreateBuild(b *library.Build) error { // https://pkg.go.dev/github.com/go-vela/types/database#Build.Validate err := build.Validate() if err != nil { - return err + return nil, err } + // crop build if any columns are too large + build = build.Crop() + // send query to the database - return e.client. - Table(constants.TableBuild). - Create(build.Crop()). - Error + result := e.client.Table(constants.TableBuild).Create(build) + + return build.ToLibrary(), result.Error } diff --git a/database/build/create_test.go b/database/build/create_test.go index 3cfe1fdf9..d6051c2dd 100644 --- a/database/build/create_test.go +++ b/database/build/create_test.go @@ -5,6 +5,7 @@ package build import ( + "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -55,7 +56,7 @@ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$ // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err := test.database.CreateBuild(_build) + got, err := test.database.CreateBuild(_build) if test.failure { if err == nil { @@ -68,6 +69,10 @@ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$ if err != nil { t.Errorf("CreateBuild for %s returned err: %v", test.name, err) } + + if !reflect.DeepEqual(got, _build) { + t.Errorf("CreateBuild for %s returned %s, want %s", test.name, got, _build) + } }) } } diff --git a/database/build/delete_test.go b/database/build/delete_test.go index 91dd1c9c7..252912b23 100644 --- a/database/build/delete_test.go +++ b/database/build/delete_test.go @@ -29,7 +29,7 @@ func TestBuild_Engine_DeleteBuild(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_build) + _, err := _sqlite.CreateBuild(_build) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/get_repo_test.go b/database/build/get_repo_test.go index f1341d960..92f86bd0c 100644 --- a/database/build/get_repo_test.go +++ b/database/build/get_repo_test.go @@ -43,7 +43,7 @@ func TestBuild_Engine_GetBuildForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_build) + _, err := _sqlite.CreateBuild(_build) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/get_test.go b/database/build/get_test.go index 7aec15f13..21ea19f94 100644 --- a/database/build/get_test.go +++ b/database/build/get_test.go @@ -34,7 +34,7 @@ func TestBuild_Engine_GetBuild(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_build) + _, err := _sqlite.CreateBuild(_build) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/interface.go b/database/build/interface.go index 6a14ad4f8..6d05fa48b 100644 --- a/database/build/interface.go +++ b/database/build/interface.go @@ -39,7 +39,7 @@ type BuildInterface interface { // CountBuildsForStatus defines a function that gets the count of builds by status. CountBuildsForStatus(string, map[string]interface{}) (int64, error) // CreateBuild defines a function that creates a new build. - CreateBuild(*library.Build) error + CreateBuild(*library.Build) (*library.Build, error) // DeleteBuild defines a function that deletes an existing build. DeleteBuild(*library.Build) error // GetBuild defines a function that gets a build by ID. @@ -59,5 +59,5 @@ type BuildInterface interface { // ListPendingAndRunningBuilds defines a function that gets a list of pending and running builds. ListPendingAndRunningBuilds(string) ([]*library.BuildQueue, error) // UpdateBuild defines a function that updates an existing build. - UpdateBuild(*library.Build) error + UpdateBuild(*library.Build) (*library.Build, error) } diff --git a/database/build/last_repo_test.go b/database/build/last_repo_test.go index 48cb32c01..853c21711 100644 --- a/database/build/last_repo_test.go +++ b/database/build/last_repo_test.go @@ -44,7 +44,7 @@ func TestBuild_Engine_LastBuildForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_build) + _, err := _sqlite.CreateBuild(_build) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/list_deployment_test.go b/database/build/list_deployment_test.go index 94983c56e..53b984bba 100644 --- a/database/build/list_deployment_test.go +++ b/database/build/list_deployment_test.go @@ -54,12 +54,12 @@ func TestBuild_Engine_ListBuildsForDeployment(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_buildOne) + _, err := _sqlite.CreateBuild(_buildOne) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildTwo) + _, err = _sqlite.CreateBuild(_buildTwo) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/list_org_test.go b/database/build/list_org_test.go index e9121e024..4e2ac91aa 100644 --- a/database/build/list_org_test.go +++ b/database/build/list_org_test.go @@ -94,12 +94,12 @@ func TestBuild_Engine_ListBuildsForOrg(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_buildOne) + _, err := _sqlite.CreateBuild(_buildOne) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildTwo) + _, err = _sqlite.CreateBuild(_buildTwo) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/list_pending_running_test.go b/database/build/list_pending_running_test.go index 8d47dc63b..3b3aa4d3e 100644 --- a/database/build/list_pending_running_test.go +++ b/database/build/list_pending_running_test.go @@ -65,12 +65,12 @@ func TestBuild_Engine_ListPendingAndRunningBuilds(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_buildOne) + _, err := _sqlite.CreateBuild(_buildOne) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildTwo) + _, err = _sqlite.CreateBuild(_buildTwo) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/list_repo_test.go b/database/build/list_repo_test.go index ac367e414..f2ccbbe9e 100644 --- a/database/build/list_repo_test.go +++ b/database/build/list_repo_test.go @@ -59,12 +59,12 @@ func TestBuild_Engine_ListBuildsForRepo(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_buildOne) + _, err := _sqlite.CreateBuild(_buildOne) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildTwo) + _, err = _sqlite.CreateBuild(_buildTwo) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/list_test.go b/database/build/list_test.go index b0e441387..958025b09 100644 --- a/database/build/list_test.go +++ b/database/build/list_test.go @@ -47,12 +47,12 @@ func TestBuild_Engine_ListBuilds(t *testing.T) { _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_buildOne) + _, err := _sqlite.CreateBuild(_buildOne) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } - err = _sqlite.CreateBuild(_buildTwo) + _, err = _sqlite.CreateBuild(_buildTwo) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } diff --git a/database/build/update.go b/database/build/update.go index d16329419..ad5c6087f 100644 --- a/database/build/update.go +++ b/database/build/update.go @@ -13,7 +13,7 @@ import ( ) // UpdateBuild updates an existing build in the database. -func (e *engine) UpdateBuild(b *library.Build) error { +func (e *engine) UpdateBuild(b *library.Build) (*library.Build, error) { e.logger.WithFields(logrus.Fields{ "build": b.GetNumber(), }).Tracef("updating build %d in the database", b.GetNumber()) @@ -28,12 +28,14 @@ func (e *engine) UpdateBuild(b *library.Build) error { // https://pkg.go.dev/github.com/go-vela/types/database#Build.Validate err := build.Validate() if err != nil { - return err + return nil, err } + // crop build if any columns are too large + build = build.Crop() + // send query to the database - return e.client. - Table(constants.TableBuild). - Save(build.Crop()). - Error + result := e.client.Table(constants.TableBuild).Save(build) + + return build.ToLibrary(), result.Error } diff --git a/database/build/update_test.go b/database/build/update_test.go index 98ac9ae16..21b2d1d5a 100644 --- a/database/build/update_test.go +++ b/database/build/update_test.go @@ -5,6 +5,7 @@ package build import ( + "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -31,7 +32,7 @@ WHERE "id" = $31`). _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() - err := _sqlite.CreateBuild(_build) + _, err := _sqlite.CreateBuild(_build) if err != nil { t.Errorf("unable to create test build for sqlite: %v", err) } @@ -57,7 +58,7 @@ WHERE "id" = $31`). // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err = test.database.UpdateBuild(_build) + got, err := test.database.UpdateBuild(_build) if test.failure { if err == nil { @@ -70,6 +71,10 @@ WHERE "id" = $31`). if err != nil { t.Errorf("UpdateBuild for %s returned err: %v", test.name, err) } + + if !reflect.DeepEqual(got, _build) { + t.Errorf("UpdateBuild for %s returned %s, want %s", test.name, got, _build) + } }) } } diff --git a/router/middleware/build/build_test.go b/router/middleware/build/build_test.go index 10af76ab7..d8a540b9c 100644 --- a/router/middleware/build/build_test.go +++ b/router/middleware/build/build_test.go @@ -95,7 +95,7 @@ func TestBuild_Establish(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(want) + _, _ = db.CreateBuild(want) // setup context gin.SetMode(gin.TestMode) diff --git a/router/middleware/perm/perm_test.go b/router/middleware/perm/perm_test.go index 34498d21d..70a88fbf8 100644 --- a/router/middleware/perm/perm_test.go +++ b/router/middleware/perm/perm_test.go @@ -448,7 +448,7 @@ func TestPerm_MustBuildAccess(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar/builds/1", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -538,7 +538,7 @@ func TestPerm_MustBuildAccess_PlatAdmin(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) _ = db.CreateUser(u) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar/builds/1", nil) @@ -623,7 +623,7 @@ func TestPerm_MustBuildToken_WrongBuild(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar/builds/1", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -707,7 +707,7 @@ func TestPerm_MustSecretAdmin_BuildToken_Repo(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) context.Request, _ = http.NewRequest(http.MethodGet, "/test/native/repo/foo/bar/baz", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -788,7 +788,7 @@ func TestPerm_MustSecretAdmin_BuildToken_Org(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) context.Request, _ = http.NewRequest(http.MethodGet, "/test/native/org/foo/*/baz", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -869,7 +869,7 @@ func TestPerm_MustSecretAdmin_BuildToken_Shared(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) context.Request, _ = http.NewRequest(http.MethodGet, "/test/native/shared/foo/*/*", nil) context.Request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tok)) @@ -1831,7 +1831,7 @@ func TestPerm_MustRead_WorkerBuildToken(t *testing.T) { db.Close() }() - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) _ = db.CreateRepo(r) context.Request, _ = http.NewRequest(http.MethodGet, "/test/foo/bar/builds/1", nil) diff --git a/router/middleware/service/service_test.go b/router/middleware/service/service_test.go index 091a3e170..cd057b471 100644 --- a/router/middleware/service/service_test.go +++ b/router/middleware/service/service_test.go @@ -85,7 +85,7 @@ func TestService_Establish(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) _ = db.CreateService(want) // setup context @@ -226,7 +226,7 @@ func TestService_Establish_NoServiceParameter(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) // setup context gin.SetMode(gin.TestMode) @@ -282,7 +282,7 @@ func TestService_Establish_InvalidServiceParameter(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) // setup context gin.SetMode(gin.TestMode) @@ -338,7 +338,7 @@ func TestService_Establish_NoService(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) // setup context gin.SetMode(gin.TestMode) diff --git a/router/middleware/step/step_test.go b/router/middleware/step/step_test.go index 2d9e7ac12..d0ad740c4 100644 --- a/router/middleware/step/step_test.go +++ b/router/middleware/step/step_test.go @@ -87,7 +87,7 @@ func TestStep_Establish(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) _ = db.CreateStep(want) // setup context @@ -228,7 +228,7 @@ func TestStep_Establish_NoStepParameter(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) // setup context gin.SetMode(gin.TestMode) @@ -284,7 +284,7 @@ func TestStep_Establish_InvalidStepParameter(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) // setup context gin.SetMode(gin.TestMode) @@ -340,7 +340,7 @@ func TestStep_Establish_NoStep(t *testing.T) { }() _ = db.CreateRepo(r) - _ = db.CreateBuild(b) + _, _ = db.CreateBuild(b) // setup context gin.SetMode(gin.TestMode) From 4e5d484be2bd4c1b9dac5303ceaace33dcf6fb39 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Fri, 16 Jun 2023 13:10:03 -0500 Subject: [PATCH 39/53] feat(worker-visibility): extend worker table with 5 fields (#772) --- api/metrics.go | 84 +++++++++++++++++++++++-- api/worker/update.go | 25 ++++++++ database/worker/create_test.go | 6 +- database/worker/get_hostname_test.go | 4 +- database/worker/list_test.go | 6 +- database/worker/table.go | 39 +++++++----- database/worker/update_test.go | 6 +- database/worker/worker_test.go | 19 +++--- go.mod | 10 +-- go.sum | 20 +++--- router/middleware/perm/perm.go | 9 +++ router/middleware/worker/worker_test.go | 5 ++ router/worker.go | 2 +- 13 files changed, 181 insertions(+), 54 deletions(-) diff --git a/api/metrics.go b/api/metrics.go index d005481b3..fc1a7f04f 100644 --- a/api/metrics.go +++ b/api/metrics.go @@ -11,6 +11,7 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/database" "github.com/go-vela/server/queue" + "github.com/go-vela/types/constants" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -56,6 +57,18 @@ type MetricsQueryParameters struct { ActiveWorkerCount bool `form:"active_worker_count"` // InactiveWorkerCount represents total number of inactive workers InactiveWorkerCount bool `form:"inactive_worker_count"` + + // IdleWorkerCount represents total number of workers with a status of idle + // where worker RunningBuildIDs.length = 0 + IdleWorkerCount bool `form:"idle_worker_count"` + // AvailableWorkerCount represents total number of workers with a status of available, + // where worker RunningBuildIDs.length > 0 and < worker BuildLimit + AvailableWorkerCount bool `form:"available_worker_count"` + // BusyWorkerCount represents total number of workers with a status of busy, + // where worker BuildLimit == worker RunningBuildIDs.length + BusyWorkerCount bool `form:"busy_worker_count"` + // ErrorWorkerCount represents total number of workers with a status of error + ErrorWorkerCount bool `form:"error_worker_count"` } // predefine Prometheus metrics else they will be regenerated @@ -180,6 +193,26 @@ var ( // description: Indicates a request for inactive worker count // type: boolean // default: false +// - in: query +// name: idle_worker_count +// description: Indicates a request for idle worker count +// type: boolean +// default: false +// - in: query +// name: available_worker_count +// description: Indicates a request for available worker count +// type: boolean +// default: false +// - in: query +// name: busy_worker_count +// description: Indicates a request for busy worker count +// type: boolean +// default: false +// - in: query +// name: error_worker_count +// description: Indicates a request for error worker count +// type: boolean +// default: false // responses: // '200': // description: Successfully retrieved the Vela metrics @@ -375,14 +408,18 @@ func recordGauges(c *gin.Context) { // add worker metrics var ( - buildLimit int64 - activeWorkers int64 - inactiveWorkers int64 + buildLimit int64 + activeWorkers int64 + inactiveWorkers int64 + idleWorkers int64 + availableWorkers int64 + busyWorkers int64 + errorWorkers int64 ) // get worker metrics based on request query parameters - // worker_build_limit, active_worker_count, inactive_worker_count - if q.WorkerBuildLimit || q.ActiveWorkerCount || q.InactiveWorkerCount { + // worker_build_limit, active_worker_count, inactive_worker_count, idle_worker_count, available_worker_count, busy_worker_count, error_worker_count + if q.WorkerBuildLimit || q.ActiveWorkerCount || q.InactiveWorkerCount || q.IdleWorkerCount || q.AvailableWorkerCount || q.BusyWorkerCount || q.ErrorWorkerCount { // send API call to capture the workers workers, err := database.FromContext(c).ListWorkers() if err != nil { @@ -391,6 +428,9 @@ func recordGauges(c *gin.Context) { // get the unix time from worker_active_interval ago before := time.Now().UTC().Add(-c.Value("worker_active_interval").(time.Duration)).Unix() + + // active, inactive counts + // idle, available, busy, error counts for _, worker := range workers { // check if the worker checked in within the last worker_active_interval if worker.GetLastCheckedIn() >= before { @@ -399,6 +439,20 @@ func recordGauges(c *gin.Context) { } else { inactiveWorkers++ } + // check if the worker checked in within the last worker_active_interval + if worker.GetLastCheckedIn() >= before { + + switch worker.GetStatus() { + case constants.WorkerStatusIdle: + idleWorkers++ + case constants.WorkerStatusAvailable: + availableWorkers++ + case constants.WorkerStatusBusy: + busyWorkers++ + case constants.WorkerStatusError: + errorWorkers++ + } + } } // apply metrics based on request query parameters @@ -416,5 +470,25 @@ func recordGauges(c *gin.Context) { if q.InactiveWorkerCount { totals.WithLabelValues("worker", "count", "inactive").Set(float64(inactiveWorkers)) } + + // idle_worker_count + if q.IdleWorkerCount { + totals.WithLabelValues("worker", "count", "idle").Set(float64(idleWorkers)) + } + + // available_worker_count + if q.AvailableWorkerCount { + totals.WithLabelValues("worker", "count", "available").Set(float64(availableWorkers)) + } + + // busy_worker_count + if q.BusyWorkerCount { + totals.WithLabelValues("worker", "count", "busy").Set(float64(busyWorkers)) + } + + // error_worker_count + if q.ErrorWorkerCount { + totals.WithLabelValues("worker", "count", "error").Set(float64(errorWorkers)) + } } } diff --git a/api/worker/update.go b/api/worker/update.go index 16d34f686..b3a8d5130 100644 --- a/api/worker/update.go +++ b/api/worker/update.go @@ -98,6 +98,31 @@ func UpdateWorker(c *gin.Context) { w.SetActive(input.GetActive()) } + if input.RunningBuildIDs != nil { + // update runningBuildIDs if set + w.SetRunningBuildIDs(input.GetRunningBuildIDs()) + } + + if len(input.GetStatus()) > 0 { + // update status if set + w.SetStatus(input.GetStatus()) + } + + if input.GetLastStatusUpdateAt() > 0 { + // update lastStatusUpdateAt if set + w.SetLastStatusUpdateAt(input.GetLastStatusUpdateAt()) + } + + if input.GetLastBuildStartedAt() > 0 { + // update lastBuildStartedAt if set + w.SetLastBuildStartedAt(input.GetLastBuildStartedAt()) + } + + if input.GetLastBuildFinishedAt() > 0 { + // update lastBuildFinishedAt if set + w.SetLastBuildFinishedAt(input.GetLastBuildFinishedAt()) + } + // send API call to update the worker err = database.FromContext(c).UpdateWorker(w) if err != nil { diff --git a/database/worker/create_test.go b/database/worker/create_test.go index 38c276d83..e4c4dc9cb 100644 --- a/database/worker/create_test.go +++ b/database/worker/create_test.go @@ -26,9 +26,9 @@ func TestWorker_Engine_CreateWorker(t *testing.T) { // ensure the mock expects the query _mock.ExpectQuery(`INSERT INTO "workers" -("hostname","address","routes","active","last_checked_in","build_limit","id") -VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING "id"`). - WithArgs("worker_0", "localhost", nil, true, nil, nil, 1). +("hostname","address","routes","active","status","last_status_update_at","running_build_ids","last_build_started_at","last_build_finished_at","last_checked_in","build_limit","id") +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) RETURNING "id"`). + WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/worker/get_hostname_test.go b/database/worker/get_hostname_test.go index 833afdc15..3dd1d4fe6 100644 --- a/database/worker/get_hostname_test.go +++ b/database/worker/get_hostname_test.go @@ -25,8 +25,8 @@ func TestWorker_Engine_GetWorkerForName(t *testing.T) { // create expected result in mock _rows := sqlmock.NewRows( - []string{"id", "hostname", "address", "routes", "active", "last_checked_in", "build_limit"}). - AddRow(1, "worker_0", "localhost", nil, true, 0, 0) + []string{"id", "hostname", "address", "routes", "active", "status", "last_status_update_at", "running_build_ids", "last_build_started_at", "last_build_finished_at", "last_checked_in", "build_limit"}). + AddRow(1, "worker_0", "localhost", nil, true, nil, 0, nil, 0, 0, 0, 0) // ensure the mock expects the query _mock.ExpectQuery(`SELECT * FROM "workers" WHERE hostname = $1 LIMIT 1`).WithArgs("worker_0").WillReturnRows(_rows) diff --git a/database/worker/list_test.go b/database/worker/list_test.go index b44c9c3d9..5eed3f94f 100644 --- a/database/worker/list_test.go +++ b/database/worker/list_test.go @@ -37,9 +37,9 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { // create expected result in mock _rows = sqlmock.NewRows( - []string{"id", "hostname", "address", "routes", "active", "last_checked_in", "build_limit"}). - AddRow(1, "worker_0", "localhost", nil, true, 0, 0). - AddRow(2, "worker_1", "localhost", nil, true, 0, 0) + []string{"id", "hostname", "address", "routes", "active", "status", "last_status_update_at", "running_build_ids", "last_build_started_at", "last_build_finished_at", "last_checked_in", "build_limit"}). + AddRow(1, "worker_0", "localhost", nil, true, nil, 0, nil, 0, 0, 0, 0). + AddRow(2, "worker_1", "localhost", nil, true, nil, 0, nil, 0, 0, 0, 0) // ensure the mock expects the query _mock.ExpectQuery(`SELECT * FROM "workers"`).WillReturnRows(_rows) diff --git a/database/worker/table.go b/database/worker/table.go index 5cecf109d..1d704674a 100644 --- a/database/worker/table.go +++ b/database/worker/table.go @@ -14,29 +14,38 @@ const ( CREATE TABLE IF NOT EXISTS workers ( - id SERIAL PRIMARY KEY, - hostname VARCHAR(250), - address VARCHAR(250), - routes VARCHAR(1000), - active BOOLEAN, - last_checked_in INTEGER, - build_limit INTEGER, + id SERIAL PRIMARY KEY, + hostname VARCHAR(250), + address VARCHAR(250), + routes VARCHAR(1000), + active BOOLEAN, + status VARCHAR(50), + last_status_update_at INTEGER, + running_build_ids VARCHAR(500), + last_build_started_at INTEGER, + last_build_finished_at INTEGER, + last_checked_in INTEGER, + build_limit INTEGER, UNIQUE(hostname) ); ` - // CreateSqliteTable represents a query to create the Sqlite workers table. CreateSqliteTable = ` CREATE TABLE IF NOT EXISTS workers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - hostname TEXT, - address TEXT, - routes TEXT, - active BOOLEAN, - last_checked_in INTEGER, - build_limit INTEGER, + id INTEGER PRIMARY KEY AUTOINCREMENT, + hostname TEXT, + address TEXT, + routes TEXT, + active BOOLEAN, + status VARCHAR(50), + last_status_update_at INTEGER, + running_build_ids VARCHAR(500), + last_build_started_at INTEGER, + last_build_finished_at INTEGER, + last_checked_in INTEGER, + build_limit INTEGER, UNIQUE(hostname) ); ` diff --git a/database/worker/update_test.go b/database/worker/update_test.go index 88644678a..0beeafa47 100644 --- a/database/worker/update_test.go +++ b/database/worker/update_test.go @@ -23,9 +23,9 @@ func TestWorker_Engine_UpdateWorker(t *testing.T) { // ensure the mock expects the query _mock.ExpectExec(`UPDATE "workers" -SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"last_checked_in"=$5,"build_limit"=$6 -WHERE "id" = $7`). - WithArgs("worker_0", "localhost", nil, true, nil, nil, 1). +SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"status"=$5,"last_status_update_at"=$6,"running_build_ids"=$7,"last_build_started_at"=$8,"last_build_finished_at"=$9,"last_checked_in"=$10,"build_limit"=$11 +WHERE "id" = $12`). + WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/worker/worker_test.go b/database/worker/worker_test.go index 48deabe54..00096c2a9 100644 --- a/database/worker/worker_test.go +++ b/database/worker/worker_test.go @@ -170,12 +170,17 @@ func testSqlite(t *testing.T) *engine { // Worker type with all fields set to their zero values. func testWorker() *library.Worker { return &library.Worker{ - ID: new(int64), - Hostname: new(string), - Address: new(string), - Routes: new([]string), - Active: new(bool), - BuildLimit: new(int64), - LastCheckedIn: new(int64), + ID: new(int64), + Hostname: new(string), + Address: new(string), + Routes: new([]string), + Active: new(bool), + Status: new(string), + LastStatusUpdateAt: new(int64), + RunningBuildIDs: new([]string), + LastBuildStartedAt: new(int64), + LastBuildFinishedAt: new(int64), + LastCheckedIn: new(int64), + BuildLimit: new(int64), } } diff --git a/go.mod b/go.mod index 445f5c13c..1e284d4f4 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.0 github.com/go-playground/assert/v2 v2.2.0 - github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c + github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-cmp v0.5.9 github.com/google/go-github/v52 v52.0.0 @@ -88,12 +88,12 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/leodido/go-urn v1.2.1 // indirect - github.com/lib/pq v1.10.8 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/microcosm-cc/bluemonday v1.0.23 // indirect + github.com/microcosm-cc/bluemonday v1.0.24 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -115,8 +115,8 @@ require ( github.com/yuin/gopher-lua v1.1.0 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/crypto v0.8.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index e705f7c03..3f90f092b 100644 --- a/go.sum +++ b/go.sum @@ -139,8 +139,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= -github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c h1:eAApIK5e5MxFF8RzZAFsvTSdwq/AzdUrdhJHOGQ0ILc= -github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c/go.mod h1:0lsuPfGyVyTWJSi2h3NS6uaEW6DgnFvIzaZu1sXYKrs= +github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af h1:Ixsa6Ha0j9Edq4v3IooDgyUoGSp08fk9FgrYKuZSML8= +github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af/go.mod h1:1ZSmKWX9MamKogwaIb53mzzRpZMV34mJFKiGfVFadFk= github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -290,8 +290,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lib/pq v1.10.8 h1:3fdt97i/cwSU83+E0hZTC/Xpc9mTZxc6UWSCRcSbxiE= -github.com/lib/pq v1.10.8/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -306,8 +306,8 @@ github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJK github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY= -github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4= +github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= +github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -481,8 +481,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -552,8 +552,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/router/middleware/perm/perm.go b/router/middleware/perm/perm.go index 812c9a8fd..da4c18013 100644 --- a/router/middleware/perm/perm.go +++ b/router/middleware/perm/perm.go @@ -100,6 +100,15 @@ func MustWorkerAuthToken() gin.HandlerFunc { "worker": cl.Subject, }).Debugf("verifying worker %s has a valid auth token", cl.Subject) + // global permissions bypass + if cl.IsAdmin { + logrus.WithFields(logrus.Fields{ + "user": cl.Subject, + }).Debugf("user %s has platform admin permissions", cl.Subject) + + return + } + switch cl.TokenType { case constants.WorkerAuthTokenType, constants.WorkerRegisterTokenType: return diff --git a/router/middleware/worker/worker_test.go b/router/middleware/worker/worker_test.go index 1cfeadfc7..58d090825 100644 --- a/router/middleware/worker/worker_test.go +++ b/router/middleware/worker/worker_test.go @@ -41,6 +41,11 @@ func TestWorker_Establish(t *testing.T) { want.SetAddress("localhost") want.SetRoutes([]string{"foo", "bar", "baz"}) want.SetActive(true) + want.SetStatus("available") + want.SetLastStatusUpdateAt(12345) + want.SetRunningBuildIDs([]string{}) + want.SetLastBuildStartedAt(12345) + want.SetLastBuildFinishedAt(12345) want.SetLastCheckedIn(12345) want.SetBuildLimit(0) diff --git a/router/worker.go b/router/worker.go index 7bb114c69..f85a84c45 100644 --- a/router/worker.go +++ b/router/worker.go @@ -32,7 +32,7 @@ func WorkerHandlers(base *gin.RouterGroup) { _worker := _workers.Group("/:worker") { _worker.GET("", wmiddleware.Establish(), worker.GetWorker) - _worker.PUT("", perm.MustPlatformAdmin(), wmiddleware.Establish(), worker.UpdateWorker) + _worker.PUT("", perm.MustWorkerAuthToken(), wmiddleware.Establish(), worker.UpdateWorker) _worker.POST("/refresh", perm.MustWorkerAuthToken(), wmiddleware.Establish(), worker.Refresh) _worker.DELETE("", perm.MustPlatformAdmin(), wmiddleware.Establish(), worker.DeleteWorker) } // end of worker endpoints From ef7d338bd36a9792f8aa7ea7023830026af2540a Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Tue, 20 Jun 2023 11:09:57 -0500 Subject: [PATCH 40/53] feat(database): add engine options (#886) --- database/close.go | 4 +- database/close_test.go | 6 +- database/context.go | 20 +- database/database.go | 79 +++++--- database/database_test.go | 35 ++-- database/driver.go | 2 +- database/opts.go | 90 +++++++++ database/opts_test.go | 409 ++++++++++++++++++++++++++++++++++++++ database/ping.go | 6 +- database/ping_test.go | 6 +- database/resource.go | 76 +++---- database/validate.go | 2 +- database/validate_test.go | 20 +- 13 files changed, 639 insertions(+), 116 deletions(-) create mode 100644 database/opts.go create mode 100644 database/opts_test.go diff --git a/database/close.go b/database/close.go index 0596b972e..0aff8e51d 100644 --- a/database/close.go +++ b/database/close.go @@ -6,10 +6,10 @@ package database // Close stops and terminates the connection to the database. func (e *engine) Close() error { - e.Logger.Tracef("closing connection to the %s database", e.Driver()) + e.logger.Tracef("closing connection to the %s database", e.Driver()) // capture database/sql database from gorm.io/gorm database - _sql, err := e.Database.DB() + _sql, err := e.client.DB() if err != nil { return err } diff --git a/database/close_test.go b/database/close_test.go index b555e78d5..fe0c55b00 100644 --- a/database/close_test.go +++ b/database/close_test.go @@ -49,15 +49,15 @@ func TestDatabase_Engine_Close(t *testing.T) { name: "failure with invalid gorm database", failure: true, database: &engine{ - Config: &Config{ + config: &config{ Driver: "invalid", }, - Database: &gorm.DB{ + client: &gorm.DB{ Config: &gorm.Config{ ConnPool: nil, }, }, - Logger: logrus.NewEntry(logrus.StandardLogger()), + logger: logrus.NewEntry(logrus.StandardLogger()), }, }, } diff --git a/database/context.go b/database/context.go index d27d73550..fd08cc371 100644 --- a/database/context.go +++ b/database/context.go @@ -43,14 +43,14 @@ func ToContext(c Setter, d Interface) { func FromCLIContext(c *cli.Context) (Interface, error) { logrus.Debug("creating database engine from CLI configuration") - return New(&Config{ - Address: c.String("database.addr"), - CompressionLevel: c.Int("database.compression.level"), - ConnectionLife: c.Duration("database.connection.life"), - ConnectionIdle: c.Int("database.connection.idle"), - ConnectionOpen: c.Int("database.connection.open"), - Driver: c.String("database.driver"), - EncryptionKey: c.String("database.encryption.key"), - SkipCreation: c.Bool("database.skip_creation"), - }) + return New( + WithAddress(c.String("database.addr")), + WithCompressionLevel(c.Int("database.compression.level")), + WithConnectionLife(c.Duration("database.connection.life")), + WithConnectionIdle(c.Int("database.connection.idle")), + WithConnectionOpen(c.Int("database.connection.open")), + WithDriver(c.String("database.driver")), + WithEncryptionKey(c.String("database.encryption.key")), + WithSkipCreation(c.Bool("database.skip_creation")), + ) } diff --git a/database/database.go b/database/database.go index 5fe3de951..c612ba3aa 100644 --- a/database/database.go +++ b/database/database.go @@ -28,8 +28,8 @@ import ( ) type ( - // Config represents the settings required to create the engine that implements the Interface. - Config struct { + // config represents the settings required to create the engine that implements the Interface. + config struct { // specifies the address to use for the database engine Address string // specifies the level of compression to use for the database engine @@ -50,9 +50,12 @@ type ( // engine represents the functionality that implements the Interface. engine struct { - Config *Config - Database *gorm.DB - Logger *logrus.Entry + // gorm.io/gorm database client used in database functions + client *gorm.DB + // engine configuration settings used in database functions + config *config + // sirupsen/logrus logger used in database functions + logger *logrus.Entry build.BuildInterface hook.HookInterface @@ -74,52 +77,64 @@ type ( // // * postgres // * sqlite3 -func New(c *Config) (Interface, error) { +func New(opts ...EngineOpt) (Interface, error) { + // create new database engine + e := new(engine) + + // create new fields + e.client = new(gorm.DB) + e.config = new(config) + e.logger = new(logrus.Entry) + + // apply all provided configuration options + for _, opt := range opts { + err := opt(e) + if err != nil { + return nil, err + } + } + // validate the configuration being provided - err := c.Validate() + err := e.config.Validate() if err != nil { return nil, err } - // create new database engine - e := &engine{ - Config: c, - Database: new(gorm.DB), - Logger: logrus.NewEntry(logrus.StandardLogger()).WithField("database", c.Driver), - } + // update the logger with additional metadata + e.logger = logrus.NewEntry(logrus.StandardLogger()).WithField("database", e.Driver()) - e.Logger.Trace("creating database engine from configuration") + e.logger.Trace("creating database engine from configuration") // process the database driver being provided - switch c.Driver { + switch e.config.Driver { case constants.DriverPostgres: // create the new Postgres database client - e.Database, err = gorm.Open(postgres.Open(e.Config.Address), &gorm.Config{}) + e.client, err = gorm.Open(postgres.Open(e.config.Address), &gorm.Config{}) if err != nil { return nil, err } case constants.DriverSqlite: // create the new Sqlite database client - e.Database, err = gorm.Open(sqlite.Open(e.Config.Address), &gorm.Config{}) + e.client, err = gorm.Open(sqlite.Open(e.config.Address), &gorm.Config{}) if err != nil { return nil, err } default: // handle an invalid database driver being provided - return nil, fmt.Errorf("invalid database driver provided: %s", c.Driver) + return nil, fmt.Errorf("invalid database driver provided: %s", e.Driver()) } // capture database/sql database from gorm.io/gorm database - db, err := e.Database.DB() + db, err := e.client.DB() if err != nil { return nil, err } // set the maximum amount of time a connection may be reused - db.SetConnMaxLifetime(e.Config.ConnectionLife) + db.SetConnMaxLifetime(e.config.ConnectionLife) // set the maximum number of connections in the idle connection pool - db.SetMaxIdleConns(e.Config.ConnectionIdle) + db.SetMaxIdleConns(e.config.ConnectionIdle) // set the maximum number of open connections to the database - db.SetMaxOpenConns(e.Config.ConnectionOpen) + db.SetMaxOpenConns(e.config.ConnectionOpen) // verify connection to the database err = e.Ping() @@ -140,14 +155,14 @@ func New(c *Config) (Interface, error) { // // This function is ONLY intended to be used for testing purposes. func NewTest() (Interface, error) { - return New(&Config{ - Address: "file::memory:?cache=shared", - CompressionLevel: 3, - ConnectionLife: 30 * time.Minute, - ConnectionIdle: 2, - ConnectionOpen: 0, - Driver: "sqlite3", - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }) + return New( + WithAddress("file::memory:?cache=shared"), + WithCompressionLevel(3), + WithConnectionLife(30*time.Minute), + WithConnectionIdle(2), + WithConnectionOpen(0), + WithDriver("sqlite3"), + WithEncryptionKey("A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW"), + WithSkipCreation(false), + ) } diff --git a/database/database_test.go b/database/database_test.go index 6d18421bc..a1cfecc8a 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -21,12 +21,12 @@ func TestDatabase_New(t *testing.T) { tests := []struct { failure bool name string - config *Config + config *config }{ { name: "failure with postgres", failure: true, - config: &Config{ + config: &config{ Driver: "postgres", Address: "postgres://foo:bar@localhost:5432/vela", CompressionLevel: 3, @@ -40,7 +40,7 @@ func TestDatabase_New(t *testing.T) { { name: "success with sqlite3", failure: false, - config: &Config{ + config: &config{ Driver: "sqlite3", Address: "file::memory:?cache=shared", CompressionLevel: 3, @@ -54,7 +54,7 @@ func TestDatabase_New(t *testing.T) { { name: "failure with invalid config", failure: true, - config: &Config{ + config: &config{ Driver: "postgres", Address: "", CompressionLevel: 3, @@ -68,7 +68,7 @@ func TestDatabase_New(t *testing.T) { { name: "failure with invalid driver", failure: true, - config: &Config{ + config: &config{ Driver: "mysql", Address: "foo:bar@tcp(localhost:3306)/vela?charset=utf8mb4&parseTime=True&loc=Local", CompressionLevel: 3, @@ -84,7 +84,16 @@ func TestDatabase_New(t *testing.T) { // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := New(test.config) + _, err := New( + WithAddress(test.config.Address), + WithCompressionLevel(test.config.CompressionLevel), + WithConnectionLife(test.config.ConnectionLife), + WithConnectionIdle(test.config.ConnectionIdle), + WithConnectionOpen(test.config.ConnectionOpen), + WithDriver(test.config.Driver), + WithEncryptionKey(test.config.EncryptionKey), + WithSkipCreation(test.config.SkipCreation), + ) if test.failure { if err == nil { @@ -105,7 +114,7 @@ func TestDatabase_New(t *testing.T) { func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { // create the engine with test configuration _engine := &engine{ - Config: &Config{ + config: &config{ CompressionLevel: 3, ConnectionLife: 30 * time.Minute, ConnectionIdle: 2, @@ -114,7 +123,7 @@ func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", SkipCreation: false, }, - Logger: logrus.NewEntry(logrus.StandardLogger()), + logger: logrus.NewEntry(logrus.StandardLogger()), } // create the new mock sql database @@ -129,7 +138,7 @@ func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { _mock.ExpectPing() // create the new mock Postgres database client - _engine.Database, err = gorm.Open( + _engine.client, err = gorm.Open( postgres.New(postgres.Config{Conn: _sql}), &gorm.Config{SkipDefaultTransaction: true}, ) @@ -146,7 +155,7 @@ func testSqlite(t *testing.T) *engine { // create the engine with test configuration _engine := &engine{ - Config: &Config{ + config: &config{ Address: "file::memory:?cache=shared", CompressionLevel: 3, ConnectionLife: 30 * time.Minute, @@ -156,12 +165,12 @@ func testSqlite(t *testing.T) *engine { EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", SkipCreation: false, }, - Logger: logrus.NewEntry(logrus.StandardLogger()), + logger: logrus.NewEntry(logrus.StandardLogger()), } // create the new mock Sqlite database client - _engine.Database, err = gorm.Open( - sqlite.Open(_engine.Config.Address), + _engine.client, err = gorm.Open( + sqlite.Open(_engine.config.Address), &gorm.Config{SkipDefaultTransaction: true}, ) if err != nil { diff --git a/database/driver.go b/database/driver.go index a81662846..1c3130094 100644 --- a/database/driver.go +++ b/database/driver.go @@ -6,5 +6,5 @@ package database // Driver outputs the configured database driver. func (e *engine) Driver() string { - return e.Config.Driver + return e.config.Driver } diff --git a/database/opts.go b/database/opts.go new file mode 100644 index 000000000..3714989c5 --- /dev/null +++ b/database/opts.go @@ -0,0 +1,90 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import "time" + +// EngineOpt represents a configuration option to initialize the database engine. +type EngineOpt func(*engine) error + +// WithAddress sets the address in the database engine. +func WithAddress(address string) EngineOpt { + return func(e *engine) error { + // set the fully qualified connection string in the database engine + e.config.Address = address + + return nil + } +} + +// WithCompressionLevel sets the compression level in the database engine. +func WithCompressionLevel(level int) EngineOpt { + return func(e *engine) error { + // set the level of compression for resources in the database engine + e.config.CompressionLevel = level + + return nil + } +} + +// WithConnectionLife sets the life of connections in the database engine. +func WithConnectionLife(connectionLife time.Duration) EngineOpt { + return func(e *engine) error { + // set the maximum duration of time for connection in the database engine + e.config.ConnectionLife = connectionLife + + return nil + } +} + +// WithConnectionIdle sets the idle connections in the database engine. +func WithConnectionIdle(connectionIdle int) EngineOpt { + return func(e *engine) error { + // set the maximum allowed idle connections in the database engine + e.config.ConnectionIdle = connectionIdle + + return nil + } +} + +// WithConnectionOpen sets the open connections in the database engine. +func WithConnectionOpen(connectionOpen int) EngineOpt { + return func(e *engine) error { + // set the maximum allowed open connections in the database engine + e.config.ConnectionOpen = connectionOpen + + return nil + } +} + +// WithDriver sets the driver in the database engine. +func WithDriver(driver string) EngineOpt { + return func(e *engine) error { + // set the database type to interact with in the database engine + e.config.Driver = driver + + return nil + } +} + +// WithEncryptionKey sets the encryption key in the database engine. +func WithEncryptionKey(encryptionKey string) EngineOpt { + return func(e *engine) error { + // set the key for encrypting resources in the database engine + e.config.EncryptionKey = encryptionKey + + return nil + } +} + +// WithSkipCreation sets the skip creation logic in the database engine. +func WithSkipCreation(skipCreation bool) EngineOpt { + return func(e *engine) error { + // set to skip creating tables and indexes in the database engine + e.config.SkipCreation = skipCreation + + return nil + } +} diff --git a/database/opts_test.go b/database/opts_test.go new file mode 100644 index 000000000..446fd5903 --- /dev/null +++ b/database/opts_test.go @@ -0,0 +1,409 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "reflect" + "testing" + "time" +) + +func TestDatabase_EngineOpt_WithAddress(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + address string + want string + }{ + { + failure: false, + name: "address set", + address: "file::memory:?cache=shared", + want: "file::memory:?cache=shared", + }, + { + failure: false, + name: "address not set", + address: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithAddress(test.address)(e) + + if test.failure { + if err == nil { + t.Errorf("WithAddress for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithAddress for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(e.config.Address, test.want) { + t.Errorf("WithAddress for %s is %v, want %v", test.name, e.config.Address, test.want) + } + }) + } +} + +func TestDatabase_EngineOpt_WithCompressionLevel(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + level int + want int + }{ + { + failure: false, + name: "compression level set to -1", + level: -1, + want: -1, + }, + { + failure: false, + name: "compression level set to 0", + level: 0, + want: 0, + }, + { + failure: false, + name: "compression level set to 1", + level: 1, + want: 1, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithCompressionLevel(test.level)(e) + + if test.failure { + if err == nil { + t.Errorf("WithCompressionLevel for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithCompressionLevel for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(e.config.CompressionLevel, test.want) { + t.Errorf("WithCompressionLevel for %s is %v, want %v", test.name, e.config.CompressionLevel, test.want) + } + }) + } +} + +func TestDatabase_EngineOpt_WithConnectionLife(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + life time.Duration + want time.Duration + }{ + { + failure: false, + name: "life of connections set", + life: 30 * time.Minute, + want: 30 * time.Minute, + }, + { + failure: false, + name: "life of connections not set", + life: 0, + want: 0, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithConnectionLife(test.life)(e) + + if test.failure { + if err == nil { + t.Errorf("WithConnectionLife for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithConnectionLife for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(e.config.ConnectionLife, test.want) { + t.Errorf("WithConnectionLife for %s is %v, want %v", test.name, e.config.ConnectionLife, test.want) + } + }) + } +} + +func TestDatabase_EngineOpt_WithConnectionIdle(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + open int + want int + }{ + { + failure: false, + name: "idle connections set", + open: 2, + want: 2, + }, + { + failure: false, + name: "idle connections not set", + open: 0, + want: 0, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithConnectionIdle(test.open)(e) + + if test.failure { + if err == nil { + t.Errorf("WithConnectionIdle for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithConnectionIdle for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(e.config.ConnectionIdle, test.want) { + t.Errorf("WithConnectionIdle for %s is %v, want %v", test.name, e.config.ConnectionIdle, test.want) + } + }) + } +} + +func TestDatabase_EngineOpt_WithConnectionOpen(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + open int + want int + }{ + { + failure: false, + name: "open connections set", + open: 2, + want: 2, + }, + { + failure: false, + name: "open connections not set", + open: 0, + want: 0, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithConnectionOpen(test.open)(e) + + if test.failure { + if err == nil { + t.Errorf("WithConnectionOpen for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithConnectionOpen for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(e.config.ConnectionOpen, test.want) { + t.Errorf("WithConnectionOpen for %s is %v, want %v", test.name, e.config.ConnectionOpen, test.want) + } + }) + } +} + +func TestDatabase_EngineOpt_WithDriver(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + driver string + want string + }{ + { + failure: false, + name: "driver set", + driver: "sqlite3", + want: "sqlite3", + }, + { + failure: false, + name: "driver not set", + driver: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithDriver(test.driver)(e) + + if test.failure { + if err == nil { + t.Errorf("WithDriver for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithDriver for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(e.config.Driver, test.want) { + t.Errorf("WithDriver for %s is %v, want %v", test.name, e.config.Driver, test.want) + } + }) + } +} + +func TestDatabase_EngineOpt_WithEncryptionKey(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + key string + want string + }{ + { + failure: false, + name: "encryption key set", + key: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + want: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + }, + { + failure: false, + name: "encryption key not set", + key: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithEncryptionKey(test.key)(e) + + if test.failure { + if err == nil { + t.Errorf("WithEncryptionKey for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithEncryptionKey for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(e.config.EncryptionKey, test.want) { + t.Errorf("WithEncryptionKey for %s is %v, want %v", test.name, e.config.EncryptionKey, test.want) + } + }) + } +} + +func TestDatabase_EngineOpt_WithSkipCreation(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + skip bool + want bool + }{ + { + failure: false, + name: "skip creation set to true", + skip: true, + want: true, + }, + { + failure: false, + name: "skip creation set to false", + skip: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithSkipCreation(test.skip)(e) + + if test.failure { + if err == nil { + t.Errorf("WithSkipCreation for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithSkipCreation returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.SkipCreation, test.want) { + t.Errorf("WithSkipCreation is %v, want %v", e.config.SkipCreation, test.want) + } + }) + } +} diff --git a/database/ping.go b/database/ping.go index 7ac1a09c2..80968c152 100644 --- a/database/ping.go +++ b/database/ping.go @@ -11,12 +11,12 @@ import ( // Ping sends a "ping" request with backoff to the database. func (e *engine) Ping() error { - e.Logger.Tracef("sending ping request to the %s database", e.Driver()) + e.logger.Tracef("sending ping request to the %s database", e.Driver()) // create a loop to attempt ping requests 5 times for i := 0; i < 5; i++ { // capture database/sql database from gorm.io/gorm database - _sql, err := e.Database.DB() + _sql, err := e.client.DB() if err != nil { return err } @@ -27,7 +27,7 @@ func (e *engine) Ping() error { // create the duration of time to sleep for before attempting to retry duration := time.Duration(i+1) * time.Second - e.Logger.Warnf("unable to ping %s database - retrying in %v", e.Driver(), duration) + e.logger.Warnf("unable to ping %s database - retrying in %v", e.Driver(), duration) // sleep for loop iteration in seconds time.Sleep(duration) diff --git a/database/ping_test.go b/database/ping_test.go index 7a34ad122..5b2fdb0c9 100644 --- a/database/ping_test.go +++ b/database/ping_test.go @@ -48,15 +48,15 @@ func TestDatabase_Engine_Ping(t *testing.T) { name: "failure with invalid gorm database", failure: true, database: &engine{ - Config: &Config{ + config: &config{ Driver: "invalid", }, - Database: &gorm.DB{ + client: &gorm.DB{ Config: &gorm.Config{ ConnPool: nil, }, }, - Logger: logrus.NewEntry(logrus.StandardLogger()), + logger: logrus.NewEntry(logrus.StandardLogger()), }, }, } diff --git a/database/resource.go b/database/resource.go index 93122eadc..1ff9c929c 100644 --- a/database/resource.go +++ b/database/resource.go @@ -24,9 +24,9 @@ func (e *engine) NewResources() error { // create the database agnostic engine for builds e.BuildInterface, err = build.New( - build.WithClient(e.Database), - build.WithLogger(e.Logger), - build.WithSkipCreation(e.Config.SkipCreation), + build.WithClient(e.client), + build.WithLogger(e.logger), + build.WithSkipCreation(e.config.SkipCreation), ) if err != nil { return err @@ -34,9 +34,9 @@ func (e *engine) NewResources() error { // create the database agnostic engine for hooks e.HookInterface, err = hook.New( - hook.WithClient(e.Database), - hook.WithLogger(e.Logger), - hook.WithSkipCreation(e.Config.SkipCreation), + hook.WithClient(e.client), + hook.WithLogger(e.logger), + hook.WithSkipCreation(e.config.SkipCreation), ) if err != nil { return err @@ -44,10 +44,10 @@ func (e *engine) NewResources() error { // create the database agnostic engine for logs e.LogInterface, err = log.New( - log.WithClient(e.Database), - log.WithCompressionLevel(e.Config.CompressionLevel), - log.WithLogger(e.Logger), - log.WithSkipCreation(e.Config.SkipCreation), + log.WithClient(e.client), + log.WithCompressionLevel(e.config.CompressionLevel), + log.WithLogger(e.logger), + log.WithSkipCreation(e.config.SkipCreation), ) if err != nil { return err @@ -55,10 +55,10 @@ func (e *engine) NewResources() error { // create the database agnostic engine for pipelines e.PipelineInterface, err = pipeline.New( - pipeline.WithClient(e.Database), - pipeline.WithCompressionLevel(e.Config.CompressionLevel), - pipeline.WithLogger(e.Logger), - pipeline.WithSkipCreation(e.Config.SkipCreation), + pipeline.WithClient(e.client), + pipeline.WithCompressionLevel(e.config.CompressionLevel), + pipeline.WithLogger(e.logger), + pipeline.WithSkipCreation(e.config.SkipCreation), ) if err != nil { return err @@ -66,10 +66,10 @@ func (e *engine) NewResources() error { // create the database agnostic engine for repos e.RepoInterface, err = repo.New( - repo.WithClient(e.Database), - repo.WithEncryptionKey(e.Config.EncryptionKey), - repo.WithLogger(e.Logger), - repo.WithSkipCreation(e.Config.SkipCreation), + repo.WithClient(e.client), + repo.WithEncryptionKey(e.config.EncryptionKey), + repo.WithLogger(e.logger), + repo.WithSkipCreation(e.config.SkipCreation), ) if err != nil { return err @@ -77,9 +77,9 @@ func (e *engine) NewResources() error { // create the database agnostic engine for schedules e.ScheduleInterface, err = schedule.New( - schedule.WithClient(e.Database), - schedule.WithLogger(e.Logger), - schedule.WithSkipCreation(e.Config.SkipCreation), + schedule.WithClient(e.client), + schedule.WithLogger(e.logger), + schedule.WithSkipCreation(e.config.SkipCreation), ) if err != nil { return err @@ -89,10 +89,10 @@ func (e *engine) NewResources() error { // // https://pkg.go.dev/github.com/go-vela/server/database/secret#New e.SecretInterface, err = secret.New( - secret.WithClient(e.Database), - secret.WithEncryptionKey(e.Config.EncryptionKey), - secret.WithLogger(e.Logger), - secret.WithSkipCreation(e.Config.SkipCreation), + secret.WithClient(e.client), + secret.WithEncryptionKey(e.config.EncryptionKey), + secret.WithLogger(e.logger), + secret.WithSkipCreation(e.config.SkipCreation), ) if err != nil { return err @@ -100,9 +100,9 @@ func (e *engine) NewResources() error { // create the database agnostic engine for services e.ServiceInterface, err = service.New( - service.WithClient(e.Database), - service.WithLogger(e.Logger), - service.WithSkipCreation(e.Config.SkipCreation), + service.WithClient(e.client), + service.WithLogger(e.logger), + service.WithSkipCreation(e.config.SkipCreation), ) if err != nil { return err @@ -110,9 +110,9 @@ func (e *engine) NewResources() error { // create the database agnostic engine for steps e.StepInterface, err = step.New( - step.WithClient(e.Database), - step.WithLogger(e.Logger), - step.WithSkipCreation(e.Config.SkipCreation), + step.WithClient(e.client), + step.WithLogger(e.logger), + step.WithSkipCreation(e.config.SkipCreation), ) if err != nil { return err @@ -120,10 +120,10 @@ func (e *engine) NewResources() error { // create the database agnostic engine for users e.UserInterface, err = user.New( - user.WithClient(e.Database), - user.WithEncryptionKey(e.Config.EncryptionKey), - user.WithLogger(e.Logger), - user.WithSkipCreation(e.Config.SkipCreation), + user.WithClient(e.client), + user.WithEncryptionKey(e.config.EncryptionKey), + user.WithLogger(e.logger), + user.WithSkipCreation(e.config.SkipCreation), ) if err != nil { return err @@ -131,9 +131,9 @@ func (e *engine) NewResources() error { // create the database agnostic engine for workers e.WorkerInterface, err = worker.New( - worker.WithClient(e.Database), - worker.WithLogger(e.Logger), - worker.WithSkipCreation(e.Config.SkipCreation), + worker.WithClient(e.client), + worker.WithLogger(e.logger), + worker.WithSkipCreation(e.config.SkipCreation), ) if err != nil { return err diff --git a/database/validate.go b/database/validate.go index c16c07cd2..18f07cb84 100644 --- a/database/validate.go +++ b/database/validate.go @@ -13,7 +13,7 @@ import ( ) // Validate verifies the required fields from the provided configuration are populated correctly. -func (c *Config) Validate() error { +func (c *config) Validate() error { logrus.Trace("validating database configuration for engine") // verify a database driver was provided diff --git a/database/validate_test.go b/database/validate_test.go index 356705fe2..60761e215 100644 --- a/database/validate_test.go +++ b/database/validate_test.go @@ -14,12 +14,12 @@ func TestDatabase_Config_Validate(t *testing.T) { tests := []struct { failure bool name string - config *Config + config *config }{ { name: "success with postgres", failure: false, - config: &Config{ + config: &config{ Driver: "postgres", Address: "postgres://foo:bar@localhost:5432/vela", CompressionLevel: 3, @@ -33,7 +33,7 @@ func TestDatabase_Config_Validate(t *testing.T) { { name: "success with sqlite3", failure: false, - config: &Config{ + config: &config{ Driver: "sqlite3", Address: "file::memory:?cache=shared", CompressionLevel: 3, @@ -47,7 +47,7 @@ func TestDatabase_Config_Validate(t *testing.T) { { name: "success with negative compression level", failure: false, - config: &Config{ + config: &config{ Driver: "postgres", Address: "postgres://foo:bar@localhost:5432/vela", CompressionLevel: -1, @@ -61,7 +61,7 @@ func TestDatabase_Config_Validate(t *testing.T) { { name: "failure with empty driver", failure: true, - config: &Config{ + config: &config{ Driver: "", Address: "postgres://foo:bar@localhost:5432/vela", CompressionLevel: 3, @@ -75,7 +75,7 @@ func TestDatabase_Config_Validate(t *testing.T) { { name: "failure with empty address", failure: true, - config: &Config{ + config: &config{ Driver: "postgres", Address: "", CompressionLevel: 3, @@ -89,7 +89,7 @@ func TestDatabase_Config_Validate(t *testing.T) { { name: "failure with invalid address", failure: true, - config: &Config{ + config: &config{ Driver: "postgres", Address: "postgres://foo:bar@localhost:5432/vela/", CompressionLevel: 3, @@ -103,7 +103,7 @@ func TestDatabase_Config_Validate(t *testing.T) { { name: "failure with invalid compression level", failure: true, - config: &Config{ + config: &config{ Driver: "postgres", Address: "postgres://foo:bar@localhost:5432/vela", CompressionLevel: 10, @@ -117,7 +117,7 @@ func TestDatabase_Config_Validate(t *testing.T) { { name: "failure with empty encryption key", failure: true, - config: &Config{ + config: &config{ Driver: "postgres", Address: "postgres://foo:bar@localhost:5432/vela", CompressionLevel: 3, @@ -131,7 +131,7 @@ func TestDatabase_Config_Validate(t *testing.T) { { name: "failure with invalid encryption key", failure: true, - config: &Config{ + config: &config{ Driver: "postgres", Address: "postgres://foo:bar@localhost:5432/vela", CompressionLevel: 3, From 3b439fccd31f746cf23d6f6a9a1e251da18c2eb5 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Tue, 20 Jun 2023 14:48:39 -0600 Subject: [PATCH 41/53] enhance(webhook): handle repository transfer events (#883) * enhance(webhook): handle repsository transfer events * use constant for transferred action * increase log level for renaming repo --------- Co-authored-by: David May <49894298+wass3rw3rk@users.noreply.github.com> --- api/webhook/post.go | 61 +++---- router/middleware/perm/perm.go | 9 +- .../hooks/repository_transferred.json | 159 ++++++++++++++++++ scm/github/webhook.go | 15 +- scm/github/webhook_test.go | 67 +++++++- util/util.go | 14 ++ 6 files changed, 285 insertions(+), 40 deletions(-) create mode 100644 scm/github/testdata/hooks/repository_transferred.json diff --git a/api/webhook/post.go b/api/webhook/post.go index c4670d8ea..7eab225f5 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -701,7 +701,7 @@ func handleRepositoryEvent(c *gin.Context, m *types.Metadata, h *library.Hook, r switch h.GetEventAction() { // if action is rename, go through rename routine - case constants.ActionRenamed: + case constants.ActionRenamed, constants.ActionTransferred: r, err := renameRepository(h, r, c, m) if err != nil { h.SetStatus(constants.StatusFailure) @@ -779,34 +779,16 @@ func handleRepositoryEvent(c *gin.Context, m *types.Metadata, h *library.Hook, r // renameRepository is a helper function that takes the old name of the repo, // queries the database for the repo that matches that name and org, and updates // that repo to its new name in order to preserve it. It also updates the secrets -// associated with that repo. +// associated with that repo as well as build links for the UI. func renameRepository(h *library.Hook, r *library.Repo, c *gin.Context, m *types.Metadata) (*library.Repo, error) { - logrus.Debugf("renaming repository from %s to %s", r.GetPreviousName(), r.GetName()) + logrus.Infof("renaming repository from %s to %s", r.GetPreviousName(), r.GetName()) // get the old name of the repo - previousName := r.GetPreviousName() - // get the repo from the database that matches the old name - dbR, err := database.FromContext(c).GetRepoForOrg(r.GetOrg(), previousName) - if err != nil { - retErr := fmt.Errorf("%s: failed to get repo %s/%s from database", baseErr, r.GetOrg(), previousName) - util.HandleError(c, http.StatusBadRequest, retErr) - - h.SetStatus(constants.StatusFailure) - h.SetError(retErr.Error()) - - return nil, retErr - } + prevOrg, prevRepo := util.SplitFullName(r.GetPreviousName()) - // update the repo name information - dbR.SetName(r.GetName()) - dbR.SetFullName(r.GetFullName()) - dbR.SetClone(r.GetClone()) - dbR.SetLink(r.GetLink()) - dbR.SetPreviousName(previousName) - - // update the repo in the database - err = database.FromContext(c).UpdateRepo(dbR) + // get the repo from the database that matches the old name + dbR, err := database.FromContext(c).GetRepoForOrg(prevOrg, prevRepo) if err != nil { - retErr := fmt.Errorf("%s: failed to update repo %s/%s in database", baseErr, r.GetOrg(), previousName) + retErr := fmt.Errorf("%s: failed to get repo %s/%s from database", baseErr, prevOrg, prevRepo) util.HandleError(c, http.StatusBadRequest, retErr) h.SetStatus(constants.StatusFailure) @@ -840,7 +822,7 @@ func renameRepository(h *library.Hook, r *library.Repo, c *gin.Context, m *types // get total number of secrets associated with repository t, err := database.FromContext(c).CountSecretsForRepo(dbR, map[string]interface{}{}) if err != nil { - return nil, fmt.Errorf("unable to get secret count for repo %s/%s: %w", r.GetOrg(), previousName, err) + return nil, fmt.Errorf("unable to get secret count for repo %s/%s: %w", prevOrg, prevRepo, err) } secrets := []*library.Secret{} @@ -849,7 +831,7 @@ func renameRepository(h *library.Hook, r *library.Repo, c *gin.Context, m *types for repoSecrets := int64(0); repoSecrets < t; repoSecrets += 100 { s, _, err := database.FromContext(c).ListSecretsForRepo(dbR, map[string]interface{}{}, page, 100) if err != nil { - return nil, fmt.Errorf("unable to get secret list for repo %s/%s: %w", r.GetOrg(), previousName, err) + return nil, fmt.Errorf("unable to get secret list for repo %s/%s: %w", prevOrg, prevRepo, err) } secrets = append(secrets, s...) @@ -859,11 +841,12 @@ func renameRepository(h *library.Hook, r *library.Repo, c *gin.Context, m *types // update secrets to point to the new repository name for _, secret := range secrets { + secret.SetOrg(r.GetOrg()) secret.SetRepo(r.GetName()) err = database.FromContext(c).UpdateSecret(secret) if err != nil { - return nil, fmt.Errorf("unable to update secret for repo %s/%s: %w", r.GetOrg(), previousName, err) + return nil, fmt.Errorf("unable to update secret for repo %s/%s: %w", prevOrg, prevRepo, err) } } @@ -890,7 +873,7 @@ func renameRepository(h *library.Hook, r *library.Repo, c *gin.Context, m *types // update build link to route to proper repo name for _, build := range builds { build.SetLink( - fmt.Sprintf("%s/%s/%d", m.Vela.WebAddress, dbR.GetFullName(), build.GetNumber()), + fmt.Sprintf("%s/%s/%d", m.Vela.WebAddress, r.GetFullName(), build.GetNumber()), ) _, err = database.FromContext(c).UpdateBuild(build) @@ -899,5 +882,25 @@ func renameRepository(h *library.Hook, r *library.Repo, c *gin.Context, m *types } } + // update the repo name information + dbR.SetName(r.GetName()) + dbR.SetOrg(r.GetOrg()) + dbR.SetFullName(r.GetFullName()) + dbR.SetClone(r.GetClone()) + dbR.SetLink(r.GetLink()) + dbR.SetPreviousName(r.GetPreviousName()) + + // update the repo in the database + err = database.FromContext(c).UpdateRepo(dbR) + if err != nil { + retErr := fmt.Errorf("%s: failed to update repo %s/%s in database", baseErr, prevOrg, prevRepo) + util.HandleError(c, http.StatusBadRequest, retErr) + + h.SetStatus(constants.StatusFailure) + h.SetError(retErr.Error()) + + return nil, retErr + } + return dbR, nil } diff --git a/router/middleware/perm/perm.go b/router/middleware/perm/perm.go index da4c18013..8028275ae 100644 --- a/router/middleware/perm/perm.go +++ b/router/middleware/perm/perm.go @@ -221,14 +221,7 @@ func MustSecretAdmin() gin.HandlerFunc { // if caller is worker with build token, verify it has access to requested secret if strings.EqualFold(cl.TokenType, constants.WorkerBuildTokenType) { - // split repo full name into org and repo - repoSlice := strings.Split(cl.Repo, "/") - if len(repoSlice) != 2 { - logger.Errorf("unable to parse repo claim in build token") - } - - org := repoSlice[0] - repo := repoSlice[1] + org, repo := util.SplitFullName(cl.Repo) switch t { case constants.SecretShared: diff --git a/scm/github/testdata/hooks/repository_transferred.json b/scm/github/testdata/hooks/repository_transferred.json new file mode 100644 index 000000000..2fdea23f6 --- /dev/null +++ b/scm/github/testdata/hooks/repository_transferred.json @@ -0,0 +1,159 @@ +{ + "action": "transferred", + "changes": { + "owner": { + "from": { + "user": { + "login": "Old-Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + } + } + } + }, + "repository": { + "id": 118, + "node_id": "MDEwOlJlcG9zaXRvcnkxMTg=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "private": false, + "owner": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + }, + "html_url": "https://octocoders.github.io/Codertocat/Hello-World", + "description": null, + "fork": false, + "url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World", + "forks_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/forks", + "keys_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/keys{/key_id}", + "collaborators_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/collaborators{/collaborator}", + "teams_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/teams", + "hooks_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/hooks", + "issue_events_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/issues/events{/number}", + "events_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/events", + "assignees_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/assignees{/user}", + "branches_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/branches{/branch}", + "tags_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/tags", + "blobs_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/git/blobs{/sha}", + "git_tags_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/git/tags{/sha}", + "git_refs_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/git/refs{/sha}", + "trees_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/git/trees{/sha}", + "statuses_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/statuses/{sha}", + "languages_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/languages", + "stargazers_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/stargazers", + "contributors_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/contributors", + "subscribers_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/subscribers", + "subscription_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/subscription", + "commits_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/commits{/sha}", + "git_commits_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/git/commits{/sha}", + "comments_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/comments{/number}", + "issue_comment_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/issues/comments{/number}", + "contents_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/contents/{+path}", + "compare_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/compare/{base}...{head}", + "merges_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/merges", + "archive_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/{archive_format}{/ref}", + "downloads_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/downloads", + "issues_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/issues{/number}", + "pulls_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/pulls{/number}", + "milestones_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/milestones{/number}", + "notifications_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/notifications{?since,all,participating}", + "labels_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/labels{/name}", + "releases_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/releases{/id}", + "deployments_url": "https://octocoders.github.io/api/v3/repos/Codertocat/Hello-World/deployments", + "created_at": "2019-05-15T19:37:07Z", + "updated_at": "2019-05-15T19:38:25Z", + "pushed_at": "2019-05-15T19:38:23Z", + "git_url": "git://octocoders.github.io/Codertocat/Hello-World.git", + "ssh_url": "git@octocoders.github.io:Codertocat/Hello-World.git", + "clone_url": "https://octocoders.github.io/Codertocat/Hello-World.git", + "svn_url": "https://octocoders.github.io/Codertocat/Hello-World", + "homepage": null, + "size": 0, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Ruby", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 2, + "license": null, + "forks": 0, + "open_issues": 2, + "watchers": 0, + "default_branch": "master" + }, + "enterprise": { + "id": 1, + "slug": "github", + "name": "GitHub", + "node_id": "MDg6QnVzaW5lc3Mx", + "avatar_url": "https://octocoders.github.io/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://octocoders.github.io/businesses/github", + "created_at": "2019-05-14T19:31:12Z", + "updated_at": "2019-05-14T19:31:12Z" + }, + "sender": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + }, + "installation": { + "id": 5, + "node_id": "MDIzOkludGVncmF0aW9uSW5zdGFsbGF0aW9uNQ==" + } + } \ No newline at end of file diff --git a/scm/github/webhook.go b/scm/github/webhook.go index ad4eda5e6..ceaa5b35d 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -465,8 +465,19 @@ func (c *client) processRepositoryEvent(h *library.Hook, payload *github.Reposit r.SetTopics(repo.Topics) // if action is renamed, then get the previous name from payload - if payload.GetAction() == "renamed" { - r.SetPreviousName(payload.GetChanges().GetRepo().GetName().GetFrom()) + if payload.GetAction() == constants.ActionRenamed { + r.SetPreviousName(repo.GetOwner().GetLogin() + "/" + payload.GetChanges().GetRepo().GetName().GetFrom()) + } + + // if action is transferred, then get the previous owner from payload + // could be a user or an org, but both are User structs + if payload.GetAction() == constants.ActionTransferred { + org := payload.GetChanges().GetOwner().GetOwnerInfo().GetOrg() + if org == nil { + org = payload.GetChanges().GetOwner().GetOwnerInfo().GetUser() + } + + r.SetPreviousName(org.GetLogin() + "/" + repo.GetName()) } h.SetEvent(constants.EventRepository) diff --git a/scm/github/webhook_test.go b/scm/github/webhook_test.go index b1f3a1f0e..1bef4c0ca 100644 --- a/scm/github/webhook_test.go +++ b/scm/github/webhook_test.go @@ -982,7 +982,72 @@ func TestGitHub_ProcessWebhook_RepositoryRename(t *testing.T) { wantRepo.SetClone("https://octocoders.github.io/Codertocat/Hello-World.git") wantRepo.SetBranch("master") wantRepo.SetPrivate(false) - wantRepo.SetPreviousName("Hello-Old-World") + wantRepo.SetPreviousName("Codertocat/Hello-Old-World") + wantRepo.SetTopics(nil) + + want := &types.Webhook{ + Comment: "", + Hook: wantHook, + Repo: wantRepo, + } + + got, err := client.ProcessWebhook(request) + + if err != nil { + t.Errorf("ProcessWebhook returned err: %v", err) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("ProcessWebhook is %v, want %v", got, want) + } +} + +func TestGitHub_ProcessWebhook_RepositoryTransfer(t *testing.T) { + // setup router + s := httptest.NewServer(http.NotFoundHandler()) + defer s.Close() + + // setup request + body, err := os.Open("testdata/hooks/repository_transferred.json") + if err != nil { + t.Errorf("unable to open file: %v", err) + } + + defer body.Close() + + request, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/test", body) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("User-Agent", "GitHub-Hookshot/a22606a") + request.Header.Set("X-GitHub-Delivery", "7bd477e4-4415-11e9-9359-0d41fdf9567e") + request.Header.Set("X-GitHub-Hook-ID", "123456") + request.Header.Set("X-GitHub-Event", "repository") + + // setup client + client, _ := NewTest(s.URL) + + // run test + wantHook := new(library.Hook) + wantHook.SetNumber(1) + wantHook.SetSourceID("7bd477e4-4415-11e9-9359-0d41fdf9567e") + wantHook.SetWebhookID(123456) + wantHook.SetCreated(time.Now().UTC().Unix()) + wantHook.SetHost("github.com") + wantHook.SetEvent(constants.EventRepository) + wantHook.SetEventAction(constants.ActionTransferred) + wantHook.SetBranch("master") + wantHook.SetStatus(constants.StatusSuccess) + wantHook.SetLink("https://github.com/Codertocat/Hello-World/settings/hooks") + + wantRepo := new(library.Repo) + wantRepo.SetActive(true) + wantRepo.SetOrg("Codertocat") + wantRepo.SetName("Hello-World") + wantRepo.SetFullName("Codertocat/Hello-World") + wantRepo.SetLink("https://octocoders.github.io/Codertocat/Hello-World") + wantRepo.SetClone("https://octocoders.github.io/Codertocat/Hello-World.git") + wantRepo.SetBranch("master") + wantRepo.SetPrivate(false) + wantRepo.SetPreviousName("Old-Codertocat/Hello-World") wantRepo.SetTopics(nil) want := &types.Webhook{ diff --git a/util/util.go b/util/util.go index 667653fbe..31cdd1104 100644 --- a/util/util.go +++ b/util/util.go @@ -64,6 +64,20 @@ func PathParameter(c *gin.Context, parameter string) string { return EscapeValue(c.Param(parameter)) } +// SplitFullName safely splits the repo.FullName field into an org and name. +func SplitFullName(value string) (string, string) { + // split repo full name into org and repo + repoSlice := strings.Split(value, "/") + if len(repoSlice) != 2 { + return "", "" + } + + org := repoSlice[0] + repo := repoSlice[1] + + return org, repo +} + // EscapeValue safely escapes any string by removing any new lines and HTML escaping it. func EscapeValue(value string) string { // replace all new lines in the value From 6f9f29e0d6b4547339c2dd1641a8629cb2f9972d Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Wed, 21 Jun 2023 08:36:08 -0600 Subject: [PATCH 42/53] chore(deps): bulk deps update (#888) --- compiler/native/compile_test.go | 2 +- compiler/registry/github/github.go | 2 +- compiler/registry/github/github_test.go | 2 +- compiler/registry/github/template.go | 2 +- go.mod | 28 ++++++------ go.sum | 58 ++++++++++++------------- scm/github/access.go | 2 +- scm/github/authentication.go | 2 +- scm/github/changeset.go | 2 +- scm/github/deployment.go | 2 +- scm/github/github.go | 2 +- scm/github/github_test.go | 2 +- scm/github/repo.go | 2 +- scm/github/webhook.go | 2 +- 14 files changed, 54 insertions(+), 56 deletions(-) diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index 07b420dde..5281dccd1 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -15,7 +15,7 @@ import ( "github.com/go-vela/types/constants" "github.com/go-vela/types/raw" - "github.com/google/go-github/v52/github" + "github.com/google/go-github/v53/github" "testing" "time" diff --git a/compiler/registry/github/github.go b/compiler/registry/github/github.go index 11bc9594a..7fb6f22b7 100644 --- a/compiler/registry/github/github.go +++ b/compiler/registry/github/github.go @@ -9,7 +9,7 @@ import ( "net/url" "strings" - "github.com/google/go-github/v52/github" + "github.com/google/go-github/v53/github" "golang.org/x/oauth2" ) diff --git a/compiler/registry/github/github_test.go b/compiler/registry/github/github_test.go index 08145430e..530b8f644 100644 --- a/compiler/registry/github/github_test.go +++ b/compiler/registry/github/github_test.go @@ -12,7 +12,7 @@ import ( "reflect" "testing" - "github.com/google/go-github/v52/github" + "github.com/google/go-github/v53/github" "golang.org/x/oauth2" ) diff --git a/compiler/registry/github/template.go b/compiler/registry/github/template.go index 59dde2708..956a7608d 100644 --- a/compiler/registry/github/template.go +++ b/compiler/registry/github/template.go @@ -13,7 +13,7 @@ import ( "github.com/go-vela/types/library" - "github.com/google/go-github/v52/github" + "github.com/google/go-github/v53/github" ) // Template captures the templated pipeline configuration from the GitHub repo. diff --git a/go.mod b/go.mod index 1e284d4f4..58439b9bc 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-cmp v0.5.9 - github.com/google/go-github/v52 v52.0.0 + github.com/google/go-github/v53 v53.2.0 github.com/google/uuid v1.3.0 github.com/goware/urlx v0.3.2 github.com/hashicorp/go-cleanhttp v0.5.2 @@ -26,18 +26,18 @@ require ( github.com/hashicorp/vault/api v1.9.2 github.com/joho/godotenv v1.5.1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/client_golang v1.16.0 github.com/redis/go-redis/v9 v9.0.5 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.9.5 github.com/urfave/cli/v2 v2.25.6 - go.starlark.net v0.0.0-20230302034142-4b1e35fe2254 - golang.org/x/oauth2 v0.7.0 - golang.org/x/sync v0.1.0 + go.starlark.net v0.0.0-20230612165344-9532f5667272 + golang.org/x/oauth2 v0.9.0 + golang.org/x/sync v0.3.0 gopkg.in/square/go-jose.v2 v2.6.0 gorm.io/driver/postgres v1.5.2 - gorm.io/driver/sqlite v1.4.4 - gorm.io/gorm v1.25.1 + gorm.io/driver/sqlite v1.5.2 + gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 k8s.io/apimachinery v0.27.2 ) @@ -53,7 +53,7 @@ require ( github.com/cenkalti/backoff/v3 v3.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/cloudflare/circl v1.1.0 // indirect + github.com/cloudflare/circl v1.3.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fatih/color v1.10.0 // indirect @@ -91,7 +91,7 @@ require ( github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.17 // indirect - github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect + github.com/mattn/go-sqlite3 v1.14.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/microcosm-cc/bluemonday v1.0.24 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect @@ -103,7 +103,7 @@ require ( github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect @@ -114,10 +114,10 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yuin/gopher-lua v1.1.0 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect - golang.org/x/crypto v0.8.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/crypto v0.10.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.30.0 // indirect diff --git a/go.sum b/go.sum index 3f90f092b..aab85f958 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,9 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -192,8 +193,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github/v52 v52.0.0 h1:uyGWOY+jMQ8GVGSX8dkSwCzlehU3WfdxQ7GweO/JP7M= -github.com/google/go-github/v52 v52.0.0/go.mod h1:WJV6VEEUPuMo5pXqqa2ZCZEdbQqua4zAk2MZTIo+m+4= +github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI= +github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -265,7 +266,6 @@ github.com/jackc/pgx/v5 v5.3.1 h1:Fcr8QJ1ZeLi5zsPZqQeUZhNhxfkkKBOgJuYkJHoBOtU= github.com/jackc/pgx/v5 v5.3.1/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -301,9 +301,8 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= -github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= +github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= @@ -333,15 +332,15 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -396,8 +395,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.starlark.net v0.0.0-20230302034142-4b1e35fe2254 h1:Ss6D3hLXTM0KobyBYEAygXzFfGcjnmfEJOBgSbemCtg= -go.starlark.net v0.0.0-20230302034142-4b1e35fe2254/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= +go.starlark.net v0.0.0-20230612165344-9532f5667272 h1:2/wtqS591wZyD2OsClsVBKRPEvBsQt/Js+fsCiYhwu8= +go.starlark.net v0.0.0-20230612165344-9532f5667272/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -410,8 +409,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -481,8 +480,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -492,8 +491,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= +golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -505,8 +504,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -552,8 +551,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -568,8 +567,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -738,11 +737,10 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/postgres v1.5.2 h1:ytTDxxEv+MplXOfFe3Lzm7SjG09fcdb3Z/c056DTBx0= gorm.io/driver/postgres v1.5.2/go.mod h1:fmpX0m2I1PKuR7mKZiEluwrP3hbs+ps7JIGMUBpCgl8= -gorm.io/driver/sqlite v1.4.4 h1:gIufGoR0dQzjkyqDyYSCvsYR6fba1Gw5YKDqKeChxFc= -gorm.io/driver/sqlite v1.4.4/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI= -gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= -gorm.io/gorm v1.25.1 h1:nsSALe5Pr+cM3V1qwwQ7rOkw+6UeLrX5O4v3llhHa64= -gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc= +gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= +gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 h1:sC1Xj4TYrLqg1n3AN10w871An7wJM0gzgcm8jkIkECQ= +gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/scm/github/access.go b/scm/github/access.go index e70693ccb..ee47cf3d1 100644 --- a/scm/github/access.go +++ b/scm/github/access.go @@ -10,7 +10,7 @@ import ( "github.com/sirupsen/logrus" "github.com/go-vela/types/library" - "github.com/google/go-github/v52/github" + "github.com/google/go-github/v53/github" ) // OrgAccess captures the user's access level for an org. diff --git a/scm/github/authentication.go b/scm/github/authentication.go index 9efe41579..3b4ccf14b 100644 --- a/scm/github/authentication.go +++ b/scm/github/authentication.go @@ -14,7 +14,7 @@ import ( "github.com/go-vela/server/random" "github.com/go-vela/types/library" - "github.com/google/go-github/v52/github" + "github.com/google/go-github/v53/github" ) // Authorize uses the given access token to authorize the user. diff --git a/scm/github/changeset.go b/scm/github/changeset.go index 4d41dd2e2..da8ca221a 100644 --- a/scm/github/changeset.go +++ b/scm/github/changeset.go @@ -10,7 +10,7 @@ import ( "github.com/sirupsen/logrus" "github.com/go-vela/types/library" - "github.com/google/go-github/v52/github" + "github.com/google/go-github/v53/github" ) // Changeset captures the list of files changed for a commit. diff --git a/scm/github/deployment.go b/scm/github/deployment.go index 9945c6303..fea3f8d03 100644 --- a/scm/github/deployment.go +++ b/scm/github/deployment.go @@ -11,7 +11,7 @@ import ( "github.com/go-vela/types/library" "github.com/go-vela/types/raw" - "github.com/google/go-github/v52/github" + "github.com/google/go-github/v53/github" ) // GetDeployment gets a deployment from the GitHub repo. diff --git a/scm/github/github.go b/scm/github/github.go index 75b1df60a..ebae7ec76 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -9,7 +9,7 @@ import ( "fmt" "net/url" - "github.com/google/go-github/v52/github" + "github.com/google/go-github/v53/github" "github.com/sirupsen/logrus" "golang.org/x/oauth2" diff --git a/scm/github/github_test.go b/scm/github/github_test.go index 2e4b5f398..1c10db60a 100644 --- a/scm/github/github_test.go +++ b/scm/github/github_test.go @@ -12,7 +12,7 @@ import ( "reflect" "testing" - "github.com/google/go-github/v52/github" + "github.com/google/go-github/v53/github" "golang.org/x/oauth2" ) diff --git a/scm/github/repo.go b/scm/github/repo.go index b8fa31c3f..682a2db94 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -15,7 +15,7 @@ import ( "github.com/go-vela/types/constants" "github.com/go-vela/types/library" - "github.com/google/go-github/v52/github" + "github.com/google/go-github/v53/github" ) // ConfigBackoff is a wrapper for Config that will retry five times if the function diff --git a/scm/github/webhook.go b/scm/github/webhook.go index ceaa5b35d..43615e8cb 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -20,7 +20,7 @@ import ( "github.com/go-vela/types" "github.com/go-vela/types/constants" "github.com/go-vela/types/library" - "github.com/google/go-github/v52/github" + "github.com/google/go-github/v53/github" ) // ProcessWebhook parses the webhook from a repo. From 7e3fda66309ef4aa2afeeb0e0e2d1261fe8496f5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 10:16:48 -0500 Subject: [PATCH 43/53] fix(deps): update module github.com/gin-gonic/gin to v1.9.1 [security] (#872) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 21 +++++++++++---------- go.sum | 49 ++++++++++++++++++++++++++++--------------------- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 58439b9bc..72d36c3c7 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/aws/aws-sdk-go v1.44.281 github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 github.com/drone/envsubst v1.0.3 - github.com/gin-gonic/gin v1.9.0 + github.com/gin-gonic/gin v1.9.1 github.com/go-playground/assert/v2 v2.2.0 github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af github.com/golang-jwt/jwt/v5 v5.0.0 @@ -49,7 +49,7 @@ require ( github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bytedance/sonic v1.8.0 // indirect + github.com/bytedance/sonic v1.9.1 // indirect github.com/cenkalti/backoff/v3 v3.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect @@ -57,14 +57,15 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fatih/color v1.10.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.11.2 // indirect - github.com/goccy/go-json v0.10.0 // indirect + github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/gomodule/redigo v2.0.0+incompatible // indirect @@ -86,11 +87,11 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect - github.com/leodido/go-urn v1.2.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-colorable v0.1.8 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-sqlite3 v1.14.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/microcosm-cc/bluemonday v1.0.24 // indirect @@ -100,7 +101,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect @@ -110,10 +111,10 @@ require ( github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.9 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yuin/gopher-lua v1.1.0 // indirect - golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect + golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.10.0 // indirect golang.org/x/net v0.11.0 // indirect golang.org/x/sys v0.9.0 // indirect diff --git a/go.sum b/go.sum index aab85f958..740315384 100644 --- a/go.sum +++ b/go.sum @@ -79,8 +79,8 @@ github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 h1:q+sMKdA6L8LyGVud github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3/go.mod h1:5hCug3EZaHXU3FdCA3gJm0YTNi+V+ooA2qNTiVpky4A= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.8.0 h1:ea0Xadu+sHlu7x5O3gKhRpQ1IKiMrSiHttPF0ybECuA= -github.com/bytedance/sonic v1.8.0/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cenkalti/backoff/v3 v3.0.0 h1:ske+9nBpD9qZsTBoF41nW5L+AIuFBKMeze18XQ3eG1c= github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -117,12 +117,14 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8= -github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -137,13 +139,13 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= -github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af h1:Ixsa6Ha0j9Edq4v3IooDgyUoGSp08fk9FgrYKuZSML8= github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af/go.mod h1:1ZSmKWX9MamKogwaIb53mzzRpZMV34mJFKiGfVFadFk= -github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= -github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= @@ -280,16 +282,17 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= @@ -299,8 +302,8 @@ github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -324,8 +327,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= -github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= @@ -370,12 +373,14 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU= -github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli/v2 v2.25.6 h1:yuSkgDSZfH3L1CjF2/5fNNg2KbM47pY2EvjBq4ESQnU= github.com/urfave/cli/v2 v2.25.6/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= @@ -397,8 +402,9 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.starlark.net v0.0.0-20230612165344-9532f5667272 h1:2/wtqS591wZyD2OsClsVBKRPEvBsQt/Js+fsCiYhwu8= go.starlark.net v0.0.0-20230612165344-9532f5667272/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -546,11 +552,12 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From 45c60077dba14925d71a58a961c2e6ab85fc5951 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Wed, 21 Jun 2023 12:55:14 -0600 Subject: [PATCH 44/53] chore(release): upgrade types to v0.20.0-rc1 (#890) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 72d36c3c7..dea6f6204 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.1 github.com/go-playground/assert/v2 v2.2.0 - github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af + github.com/go-vela/types v0.20.0-rc1 github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-cmp v0.5.9 github.com/google/go-github/v53 v53.2.0 diff --git a/go.sum b/go.sum index 740315384..2465b7cca 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= -github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af h1:Ixsa6Ha0j9Edq4v3IooDgyUoGSp08fk9FgrYKuZSML8= -github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af/go.mod h1:1ZSmKWX9MamKogwaIb53mzzRpZMV34mJFKiGfVFadFk= +github.com/go-vela/types v0.20.0-rc1 h1:t4tz9YjExtrFMFTq6w+0xWens8b0UPC1kcI642Ta3yc= +github.com/go-vela/types v0.20.0-rc1/go.mod h1:1ZSmKWX9MamKogwaIb53mzzRpZMV34mJFKiGfVFadFk= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= From 9f92c243082e8a4f599cc8991dd7772a9ce3ccbb Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 26 Jun 2023 14:17:45 -0600 Subject: [PATCH 45/53] fix(schedules): updated_at automatically updates to any change to row (#894) * fix(schedules): updated_at automatically updates to any change to row * revert docker compose * change config to fields and improve comment * what is your why --- api/schedule/create.go | 2 +- api/schedule/update.go | 2 +- cmd/vela-server/schedule.go | 2 +- database/schedule/interface.go | 2 +- database/schedule/update.go | 19 +++++--- database/schedule/update_test.go | 76 +++++++++++++++++++++++++++++++- 6 files changed, 90 insertions(+), 13 deletions(-) diff --git a/api/schedule/create.go b/api/schedule/create.go index c8eb92741..365df170e 100644 --- a/api/schedule/create.go +++ b/api/schedule/create.go @@ -170,7 +170,7 @@ func CreateSchedule(c *gin.Context) { dbSchedule.SetActive(true) // send API call to update the schedule - err = database.FromContext(c).UpdateSchedule(dbSchedule) + err = database.FromContext(c).UpdateSchedule(dbSchedule, true) if err != nil { retErr := fmt.Errorf("unable to set schedule %s to active: %w", dbSchedule.GetName(), err) diff --git a/api/schedule/update.go b/api/schedule/update.go index 646e55fc2..b0862132a 100644 --- a/api/schedule/update.go +++ b/api/schedule/update.go @@ -123,7 +123,7 @@ func UpdateSchedule(c *gin.Context) { } // update the schedule within the database - err = database.FromContext(c).UpdateSchedule(s) + err = database.FromContext(c).UpdateSchedule(s, true) if err != nil { retErr := fmt.Errorf("unable to update scheduled %s: %w", scheduleName, err) diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index ab0a56b90..a9fd38234 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -350,7 +350,7 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat } // send API call to update schedule for ensuring scheduled_at field is set - err = database.UpdateSchedule(s) + err = database.UpdateSchedule(s, false) if err != nil { return fmt.Errorf("unable to update schedule %s/%s: %w", r.GetFullName(), s.GetName(), err) } diff --git a/database/schedule/interface.go b/database/schedule/interface.go index 8aa14efb5..afa163063 100644 --- a/database/schedule/interface.go +++ b/database/schedule/interface.go @@ -45,5 +45,5 @@ type ScheduleInterface interface { // ListSchedulesForRepo defines a function that gets a list of schedules by repo ID. ListSchedulesForRepo(*library.Repo, int, int) ([]*library.Schedule, int64, error) // UpdateSchedule defines a function that updates an existing schedule. - UpdateSchedule(*library.Schedule) error + UpdateSchedule(*library.Schedule, bool) error } diff --git a/database/schedule/update.go b/database/schedule/update.go index b69f8eff0..dd03f62c5 100644 --- a/database/schedule/update.go +++ b/database/schedule/update.go @@ -2,7 +2,6 @@ // // Use of this source code is governed by the LICENSE file in this repository. -//nolint:dupl // ignore similar code with create.go package schedule import ( @@ -13,7 +12,7 @@ import ( ) // UpdateSchedule updates an existing schedule in the database. -func (e *engine) UpdateSchedule(s *library.Schedule) error { +func (e *engine) UpdateSchedule(s *library.Schedule, fields bool) error { e.logger.WithFields(logrus.Fields{ "schedule": s.GetName(), }).Tracef("updating schedule %s in the database", s.GetName()) @@ -27,9 +26,15 @@ func (e *engine) UpdateSchedule(s *library.Schedule) error { return err } - // send query to the database - return e.client. - Table(constants.TableSchedule). - Save(schedule). - Error + // If "fields" is true, update entire record; otherwise, just update scheduled_at (part of processSchedule) + // + // we do this because Gorm will automatically set `updated_at` with the Save function + // and the `updated_at` field should reflect the last time a user updated the record, rather than the scheduler + if fields { + err = e.client.Table(constants.TableSchedule).Save(schedule).Error + } else { + err = e.client.Table(constants.TableSchedule).Model(schedule).UpdateColumn("scheduled_at", s.GetScheduledAt()).Error + } + + return err } diff --git a/database/schedule/update_test.go b/database/schedule/update_test.go index 05fff4286..2554cefdc 100644 --- a/database/schedule/update_test.go +++ b/database/schedule/update_test.go @@ -11,7 +11,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" ) -func TestSchedule_Engine_UpdateSchedule(t *testing.T) { +func TestSchedule_Engine_UpdateSchedule_Config(t *testing.T) { _repo := testRepo() _repo.SetID(1) _repo.SetOrg("foo") @@ -67,7 +67,79 @@ WHERE "id" = $10`). // run tests for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err = test.database.UpdateSchedule(_schedule) + err = test.database.UpdateSchedule(_schedule, true) + + if test.failure { + if err == nil { + t.Errorf("UpdateSchedule for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("UpdateSchedule for %s returned err: %v", test.name, err) + } + }) + } +} + +func TestSchedule_Engine_UpdateSchedule_NotConfig(t *testing.T) { + _repo := testRepo() + _repo.SetID(1) + _repo.SetOrg("foo") + _repo.SetName("bar") + _repo.SetFullName("foo/bar") + + _schedule := testSchedule() + _schedule.SetID(1) + _schedule.SetRepoID(1) + _schedule.SetName("nightly") + _schedule.SetEntry("0 0 * * *") + _schedule.SetCreatedAt(1) + _schedule.SetCreatedBy("user1") + _schedule.SetUpdatedAt(1) + _schedule.SetUpdatedBy("user2") + _schedule.SetScheduledAt(1) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`UPDATE "schedules" SET "scheduled_at"=$1 WHERE "id" = $2`). + WithArgs(1, 1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateSchedule(_schedule) + if err != nil { + t.Errorf("unable to create test schedule for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err = test.database.UpdateSchedule(_schedule, false) if test.failure { if err == nil { From a6277aba3c8b09f7d8b2539e84594a6eeb491255 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Thu, 29 Jun 2023 09:37:29 -0600 Subject: [PATCH 46/53] fix(compose): use hashicorp/vault docker repo (#897) --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d030743e6..09e729ee8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -143,7 +143,7 @@ services: # # https://www.vaultproject.io/ vault: - image: vault:latest + image: hashicorp/vault:latest container_name: vault command: server -dev networks: From 900608b940587c979297c018d831a7df361818b2 Mon Sep 17 00:00:00 2001 From: David May <49894298+wass3rw3rk@users.noreply.github.com> Date: Thu, 6 Jul 2023 16:31:44 -0500 Subject: [PATCH 47/53] fix(api): support schedule in filter (#900) --- api/build/list_org.go | 29 ++++++++++++++++++++++++++++- api/build/list_repo.go | 9 +++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/api/build/list_org.go b/api/build/list_org.go index 1ccd8b45b..9758bbc6a 100644 --- a/api/build/list_org.go +++ b/api/build/list_org.go @@ -35,6 +35,33 @@ import ( // required: true // type: string // - in: query +// name: event +// description: Filter by build event +// type: string +// enum: +// - comment +// - deployment +// - pull_request +// - push +// - schedule +// - tag +// - in: query +// name: branch +// description: Filter builds by branch +// type: string +// - in: query +// name: status +// description: Filter by build status +// type: string +// enum: +// - canceled +// - error +// - failure +// - killed +// - pending +// - running +// - success +// - in: query // name: page // description: The page of results to retrieve // type: integer @@ -109,7 +136,7 @@ func ListBuildsForOrg(c *gin.Context) { // verify the event provided is a valid event type if event != constants.EventComment && event != constants.EventDeploy && event != constants.EventPush && event != constants.EventPull && - event != constants.EventTag { + event != constants.EventTag && event != constants.EventSchedule { retErr := fmt.Errorf("unable to process event %s: invalid event type provided", event) util.HandleError(c, http.StatusBadRequest, retErr) diff --git a/api/build/list_repo.go b/api/build/list_repo.go index 2dd71fc1a..f11d35b0b 100644 --- a/api/build/list_repo.go +++ b/api/build/list_repo.go @@ -45,11 +45,12 @@ import ( // description: Filter by build event // type: string // enum: -// - push +// - comment +// - deployment // - pull_request +// - push +// - schedule // - tag -// - deployment -// - comment // - in: query // name: commit // description: Filter builds based on the commit hash @@ -159,7 +160,7 @@ func ListBuildsForRepo(c *gin.Context) { // verify the event provided is a valid event type if event != constants.EventComment && event != constants.EventDeploy && event != constants.EventPush && event != constants.EventPull && - event != constants.EventTag { + event != constants.EventTag && event != constants.EventSchedule { retErr := fmt.Errorf("unable to process event %s: invalid event type provided", event) util.HandleError(c, http.StatusBadRequest, retErr) From 71a484dd236d606a9f72643b0a6881137f45d620 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Fri, 7 Jul 2023 14:21:41 -0600 Subject: [PATCH 48/53] fix(api/schedule): make validateEntry more strict and set updated_by using claims (#901) * fix(api/schedule): make validateEntry more strict and set updated_by using claims * rm docker compose update * add test cases to validateEntry --- api/schedule/create.go | 36 +++++++++++++++++++++++------------- api/schedule/create_test.go | 16 ++++++++++++++++ api/schedule/update.go | 5 +++++ 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/api/schedule/create.go b/api/schedule/create.go index 365df170e..f03fa0cea 100644 --- a/api/schedule/create.go +++ b/api/schedule/create.go @@ -209,21 +209,31 @@ func validateEntry(minimum time.Duration, entry string) error { return fmt.Errorf("invalid entry of %s", entry) } - // check the previous occurrence of the entry - prevTime, err := gronx.PrevTick(entry, true) - if err != nil { - return err - } + // iterate 5 times through ticks in an effort to catch scalene entries + tickForward := 5 - // check the next occurrence of the entry - nextTime, err := gronx.NextTick(entry, true) - if err != nil { - return err - } + // start with now + t := time.Now().UTC() + + for i := 0; i < tickForward; i++ { + // check the previous occurrence of the entry + prevTime, err := gronx.PrevTickBefore(entry, t, true) + if err != nil { + return err + } + + // check the next occurrence of the entry + nextTime, err := gronx.NextTickAfter(entry, t, false) + if err != nil { + return err + } + + // ensure the time between previous and next schedule exceeds the minimum duration + if nextTime.Sub(prevTime) < minimum { + return fmt.Errorf("entry needs to occur less frequently than every %s", minimum) + } - // ensure the time between previous and next schedule exceeds the minimum duration - if nextTime.Sub(prevTime) < minimum { - return fmt.Errorf("entry needs to occur less frequently than every %s", minimum) + t = nextTime } return nil diff --git a/api/schedule/create_test.go b/api/schedule/create_test.go index 0ca425e32..a2956f6ca 100644 --- a/api/schedule/create_test.go +++ b/api/schedule/create_test.go @@ -35,6 +35,14 @@ func Test_validateEntry(t *testing.T) { }, wantErr: true, }, + { + name: "exceeds minimum frequency with scalene entry pattern", + args: args{ + minimum: 30 * time.Minute, + entry: "1,2,45 * * * *", + }, + wantErr: true, + }, { name: "meets minimum frequency", args: args{ @@ -51,6 +59,14 @@ func Test_validateEntry(t *testing.T) { }, wantErr: false, }, + { + name: "meets minimum frequency with comma entry pattern", + args: args{ + minimum: 15 * time.Minute, + entry: "0,15,30,45 * * * *", + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/api/schedule/update.go b/api/schedule/update.go index b0862132a..5221d222c 100644 --- a/api/schedule/update.go +++ b/api/schedule/update.go @@ -13,6 +13,7 @@ import ( "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/schedule" + "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" "github.com/go-vela/types/library" "github.com/sirupsen/logrus" @@ -73,6 +74,7 @@ func UpdateSchedule(c *gin.Context) { // capture middleware values r := repo.Retrieve(c) s := schedule.Retrieve(c) + u := user.Retrieve(c) scheduleName := util.PathParameter(c, "schedule") minimumFrequency := c.Value("scheduleminimumfrequency").(time.Duration) @@ -122,6 +124,9 @@ func UpdateSchedule(c *gin.Context) { s.SetEntry(input.GetEntry()) } + // set the updated by field using claims + s.SetUpdatedBy(u.GetName()) + // update the schedule within the database err = database.FromContext(c).UpdateSchedule(s, true) if err != nil { From 563f22636bea9c6e70e8d8980a4de7da1d0d1567 Mon Sep 17 00:00:00 2001 From: Jordan Brockopp Date: Mon, 10 Jul 2023 09:29:41 -0500 Subject: [PATCH 49/53] fix(schedules): criteria for triggering a build (#893) * fix(schedules): ignore trigger for first time schedule * fix(schedules): determine trigger off current UTC time * chore: save work * cleanup: ignore inactive schedules * feat: add interval for schedules * chore: address slack feedback * chore: fix typos * fix: processing timed schedules * fix: processing schedules * fix: typo in comment * chore: address review feedback * temp: add test docker compose * fix: finalize * revert: add test docker compose --------- Co-authored-by: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Co-authored-by: David May <49894298+wass3rw3rk@users.noreply.github.com> --- cmd/vela-server/main.go | 8 ++- cmd/vela-server/schedule.go | 123 +++++++++++++++++++++--------------- cmd/vela-server/server.go | 29 +++++---- 3 files changed, 98 insertions(+), 62 deletions(-) diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 19463fe0b..788f6c26b 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -213,9 +213,15 @@ func main() { &cli.DurationFlag{ EnvVars: []string{"VELA_SCHEDULE_MINIMUM_FREQUENCY", "SCHEDULE_MINIMUM_FREQUENCY"}, Name: "schedule-minimum-frequency", - Usage: "minimum time between each schedule entry", + Usage: "minimum time allowed between each build triggered for a schedule", Value: 1 * time.Hour, }, + &cli.DurationFlag{ + EnvVars: []string{"VELA_SCHEDULE_INTERVAL", "SCHEDULE_INTERVAL"}, + Name: "schedule-interval", + Usage: "interval at which schedules will be processed by the server to trigger builds", + Value: 5 * time.Minute, + }, &cli.StringSliceFlag{ EnvVars: []string{"VELA_SCHEDULE_ALLOWLIST"}, Name: "vela-schedule-allowlist", diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index a9fd38234..08e7867cc 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -24,9 +24,13 @@ import ( "k8s.io/apimachinery/pkg/util/wait" ) -const baseErr = "unable to schedule build" +const ( + scheduleErr = "unable to trigger build for schedule" -func processSchedules(compiler compiler.Engine, database database.Interface, metadata *types.Metadata, queue queue.Service, scm scm.Service) error { + scheduleWait = "waiting to trigger build for schedule" +) + +func processSchedules(start time.Time, compiler compiler.Engine, database database.Interface, metadata *types.Metadata, queue queue.Service, scm scm.Service) error { logrus.Infof("processing active schedules to create builds") // send API call to capture the list of active schedules @@ -37,6 +41,13 @@ func processSchedules(compiler compiler.Engine, database database.Interface, met // iterate through the list of active schedules for _, s := range schedules { + // sleep for 1s - 2s before processing the active schedule + // + // This should prevent multiple servers from processing a schedule at the same time by + // leveraging a base duration along with a standard deviation of randomness a.k.a. + // "jitter". To create the jitter, we use a base duration of 1s with a scale factor of 1.0. + time.Sleep(wait.Jitter(time.Second, 1.0)) + // send API call to capture the schedule // // This is needed to ensure we are not dealing with a stale schedule since we fetch @@ -44,52 +55,79 @@ func processSchedules(compiler compiler.Engine, database database.Interface, met // amount of time to get to the end of the list. schedule, err := database.GetSchedule(s.GetID()) if err != nil { - logrus.WithError(err).Warnf("%s for %s", baseErr, schedule.GetName()) + logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) continue } - // create a variable to track if a build should be triggered based off the schedule - trigger := false + // ignore triggering a build if the schedule is no longer active + if !schedule.GetActive() { + logrus.Tracef("skipping to trigger build for inactive schedule %s", schedule.GetName()) - // check if a build has already been triggered for the schedule - if schedule.GetScheduledAt() == 0 { - // trigger a build for the schedule since one has not already been scheduled - trigger = true - } else { - // parse the previous occurrence of the entry for the schedule - prevTime, err := gronx.PrevTick(schedule.GetEntry(), true) - if err != nil { - logrus.WithError(err).Warnf("%s for %s", baseErr, schedule.GetName()) + continue + } - continue - } + // capture the last time a build was triggered for the schedule in UTC + scheduled := time.Unix(schedule.GetScheduledAt(), 0).UTC() - // parse the next occurrence of the entry for the schedule - nextTime, err := gronx.NextTick(schedule.GetEntry(), true) - if err != nil { - logrus.WithError(err).Warnf("%s for %s", baseErr, schedule.GetName()) + // capture the previous occurrence of the entry rounded to the nearest whole interval + // + // i.e. if it's 4:02 on five minute intervals, this will be 4:00 + prevTime, err := gronx.PrevTick(schedule.GetEntry(), true) + if err != nil { + logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) - continue - } + continue + } - // parse the UNIX timestamp from when the last build was triggered for the schedule - t := time.Unix(schedule.GetScheduledAt(), 0).UTC() + // capture the next occurrence of the entry after the last schedule rounded to the nearest whole interval + // + // i.e. if it's 4:02 on five minute intervals, this will be 4:05 + nextTime, err := gronx.NextTickAfter(schedule.GetEntry(), scheduled, true) + if err != nil { + logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) - // check if the time since the last triggered build is greater than the entry duration for the schedule - if time.Since(t) > nextTime.Sub(prevTime) { - // trigger a build for the schedule since it has not previously ran - trigger = true - } + continue } - if trigger && schedule.GetActive() { - err = processSchedule(schedule, compiler, database, metadata, queue, scm) - if err != nil { - logrus.WithError(err).Warnf("%s for %s", baseErr, schedule.GetName()) + // check if we should wait to trigger a build for the schedule + // + // The current time must be after the next occurrence of the schedule. + if !time.Now().After(nextTime) { + logrus.Tracef("%s %s: current time not past next occurrence", scheduleWait, schedule.GetName()) - continue - } + continue + } + + // check if we should wait to trigger a build for the schedule + // + // The previous occurrence of the schedule must be after the starting time of processing schedules. + if !prevTime.After(start) { + logrus.Tracef("%s %s: previous occurence not after starting point", scheduleWait, schedule.GetName()) + + continue + } + + // update the scheduled_at field with the current timestamp + // + // This should help prevent multiple servers from processing a schedule at the same time + // by updating the schedule with a new timestamp to reflect the current state. + schedule.SetScheduledAt(time.Now().UTC().Unix()) + + // send API call to update schedule for ensuring scheduled_at field is set + err = database.UpdateSchedule(schedule, false) + if err != nil { + logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) + + continue + } + + // process the schedule and trigger a new build + err = processSchedule(schedule, compiler, database, metadata, queue, scm) + if err != nil { + logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) + + continue } } @@ -98,13 +136,6 @@ func processSchedules(compiler compiler.Engine, database database.Interface, met //nolint:funlen // ignore function length and number of statements func processSchedule(s *library.Schedule, compiler compiler.Engine, database database.Interface, metadata *types.Metadata, queue queue.Service, scm scm.Service) error { - // sleep for 1s - 3s before processing the schedule - // - // This should prevent multiple servers from processing a schedule at the same time by - // leveraging a base duration along with a standard deviation of randomness a.k.a. - // "jitter". To create the jitter, we use a base duration of 1s with a scale factor of 3.0. - time.Sleep(wait.Jitter(time.Second, 3.0)) - // send API call to capture the repo for the schedule r, err := database.GetRepo(s.GetRepoID()) if err != nil { @@ -337,8 +368,6 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat return err } - s.SetScheduledAt(time.Now().UTC().Unix()) - // break the loop because everything was successful break } // end of retry loop @@ -349,12 +378,6 @@ func processSchedule(s *library.Schedule, compiler compiler.Engine, database dat return fmt.Errorf("unable to update repo %s: %w", r.GetFullName(), err) } - // send API call to update schedule for ensuring scheduled_at field is set - err = database.UpdateSchedule(s, false) - if err != nil { - return fmt.Errorf("unable to update schedule %s/%s: %w", r.GetFullName(), s.GetName(), err) - } - // send API call to capture the triggered build b, err = database.GetBuildForRepo(r, b.GetNumber()) if err != nil { diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 0d95e88e5..689ba03c2 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -176,23 +176,30 @@ func server(c *cli.Context) error { g.Go(func() error { logrus.Info("starting scheduler") for { - // cut the configured minimum frequency duration for schedules in half + // track the starting time for when the server begins processing schedules // - // We need to sleep for some amount of time before we attempt to process schedules - // setup in the database. Since the minimum frequency is configurable, we cut it in - // half and use that as the base duration to determine how long to sleep for. - base := c.Duration("schedule-minimum-frequency") / 2 - logrus.Infof("sleeping for %v before scheduling builds", base) + // This will be used to control which schedules will have a build triggered based + // off the configured entry and last time a build was triggered for the schedule. + start := time.Now().UTC() - // sleep for a duration of time before processing schedules + // capture the interval of time to wait before processing schedules // + // We need to sleep for some amount of time before we attempt to process schedules + // setup in the database. Since the schedule interval is configurable, we use that + // as the base duration to determine how long to sleep for. + interval := c.Duration("schedule-interval") + // This should prevent multiple servers from processing schedules at the same time by // leveraging a base duration along with a standard deviation of randomness a.k.a. - // "jitter". To create the jitter, we use the configured minimum frequency duration - // along with a scale factor of 0.1. - time.Sleep(wait.Jitter(base, 0.1)) + // "jitter". To create the jitter, we use the configured schedule interval duration + // along with a scale factor of 0.5. + jitter := wait.Jitter(interval, 0.5) + + logrus.Infof("sleeping for %v before scheduling builds", jitter) + // sleep for a duration of time before processing schedules + time.Sleep(jitter) - err = processSchedules(compiler, database, metadata, queue, scm) + err = processSchedules(start, compiler, database, metadata, queue, scm) if err != nil { logrus.WithError(err).Warn("unable to process schedules") } else { From 3f0c184e0ce446e84c9b55ca5518659f76032b42 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 10 Jul 2023 10:27:35 -0600 Subject: [PATCH 50/53] fix(db tests): use lenient timestamp check for time.Now query matching (#902) Co-authored-by: dave vader <48764154+plyr4@users.noreply.github.com> --- database/build/build_test.go | 15 +++++++++++++++ database/build/clean_test.go | 3 +-- database/schedule/schedule_test.go | 20 ++++++++++++++++++++ database/schedule/update_test.go | 3 +-- database/secret/secret_test.go | 15 +++++++++++++++ database/secret/update_test.go | 7 +++---- database/service/clean_test.go | 3 +-- database/service/service_test.go | 22 ++++++++++++++++++++++ database/step/clean_test.go | 3 +-- database/step/step_test.go | 22 ++++++++++++++++++++++ 10 files changed, 101 insertions(+), 12 deletions(-) diff --git a/database/build/build_test.go b/database/build/build_test.go index 6304cd167..070744260 100644 --- a/database/build/build_test.go +++ b/database/build/build_test.go @@ -8,6 +8,7 @@ import ( "database/sql/driver" "reflect" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/types/library" @@ -266,3 +267,17 @@ type AnyArgument struct{} func (a AnyArgument) Match(v driver.Value) bool { return true } + +// NowTimestamp is used to test whether timestamps get updated correctly to the current time with lenience. +type NowTimestamp struct{} + +// Match satisfies sqlmock.Argument interface. +func (t NowTimestamp) Match(v driver.Value) bool { + ts, ok := v.(int64) + if !ok { + return false + } + now := time.Now().Unix() + + return now-ts < 10 +} diff --git a/database/build/clean_test.go b/database/build/clean_test.go index 05b606b3f..ff6e36556 100644 --- a/database/build/clean_test.go +++ b/database/build/clean_test.go @@ -7,7 +7,6 @@ package build import ( "reflect" "testing" - "time" "github.com/DATA-DOG/go-sqlmock" ) @@ -48,7 +47,7 @@ func TestBuild_Engine_CleanBuilds(t *testing.T) { // ensure the mock expects the name query _mock.ExpectExec(`UPDATE "builds" SET "status"=$1,"error"=$2,"finished"=$3,"deploy_payload"=$4 WHERE created < $5 AND (status = 'running' OR status = 'pending')`). - WithArgs("error", "msg", time.Now().UTC().Unix(), AnyArgument{}, 3). + WithArgs("error", "msg", NowTimestamp{}, AnyArgument{}, 3). WillReturnResult(sqlmock.NewResult(1, 2)) _sqlite := testSqlite(t) diff --git a/database/schedule/schedule_test.go b/database/schedule/schedule_test.go index c86f1f353..fc4b4c6e1 100644 --- a/database/schedule/schedule_test.go +++ b/database/schedule/schedule_test.go @@ -5,8 +5,10 @@ package schedule import ( + "database/sql/driver" "reflect" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/types/library" @@ -210,3 +212,21 @@ func testRepo() *library.Repo { AllowComment: new(bool), } } + +// This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values +// that are otherwise not easily compared. These typically would be values generated +// before adding or updating them in the database. +// +// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime +type NowTimestamp struct{} + +// Match satisfies sqlmock.Argument interface. +func (t NowTimestamp) Match(v driver.Value) bool { + ts, ok := v.(int64) + if !ok { + return false + } + now := time.Now().Unix() + + return now-ts < 10 +} diff --git a/database/schedule/update_test.go b/database/schedule/update_test.go index 2554cefdc..07cbfbc58 100644 --- a/database/schedule/update_test.go +++ b/database/schedule/update_test.go @@ -6,7 +6,6 @@ package schedule import ( "testing" - "time" "github.com/DATA-DOG/go-sqlmock" ) @@ -35,7 +34,7 @@ func TestSchedule_Engine_UpdateSchedule_Config(t *testing.T) { _mock.ExpectExec(`UPDATE "schedules" SET "repo_id"=$1,"active"=$2,"name"=$3,"entry"=$4,"created_at"=$5,"created_by"=$6,"updated_at"=$7,"updated_by"=$8,"scheduled_at"=$9 WHERE "id" = $10`). - WithArgs(1, false, "nightly", "0 0 * * *", 1, "user1", time.Now().UTC().Unix(), "user2", nil, 1). + WithArgs(1, false, "nightly", "0 0 * * *", 1, "user1", NowTimestamp{}, "user2", nil, 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/secret/secret_test.go b/database/secret/secret_test.go index 228567344..0d6a61c25 100644 --- a/database/secret/secret_test.go +++ b/database/secret/secret_test.go @@ -8,6 +8,7 @@ import ( "database/sql/driver" "reflect" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/types/library" @@ -238,3 +239,17 @@ type AnyArgument struct{} func (a AnyArgument) Match(_ driver.Value) bool { return true } + +// NowTimestamp is used to test whether timestamps get updated correctly to the current time with lenience. +type NowTimestamp struct{} + +// Match satisfies sqlmock.Argument interface. +func (t NowTimestamp) Match(v driver.Value) bool { + ts, ok := v.(int64) + if !ok { + return false + } + now := time.Now().Unix() + + return now-ts < 10 +} diff --git a/database/secret/update_test.go b/database/secret/update_test.go index e719e220e..fd9a24ff6 100644 --- a/database/secret/update_test.go +++ b/database/secret/update_test.go @@ -6,7 +6,6 @@ package secret import ( "testing" - "time" "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/types/library" @@ -57,21 +56,21 @@ func TestSecret_Engine_UpdateSecret(t *testing.T) { _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"events"=$8,"allow_command"=$9,"created_at"=$10,"created_by"=$11,"updated_at"=$12,"updated_by"=$13 WHERE "id" = $14`). - WithArgs("foo", "bar", nil, "baz", AnyArgument{}, "repo", nil, nil, false, 1, "user", time.Now().UTC().Unix(), "user2", 1). + WithArgs("foo", "bar", nil, "baz", AnyArgument{}, "repo", nil, nil, false, 1, "user", AnyArgument{}, "user2", 1). WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the org query _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"events"=$8,"allow_command"=$9,"created_at"=$10,"created_by"=$11,"updated_at"=$12,"updated_by"=$13 WHERE "id" = $14`). - WithArgs("foo", "*", nil, "bar", AnyArgument{}, "org", nil, nil, false, 1, "user", time.Now().UTC().Unix(), "user2", 2). + WithArgs("foo", "*", nil, "bar", AnyArgument{}, "org", nil, nil, false, 1, "user", AnyArgument{}, "user2", 2). WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the shared query _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"events"=$8,"allow_command"=$9,"created_at"=$10,"created_by"=$11,"updated_at"=$12,"updated_by"=$13 WHERE "id" = $14`). - WithArgs("foo", nil, "bar", "baz", AnyArgument{}, "shared", nil, nil, false, 1, "user", time.Now().UTC().Unix(), "user2", 3). + WithArgs("foo", nil, "bar", "baz", AnyArgument{}, "shared", nil, nil, false, 1, "user", NowTimestamp{}, "user2", 3). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/service/clean_test.go b/database/service/clean_test.go index f30939880..3bd9baf15 100644 --- a/database/service/clean_test.go +++ b/database/service/clean_test.go @@ -7,7 +7,6 @@ package service import ( "reflect" "testing" - "time" "github.com/DATA-DOG/go-sqlmock" ) @@ -59,7 +58,7 @@ func TestService_Engine_CleanService(t *testing.T) { // ensure the mock expects the name query _mock.ExpectExec(`UPDATE "services" SET "status"=$1,"error"=$2,"finished"=$3 WHERE created < $4 AND (status = 'running' OR status = 'pending')`). - WithArgs("error", "msg", time.Now().UTC().Unix(), 3). + WithArgs("error", "msg", NowTimestamp{}, 3). WillReturnResult(sqlmock.NewResult(1, 2)) _sqlite := testSqlite(t) diff --git a/database/service/service_test.go b/database/service/service_test.go index 7749f43d3..c9d658769 100644 --- a/database/service/service_test.go +++ b/database/service/service_test.go @@ -5,8 +5,10 @@ package service import ( + "database/sql/driver" "reflect" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/types/library" @@ -222,3 +224,23 @@ func testService() *library.Service { Distribution: new(string), } } + +// This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values +// that are otherwise not easily compared. These typically would be values generated +// before adding or updating them in the database. +// +// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime + +// NowTimestamp is used to test whether timestamps get updated correctly to the current time with lenience. +type NowTimestamp struct{} + +// Match satisfies sqlmock.Argument interface. +func (t NowTimestamp) Match(v driver.Value) bool { + ts, ok := v.(int64) + if !ok { + return false + } + now := time.Now().Unix() + + return now-ts < 10 +} diff --git a/database/step/clean_test.go b/database/step/clean_test.go index 22709749a..4d0c68e07 100644 --- a/database/step/clean_test.go +++ b/database/step/clean_test.go @@ -7,7 +7,6 @@ package step import ( "reflect" "testing" - "time" "github.com/DATA-DOG/go-sqlmock" ) @@ -59,7 +58,7 @@ func TestStep_Engine_CleanStep(t *testing.T) { // ensure the mock expects the name query _mock.ExpectExec(`UPDATE "steps" SET "status"=$1,"error"=$2,"finished"=$3 WHERE created < $4 AND (status = 'running' OR status = 'pending')`). - WithArgs("error", "msg", time.Now().UTC().Unix(), 3). + WithArgs("error", "msg", NowTimestamp{}, 3). WillReturnResult(sqlmock.NewResult(1, 2)) _sqlite := testSqlite(t) diff --git a/database/step/step_test.go b/database/step/step_test.go index 26edfa5e1..136d5ca40 100644 --- a/database/step/step_test.go +++ b/database/step/step_test.go @@ -5,8 +5,10 @@ package step import ( + "database/sql/driver" "reflect" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/types/library" @@ -223,3 +225,23 @@ func testStep() *library.Step { Distribution: new(string), } } + +// This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values +// that are otherwise not easily compared. These typically would be values generated +// before adding or updating them in the database. +// +// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime + +// NowTimestamp is used to test whether timestamps get updated correctly to the current time with lenience. +type NowTimestamp struct{} + +// Match satisfies sqlmock.Argument interface. +func (t NowTimestamp) Match(v driver.Value) bool { + ts, ok := v.(int64) + if !ok { + return false + } + now := time.Now().Unix() + + return now-ts < 10 +} From 8a25e311b691aeba64fdb09608cd54c3872a0da8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 11:37:03 -0500 Subject: [PATCH 51/53] fix(deps): update deps (patch) (#895) * fix(deps): update deps (patch) * fix(deps): update deps (patch) --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: ecrupper --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index dea6f6204..48eba88e8 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 github.com/adhocore/gronx v1.6.3 - github.com/alicebob/miniredis/v2 v2.30.3 - github.com/aws/aws-sdk-go v1.44.281 + github.com/alicebob/miniredis/v2 v2.30.4 + github.com/aws/aws-sdk-go v1.44.298 github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.1 @@ -30,7 +30,7 @@ require ( github.com/redis/go-redis/v9 v9.0.5 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.9.5 - github.com/urfave/cli/v2 v2.25.6 + github.com/urfave/cli/v2 v2.25.7 go.starlark.net v0.0.0-20230612165344-9532f5667272 golang.org/x/oauth2 v0.9.0 golang.org/x/sync v0.3.0 @@ -38,7 +38,7 @@ require ( gorm.io/driver/postgres v1.5.2 gorm.io/driver/sqlite v1.5.2 gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 - k8s.io/apimachinery v0.27.2 + k8s.io/apimachinery v0.27.3 ) require ( diff --git a/go.sum b/go.sum index 2465b7cca..f1d227814 100644 --- a/go.sum +++ b/go.sum @@ -63,11 +63,11 @@ github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGn github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis/v2 v2.11.1/go.mod h1:UA48pmi7aSazcGAvcdKcBB49z521IC9VjTTRz2nIaJE= -github.com/alicebob/miniredis/v2 v2.30.3 h1:hrqDB4cHFSHQf4gO3xu6YKQg8PqJpNjLYsQAFYHstqw= -github.com/alicebob/miniredis/v2 v2.30.3/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg= +github.com/alicebob/miniredis/v2 v2.30.4 h1:8S4/o1/KoUArAGbGwPxcwf0krlzceva2XVOSchFS7Eo= +github.com/alicebob/miniredis/v2 v2.30.4/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.44.281 h1:z/ptheJvINaIAsKXthxONM+toTKw2pxyk700Hfm6yUw= -github.com/aws/aws-sdk-go v1.44.281/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.298 h1:5qTxdubgV7PptZJmp/2qDwD2JL187ePL7VOxsSh1i3g= +github.com/aws/aws-sdk-go v1.44.298/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -381,8 +381,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/urfave/cli/v2 v2.25.6 h1:yuSkgDSZfH3L1CjF2/5fNNg2KbM47pY2EvjBq4ESQnU= -github.com/urfave/cli/v2 v2.25.6/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -755,8 +755,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/apimachinery v0.27.2 h1:vBjGaKKieaIreI+oQwELalVG4d8f3YAMNpWLzDXkxeg= -k8s.io/apimachinery v0.27.2/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= +k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM= +k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= From 73674fc4de2263bfd4a801389bc9419c0b1ea7d3 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 17 Jul 2023 10:23:28 -0600 Subject: [PATCH 52/53] chore(release): upgrade types to v0.20.0 and upgrade other deps (#907) * chore(release): upgrade types to v0.20.0 + gorm and starlark upgrade * update Dockerfile to pin to alpine version for certs * go get correct docker version syntax * every day I surprise myself with my foolishness * pin gorm to an actual tag --- Dockerfile | 2 +- go.mod | 6 +++--- go.sum | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 98ecc5101..79beef6fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # # Use of this source code is governed by the LICENSE file in this repository. -FROM alpine as certs +FROM alpine:3.18.2@sha256:25fad2a32ad1f6f510e528448ae1ec69a28ef81916a004d3629874104f8a7f70 as certs RUN apk add --update --no-cache ca-certificates diff --git a/go.mod b/go.mod index 48eba88e8..9286615af 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.1 github.com/go-playground/assert/v2 v2.2.0 - github.com/go-vela/types v0.20.0-rc1 + github.com/go-vela/types v0.20.0 github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-cmp v0.5.9 github.com/google/go-github/v53 v53.2.0 @@ -31,13 +31,13 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.9.5 github.com/urfave/cli/v2 v2.25.7 - go.starlark.net v0.0.0-20230612165344-9532f5667272 + go.starlark.net v0.0.0-20230712173630-2226322290fc golang.org/x/oauth2 v0.9.0 golang.org/x/sync v0.3.0 gopkg.in/square/go-jose.v2 v2.6.0 gorm.io/driver/postgres v1.5.2 gorm.io/driver/sqlite v1.5.2 - gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 + gorm.io/gorm v1.25.2 k8s.io/apimachinery v0.27.3 ) diff --git a/go.sum b/go.sum index f1d227814..978abf40d 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= -github.com/go-vela/types v0.20.0-rc1 h1:t4tz9YjExtrFMFTq6w+0xWens8b0UPC1kcI642Ta3yc= -github.com/go-vela/types v0.20.0-rc1/go.mod h1:1ZSmKWX9MamKogwaIb53mzzRpZMV34mJFKiGfVFadFk= +github.com/go-vela/types v0.20.0 h1:u/wHwc6ElVbIEI+q9TaVl9Iai1EoEr4Lwis6mikOte8= +github.com/go-vela/types v0.20.0/go.mod h1:1ZSmKWX9MamKogwaIb53mzzRpZMV34mJFKiGfVFadFk= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -400,8 +400,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.starlark.net v0.0.0-20230612165344-9532f5667272 h1:2/wtqS591wZyD2OsClsVBKRPEvBsQt/Js+fsCiYhwu8= -go.starlark.net v0.0.0-20230612165344-9532f5667272/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= +go.starlark.net v0.0.0-20230712173630-2226322290fc h1:x7dWtxLF8z8E5/+KkK3MJJTK/kBZhTCLmYCk75rhKxk= +go.starlark.net v0.0.0-20230712173630-2226322290fc/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= @@ -746,8 +746,8 @@ gorm.io/driver/postgres v1.5.2 h1:ytTDxxEv+MplXOfFe3Lzm7SjG09fcdb3Z/c056DTBx0= gorm.io/driver/postgres v1.5.2/go.mod h1:fmpX0m2I1PKuR7mKZiEluwrP3hbs+ps7JIGMUBpCgl8= gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc= gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= -gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 h1:sC1Xj4TYrLqg1n3AN10w871An7wJM0gzgcm8jkIkECQ= -gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho= +gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From af551916026501435a453d0654e226e6daf3cfdc Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 17 Jul 2023 10:31:17 -0600 Subject: [PATCH 53/53] chore(docker-alpine): pin to alpine tag and sha (#909) --- Dockerfile-alpine | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-alpine b/Dockerfile-alpine index f11d181f3..38ce8fded 100644 --- a/Dockerfile-alpine +++ b/Dockerfile-alpine @@ -2,7 +2,7 @@ # # Use of this source code is governed by the LICENSE file in this repository. -FROM alpine +FROM alpine:3.18.2@sha256:25fad2a32ad1f6f510e528448ae1ec69a28ef81916a004d3629874104f8a7f70 RUN apk add --update --no-cache ca-certificates